FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(moe): serve experts of several formats through per-format slot pools (#700)
Xiaoze Fane8c6bf11235d
diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.pyindex 054fade4..5551759f 100644--- a/python/freetoken/checkpoint/convert.py+++ b/python/freetoken/checkpoint/convert.py@@ -203,14 +203,14 @@ def convert_checkpoint(# every MoE strategy reads the routed experts from the banks, fused includedoffload = getattr(mc, "is_moe", False)include_moe_experts = not offload- method = None+ method = layer_methods = Noneif offload:- from freetoken.engine.engine import offload_expert_method+ from freetoken.engine.engine import offload_expert_methodsfrom freetoken.layers import set_rope_deviceset_rope_device(dev)object.__setattr__(cfg, "moe_strategy", "offload")- method = offload_expert_method(cfg)+ method, layer_methods = offload_expert_methods(cfg)from freetoken.utils.progress import byte_bar, count_bar@@ -239,6 +239,7 @@ def convert_checkpoint(# 2) offload expert banks (post-repack) + alpha scales (slow path auto-picks parallel/serial)quant_format = None+ layer_quant_formats = Nonenum_layers = Noneif offload:# every method-packed format streams each layer to its own FTW entry as it completes (via the sink); the GGUF provider reports through ExpertBanks.streamed whether it engaged the sink or materialized the whole bank set first@@ -247,9 +248,11 @@ def convert_checkpoint(names = legacy_bank_names(legacy_format_for(method.kind, method.kernel.name)) if method is not None else {}sink = _ConvertSink(writer, names=names)banks = load_expert_banks(- model_path, mc, method=method, device=dev, layer_sink=sink+ model_path, mc, method=method, device=dev, layer_sink=sink, layer_methods=layer_methods)quant_format = banks.quant_format+ layer_quant_formats = banks.layer_formats+ assert layer_quant_formats is None or banks.streamed, "experts of several formats always stream to the sink"if banks.streamed:sink.close()num_layers = sink.num_layers # however many distinct layers the sink actually saw@@ -313,6 +316,9 @@ def convert_checkpoint(# read back at load (ftw.load_ftw_banks). dtype/moe_backend were dropped: each# tensor already carries its own dtype, and nothing reads a model-level backend."quant_format": quant_format,+ # Experts of several formats: each MoE layer's quant_format (quant_format above is the most common). A layer only+ # stores the banks its kernel reads; the reader gives it zero-byte placeholders for the others.+ "layer_quant_formats": layer_quant_formats,# The reader takes num_layers from the model config (copied into this# checkpoint); recording it here too gives load_ftw_banks a cross-check that# the banks match the config they ship with. None for non-offload checkpoints.diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.pyindex e31e9d82..ae97a98d 100644--- a/python/freetoken/checkpoint/ftw.py+++ b/python/freetoken/checkpoint/ftw.py@@ -461,11 +461,18 @@ def _split_bank_entries(reader: FTWReader, path: str, num_layers: int):mixed = {e["name"] for e in flat_entries} & per_layer_groups.keys()assert not mixed, f"FTW bank(s) mix flat and per-layer row layouts: {sorted(mixed)}"+ # experts of several formats: a layer stores only the banks its kernel reads+ several = reader.meta("layer_quant_formats") is not None+ assert not (several and flat_entries), "experts of several formats are stored per layer"for base, by_layer in per_layer_groups.items():- assert sorted(by_layer) == list(range(num_layers)), (- f"FTW bank {base!r} has per-layer entries for layers {sorted(by_layer)}, "+ layers = sorted(by_layer)+ assert (set(layers) <= set(range(num_layers))) if several else layers == list(range(num_layers)), (+ f"FTW bank {base!r} has per-layer entries for layers {layers}, "f"expected exactly range({num_layers})")+ if several:+ covered = set().union(*(by_layer.keys() for by_layer in per_layer_groups.values()))+ assert covered == set(range(num_layers)), f"FTW expert banks have no entries for layers {sorted(set(range(num_layers)) - covered)}"return bank_entries, alpha_entries, flat_entries, per_layer_groups@@ -562,11 +569,16 @@ def load_ftw_banks(row_view_args[name].append((head_pad, layer_bytes, num_experts, tuple(row_shape), dtype))row_jobs.append((name, bank, win_off, win_end - win_off, layer_bytes, layer_id))+ num_experts = next((e["shape"][0] for by_layer in per_layer_groups.values() for e in by_layer.values()), None)for base, by_layer in per_layer_groups.items():row_hb[base] = []row_view_args[base] = []for layer_id in range(num_layers):- e = by_layer[layer_id]+ e = by_layer.get(layer_id)+ if e is None: # experts of several formats: this layer's kernel reads no such bank+ row_hb[base].append(None)+ row_view_args[base].append("placeholder")+ continueassert e["global_off"] % ALIGN == 0, (base, layer_id, e["global_off"]) # writer invariantbank = HostBank(tuple(e["shape"]), _dtype_of(e["dtype"]), backing=_backing(layer_id))row_hb[base].append(bank)@@ -611,10 +623,15 @@ def load_ftw_banks(bar.close()reader.close()+ from freetoken.moe.offload_cache import placeholder_bank+sources: dict[str, list] = {}for name, banks in row_hb.items():views = []for bank, view_args in zip(banks, row_view_args[name]):+ if view_args == "placeholder":+ views.append(placeholder_bank(num_experts))+ continueif view_args is None: # per-layer entry: already shaped [num_experts, ...]views.append(bank.tensor)continue@@ -635,6 +652,8 @@ def load_ftw_banks(applied = list(residency)for banks in row_hb.values():for layer_id, bank in enumerate(banks):+ if bank is None:+ continueif (applied[layer_id] == HostResidency.LOCKED.valueand bank.residency is not HostResidency.LOCKED):applied[layer_id] = HostResidency.PAGEABLE.value@@ -643,7 +662,7 @@ def load_ftw_banks(by_layer = [0] * num_layersfor name, banks in row_hb.items():for layer_id, bank in enumerate(banks):- by_layer[layer_id] += bank.nbytes+ by_layer[layer_id] += bank.nbytes if bank is not None else 0locked = [i for i in unpinned if applied[i] == HostResidency.LOCKED.value]pageable = [i for i in unpinned if i not in set(locked)]pinned_b = sum(b for i, b in enumerate(by_layer) if i not in set(unpinned))@@ -666,9 +685,11 @@ def load_ftw_banks(# alphas are the small per-expert scale vectors, distinguished by their reserved names# (not a separate kind); everything else under experts_bank is a weight source.alpha_kw = {n: alpha_hb[n].tensor for n in alpha_hb}+ layer_formats = reader.meta("layer_quant_formats")return ExpertBanks(quant_format, sources, **alpha_kw,layer_residency=applied, kind=kind, kernel=kernel,+ layer_tags=layer_formats, layer_formats=layer_formats,)@@ -683,6 +704,8 @@ def load_ftw_banks_to_device(path: str, *, num_layers: int, device: torch.devicesplit = _split_bank_entries(reader, path, num_layers)if split is None:return None+ if reader.meta("layer_quant_formats") is not None:+ raise ValueError(f"{path!r} holds experts of several formats, which are served from the offload slot cache only")bank_entries, alpha_entries, flat_entries, per_layer_groups = split# (name, layer_id or None for an alpha, read offset, read length, head pad, shape, dtype)diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 81f42dba..980bb043 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -606,6 +606,7 @@ class Engine:dummy=config.use_dummy_weight,parallel={"serial": False, "parallel": True}.get(config.expert_load, None),resident=True,+ layer_methods=[layer.quant_method for layer in sorted(layers, key=lambda layer: layer.layer_id)],)attach_resident_banks(layers, banks)@@ -670,10 +671,21 @@ class Engine:)def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:+ from freetoken.moe.expert_banks import is_mixed+method = shared_offload_method(self.model)num_moe_layers = config.model_config.num_moe_layers+ # experts of several formats: one method per MoE layer, one slot pool per format+ layer_methods = offload_layer_methods(self.model)+ if not is_mixed(layer_methods):+ layer_methods = None+ elif config.moe_strategy == "cpu" or config.moe_cpu_layers:+ raise ValueError(+ "this model's experts come in several formats, which are served from per-format GPU slot pools; "+ "they do not combine with --moe-strategy cpu or --moe-cpu-layers (use offload or hybrid)"+ )cpu_layer_ids = _resolve_cpu_layers(config, num_moe_layers, reserved=self._host_tables_bytes, method=method)- _check_pin_budget(config, reserved=self._host_tables_bytes, method=method)+ _check_pin_budget(config, reserved=self._host_tables_bytes, method=method, layer_methods=layer_methods)# the kernels were picked for model_config.decode_target; --moe-cpu-layers auto may still find that every bank fits the pin budgetdecode_target = config.model_config.decode_targetif decode_target == "cpu" and not cpu_layer_ids:@@ -737,6 +749,7 @@ class Engine:dummy=config.use_dummy_weight,parallel=expert_parallel,layer_residency=requested_residency,+ layer_methods=layer_methods,)except PinFailed as exc:raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc@@ -755,7 +768,7 @@ class Engine:f"--moe-cache-split {config.moe_cache_split} is ignored: all MoE layers share one slot pool "f"(--moe-cache-scope per-layer gives each layer its own)")- self._moe_slot_limit = method.slot_limit() if method is not None else None+ self._moe_slot_limit = _slot_limit(layer_methods) if layer_methods is not None else (method.slot_limit() if method is not None else None)# with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one token per steppools = self._moe_pools(1)size_cap = sum(p.max_num_slots for p in pools)@@ -798,8 +811,17 @@ class Engine:)if decode_target == "gpu":_clamp_decode_batch_to_pools(config, pool_sizes)- layout = max_slots = None- if method is not None:+ layout = layer_layouts = max_slots = None+ if layer_methods is not None:+ _check_layer_formats(banks, layer_methods)+ layer_layouts = [m.layout() for m in layer_methods]+ max_slots = self._moe_slot_limit+ elif method is not None:+ if banks.layer_formats is not None:+ raise ValueError(+ f"expert banks hold experts of several formats ({sorted(set(banks.layer_formats))}) but the model "+ f"binds {method.kind} / {method.kernel.name} on every MoE layer; reconvert the checkpoint"+ )if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):raise ValueError(f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "@@ -822,6 +844,7 @@ class Engine:decode_target=decode_target,hybrid_max_fetch=config.moe_hybrid_max_fetch,layout=layout,+ layer_layouts=layer_layouts,max_slots=max_slots,pool_of_layer=pool_of_layer(pools, config.model_config.num_moe_layers),pool_sizes=pool_sizes,@@ -903,10 +926,8 @@ class Engine:# Decode batches never exceed max_running_req, but CUDA-graph padding can# round a batch up to the largest captured size; cover both.max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)- method = sample.quant_method- formats = {layer.quant_method.cpu_format for layer in layers}- if len(formats) > 1:- raise NotImplementedError(f"CPU MoE backend needs one expert format across the MoE layers, got {sorted(map(str, formats))}")+ by_id = sorted(layers, key=lambda layer: layer.layer_id)+ layer_formats = [layer.quant_method.cpu_spec() for layer in by_id]executor = CpuMoeExecutor(cache,top_k=sample.top_k,@@ -917,8 +938,7 @@ class Engine:device=self.device,swiglu_alpha=float(sample.alpha),swiglu_limit=sample.limit,- fmt=method.cpu_format,- act_block=method.cfg.act_block,+ layer_formats=layer_formats,)cache.set_cpu_executor(executor)self.cpu_moe_executor = executor@@ -1475,10 +1495,10 @@ def _pin_budget_bytes(reserved: int = 0) -> int | None:return max(0, cap - reserved)-def _bank_bytes(config: EngineConfig, method=None) -> int | None:+def _bank_bytes(config: EngineConfig, method=None, layer_methods=None) -> int | None:from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes- return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)+ return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method, layer_methods)def _pin_hint(reserved: int) -> str:@@ -1490,12 +1510,12 @@ def _pin_hint(reserved: int) -> str:)-def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None) -> None:+def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None, layer_methods=None) -> None:"""Stop a plain offload boot whose banks exceed a known pin budget before any bank is read."""if config.moe_cpu_layers or config.moe_strategy not in ("offload", "hybrid"):returnbudget = _pin_budget_bytes(reserved)- bank_bytes = _bank_bytes(config, method) if budget is not None else None+ bank_bytes = _bank_bytes(config, method, layer_methods) if budget is not None else Noneif bank_bytes and bank_bytes > budget:raise ValueError(f"expert banks need {bank_bytes / 2**30:.1f} GiB of pinned host RAM but the pin budget is "@@ -1574,27 +1594,58 @@ def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> Qreturn QuantBackend(quant_backend.items + (((LayerKind.MOE, kind), kernel),))-def shared_offload_method(model):- """The expert method every offload MoE layer of ``model`` uses, or None for models whose MoE layers carry none (GGUF).-- The offload cache holds one bank layout, so the layers must agree on (kind, kernel)."""+def offload_layer_methods(model) -> list | None:+ """The expert method of each offload MoE layer of ``model``, by MoE-layer id; None for models whose MoE layers carry none (GGUF)."""layers = [l for l in iter_offload_moe_layers(model) if getattr(l, "quant_method", None) is not None]if not layers:return None- keys = {(layer.quant_method.kind, layer.quant_method.kernel.name) for layer in layers}- if len(keys) != 1:- found = sorted(f"{kind} / {kernel}" for kind, kernel in keys)- raise ValueError(f"expert layers disagree on format / kernel: {found}; per-layer mixed expert formats are not supported")- kind, kernel = keys.pop()- logger.info_rank0(f"MoE experts: {kind} via {kernel}")- return layers[0].quant_method+ layers.sort(key=lambda layer: layer.layer_id)+ assert [layer.layer_id for layer in layers] == list(range(len(layers))), [layer.layer_id for layer in layers]+ return [layer.quant_method for layer in layers]-def offload_expert_method(config: EngineConfig):- """The offload expert method of ``config``'s model, from a meta-device build.+def shared_offload_method(model):+ """The expert method the offload MoE layers of ``model`` use, or None for models whose MoE layers carry none (GGUF).++ With experts of several formats (a checkpoint that keeps some layers unquantized) this is the format of the most+ layers; the cache then takes every layer's method (``offload_layer_methods``) and keeps one slot pool per format."""+ from freetoken.moe.expert_banks import method_groups++ methods = offload_layer_methods(model)+ if methods is None:+ return None+ groups = method_groups(methods)+ found = ", ".join(f"{m.kind} via {m.kernel.name}" + (f" on {len(ids)} layers" if len(groups) > 1 else "") for m, ids in groups)+ logger.info_rank0(f"MoE experts: {found}")+ return groups[0][0]+++def _slot_limit(layer_methods) -> int | None:+ """The tightest slot cap of the layers' expert kernels (a pool never mixes kernels, but the limit is checked per pool against the largest)."""+ limits = [limit for limit in (m.slot_limit() for m in layer_methods) if limit is not None]+ return min(limits) if limits else None+++def _check_layer_formats(banks, layer_methods) -> None:+ from freetoken.moe.legacy_format import legacy_format_for++ want = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]+ have = banks.layer_formats or [banks.quant_format] * len(want)+ bad = [layer for layer, (w, h) in enumerate(zip(want, have)) if w != h]+ if bad:+ raise ValueError(+ f"expert banks of MoE layers {bad[:8]} were packed as {[have[l] for l in bad[:8]]} but the model binds "+ f"{[want[l] for l in bad[:8]]}; reconvert the checkpoint or select those kernels"+ )+++def offload_expert_methods(config: EngineConfig) -> tuple:+ """``(method, layer_methods)`` of ``config``'s model, from a meta-device build: the most common expert method and,+ for experts of several formats, one per MoE layer (else None).For tools that load expert banks without an engine (the FTW converter); they pack for GPU decode."""from freetoken.models import create_model+ from freetoken.moe.expert_banks import is_mixedfrom freetoken.utils.torch_utils import torch_dtypeset_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))@@ -1602,7 +1653,13 @@ def offload_expert_method(config: EngineConfig):object.__setattr__(config.model_config, "decode_target", "gpu")with torch.device("meta"), torch_dtype(config.dtype):model = create_model(config.model_config)- return shared_offload_method(model)+ layer_methods = offload_layer_methods(model)+ return shared_offload_method(model), (layer_methods if is_mixed(layer_methods) else None)+++def offload_expert_method(config: EngineConfig):+ """The (most common) offload expert method of ``config``'s model; see ``offload_expert_methods``."""+ return offload_expert_methods(config)[0]def _adjust_config(config: EngineConfig):diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex c8b4d1b4..6236816e 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -21,6 +21,7 @@#include <stdexcept>#include <string>#include <thread>+#include <tuple>#include <utility>#include <vector>@@ -103,6 +104,9 @@ std::unique_ptr<MoeFormat> make_format(const std::string& name, const MoeParams&throw std::invalid_argument("unknown CPU MoE weight format " + name);}+// One format instance as cpu_executor.py passes it: name, bank tables by role, integer options.+using FormatSpec = std::tuple<std::string, std::map<std::string, uint64_t>, std::map<std::string, int64_t>>;+struct CpuMoeExecutor {// A host-function callback gets one pointer, so it carries the executor with the task.struct HostTask {@@ -112,10 +116,9 @@ struct CpuMoeExecutor {const int num_layers, num_experts;const bool apply_on_input;- MoeParams params; // the format keeps a reference; declared before fmt- std::unique_ptr<MoeFormat> fmt;- bool preps_inter = false;- bool rounds_routes = false;+ MoeParams params; // the formats keep a reference; declared before fmts+ std::vector<std::unique_ptr<MoeFormat>> fmts;+ std::vector<int> layer_fmt; // [num_layers], index into fmtsint n_iblk = 0, n_hblk = 0; // kRowBlock blocks per intermediate row and per output row// Pinned IO shared by every layer and batch size (one task in flight): [max_tokens, H] x and y, [max_tokens, top_k] ids and w.@@ -167,9 +170,8 @@ struct CpuMoeExecutor {CpuMoeExecutor(int num_threads_, int num_layers_, int num_experts_, int top_k_,int hidden_size, int inter_size, int max_tokens_, int activation_id,- int apply_router_weight_on_input, const std::string& weight_format,- std::map<std::string, uint64_t> banks, std::map<std::string, int64_t> options,- double swiglu_alpha_, double swiglu_limit_,+ int apply_router_weight_on_input, std::vector<FormatSpec> formats,+ std::vector<int> layer_formats, double swiglu_alpha_, double swiglu_limit_,std::vector<int> core_ids_, uintptr_t x_ptr, uintptr_t ids_ptr, uintptr_t w_ptr,uintptr_t y_ptr): num_layers(num_layers_),@@ -184,9 +186,13 @@ struct CpuMoeExecutor {num_threads(num_threads_ > 0 ? num_threads_ : 1),core_ids(std::move(core_ids_)) {if (params.top_k > kMaxTopK) throw std::invalid_argument("CPU MoE supports top_k <= " + std::to_string(kMaxTopK));- fmt = make_format(weight_format, params, FormatArgs{weight_format, std::move(banks), std::move(options)});- preps_inter = fmt->preps_inter();- rounds_routes = fmt->rounds_routes();+ for (auto& [name, banks, options] : formats)+ fmts.push_back(make_format(name, params, FormatArgs{name, std::move(banks), std::move(options)}));+ if (static_cast<int>(layer_formats.size()) != num_layers)+ throw std::invalid_argument("CPU MoE needs the format of every layer");+ for (const int f : layer_formats)+ if (f < 0 || f >= static_cast<int>(fmts.size())) throw std::out_of_range("CPU MoE layer format index out of range");+ layer_fmt = std::move(layer_formats);n_iblk = (params.I + kRowBlock - 1) / kRowBlock;n_hblk = (params.H + kRowBlock - 1) / kRowBlock;g_scratch.assign(static_cast<size_t>(params.max_tokens) * params.top_k * params.I, 0);@@ -217,7 +223,16 @@ struct CpuMoeExecutor {return MoeTask{static_cast<uint64_t>(layer_id), static_cast<uint64_t>(num_tokens)};}- std::string isa_name() const { return fmt->isa_name(); }+ std::string isa_name() const {+ std::vector<std::string> names;+ for (const auto& f : fmts)+ if (const std::string n = f->isa_name(); std::find(names.begin(), names.end(), n) == names.end()) names.push_back(n);+ std::string out;+ for (const auto& n : names) out += (out.empty() ? "" : ",") + n;+ return out;+ }++ const MoeFormat& format_of(const MoeTask& t) const { return *fmts[layer_fmt[t.layer]]; }// Whether any route of the task names an expert; do_pass1 and do_pass2 skip the others.bool has_routes(const MoeTask& t) const {@@ -249,8 +264,8 @@ struct CpuMoeExecutor {const float w_in = apply_on_input ? io_w[static_cast<size_t>(tok) * top_k + k] : 1.0f;const int i0 = static_cast<int>(ib) * kRowBlock;const int i1 = std::min(I, i0 + kRowBlock);- fmt->gate_up(static_cast<int>(t.layer), e, tok, i0, i1, io_x + (size_t)tok * H, w_in,- g_scratch.data() + ((size_t)tok * top_k + k) * I);+ format_of(t).gate_up(static_cast<int>(t.layer), e, tok, i0, i1, io_x + (size_t)tok * H, w_in,+ g_scratch.data() + ((size_t)tok * top_k + k) * I);}// Pass 2 item p = one kRowBlock block of one token's output row, summed over its routes.@@ -272,7 +287,9 @@ struct CpuMoeExecutor {++n;}float part[kMaxTopK * kRowBlock];- if (n > 0) fmt->down(static_cast<int>(t.layer), h0, h1, routes, n, part);+ const MoeFormat& f = format_of(t);+ if (n > 0) f.down(static_cast<int>(t.layer), h0, h1, routes, n, part);+ const bool rounds_routes = f.rounds_routes();bf16_t* y_row = io_y + (size_t)tok * H;for (int h = h0; h < h1; ++h) {float acc = 0.0f;@@ -294,11 +311,12 @@ struct CpuMoeExecutor {barrier(local_sense);// Prepare the intermediate rows (per token,route) for the down GEMV, see prep_inter.// Needs all of pass1 done (a full row spans every iblk).- if (preps_inter) {+ if (prt_total > 0) {+ MoeFormat& f = *fmts[layer_fmt[t.layer]];for (;;) {int64_t r = prt_next.fetch_add(1, std::memory_order_relaxed);if (r >= prt_total) break;- fmt->prep_inter(r, g_scratch.data() + (size_t)r * params.I);+ f.prep_inter(r, g_scratch.data() + (size_t)r * params.I);}barrier(local_sense);}@@ -348,9 +366,10 @@ struct CpuMoeExecutor {void submit(MoeTask t) {const int num_tokens = static_cast<int>(t.num_tokens);+ MoeFormat& fmt = *fmts[layer_fmt[t.layer]];p1_total = static_cast<int64_t>(num_tokens) * params.top_k * n_iblk;p2_total = static_cast<int64_t>(num_tokens) * n_hblk;- prt_total = preps_inter ? static_cast<int64_t>(num_tokens) * params.top_k : 0;+ prt_total = fmt.preps_inter() ? static_cast<int64_t>(num_tokens) * params.top_k : 0;p1_next.store(0, std::memory_order_relaxed);p2_next.store(0, std::memory_order_relaxed);prt_next.store(0, std::memory_order_relaxed);@@ -359,7 +378,7 @@ struct CpuMoeExecutor {bar_sense.store(0, std::memory_order_relaxed);// Prepare the per-token input once, up front.// Single-threaded: tiny for decode, and done before the workers are woken below.- for (int tok = 0; tok < num_tokens; ++tok) fmt->prep_input(tok, io_x + (size_t)tok * params.H);+ for (int tok = 0; tok < num_tokens; ++tok) fmt.prep_input(tok, io_x + (size_t)tok * params.H);{std::lock_guard<std::mutex> lk(task_mtx);cur_task = t;@@ -510,14 +529,14 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {using freetoken::cpu_moe::CpuMoeExecutor;using freetoken::cpu_moe::GpuSync;py::class_<CpuMoeExecutor>(m, "CpuMoeExecutor")- .def(py::init<int, int, int, int, int, int, int, int, int, std::string,- std::map<std::string, uint64_t>, std::map<std::string, int64_t>,- double, double, std::vector<int>, uintptr_t, uintptr_t, uintptr_t, uintptr_t>(),+ .def(py::init<int, int, int, int, int, int, int, int, int, std::vector<freetoken::cpu_moe::FormatSpec>,+ std::vector<int>, double, double, std::vector<int>, uintptr_t, uintptr_t, uintptr_t,+ uintptr_t>(),py::arg("num_threads"), py::arg("num_layers"), py::arg("num_experts"),py::arg("top_k"), py::arg("hidden_size"), py::arg("inter_size"),py::arg("max_tokens"), py::arg("activation_id"),- py::arg("apply_router_weight_on_input"), py::arg("weight_format"),- py::arg("banks"), py::arg("options"), py::arg("swiglu_alpha"), py::arg("swiglu_limit"),+ py::arg("apply_router_weight_on_input"), py::arg("formats"), py::arg("layer_formats"),+ py::arg("swiglu_alpha"), py::arg("swiglu_limit"),py::arg("core_ids"), py::arg("x_ptr"), py::arg("ids_ptr"), py::arg("w_ptr"),py::arg("y_ptr")).def("submit_with_cuda_stream", &CpuMoeExecutor::submit_with_cuda_stream,diff --git a/python/freetoken/layers/quantization/moe/base.py b/python/freetoken/layers/quantization/moe/base.pyindex de1ce004..5ae89c97 100644--- a/python/freetoken/layers/quantization/moe/base.py+++ b/python/freetoken/layers/quantization/moe/base.py@@ -151,6 +151,10 @@ class MoEKernel(ABC):def worth_it(self, cfg: MoEConfig) -> bool:return True+ def cpu_options(self, cfg: MoEConfig) -> dict[str, int]:+ """Integer options of ``cpu_format`` for these experts."""+ return {}+def slot_limit(self, cfg: MoEConfig) -> int | None:"""Most GPU cache slots the kernel can address for ``cfg``; None for no limit."""return self.max_slots@@ -196,6 +200,10 @@ class MoEMethod(QuantMethod):def cpu_format(self) -> str | None:return self.kernel.cpu_format+ def cpu_spec(self) -> tuple[str | None, dict[str, int]]:+ """The CPU executor format and options of this layer's experts."""+ return self.cpu_format, self.kernel.cpu_options(self.cfg)+def create_weights(self, layer: Any) -> None:# resident experts stay out of the state dict; attach_resident_banks fills these after the dense weights loadlayer._expert_banks = Nonediff --git a/python/freetoken/layers/quantization/moe/mxfp4.py b/python/freetoken/layers/quantization/moe/mxfp4.pyindex 1af6d9f4..a17b5bbe 100644--- a/python/freetoken/layers/quantization/moe/mxfp4.py+++ b/python/freetoken/layers/quantization/moe/mxfp4.py@@ -28,6 +28,9 @@ class TritonMxfp4MoEKernel(MoEKernel):return "W4A8 ds_fp4 experts need the fp8 activation quant block"return self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)+ def cpu_options(self, cfg: MoEConfig) -> dict[str, int]:+ return {"act_block": int(cfg.act_block)}+def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:i, h = cfg.intermediate, cfg.hiddenreturn {diff --git a/python/freetoken/models/deepseek_v4/weight.py b/python/freetoken/models/deepseek_v4/weight.pyindex fbcddb1c..8bc4c1f0 100644--- a/python/freetoken/models/deepseek_v4/weight.py+++ b/python/freetoken/models/deepseek_v4/weight.py@@ -237,7 +237,7 @@ _PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}_KIND_SUFFIX = {"weight": "", "scale": "_scale"}-def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):"""Routed experts, one piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0``_scale`` companions (``w1`` / ``w3`` / ``w2``). The MTP layer's experts are skipped."""if kind is not QuantKind.MXFP4:@@ -249,10 +249,11 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: boargs = load_args(model_path, max_batch_size=1)L, E = args.n_layers, args.n_routed_experts+ keep = None if layers is None else set(layers)def locate(raw_name: str):m = _EXPERT_RE.match(raw_name)- if m is None or int(m["layer"]) >= L:+ if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):return Nonereturn int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]@@ -263,7 +264,7 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bodef _serial():reader = _ShardReader(model_path, _weight_map(model_path), torch.device("cpu"))try:- for li in tqdm(range(L), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):+ for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):for e in range(E):base = f"layers.{li}.ffn.experts.{e}"for proj in ("w1", "w3", "w2"):diff --git a/python/freetoken/models/deepseek_v41/weight.py b/python/freetoken/models/deepseek_v41/weight.pyindex 1ccd23ea..962bd195 100644--- a/python/freetoken/models/deepseek_v41/weight.py+++ b/python/freetoken/models/deepseek_v41/weight.py@@ -200,7 +200,7 @@ _PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}_KIND_SUFFIX = {"weight": "", "scale": "_scale"}-def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):"""One piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0 ``_scale`` companions.The DSpark stage's experts (``mtp.*``) never match."""if kind is not QuantKind.MXFP4:@@ -212,10 +212,11 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: boargs = load_args(model_path)L, E = args.n_layers, args.n_routed_experts+ keep = None if layers is None else set(layers)def locate(raw_name: str):m = _EXPERT_RE.match(raw_name)- if m is None or int(m["layer"]) >= L:+ if m is None or int(m["layer"]) >= L or (keep is not None and int(m["layer"]) not in keep):return Nonereturn int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]@@ -226,7 +227,7 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bodef _serial():reader = _ShardReader(model_path, weight_map(model_path), torch.device("cpu"))try:- for li in tqdm(range(L), desc="Loading DeepSeek-V4.1 experts (serial)", disable=not get_tp_info().is_primary()):+ for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading DeepSeek-V4.1 experts (serial)", disable=not get_tp_info().is_primary()):for e in range(E):base = f"layers.{li}.ffn.experts.{e}"for proj in ("w1", "w3", "w2"):diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.pyindex 6f967dd8..09f91694 100644--- a/python/freetoken/models/glm5_next/weight.py+++ b/python/freetoken/models/glm5_next/weight.py@@ -250,7 +250,7 @@ def iter_vision_weights(model_path: str, device: torch.device) -> Iterator[tuplereader.close()-def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):+def iter_expert_pieces(model_path, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):"""Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their ``_scale`` companions; other kinds use the generic readers."""if kind is not QuantKind.FP8_BLOCK:return None@@ -260,13 +260,14 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |from freetoken.moe.expert_pieces import per_expert_piecessuffix = {"weight": "", "weight_scale_inv": "_scale"}+ keep = None if layers is None else set(layers)def locate(raw_name: str):m = _FP8_EXPERT_RE.match(raw_name)if m is None:return Nonebank = _layer_to_bank(int(m["layer"]), config)- if bank is None:+ if bank is None or (keep is not None and bank not in keep):return Nonereturn bank, int(m["expert"]), m["proj"] + suffix[m["kind"]]@@ -284,8 +285,11 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |weight_map = json.load(f)["weight_map"]reader = _ShardReader(folder, weight_map, torch.device("cpu"))try:- layers = range(config.first_k_dense_replace, config.num_layers)- for layer in tqdm(layers, desc="Loading GLM-5.3 fp8 experts (serial)", disable=not get_tp_info().is_primary()):+ ckpt_layers = [+ layer for layer in range(config.first_k_dense_replace, config.num_layers)+ if keep is None or _layer_to_bank(layer, config) in keep+ ]+ for layer in tqdm(ckpt_layers, desc="Loading GLM-5.3 fp8 experts (serial)", disable=not get_tp_info().is_primary()):for e in range(config.num_experts):base = f"{_CKPT}.layers.{layer}.mlp.experts.{e}"for proj in ("gate", "up", "down"):diff --git a/python/freetoken/models/gpt_oss/weight.py b/python/freetoken/models/gpt_oss/weight.pyindex 686f0246..cb5830c3 100644--- a/python/freetoken/models/gpt_oss/weight.py+++ b/python/freetoken/models/gpt_oss/weight.py@@ -288,7 +288,7 @@ def _source_slices(config, tp_info) -> dict[str, tuple[str, tuple[slice, ...]]]:}-def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20):+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, layers=None, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20):"""gpt-oss experts as HF ships them, one piece per layer: stacked ``[E, ...]`` mxfp4``_blocks`` / ``_scales`` / ``_bias`` tensors sliced to this rank's intermediate range."""if kind is not QuantKind.MXFP4:@@ -298,10 +298,11 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: botp_info = get_tp_info()slices = _source_slices(config, tp_info)num_layers, num_experts = config.num_layers, config.num_experts+ keep = None if layers is None else set(layers)def _is_expert(name: str) -> bool:info = _expert_layer_and_name(name)- return info is not None and 0 <= info[0] < num_layers+ return info is not None and 0 <= info[0] < num_layers and (keep is None or info[0] in keep)def _tensors():if parallel:@@ -320,6 +321,8 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bolayer_id, source = infoif not 0 <= layer_id < num_layers:raise ValueError(f"Unexpected GPT-OSS expert layer in checkpoint: {name}")+ if keep is not None and layer_id not in keep:+ continueif source not in slices:raise ValueError(f"Unexpected GPT-OSS expert source: {name}")yield name, _read_safetensor_slice(f, name, slices[source][1])diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.pyindex 2dfdaaa3..04da0509 100644--- a/python/freetoken/models/nvfp4_banks.py+++ b/python/freetoken/models/nvfp4_banks.py@@ -4,7 +4,7 @@ import collectionsimport osimport refrom dataclasses import dataclass-from typing import Callable+from typing import Callable, Collectionimport safetensorsimport torch@@ -68,6 +68,7 @@ def iter_nvfp4_expert_pieces(config,spec: Nvfp4ExpertSourceSpec,*,+ layers: Collection[int] | None = None,parallel: bool = False,workers: int = 8,chunk: int = 8 << 20,@@ -80,6 +81,7 @@ def iter_nvfp4_expert_pieces(Serial reads walk the shards in order; ``parallel`` uses the chunked O_DIRECT reader. Eitherway tensors of one expert may span shards, so they are grouped by (layer, expert) as they land.+ ``layers`` (bank layer ids, default all) reads only those layers' experts."""from freetoken.models.loader import drop_page_cache as _dropfrom freetoken.models.loader import safetensors_weight_map@@ -89,13 +91,14 @@ def iter_nvfp4_expert_pieces(folder = download_hf_weight(model_path)weight_map = safetensors_weight_map(folder)+ keep = None if layers is None else set(layers)wanted: dict[str, tuple[int, int, str]] = {}for name in weight_map:match = spec.key_pattern.match(name)if match is None:continuebank_layer = _bank_layer(spec, int(match.group("layer")), config)- if bank_layer is None:+ if bank_layer is None or (keep is not None and bank_layer not in keep):continueproj = match.group("proj")if proj not in spec.proj_to_role:@@ -104,7 +107,7 @@ def iter_nvfp4_expert_pieces(if kind not in ("weight", "weight_scale", "weight_scale_2"):raise ValueError(f"{spec.desc}: unknown NVFP4 expert tensor kind {kind!r}")wanted[name] = (bank_layer, int(match.group("expert")), spec.proj_to_role[proj] + _kind_suffix(kind))- expected = _num_moe_layers(config) * config.num_experts * 9+ expected = (_num_moe_layers(config) if keep is None else len(keep)) * config.num_experts * 9if len(wanted) != expected:raise ValueError(f"{spec.desc}: found {len(wanted)} expert tensors, expected {expected}")diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex de5b9e89..724ad153 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -344,7 +344,7 @@ def _moe_dims(model_config):)-def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):+def iter_expert_pieces(model_path, config, kind: QuantKind, *, layers=None, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):"""Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their``_scale`` (block scale) companions, named as the checkpoint's dialect stores them. Other expert kinds use the generic readers."""if kind is not QuantKind.FP8_BLOCK:@@ -355,6 +355,7 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |from freetoken.moe.expert_pieces import per_expert_piecesL, E, H, I, dense = _moe_dims(config)+ keep = None if layers is None else set(layers)scale = get_quant_config().stored_tensors(QuantKind.FP8_BLOCK)["weight_scale_inv"].namekey_re = re.compile(_FP8_EXPERT_KEY_RE.format(scale=re.escape(scale)))suffix = {"weight": "", scale: "_scale"}@@ -366,20 +367,22 @@ def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool |li = int(m["layer"]) - denseif not 0 <= li < L:raise ValueError(f"unexpected routed-expert layer in {raw_name}")+ if keep is not None and li not in keep:+ return Nonereturn li, int(m["expert"]), m["proj"] + suffix[m["kind"]]if parallel is None:parallel = experts_scattered(model_path)if parallel:tensors = iter_expert_tensors_parallel(- model_path, lambda n: key_re.match(n) is not None, workers=workers, chunk=chunk+ model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk)return per_expert_pieces(tensors, locate, tensors_per_expert=6)def _serial():reader = ShardReader(model_path, torch.device("cpu"))try:- for li in tqdm(range(L), desc="Loading fp8 experts (serial)", disable=not get_tp_info().is_primary()):+ for li in tqdm(range(L) if keep is None else sorted(keep), desc="Loading fp8 experts (serial)", disable=not get_tp_info().is_primary()):for e in range(E):base = f"model.language_model.layers.{dense + li}.mlp.experts.{e}"for proj in ("gate", "up", "down"):diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex cd260cb2..86a03f41 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -476,7 +476,7 @@ def _build_cpu_moe_executor(fmt: str, wl: Workload, banks: dict, num_threads: incache, top_k=wl.top_k, activation=wl.activation,apply_router_weight_on_input=False, num_threads=num_threads, max_tokens=1,device=torch.device("cuda"), swiglu_alpha=wl.swiglu_alpha,- swiglu_limit=wl.swiglu_limit, act_block=wl.act_block,+ swiglu_limit=wl.swiglu_limit, layer_formats=[(fmt, {"act_block": wl.act_block} if wl.act_block is not None else {})] * cache.num_layers,)diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex bb9ee563..5d9bd3c9 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -163,28 +163,34 @@ class CpuMoeExecutor:device: torch.device,swiglu_alpha: float = 1.702,swiglu_limit: float | None = None,- fmt: str | None = None,- act_block: int | None = None,+ layer_formats: list[tuple[str, dict[str, int]]] | None = None,) -> None:from freetoken.kernel import _cpu_moefrom freetoken.moe.legacy_format import canonical_role- fmt = fmt or cache.quant_format- if fmt not in _cpu_moe.weight_formats():- raise NotImplementedError(- f"--moe-strategy cpu/hybrid computes experts on the CPU and supports "- f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are "- f"{fmt!r}; use --moe-strategy offload (GPU-side dequant) instead."- )+ # layer_formats[l] = (format, options) of layer l; None runs every layer in cache.quant_format+ if layer_formats is None:+ layer_formats = [(cache.quant_format, {})] * int(cache.num_layers)+ assert len(layer_formats) == cache.num_layers, (len(layer_formats), cache.num_layers)+ keys = [(name, tuple(sorted(opts.items()))) for name, opts in layer_formats]+ specs = list(dict.fromkeys(keys))+ names = sorted({name for name, _ in specs})+ for f in names:+ if f not in _cpu_moe.weight_formats():+ raise NotImplementedError(+ f"--moe-strategy cpu/hybrid computes experts on the CPU and supports "+ f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are "+ f"{f!r}; use --moe-strategy offload (GPU-side dequant) instead."+ )if activation not in _ACT_IDS:raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")- if apply_router_weight_on_input and fmt in ("mxfp4_triton", "ds_fp4"):- raise NotImplementedError(f"CPU MoE backend: {fmt} experts apply the router weight on the output only")+ if apply_router_weight_on_input and (output_only := {"mxfp4_triton", "ds_fp4"} & set(names)):+ raise NotImplementedError(f"CPU MoE backend: {sorted(output_only)} experts apply the router weight on the output only")# ABI probe: a stale prebuilt _cpu_moe.so accepts newer act ids without# error and silently computes the wrong activation in the generic# epilogue -- fail loudly with the rebuild instruction instead. (mxfp4# handles its act inside the kernel and predates the marker.)- if _ACT_IDS[activation] >= 3 and fmt != "mxfp4_triton":+ if _ACT_IDS[activation] >= 3 and names != ["mxfp4_triton"]:supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()if _ACT_IDS[activation] > supported:raise RuntimeError(@@ -197,16 +203,24 @@ class CpuMoeExecutor:self.num_layers = int(cache.num_layers)self.num_experts = int(cache.num_experts)self.top_k = int(top_k)- self.quant_format = fmt+ self.quant_format = ",".join(names)self.device = deviceself.max_tokens = int(max_tokens)self.apply_router_weight_on_input = bool(apply_router_weight_on_input)# The per-layer tensors and their pointer tables must outlive the executor# (C++ holds raw addresses into both).self._banks: list[torch.Tensor] = []- banks, options, (self.H, self.I) = self._resolve_banks(- {canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()}, fmt- )+ by_role = {canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()}+ # every format's tables address every layer, but a format reads only its own layers' rows+ formats, shapes = [], set()+ for name, opts in specs:+ banks, options, shape = self._resolve_banks(by_role, name, keys.index((name, opts)), dict(opts))+ formats.append((name, banks, {**dict(opts), **options}))+ shapes.add(shape)+ if len(shapes) != 1:+ raise NotImplementedError(f"CPU MoE formats disagree on the expert shape (hidden, intermediate): {sorted(shapes)}")+ (self.H, self.I), = shapes+ layer_counts = {spec: keys.count(spec) for spec in specs}# ds_fp4: the reference FP8 activation round-trip is a scalar per-element chain# that the C++ side runs single-threaded on the CUDA host-callback thread --@@ -215,12 +229,15 @@ class CpuMoeExecutor:# round-trip as a captured GPU kernel BEFORE the D2H (see decode_submit) and tell# the C++ side to skip its own. Measured on DeepSeek-V4-Flash bs=1 decode:# 12.85 -> 15.65 tok/s, output bit-identical (tests/moe/test_dsfp4_prequant.py).- self._gpu_prequant = fmt == "ds_fp4" and device.type == "cuda" and _DSFP4_GPU_PREQUANT# the W4A8 activation round-trip block follows the checkpoint's fp8 block (128 on V4, 32 on V4.1)- self._act_block = act_block- if fmt == "ds_fp4":- assert act_block is not None, "ds_fp4 experts need act_block, the checkpoint's fp8 activation quant block"- options.update(act_block=int(act_block), input_prequant=int(self._gpu_prequant))+ blocks = {options.get("act_block") for name, _, options in formats if name == "ds_fp4"}+ assert None not in blocks, "ds_fp4 experts need the act_block option, the checkpoint's fp8 activation quant block"+ # the GPU round-trip rewrites the whole input, so only when every layer is ds_fp4 with one block+ self._gpu_prequant = names == ["ds_fp4"] and len(blocks) == 1 and device.type == "cuda" and _DSFP4_GPU_PREQUANT+ self._act_block = next(iter(blocks)) if self._gpu_prequant else None+ for name, _, options in formats:+ if name == "ds_fp4":+ options["input_prequant"] = int(self._gpu_prequant)if self._gpu_prequant:logger.info_rank0("ds_fp4: input FP8 round-trip moved to the GPU "@@ -271,9 +288,8 @@ class CpuMoeExecutor:max_tokens=self.max_tokens,activation_id=_ACT_IDS[activation],apply_router_weight_on_input=1 if apply_router_weight_on_input else 0,- weight_format=fmt,- banks=banks,- options=options,+ formats=formats,+ layer_formats=[specs.index(k) for k in keys],swiglu_alpha=float(swiglu_alpha),swiglu_limit=float(swiglu_limit) if swiglu_limit is not None else float("inf"),core_ids=core_ids,@@ -332,9 +348,11 @@ class CpuMoeExecutor:self._watchdog.start()+ fmt_label = "; ".join(f"{name}{dict(opts) if opts else ''}: {n} layers" for (name, opts), n in layer_counts.items())logger.info_rank0(f"CPU MoE executor ready: threads={nthreads} (pinned to cores "- f"{core_ids[0]}..{core_ids[-1]}) isa={self.isa} fmt={fmt} "+ f"{core_ids[0]}..{core_ids[-1]}) isa={self.isa} "+ f"fmt={fmt_label} "f"H={self.H} I={self.I} experts={self.num_experts} layers={self.num_layers} "f"top_k={self.top_k} act={activation} max_tokens={self.max_tokens} "f"spin_wait={self.spin_wait} "@@ -360,23 +378,23 @@ class CpuMoeExecutor:"""Per-layer table addresses keyed by canonical bank role."""return {role: self._make_table(layers).data_ptr() for role, layers in roles.items()}- def _resolve_fp8_block_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:+ def _resolve_fp8_block_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:"""Block-FP8 schema: row-major e4m3 + one bf16 scale per 128x128 block; the scale banks pad their last dim, so C++ gets the real row stride."""gu, gus = banks["gate_up"], banks["gate_up_scale"]dn, dns = banks["down"], banks["down_scale"]- assert gu[0].element_size() == 1 and dn[0].element_size() == 1, (gu[0].dtype, dn[0].dtype)- assert gus[0].dtype == torch.bfloat16 and dns[0].dtype == torch.bfloat16, (gus[0].dtype, dns[0].dtype)- I = int(gu[0].shape[1] // 2)- H = int(gu[0].shape[2])- assert gu[0].shape[1] == 2 * I- assert tuple(dn[0].shape[1:]) == (H, I), (dn[0].shape, H, I)- assert gus[0].shape[1] == (2 * I + 127) // 128 and gus[0].shape[2] >= (H + 127) // 128, (gus[0].shape, H, I)- assert dns[0].shape[1] == (H + 127) // 128 and dns[0].shape[2] >= (I + 127) // 128, (dns[0].shape, H, I)+ assert gu[rep].element_size() == 1 and dn[rep].element_size() == 1, (gu[rep].dtype, dn[rep].dtype)+ assert gus[rep].dtype == torch.bfloat16 and dns[rep].dtype == torch.bfloat16, (gus[rep].dtype, dns[rep].dtype)+ I = int(gu[rep].shape[1] // 2)+ H = int(gu[rep].shape[2])+ assert gu[rep].shape[1] == 2 * I+ assert tuple(dn[rep].shape[1:]) == (H, I), (dn[rep].shape, H, I)+ assert gus[rep].shape[1] == (2 * I + 127) // 128 and gus[rep].shape[2] >= (H + 127) // 128, (gus[rep].shape, H, I)+ assert dns[rep].shape[1] == (H + 127) // 128 and dns[rep].shape[2] >= (I + 127) // 128, (dns[rep].shape, H, I)banks = self._tables(gate_up=gu, down=dn, gate_up_scale=gus, down_scale=dns)- options = dict(gu_scale_stride=int(gus[0].stride(1)), dn_scale_stride=int(dns[0].stride(1)))+ options = dict(gu_scale_stride=int(gus[rep].stride(1)), dn_scale_stride=int(dns[rep].stride(1)))return banks, options, (H, I)- def _resolve_banks(self, banks: dict, fmt: str) -> tuple[dict, dict, tuple[int, int]]:+ def _resolve_banks(self, banks: dict, fmt: str, rep: int = 0, opts: dict | None = None) -> tuple[dict, dict, tuple[int, int]]:"""Return (banks, options, (H, I)) for the C++ ctor: banks maps each canonical rolethe format reads to its per-layer table's address, options holds its integer settings.@@ -389,102 +407,102 @@ class CpuMoeExecutor:if fmt == "bf16":gate_up = banks["gate_up"]down = banks["down"]- if gate_up[0].dtype != torch.bfloat16 or down[0].dtype != torch.bfloat16:+ if gate_up[rep].dtype != torch.bfloat16 or down[rep].dtype != torch.bfloat16:raise NotImplementedError(- f"bf16 CPU MoE requires bf16 banks, got {gate_up[0].dtype}/{down[0].dtype}"+ f"bf16 CPU MoE requires bf16 banks, got {gate_up[rep].dtype}/{down[rep].dtype}")- H = int(gate_up[0].shape[2])- I = int(gate_up[0].shape[1] // 2)- assert gate_up[0].shape[1] == 2 * I- assert tuple(down[0].shape[1:]) == (H, I), (down[0].shape, H, I)+ H = int(gate_up[rep].shape[2])+ I = int(gate_up[rep].shape[1] // 2)+ assert gate_up[rep].shape[1] == 2 * I+ assert tuple(down[rep].shape[1:]) == (H, I), (down[rep].shape, H, I)banks = self._tables(gate_up=gate_up, down=down)return banks, {}, (H, I)if fmt == "q4_0":- return self._resolve_q4_0_banks(banks)+ return self._resolve_q4_0_banks(banks, rep)if fmt == "mxfp4_triton":- return self._resolve_mxfp4_banks(banks)+ return self._resolve_mxfp4_banks(banks, rep)if fmt == "ds_fp4":- return self._resolve_dsfp4_banks(banks)+ return self._resolve_dsfp4_banks(banks, rep)if fmt == "fp8_block":- return self._resolve_fp8_block_banks(banks)+ return self._resolve_fp8_block_banks(banks, rep)# nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]- assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)- assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"- assert gug[0].dtype == torch.float16 and dng[0].dtype == torch.float16, (gug[0].dtype, dng[0].dtype)- I = int(gup[0].shape[1] // 2)- H = int(gup[0].shape[2] * 2)- assert gup[0].shape[1] == 2 * I+ assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype)+ assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte"+ assert gug[rep].dtype == torch.float16 and dng[rep].dtype == torch.float16, (gug[rep].dtype, dng[rep].dtype)+ I = int(gup[rep].shape[1] // 2)+ H = int(gup[rep].shape[2] * 2)+ assert gup[rep].shape[1] == 2 * Iassert H % 16 == 0 and I % 16 == 0, (H, I)- assert tuple(dnp[0].shape[1:]) == (H, I // 2), (dnp[0].shape, H, I)- assert tuple(gus[0].shape[1:]) == (2 * I, H // 16), (gus[0].shape, I, H)- assert tuple(dns[0].shape[1:]) == (H, I // 16), (dns[0].shape, H, I)- assert tuple(gug[0].shape[1:]) == (2 * I,) and tuple(dng[0].shape[1:]) == (H,)+ assert tuple(dnp[rep].shape[1:]) == (H, I // 2), (dnp[rep].shape, H, I)+ assert tuple(gus[rep].shape[1:]) == (2 * I, H // 16), (gus[rep].shape, I, H)+ assert tuple(dns[rep].shape[1:]) == (H, I // 16), (dns[rep].shape, H, I)+ assert tuple(gug[rep].shape[1:]) == (2 * I,) and tuple(dng[rep].shape[1:]) == (H,)banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, gate_up_global=gug, down_scale=dns, down_global=dng)return banks, {}, (H, I)- def _resolve_q4_0_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:+ def _resolve_q4_0_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:"""Native GGUF Q4_0 schema (gemma4 GGUF): per-32 blocks (fp16 scale + 16 nibblebytes), row-major over K -- the *same* packed banks the GPU offload path streams.gate_up is [S, 2I, H//32*18], down is [S, H, I//32*18]; the C++ W4A16 GEMV reads arow in place (18 bytes / 32 K) and dequantizes weights inside the K-loop."""gate_up, down = banks["gate_up"], banks["down"]- assert gate_up[0].dtype == torch.uint8 and down[0].dtype == torch.uint8, (- gate_up[0].dtype, down[0].dtype,+ assert gate_up[rep].dtype == torch.uint8 and down[rep].dtype == torch.uint8, (+ gate_up[rep].dtype, down[rep].dtype,)- I = int(gate_up[0].shape[1] // 2)- H = int(down[0].shape[1])- assert gate_up[0].shape[1] == 2 * I+ I = int(gate_up[rep].shape[1] // 2)+ H = int(down[rep].shape[1])+ assert gate_up[rep].shape[1] == 2 * Iassert H % 32 == 0 and I % 32 == 0, (H, I)- assert int(gate_up[0].shape[2]) == (H // 32) * 18, (gate_up[0].shape, H)- assert int(down[0].shape[2]) == (I // 32) * 18, (down[0].shape, I)+ assert int(gate_up[rep].shape[2]) == (H // 32) * 18, (gate_up[rep].shape, H)+ assert int(down[rep].shape[2]) == (I // 32) * 18, (down[rep].shape, I)banks = self._tables(gate_up=gate_up, down=down)return banks, {}, (H, I)- def _resolve_mxfp4_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:+ def _resolve_mxfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:"""gpt-oss mxfp4 ``mxfp4_triton`` schema: transposed split-K blocks/scales(N innermost) + per-output-row biases. The C++ kernel streams K andaccumulates a contiguous N-block, so the GPU-tiled layout is read in place(no repack, no extra host memory). Block scales are e8m0 (1 byte / 32 K)."""gub, gus, gob = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_bias"]dnb, dns, dob = banks["down"], banks["down_scale"], banks["down_bias"]- assert gub[0].dtype == torch.uint8 and dnb[0].dtype == torch.uint8, (gub[0].dtype, dnb[0].dtype)- assert gus[0].dtype == torch.uint8 and dns[0].dtype == torch.uint8, (gus[0].dtype, dns[0].dtype)- assert gob[0].dtype == torch.bfloat16 and dob[0].dtype == torch.bfloat16, (gob[0].dtype, dob[0].dtype)+ assert gub[rep].dtype == torch.uint8 and dnb[rep].dtype == torch.uint8, (gub[rep].dtype, dnb[rep].dtype)+ assert gus[rep].dtype == torch.uint8 and dns[rep].dtype == torch.uint8, (gus[rep].dtype, dns[rep].dtype)+ assert gob[rep].dtype == torch.bfloat16 and dob[rep].dtype == torch.bfloat16, (gob[rep].dtype, dob[rep].dtype)# gate_up_blocks [E, H//2, 2I]; down_blocks [E, I//2, H]- H = int(gub[0].shape[1] * 2)- I = int(gub[0].shape[2] // 2)- assert gub[0].shape[2] == 2 * I+ H = int(gub[rep].shape[1] * 2)+ I = int(gub[rep].shape[2] // 2)+ assert gub[rep].shape[2] == 2 * Iassert H % 32 == 0 and I % 32 == 0, (H, I)- assert tuple(dnb[0].shape[1:]) == (I // 2, H), (dnb[0].shape, H, I)- assert tuple(gus[0].shape[1:]) == (H // 32, 2 * I), (gus[0].shape, H, I)- assert tuple(dns[0].shape[1:]) == (I // 32, H), (dns[0].shape, H, I)- assert tuple(gob[0].shape[1:]) == (2 * I,) and tuple(dob[0].shape[1:]) == (H,)+ assert tuple(dnb[rep].shape[1:]) == (I // 2, H), (dnb[rep].shape, H, I)+ assert tuple(gus[rep].shape[1:]) == (H // 32, 2 * I), (gus[rep].shape, H, I)+ assert tuple(dns[rep].shape[1:]) == (I // 32, H), (dns[rep].shape, H, I)+ assert tuple(gob[rep].shape[1:]) == (2 * I,) and tuple(dob[rep].shape[1:]) == (H,)banks = self._tables(gate_up=gub, down=dnb, gate_up_scale=gus, down_scale=dns, gate_up_bias=gob, down_bias=dob)return banks, {}, (H, I)- def _resolve_dsfp4_banks(self, banks: dict) -> tuple[dict, dict, tuple[int, int]]:+ def _resolve_dsfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:"""DeepSeek-V4 ``ds_fp4`` schema: row-major e2m1 (2/byte) + e8m0 per-32 blockscales, no global, no bias. Layout matches nvfp4 (K contiguous per output row),so the C++ GEMV reads it in place. The kernel additionally FP8-round-trips theactivations (the checkpoint's fp8 block) to match the DeepSeek W4A8 reference."""gup, gus = banks["gate_up"], banks["gate_up_scale"]dnp, dns = banks["down"], banks["down_scale"]- assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)- assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"- I = int(gup[0].shape[1] // 2)- H = int(gup[0].shape[2] * 2)- assert gup[0].shape[1] == 2 * I+ assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype)+ assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte"+ I = int(gup[rep].shape[1] // 2)+ H = int(gup[rep].shape[2] * 2)+ assert gup[rep].shape[1] == 2 * Iassert H % 32 == 0 and I % 32 == 0, (H, I) # e8m0 scales per 32 along K- assert tuple(dnp[0].shape[1:]) == (H, I // 2), (dnp[0].shape, H, I)- assert tuple(gus[0].shape[1:]) == (2 * I, H // 32), (gus[0].shape, I, H)- assert tuple(dns[0].shape[1:]) == (H, I // 32), (dns[0].shape, H, I)+ assert tuple(dnp[rep].shape[1:]) == (H, I // 2), (dnp[rep].shape, H, I)+ assert tuple(gus[rep].shape[1:]) == (2 * I, H // 32), (gus[rep].shape, I, H)+ assert tuple(dns[rep].shape[1:]) == (H, I // 32), (dns[rep].shape, H, I)banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, down_scale=dns)return banks, {}, (H, I)diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.pyindex 03ce4b75..7e25e58f 100644--- a/python/freetoken/moe/expert_banks.py+++ b/python/freetoken/moe/expert_banks.py@@ -18,8 +18,8 @@ import torchfrom freetoken.layers.quantization import QuantKindfrom freetoken.utils import init_logger-from .host_banks import alloc_layer_banks-from .offload_cache import _BANK_BYTES_PER_EXPERT+from .host_banks import HostBank+from .offload_cache import _BANK_BYTES_PER_EXPERT, placeholder_banklogger = init_logger(__name__)@@ -52,6 +52,8 @@ class ExpertBanks:layer_tags: list | None = None# per bank, per layer: bytes past an expert matrix's last row that the kernel reads and that must read as zeros (ggml MMQ); None = no such readsrow_tail_bytes: dict[str, list[int]] | None = None+ # per-layer quant_format of experts in several formats (``quant_format``/``kind``/``kernel`` then name the most common one); None = every layer is ``quant_format``+ layer_formats: list[str] | None = Nonedef _dummy_fill(role: str, tensor: torch.Tensor) -> None:@@ -80,6 +82,7 @@ def build_expert_banks(layer_sink=None,dummy: bool = False,resident: bool = False,+ layers: list[int] | None = None,) -> ExpertBanks:"""Fill host banks in the kernel's layout from a stream of expert pieces.@@ -89,6 +92,8 @@ def build_expert_banks(are pinned in the background, otherwise the sink receives them (converter). ``dummy``skips the pieces and fills the banks with finite random contents.``resident`` packs into banks on ``device`` instead of host banks (the resident 'fused' experts).+ ``layers`` fills only those bank layers (the ones ``method`` serves, for experts of several+ formats): pieces of other layers are skipped and the other layers' sources are None."""from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, pin_banksfrom freetoken.moe.legacy_format import legacy_format_for@@ -96,29 +101,34 @@ def build_expert_banks(kernel = method.kernellayout = method.layout()E = method.cfg.num_experts+ ids = list(range(num_layers)) if layers is None else sorted(layers)+ filled = set(ids)specs = {role: ((E, *spec.shape), spec.dtype) for role, spec in layout.items() if not spec.resident}if resident:assert layer_sink is Nonehb = None- banks = {role: [torch.empty(shape, dtype=dtype, device=device) for _ in range(num_layers)] for role, (shape, dtype) in specs.items()}+ banks = {role: {l: torch.empty(shape, dtype=dtype, device=device) for l in ids} for role, (shape, dtype) in specs.items()}else:- hb = alloc_layer_banks(specs, num_layers)- banks = {role: [b.tensor for b in hb[role]] for role in specs}+ hb = {role: {l: HostBank(shape, dtype) for l in ids} for role, (shape, dtype) in specs.items()}+ banks = {role: {l: b.tensor for l, b in per_layer.items()} for role, per_layer in hb.items()}alphas = {role: torch.empty(num_layers * E, dtype=spec.dtype, device=device)for role, spec in layout.items() if spec.resident}+ def _sources() -> dict[str, list]:+ return {role: [per_layer.get(l) for l in range(num_layers)] for role, per_layer in banks.items()}+if dummy:for role, per_layer in banks.items():- for tensor in per_layer:+ for tensor in per_layer.values():_dummy_fill(role, tensor)for alpha in alphas.values():alpha.fill_(1.0)if hb is not None and torch.cuda.is_available():pin_banks(hb)return ExpertBanks(- legacy_format_for(method.kind, kernel.name), banks,+ legacy_format_for(method.kind, kernel.name), _sources(),gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),kind=method.kind, kernel=kernel.name, layout=layout,)@@ -130,6 +140,8 @@ def build_expert_banks(for layer_id, e0, e1, piece in pieces:if not (0 <= layer_id < num_layers and 0 <= e0 < e1 <= E):raise ValueError(f"expert piece out of range: layer {layer_id}, experts {e0}:{e1} of {num_layers} x {E}")+ if layer_id not in filled:+ continue# refuse before writing: a duplicate row would also complete the layer early and hand the sink a half-filled bankif written[layer_id, e0:e1].any():raise ValueError(f"expert rows written more than once: layer {layer_id}, experts {e0}:{e1}")@@ -141,8 +153,9 @@ def build_expert_banks(if tracker is not None:for _ in range(e1 - e0):tracker.note(layer_id)- missing = (written == 0).nonzero().tolist()+ missing = (written[ids] == 0).nonzero().tolist()if missing:+ missing = [[ids[l], e] for l, e in missing]raise ValueError(f"expert banks were not filled: {len(missing)} (layer, expert) rows missing (first {missing[:4]})")if layer_sink is not None:@@ -154,12 +167,74 @@ def build_expert_banks(_fill(None)return ExpertBanks(- legacy_format_for(method.kind, kernel.name), banks,+ legacy_format_for(method.kind, kernel.name), _sources(),gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),streamed=layer_sink is not None, kind=method.kind, kernel=kernel.name, layout=layout,)+def method_groups(layer_methods) -> list[tuple]:+ """``[(method, [bank layers])]`` of the layers' expert methods, one entry per (kind, kernel), most layers first."""+ groups: dict = {}+ for layer, method in enumerate(layer_methods):+ groups.setdefault((method.kind, method.kernel.name), (method, []))[1].append(layer)+ return sorted(groups.values(), key=lambda g: (-len(g[1]), g[1][0]))+++def is_mixed(layer_methods) -> bool:+ return layer_methods is not None and len(method_groups(layer_methods)) > 1+++def merge_expert_banks(parts: list[ExpertBanks], layer_methods, num_experts: int) -> ExpertBanks:+ """One ``ExpertBanks`` over the per-format ``parts`` (each filled only on its layers): the banks are the+ union of the roles, a layer without a role gets a zero-byte placeholder (see ``OffloadMoeCache.layer_layouts``)."""+ from freetoken.moe.legacy_format import legacy_format_for++ num_layers = len(layer_methods)+ sources: dict[str, list] = {}+ for part in parts:+ if part.gate_up_alpha is not None or part.down_alpha is not None:+ raise ValueError(f"experts of several formats cannot use the {part.kernel} kernel, which keeps GPU-resident alphas")+ for role, per_layer in part.sources.items():+ merged = sources.setdefault(role, [None] * num_layers)+ for layer, tensor in enumerate(per_layer):+ if tensor is not None:+ assert merged[layer] is None, (role, layer)+ merged[layer] = tensor+ for per_layer in sources.values():+ for layer, tensor in enumerate(per_layer):+ if tensor is None:+ per_layer[layer] = placeholder_bank(num_experts)+ formats = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]+ primary = parts[0]+ return ExpertBanks(+ primary.quant_format, sources, streamed=all(p.streamed for p in parts),+ kind=primary.kind, kernel=primary.kernel, layer_tags=formats, layer_formats=formats,+ )+++def _mixed_expert_banks(model_path, model_config, layer_methods, device, dummy, parallel, workers, chunk, layer_sink=None) -> ExpertBanks:+ from freetoken.moe.expert_pieces import iter_expert_pieces++ num_layers = model_config.num_moe_layers+ assert len(layer_methods) == num_layers, (len(layer_methods), num_layers)+ groups = method_groups(layer_methods)+ # resolve every format's reader before any bank exists, so a missing parallel reader falls back to serial with nothing allocated+ readers = [+ None if dummy else iter_expert_pieces(+ model_path, model_config, method.kind, layers=layers, parallel=parallel, workers=workers, chunk=chunk+ )+ for method, layers in groups+ ]+ parts = []+ for (method, layers), pieces in zip(groups, readers):+ parts.append(build_expert_banks(+ method, num_layers, pieces, device=device, layer_sink=layer_sink, dummy=dummy, layers=layers,+ ))+ logger.info_rank0(f"expert banks: {method.kind} / {method.kernel.name} on {len(layers)} of {num_layers} MoE layers")+ return merge_expert_banks(parts, layer_methods, model_config.num_experts)++_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader@@ -216,19 +291,24 @@ def ftw_bank_bytes(model_path: str) -> int | None:return sum(t["nbytes"] for t in tensors if t.get("kind") == "experts_bank")-def bank_bytes_estimate(model_config, method=None) -> int | None:+def _method_expert_bytes(method) -> int:+ return method.cfg.num_experts * sum(+ math.prod(spec.shape) * torch.empty((), dtype=spec.dtype).element_size()+ for spec in method.layout().values() if not spec.resident+ )+++def bank_bytes_estimate(model_config, method=None, layer_methods=None) -> int | None:"""Estimated total expert-bank bytes of a raw checkpoint before loading it.- With a bound expert ``method`` the kernel's layout gives the exact host bytes; otherwise the- format-tag table sizes the format. ``None`` for unknown formats or missing dims- (callers then skip the pre-load sizing)."""+ With a bound expert ``method`` the kernel's layout gives the exact host bytes (``layer_methods``,+ one per MoE layer, for experts of several formats); otherwise the format-tag table sizes the+ format. ``None`` for unknown formats or missing dims (callers then skip the pre-load sizing)."""layers = getattr(model_config, "num_moe_layers", None)+ if layer_methods is not None:+ return sum(_method_expert_bytes(m) for m in layer_methods)if method is not None and layers:- per_expert = sum(- math.prod(spec.shape) * torch.empty((), dtype=spec.dtype).element_size()- for spec in method.layout().values() if not spec.resident- )- return layers * method.cfg.num_experts * per_expert+ return layers * _method_expert_bytes(method)expert_quant = getattr(model_config, "expert_quant", "none")fmt = expert_quant if expert_quant != "none" else (getattr(model_config, "moe_weight_format", None) or "bf16"@@ -256,6 +336,7 @@ def load_expert_banks(layer_sink=None,layer_residency: list[str] | None = None,resident: bool = False,+ layer_methods: list | None = None,) -> ExpertBanks:"""Load (or fabricate, with ``dummy=True``) the expert banks. Two paths, both returningthe same normalized ``ExpertBanks`` and both pinning after fill:@@ -282,9 +363,15 @@ def load_expert_banks(Applied labels are echoed on ``ExpertBanks.layer_residency``; a loader that settles some other way leaves it ``None`` (CPU-layer decode still works on pinned banks, it just saves no pin quota).``resident`` (with ``method``): the banks land on ``device`` for the resident 'fused' experts; no host bank outlives the load.++ ``layer_methods`` (one bound method per MoE layer) serves experts of several formats: each+ format fills its own layers and the banks merge over the union of roles (``merge_expert_banks``)."""from freetoken.checkpoint.ftw import is_ftw_checkpoint, load_ftw_banks, load_ftw_banks_to_device+ mixed = is_mixed(layer_methods)+ if mixed and (resident or layer_residency is not None):+ raise ValueError("experts of several formats are served from the offload slot cache only (no resident experts, no --moe-cpu-layers)")if resident:assert method is not None and layer_sink is None and layer_residency is Noneif model_path and is_ftw_checkpoint(model_path) and not dummy:@@ -330,6 +417,8 @@ def load_expert_banks(from freetoken.moe.host_banks import requested_residencydef _build(par: bool) -> ExpertBanks:+ if mixed:+ return _mixed_expert_banks(model_path, model_config, layer_methods, device, dummy, par, workers, chunk, layer_sink)if method is not None and method.kind is QuantKind.GGUF:from freetoken.models.gguf.experts import load_gguf_expert_banksdiff --git a/python/freetoken/moe/expert_pieces.py b/python/freetoken/moe/expert_pieces.pyindex 79f655a9..917d84bb 100644--- a/python/freetoken/moe/expert_pieces.py+++ b/python/freetoken/moe/expert_pieces.py@@ -8,7 +8,7 @@ their ``_scale`` / ``_global`` companions, or an already fused ``gate_up``). Thefrom __future__ import annotations-from typing import Callable, Iterable, Iterator+from typing import Callable, Collection, Iterable, Iteratorimport torchfrom freetoken.layers.quantization import QuantKind@@ -38,24 +38,34 @@ def _model_hook(spec, name: str):def iter_expert_pieces(- model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20+ model_path: str,+ config,+ kind: QuantKind,+ *,+ layers: Collection[int] | None = None,+ parallel: bool = False,+ workers: int = 8,+ chunk: int = 8 << 20,) -> Iterator[Piece]:"""The pieces of ``model_path``'s routed experts, stored as ``kind``.- A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, parallel, workers,- chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16+ A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, layers, parallel,+ workers, chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16experts come from the family's stacked ``iter_weights`` and NVFP4 experts from its``nvfp4_expert_spec``. The reader is resolved here, before any bank is allocated, so amissing parallel reader raises ``NotImplementedError`` while a serial fallback is still cheap.++ ``layers`` (bank layer ids, default all) limits the pieces to those layers, for experts of several+ formats: a reader skips the other layers' tensors and checks only these layers for completeness."""spec = get_model_spec(config.architectures[0])hook = _model_hook(spec, "iter_expert_pieces")if hook is not None:- pieces = hook(model_path, config, kind, parallel=parallel, workers=workers, chunk=chunk)+ pieces = hook(model_path, config, kind, layers=layers, parallel=parallel, workers=workers, chunk=chunk)if pieces is not None:return piecesif kind is QuantKind.NONE:- return _bf16_pieces(model_path, config, spec, parallel=parallel, workers=workers, chunk=chunk)+ return _bf16_pieces(model_path, config, spec, layers=layers, parallel=parallel, workers=workers, chunk=chunk)if kind is QuantKind.NVFP4:spec_hook = _model_hook(spec, "nvfp4_expert_spec")if spec_hook is None:@@ -63,7 +73,7 @@ def iter_expert_pieces(from freetoken.models.nvfp4_banks import iter_nvfp4_expert_piecesreturn iter_nvfp4_expert_pieces(- model_path, config, spec_hook(model_path, config), parallel=parallel, workers=workers, chunk=chunk+ model_path, config, spec_hook(model_path, config), layers=layers, parallel=parallel, workers=workers, chunk=chunk)raise NotImplementedError(f"{spec.module} provides no expert reader for {kind!r} experts")@@ -81,18 +91,31 @@ def packed_expert_source_info(key: str) -> tuple[int, str] | None:return None-def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config) -> Iterator[Piece]:- """Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer."""+def _source_layer(key: str) -> int | None:+ """Checkpoint layer of a ``model.layers.N....`` key."""+ parts = key.split(".")+ return int(parts[2]) if len(parts) > 2 and parts[:2] == ["model", "layers"] and parts[2].isdigit() else None+++def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config, *, layers: Collection[int] | None = None) -> Iterator[Piece]:+ """Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer, for ``layers`` (bank layer ids, default all)."""+ keep = None if layers is None else set(layers)pending: dict[int, dict[str, torch.Tensor]] = {}num_experts = config.num_expertsfor name, tensor in tensors:info = packed_expert_source_info(name)if info is None:+ # another format's layer stores its experts in another form+ layer = _source_layer(name)+ if keep is not None and layer is not None and bank_layer_of(config, layer) not in keep:+ continueraise ValueError(f"Unexpected expert weight key: {name}")layer, packed_name = infobank_layer = bank_layer_of(config, layer)if bank_layer is None:raise ValueError(f"Unexpected MoE expert layer {layer}; expected a routed-expert layer")+ if keep is not None and bank_layer not in keep:+ continueif tensor.size(0) != num_experts:raise ValueError(f"Unexpected {packed_name} expert count {tensor.size(0)}; expected {num_experts}")piece = pending.setdefault(bank_layer, {})@@ -104,7 +127,7 @@ def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config) -raise ValueError(f"Missing MoE expert source layers: {sorted(pending)}")-def _bf16_pieces(model_path: str, config, spec, *, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:+def _bf16_pieces(model_path: str, config, spec, *, layers: Collection[int] | None, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:device = torch.device("cpu")if parallel:iter_weights = _model_hook(spec, "iter_weights_parallel")@@ -116,7 +139,7 @@ def _bf16_pieces(model_path: str, config, spec, *, parallel: bool, workers: int,else:iter_weights = _load_attr(spec.module, spec.iter_weights)tensors = iter_weights(model_path, device, include_moe_experts=True, include_non_moe=False)- return stacked_expert_pieces(tensors, config)+ return stacked_expert_pieces(tensors, config, layers=layers)def per_expert_pieces(diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.pyindex 91e26487..95562a88 100644--- a/python/freetoken/moe/host_banks.py+++ b/python/freetoken/moe/host_banks.py@@ -204,18 +204,6 @@ def alloc_banks(specs: dict[str, tuple[tuple[int, ...], torch.dtype]]) -> dict[sreturn {name: HostBank(shape, dtype) for name, (shape, dtype) in specs.items()}-def alloc_layer_banks(- specs: dict[str, tuple[tuple[int, ...], torch.dtype]], num_layers: int-) -> dict[str, list[HostBank]]:- """Allocate per-layer host banks: ``{name: ([num_experts, ...] row shape, dtype)}``- -> one independently allocated (page-aligned, independently pin/lock-able)- ``HostBank`` per layer per name."""- return {- name: [HostBank(shape, dtype) for _ in range(num_layers)]- for name, (shape, dtype) in specs.items()- }--class _ResidencyPlan:"""Per-layer ``HostResidency`` labels, ambiently visible to the bank settle points.@@ -265,13 +253,13 @@ def _settle(bank: HostBank, residency: str) -> None:bank.lock()-def pin_banks(banks: dict[str, HostBank | list[HostBank]]) -> None:+def pin_banks(banks: dict[str, HostBank | list[HostBank] | dict[int, HostBank]]) -> None:"""Settle every bank after it has been filled -- pin-after-fill by default.- List-valued entries are per-layer and honor the ambient :func:`requested_residency` plan; scalar banks always pin."""+ List-valued entries (indexed by layer id) and dict-valued ones (layer id -> bank) are per-layer and honor the ambient :func:`requested_residency` plan; scalar banks always pin."""plan = _requested_residencyfor bank in banks.values():- if isinstance(bank, list):- for layer_id, layer_bank in enumerate(bank):+ if isinstance(bank, (list, dict)):+ for layer_id, layer_bank in bank.items() if isinstance(bank, dict) else enumerate(bank):residency = (HostResidency.PINNED.value if plan is Noneelse plan.residency_for(layer_id)@@ -479,7 +467,6 @@ __all__ = ["LayerCompletionTracker","PinPipeline","alloc_banks",- "alloc_layer_banks","born_pinned_default","pin_banks","read_file_into",diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex a209f9a9..23ba3251 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -102,6 +102,11 @@ _BANK_BYTES_PER_EXPERT = {MARLIN_MAX_CACHE_SIZE = 992+def placeholder_bank(num_experts: int) -> torch.Tensor:+ """The source of a bank whose role a layer's expert kernel does not read (experts of several formats): zero bytes per row."""+ return torch.empty((num_experts, 0), dtype=torch.uint8)++def bank_layouts(sources: dict[str, list[torch.Tensor]], layer_tags: list | None = None) -> list[tuple]:"""Each layer's layout key: its quant tag and every bank's (dtype, row shape). A slot pool holds layers of one key."""num_layers = len(next(iter(sources.values())))@@ -154,6 +159,9 @@ class OffloadMoeCache:# bank layout from the expert kernel (a BankSpec per role); when given it replaces the _BANK_SCHEMAS lookup and the slot cap comes from max_slotslayout: dict | None = Nonemax_slots: int | None = None+ # Per-layer kernel layouts for experts of several formats (replaces ``layout``). The banks are the union of their roles;+ # a layer whose kernel reads no such role carries a zero-byte [num_experts, 0] uint8 placeholder (see placeholder_bank).+ layer_layouts: list[dict] | None = None# Each layer belongs to one pool: a contiguous slot range with its own LRU scope; pool p owns slots [sum(pool_sizes[:p]), +pool_sizes[p]). Slot ids in slot_for_id are local to the pool. None = one pool for all layers.pool_of_layer: list[int] | None = Nonepool_sizes: list[int] | None = None@@ -162,7 +170,12 @@ class OffloadMoeCache:policy_ids = {"lru": 0}assert self.cache_policy in policy_idsassert self.decode_target in ("gpu", "cpu", "hybrid"), self.decode_target- if self.layout is None:+ if self.layer_layouts is not None:+ assert self.layout is None, "pass layout or layer_layouts, not both"+ assert len(self.layer_layouts) == self.num_layers, (len(self.layer_layouts), self.num_layers)+ if any(spec.resident for layout in self.layer_layouts for spec in layout.values()):+ raise ValueError("experts of several formats cannot use a kernel with GPU-resident alphas (marlin, b12x)")+ elif self.layout is None:assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"if self.pool_sizes is None:self.pool_of_layer = [0] * self.num_layers@@ -231,7 +244,11 @@ class OffloadMoeCache:# carry independent host attributes -- see layer_residency) and their GPU# slot caches, keyed by the format's bank schema (attached by# set_bank_sources). The GPU slot cache stays one unified pool per bank.- if self.layout is not None:+ if self.layer_layouts is not None:+ self.bank_schema = tuple(dict.fromkeys(+ role for layout in self.layer_layouts for role, spec in layout.items() if not spec.resident+ ))+ elif self.layout is not None:self.bank_schema = tuple(role for role, spec in self.layout.items() if not spec.resident)else:self.bank_schema = _BANK_SCHEMAS[self.quant_format]@@ -371,9 +388,9 @@ class OffloadMoeCache:for s, r, d in zip(self.pool_sizes[:-1], row, pad):offsets[-1].append(offsets[-1][-1] + s * r + d)self._pool_offsets = offsets- # local slots below this index overlap the prefill staging area in some bank+ # local slots below this index overlap the prefill staging area in some bank; a pool's zero-byte rows (a role its kernel does not read) overlap nothingself._pool_stable_from = [- max(min(max(-(-(2 * stage[k] - offsets[k][p]) // rows[k][p]), 0), s) for k in range(len(rows)))+ max((min(max(-(-(2 * stage[k] - offsets[k][p]) // rows[k][p]), 0), s) for k in range(len(rows)) if rows[k][p]), default=0)for p, s in enumerate(self.pool_sizes)]self._inval = []@@ -382,6 +399,8 @@ class OffloadMoeCache:for k in range(len(rows)):lo, hi = b * stage[k], (b + 1) * stage[k]for p, (base, s) in enumerate(spans):+ if not rows[k][p]:+ continuefirst = max((lo - offsets[k][p]) // rows[k][p], 0)end = min(-(-(hi - offsets[k][p]) // rows[k][p]), s)mask[base + first : base + max(first, end)] = True@@ -488,8 +507,12 @@ class OffloadMoeCache:for layer_id, source in enumerate(per_layer):assert source.is_contiguous(), f"bank {name!r} layer {layer_id} must be contiguous"assert source.size(0) == self.num_experts, (name, layer_id, source.shape)- if self.layout is not None:- spec = self.layout[name]+ spec = self.layout[name] if self.layout is not None else None+ if self.layer_layouts is not None:+ spec = self.layer_layouts[layer_id].get(name)+ if spec is None and source.numel() != 0:+ raise ValueError(f"layer {layer_id}'s expert kernel reads no {name!r} bank, so it takes a zero-byte placeholder")+ if spec is not None:if tuple(source.shape[1:]) != tuple(spec.shape) or source.dtype != spec.dtype:raise ValueError(f"bank {name!r} rows are {tuple(source.shape[1:])} {source.dtype} but the expert kernel's layout "@@ -586,6 +609,9 @@ class OffloadMoeCache:# a 0 placeholder keeps the descriptor shapelayer_src_ptrs[layer_id].append(0)continue+ if source.numel() == 0:+ layer_src_ptrs[layer_id].append(0) # placeholder bank: zero bytes per row, never dereferenced+ continue# The kernel dereferences these on the GPU, so store each host bank's# device alias (== data_ptr() under UVA identity; differs on# Windows/WDDM).@@ -980,6 +1006,8 @@ class OffloadMoeCache:dst, src, nbytes = [], [], []for b, feat in enumerate(self._row_bytes[layout]):stage = self._copy_dst_ptrs_host[b] + buffer_id * self._stage_bytes[b]+ if feat == 0:+ continueif feat < _SMALL_BANK_FEAT_BYTES:# Whole layer as one entry, EVEN with zero misses: it keeps every# batch entry above the driver's async floor and covers the hit@@ -1229,6 +1257,8 @@ class OffloadMoeCache:views = self.stage_views(layer_id, 0) if whole else self.pool_bank_views(layer_id)for (per_layer, _), cache in zip(self.banks, views):+ if cache.numel() == 0:+ continuefast_index_copy_jit(cache,self.evict_slots,diff --git a/tests/models/test_models_loader.py b/tests/models/test_models_loader.pyindex b0ac2b13..4f634c4c 100644--- a/tests/models/test_models_loader.py+++ b/tests/models/test_models_loader.py@@ -201,3 +201,22 @@ def test_stacked_expert_pieces_pair_each_layer_in_arrival_order():assert torch.equal(pieces[1][3]["gate_up"], torch.full((2, 3, 4), 2.0))with pytest.raises(ValueError, match="Missing MoE expert source layers"):list(stacked_expert_pieces(tensors[:3], config))+++def test_stacked_expert_pieces_skip_the_layers_of_other_formats():+ from freetoken.moe.expert_pieces import stacked_expert_pieces++ config = SimpleNamespace(num_layers=3, num_experts=2)+ tensors = [+ ("model.layers.0.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 2.0)),+ ("model.layers.1.mlp.experts.0.gate_proj.weight_scale", torch.zeros(3, 1)),+ ("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),+ ("model.layers.2.mlp.experts.gate_up_proj", torch.full((2, 3, 4), 3.0)),+ ("model.layers.2.mlp.experts.down_proj", torch.full((2, 4, 3), 11.0)),+ ]++ pieces = list(stacked_expert_pieces(tensors, config, layers=[0]))++ assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(0, 0, 2)]+ with pytest.raises(ValueError, match="Unexpected expert weight key"):+ list(stacked_expert_pieces(tensors, config))diff --git a/tests/models/test_qwen3_5_moe_weight.py b/tests/models/test_qwen3_5_moe_weight.pyindex 158f1f45..16f439a9 100644--- a/tests/models/test_qwen3_5_moe_weight.py+++ b/tests/models/test_qwen3_5_moe_weight.py@@ -507,6 +507,8 @@ def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):spec = nvfp4_expert_spec(folder, config)pieces = list(iter_nvfp4_expert_pieces(folder, config, spec))assert len(pieces) == 2 * E+ only = list(iter_nvfp4_expert_pieces(folder, config, spec, layers=[1]))+ assert len(only) == E and {p[0] for p in only} == {1}layer, e0, e1, piece = next(p for p in pieces if p[0] == 1 and p[1] == 2)assert (e0, e1) == (2, 3)base = f"{LM}.layers.1.mlp.experts.2.gate_proj"@@ -529,6 +531,8 @@ def test_block_fp8_expert_pieces_read_either_dialect(checkpoint, parallel):config = parse_config(cached_load_hf_config(folder))pieces = list(iter_expert_pieces(folder, config, QuantKind.FP8_BLOCK, parallel=parallel))assert len(pieces) == 2 * E+ only = list(iter_expert_pieces(folder, config, QuantKind.FP8_BLOCK, layers=[1], parallel=parallel))+ assert len(only) == E and {p[0] for p in only} == {1}layer, e0, e1, piece = next(p for p in pieces if p[0] == 1 and p[1] == 2)base = f"{LM}.layers.1.mlp.experts.2"scale = "weight_scale_inv" if name == "fp8_block" else "weight_scale"diff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.pyindex 38e79b27..56c6c18a 100644--- a/tests/moe/test_cpu_moe.py+++ b/tests/moe/test_cpu_moe.py@@ -391,7 +391,7 @@ def test_cpu_decode_dsfp4_matches_gpu(bs):max_tokens=bs,device=dev,swiglu_limit=swiglu_limit,- act_block=128,+ layer_formats=[("ds_fp4", {"act_block": 128})] * cache.num_layers,)hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)@@ -689,7 +689,7 @@ def test_cpu_moe_decode_cuda_graph_replay_dsfp4():ex = CpuMoeExecutor(cache, top_k=top_k, activation="silu",apply_router_weight_on_input=False, num_threads=8, max_tokens=bs, device=dev,- swiglu_limit=limit, act_block=128,+ swiglu_limit=limit, layer_formats=[("ds_fp4", {"act_block": 128})] * cache.num_layers,)b = cache.bank_sourceslayer_banks = [b[n][layer].to(dev) for n in (diff --git a/tests/moe/test_dsfp4_prequant.py b/tests/moe/test_dsfp4_prequant.pyindex 8187402d..1212c166 100644--- a/tests/moe/test_dsfp4_prequant.py+++ b/tests/moe/test_dsfp4_prequant.py@@ -49,7 +49,7 @@ def test_dsfp4_gpu_prequant_bit_parity(monkeypatch):monkeypatch.setattr(cpu_executor, "_DSFP4_GPU_PREQUANT", not cpu_roundtrip)ex = CpuMoeExecutor(cache, top_k=TOPK, activation="silu", apply_router_weight_on_input=False,- num_threads=4, max_tokens=2, device=dev, swiglu_limit=10.0, act_block=128,+ num_threads=4, max_tokens=2, device=dev, swiglu_limit=10.0, layer_formats=[("ds_fp4", {"act_block": 128})] * cache.num_layers,)assert ex._gpu_prequant != cpu_roundtrip, "ds_fp4 on CUDA should enable GPU prequant"out = ex.decode(0, x, wts, ids.clone()).clone()diff --git a/tests/moe/test_dsfp4_reference_semantics.py b/tests/moe/test_dsfp4_reference_semantics.pyindex 2f89cf33..f537b248 100644--- a/tests/moe/test_dsfp4_reference_semantics.py+++ b/tests/moe/test_dsfp4_reference_semantics.py@@ -105,7 +105,7 @@ def test_cpu_executor_matches_the_transcription(block):cache = SimpleNamespace(quant_format="ds_fp4", bank_sources={k: [v] for k, v in pinned.items()}, num_layers=1, num_experts=E)dev = torch.device("cuda", 0)x, slots, w = _inputs(2, dev)- ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=dev, swiglu_limit=LIMIT, act_block=block)+ ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=dev, swiglu_limit=LIMIT, layer_formats=[("ds_fp4", {"act_block": block})] * cache.num_layers)got = ex.decode(0, x, w, slots.clone()).clone().float()torch.cuda.synchronize()del ex@@ -162,7 +162,7 @@ def test_inactive_routes_contribute_zero_without_reading_their_slots(block):buf.copy_(t.cpu())pinned[name] = bufcache = SimpleNamespace(quant_format="ds_fp4", bank_sources={k: [v] for k, v in pinned.items()}, num_layers=1, num_experts=E)- ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=torch.device("cuda", 0), swiglu_limit=LIMIT, act_block=block)+ ex = CpuMoeExecutor(cache, top_k=TOP_K, activation="silu", apply_router_weight_on_input=False, num_threads=4, max_tokens=2, device=torch.device("cuda", 0), swiglu_limit=LIMIT, layer_formats=[("ds_fp4", {"act_block": block})] * cache.num_layers)cpu_ids = torch.where(on_gpu, -1, slots)cpu_out = ex.decode(0, x, w, cpu_ids.clone()).clone()torch.cuda.synchronize()diff --git a/tests/moe/test_mixed_expert_formats.py b/tests/moe/test_mixed_expert_formats.pynew file mode 100644index 00000000..c9301dc1--- /dev/null+++ b/tests/moe/test_mixed_expert_formats.py@@ -0,0 +1,288 @@+"""Experts of several formats in one model: bf16 layers next to NVFP4 layers.++A checkpoint may leave some MoE layers unquantized (Muse Spark keeps its first and last layer's+experts bf16). Each format fills its own layers' banks, the banks merge over the union of roles with+zero-byte placeholders, the slot cache keeps one pool per format, the FTW stores each layer's own+banks, and the CPU executor computes each layer in its format.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++from freetoken.distributed import set_tp_info, try_get_tp_info++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++E, H, I, TOPK = 8, 256, 128, 2+FORMATS = ["bf16", "nvfp4", "nvfp4", "nvfp4", "bf16"]+L = len(FORMATS)++_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0])+++def _init_tp():+ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+++def _layer(layer_id: int, fmt: str):+ from freetoken.layers.moe import OffloadMoELayer+ from freetoken.layers.quantization import NoQuantConfig, QuantBackend, QuantConfig, set_quant_backend++ _init_tp()+ if fmt == "bf16":+ quant = NoQuantConfig()+ else:+ set_quant_backend(QuantBackend.parse("moe.nvfp4=triton"))+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})+ return OffloadMoELayer(layer_id, E, TOPK, H, I, quant_config=quant, prefix=f"model.layers.{layer_id}.mlp.experts")+++def _dequant(packed, scale, row_global):+ n, k2 = packed.shape+ codes = torch.stack([packed & 0xF, packed >> 4], dim=-1).view(n, 2 * k2).long()+ return _E2M1.to(packed.device)[codes] * scale.float().repeat_interleave(16, dim=1) * row_global.float().unsqueeze(1)+++def _pieces(seed: int = 0):+ """The checkpoint side: per layer, bf16 or ModelOpt NVFP4 pieces in the layer's format."""+ g = torch.Generator().manual_seed(seed)+ pieces = {}+ for layer, fmt in enumerate(FORMATS):+ if fmt == "bf16":+ pieces[layer] = {+ "gate_up": (torch.randn(E, 2 * I, H, generator=g) / 16).to(torch.bfloat16),+ "down": (torch.randn(E, H, I, generator=g) / 16).to(torch.bfloat16),+ }+ else:+ pieces[layer] = {+ "gate_up": torch.randint(0, 256, (E, 2 * I, H // 2), dtype=torch.uint8, generator=g),+ "gate_up_scale": (torch.rand(E, 2 * I, H // 16, generator=g) + 0.25).to(torch.float8_e4m3fn),+ "gate_up_global": torch.full((E, 2 * I), 0.02, dtype=torch.float16),+ "down": torch.randint(0, 256, (E, H, I // 2), dtype=torch.uint8, generator=g),+ "down_scale": (torch.rand(E, H, I // 16, generator=g) + 0.25).to(torch.float8_e4m3fn),+ "down_global": torch.full((E, H), 0.02, dtype=torch.float16),+ }+ return pieces+++def _reader(pieces, kind):+ """A family reader for one storage form: it yields only the layers stored that way."""+ from freetoken.layers.quantization import QuantKind++ want = "bf16" if kind is QuantKind.NONE else "nvfp4"+ for layer, fmt in enumerate(FORMATS):+ if fmt == want:+ yield layer, 0, E, pieces[layer]+++def _weights(pieces, layer):+ """Dense fp32 (gate_up [E, 2I, H], down [E, H, I]) of one layer, the reference."""+ p = pieces[layer]+ if FORMATS[layer] == "bf16":+ return p["gate_up"].float(), p["down"].float()+ gu = torch.stack([_dequant(p["gate_up"][e], p["gate_up_scale"][e], p["gate_up_global"][e]) for e in range(E)])+ dn = torch.stack([_dequant(p["down"][e], p["down_scale"][e], p["down_global"][e]) for e in range(E)])+ return gu, dn+++def _reference(pieces, layer, hidden, weights, ids):+ gu, dn = _weights(pieces, layer)+ out = torch.zeros(hidden.shape, dtype=torch.float32)+ x = hidden.float().cpu()+ for t in range(hidden.shape[0]):+ for k in range(ids.shape[1]):+ e = int(ids[t, k])+ h = gu[e] @ x[t]+ out[t] += float(weights[t, k]) * (dn[e] @ (torch.nn.functional.silu(h[:I]) * h[I:]))+ return out+++def _assert_close(out, ref):+ torch.testing.assert_close(out.float().cpu(), ref, rtol=3e-2, atol=0.03 * float(ref.abs().max()))+++def _mixed_banks(pieces, device, layer_sink=None):+ from freetoken.moe.expert_banks import build_expert_banks, merge_expert_banks, method_groups++ methods = [_layer(layer, fmt).quant_method for layer, fmt in enumerate(FORMATS)]+ parts = [+ build_expert_banks(method, L, _reader(pieces, method.kind), device=device, layer_sink=layer_sink, layers=layers)+ for method, layers in method_groups(methods)+ ]+ return merge_expert_banks(parts, methods, E), methods+++def _mixed_cache(device, pieces, *, decode_target="gpu", prefill_overlap=False):+ from freetoken.moe.offload_cache import OffloadMoeCache, bank_layouts+ from freetoken.moe.pool_policy import group_layers, make_pools, pool_of_layer, split_slots+ from freetoken.engine.cache_budget import layer_slot_bytes++ banks, methods = _mixed_banks(pieces, device)+ slot_bytes = layer_slot_bytes(banks.sources)+ groups = group_layers("unified", bank_layouts(banks.sources, banks.layer_tags), slot_bytes)+ pools = make_pools(groups, E, TOPK, None)+ sizes = split_slots(4 * E, pools)+ cache = OffloadMoeCache(+ num_layers=L, num_experts=E, cache_size=sum(sizes), device=device, quant_format=banks.quant_format,+ decode_target=decode_target, hybrid_max_fetch=1, prefill_overlap=prefill_overlap,+ layer_layouts=[m.layout() for m in methods], pool_of_layer=pool_of_layer(pools, L), pool_sizes=sizes,+ )+ cache.set_bank_sources(banks.sources, layer_tags=banks.layer_tags)+ cache.reset()+ layers = [_layer(layer, fmt) for layer, fmt in enumerate(FORMATS)]+ for layer in layers:+ layer.offload_cache = cache+ return cache, layers, banks+++def test_engine_binds_the_most_common_format_and_checks_every_layer():+ from freetoken.engine.engine import _check_layer_formats, offload_layer_methods, shared_offload_method+ from freetoken.layers.base import BaseOP+ from freetoken.layers.quantization import QuantKind+ from freetoken.moe.expert_banks import ExpertBanks++ class Model(BaseOP):+ def __init__(self):+ self.layers = [_layer(layer, fmt) for layer, fmt in reversed(list(enumerate(FORMATS)))]++ model = Model()+ methods = offload_layer_methods(model)+ assert [m.kind for m in methods] == [QuantKind.NONE if f == "bf16" else QuantKind.NVFP4 for f in FORMATS]+ assert shared_offload_method(model).kind is QuantKind.NVFP4+ _check_layer_formats(ExpertBanks("nvfp4", {}, layer_formats=list(FORMATS)), methods)+ with pytest.raises(ValueError, match=r"MoE layers \[0, 4\]"):+ _check_layer_formats(ExpertBanks("nvfp4", {}), methods)+++def test_mixed_dummy_banks_merge_the_roles_with_placeholders():+ from freetoken.moe.expert_banks import load_expert_banks++ methods = [_layer(layer, fmt).quant_method for layer, fmt in enumerate(FORMATS)]+ config = SimpleNamespace(num_moe_layers=L, num_experts=E)+ banks = load_expert_banks("", config, method=methods[1], device=torch.device("cpu"), dummy=True, layer_methods=methods)+ assert banks.layer_formats == FORMATS and banks.quant_format == "nvfp4"+ assert set(banks.sources) == {"gate_up", "gate_up_scale", "gate_up_global", "down", "down_scale", "down_global"}+ for layer, fmt in enumerate(FORMATS):+ if fmt == "bf16":+ assert banks.sources["gate_up"][layer].shape == (E, 2 * I, H) and banks.sources["gate_up"][layer].dtype == torch.bfloat16+ assert banks.sources["down_scale"][layer].shape == (E, 0)+ else:+ assert banks.sources["gate_up"][layer].shape == (E, 2 * I, H // 2) and banks.sources["gate_up"][layer].dtype == torch.uint8+ with pytest.raises(ValueError, match="several formats"):+ load_expert_banks("", config, method=methods[1], device=torch.device("cpu"), dummy=True,+ layer_methods=methods, layer_residency=["pinned"] * L)+++@cuda+def test_mixed_banks_give_each_reader_its_layers_and_resolve_every_reader_first(monkeypatch):+ from freetoken.moe import expert_pieces+ from freetoken.moe.expert_banks import _mixed_expert_banks++ pieces = _pieces(seed=2)+ methods = [_layer(layer, fmt).quant_method for layer, fmt in enumerate(FORMATS)]+ events = []++ def fake_reader(model_path, config, kind, *, layers=None, parallel=False, workers=8, chunk=0):+ events.append(("resolve", sorted(layers)))++ def read():+ events.append(("read", sorted(layers)))+ for layer in layers:+ yield layer, 0, E, pieces[layer]++ return read()++ monkeypatch.setattr(expert_pieces, "iter_expert_pieces", fake_reader)+ config = SimpleNamespace(num_moe_layers=L, num_experts=E)+ banks = _mixed_expert_banks("", config, methods, torch.device("cpu"), False, False, 8, 0)+ assert sorted(layers for step, layers in events if step == "resolve") == [[0, 4], [1, 2, 3]]+ assert [step for step, _ in events[:2]] == ["resolve", "resolve"]+ assert banks.layer_formats == FORMATS+++@cuda+def test_mixed_layers_decode_and_prefill_through_their_pools():+ torch.manual_seed(0)+ device = torch.device("cuda")+ pieces = _pieces()+ cache, layers, banks = _mixed_cache(device, pieces, prefill_overlap=True)+ assert len(cache.pool_sizes) == 2 and cache._raw_banks+ # a staging buffer holds one bf16 layer (the widest rows); two of them still fit the 13 + 19 slots+ assert cache.prefill_overlap+ hidden = torch.randn(1, H, dtype=torch.bfloat16, device=device) / 4+ weights = torch.rand(1, TOPK, device=device)+ for step in range(6):+ for layer in layers:+ ids = torch.randperm(E)[:TOPK].to(torch.int32).view(1, TOPK)+ out = layer._decode_routed(hidden, weights, ids.clone().to(device))+ _assert_close(out, _reference(pieces, layer.layer_id, hidden, weights, ids))++ M = 16+ hidden = torch.randn(M, H, dtype=torch.bfloat16, device=device) / 4+ ids = torch.stack([torch.randperm(E)[:TOPK] for _ in range(M)]).to(torch.int32)+ weights = torch.rand(M, TOPK, device=device)+ for layer in layers:+ # the bf16 prefill kernel overwrites its input+ out = layer._prefill_routed(hidden.clone(), weights, ids.to(device))+ _assert_close(out, _reference(pieces, layer.layer_id, hidden, weights, ids))+ # decode after the prefill stomped the staging slots must reload, not read another format's bytes+ for layer in layers:+ ids = torch.randperm(E)[:TOPK].to(torch.int32).view(1, TOPK)+ out = layer._decode_routed(hidden[:1], weights[:1], ids.clone().to(device))+ _assert_close(out, _reference(pieces, layer.layer_id, hidden[:1], weights[:1], ids))+++@cuda+def test_cpu_executor_computes_each_layer_in_its_format():+ pytest.importorskip("freetoken.kernel._cpu_moe")+ from freetoken.moe.cpu_executor import CpuMoeExecutor++ torch.manual_seed(1)+ device = torch.device("cuda")+ pieces = _pieces(seed=1)+ cache, layers, _ = _mixed_cache(device, pieces, decode_target="hybrid")+ ex = CpuMoeExecutor(+ cache, top_k=TOPK, activation="silu", apply_router_weight_on_input=False, num_threads=2,+ max_tokens=4, device=device, layer_formats=[(f, {}) for f in FORMATS],+ )+ cache.set_cpu_executor(ex)+ bs = 3+ hidden = torch.randn(bs, H, dtype=torch.bfloat16, device=device) / 4+ weights = torch.rand(bs, TOPK, device=device)+ for layer in range(L):+ ids = torch.stack([torch.randperm(E)[:TOPK] for _ in range(bs)]).to(torch.int32)+ _assert_close(ex.decode(layer, hidden, weights, ids.to(device)), _reference(pieces, layer, hidden, weights, ids))+ # hybrid: capped PCIe fetch on the GPU, the overflow misses on the CPU, per layer format+ for _ in range(3):+ for layer in layers:+ ids = torch.stack([torch.randperm(E)[:TOPK] for _ in range(bs)]).to(torch.int32)+ out = layer._decode_routed(hidden, weights, ids.clone().to(device))+ _assert_close(out, _reference(pieces, layer.layer_id, hidden, weights, ids))+++@cuda+def test_ftw_stores_each_layers_own_banks(tmp_path):+ from freetoken.checkpoint.convert import _ConvertSink+ from freetoken.checkpoint.ftw import FTWWriter, load_ftw_banks++ pieces = _pieces(seed=2)+ reference, _ = _mixed_banks(pieces, torch.device("cuda"))+ writer = FTWWriter(str(tmp_path))+ sink = _ConvertSink(writer, names={"gate_up": "gate_up_packed", "down": "down_packed"})+ banks, _ = _mixed_banks(pieces, torch.device("cuda"), layer_sink=sink)+ assert banks.streamed and sink.num_layers == L+ writer.finalize({"quant_format": banks.quant_format, "layer_quant_formats": banks.layer_formats, "expert_bank_num_layers": L})++ loaded = load_ftw_banks(str(tmp_path), num_layers=L)+ assert loaded.layer_formats == FORMATS and loaded.layer_tags == FORMATS and loaded.quant_format == "nvfp4"+ assert set(loaded.sources) == set(reference.sources)+ for role, per_layer in reference.sources.items():+ for layer, tensor in enumerate(per_layer):+ got = loaded.sources[role][layer]+ assert got.shape == tensor.shape and got.dtype == tensor.dtype, (role, layer)+ assert torch.equal(got.view(torch.uint8), tensor.view(torch.uint8)), (role, layer)