FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
refactor(quant): config, scheme and method layers for quantization (#418)
Xiaoze Fan477c8601b248
diff --git a/benchmarks/bench_load_weight_generic.py b/benchmarks/bench_load_weight_generic.pyindex 0e1905a0..4e4c5fe0 100644--- a/benchmarks/bench_load_weight_generic.py+++ b/benchmarks/bench_load_weight_generic.py@@ -145,7 +145,7 @@ def _model_config(model_path: str):dev = bind_assigned_gpu()torch.zeros(1, device=dev) # init CUDA context (pinning / nvfp4 backend pick)cfg = EngineConfig(model_path=model_path, tp_info=DistributedInfo(0, 1),- dtype=torch.bfloat16, moe_backend="offload")+ dtype=torch.bfloat16, moe_strategy="offload")return cfg.model_configdiff --git a/docs/cli.md b/docs/cli.mdindex ff4af382..37e44335 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -75,15 +75,17 @@ ft serve --model ... --gpu GPU-9e8d7c6b # the same card by UUID (a unique prefi### MoE offload-See [models.md](models.md#moe-backends) for what each backend does.+See [models.md](models.md#moe-strategies) for what each strategy does.| Flag | Default | Meaning ||---|---|---|-| `--moe-backend` | auto | `fused`/`offload`/`cpu`/`hybrid`; auto → offload, or hybrid with a `ft bench bw` profile |-| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family backends) |+| `--moe-strategy` | auto | `fused`/`offload`/`cpu`/`hybrid`; auto → offload, or hybrid with a `ft bench bw` profile. `--moe-backend` is the deprecated old spelling |+| `--quant-backend` | auto | Kernel per quantized layer type, `layer[.kind]=name` entries: `linear=marlin,moe=b12x` or `moe.nvfp4=triton`. A layer-level entry applies to every kind whose table lists the name |+| `--nvfp4-backend` | — | Deprecated: stands in for `--quant-backend moe.nvfp4=<marlin\|b12x\|triton>` (`flashinfer` means b12x); cannot be combined with `--quant-backend` |+| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) || `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts || `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor |-| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, or a fraction) |+| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, nvfp4, mxfp4), so fp8 experts cannot use it || `--moe-hybrid-max-fetch` | auto | With `hybrid`: max experts fetched over PCIe per layer per step; rest computed on CPU || `--moe-prefill-hit-d2d` | off | Prefill: copy cache-hit experts device-side, stream only misses (CUDA >= 13) || `--disable-moe-prefill-overlap` | overlap on | Disable the two-buffer prefill copy overlap |@@ -146,7 +148,7 @@ environment so the agent cannot silently fall back to a paid endpoint.## ft checkpoint```bash-ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8] [--gpu <uuid-or-index>]+ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backend offload] [--quant-backend moe.nvfp4=b12x] [--shard-gib 8] [--gpu <uuid-or-index>]```Converts an HF safetensors checkpoint to FTW, FreeToken's self-contained@@ -164,7 +166,7 @@ ft bench bw --gpu 1 # a specific GPU (UUID or nvidia-smi index, as```Measures host-RAM vs PCIe bandwidth with the real cpu/offload MoE kernels and writes a-profile that `ft serve --moe-backend auto` and `--moe-hybrid-max-fetch -1` then read.+profile that `ft serve --moe-strategy auto` and `--moe-hybrid-max-fetch -1` then read.- One profile per GPU, at `~/.cache/freetoken/benchbw/<gpu-uuid>.json`.- Keyed on expert format + GPU, so a profile from other hardware is ignored rather thandiff --git a/docs/models.md b/docs/models.mdindex c9499163..ca3c1853 100644--- a/docs/models.md+++ b/docs/models.md@@ -1,7 +1,6 @@# Supported models-FreeToken loads HF safetensors checkpoints directly (plus native GGUF for-Gemma-4). The checkpoints below are known-good — the prebuilt kernels are tuned+FreeToken loads HF safetensors checkpoints directly. The checkpoints below are known-good — the prebuilt kernels are tunedfor them; other checkpoints of the same architectures work too.| Model | HF checkpoints |@@ -19,9 +18,9 @@ for them; other checkpoints of the same architectures work too.| MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) || Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) |-## MoE backends+## MoE strategies-`ft serve --moe-backend {auto,fused,offload,cpu,hybrid}`:+`ft serve --moe-strategy {auto,fused,offload,cpu,hybrid}` (`--moe-backend` is the deprecated old spelling):- **fused** — experts resident on GPU (needs the VRAM); never auto-selected.- **offload** — experts live in host RAM, an LRU cache of expert slots on GPU;diff --git a/python/freetoken/checkpoint/__main__.py b/python/freetoken/checkpoint/__main__.pyindex 2786173f..2b7e35f1 100644--- a/python/freetoken/checkpoint/__main__.py+++ b/python/freetoken/checkpoint/__main__.py@@ -1,7 +1,7 @@"""CLI: convert an HF safetensors checkpoint to a FreeToken Weight (FTW) checkpoint.ft checkpoint --model <hf_dir> --out <ftw_dir> \- [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8] [--gpu <uuid-or-index>]+ [--dtype bfloat16] [--moe-backend offload] [--quant-backend moe.nvfp4=b12x] [--shard-gib 8] [--gpu <uuid-or-index>]The output dir is self-contained: point the server's ``--model`` at it to load via the FTWfast path (auto-detected).@@ -21,6 +21,16 @@ from .convert import convert_checkpoint_DTYPES = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}+def _parse_quant_backend(value: str) -> str:+ from freetoken.layers.quantization import QuantBackend++ try:+ QuantBackend.parse(value)+ except ValueError as exc:+ raise argparse.ArgumentTypeError(str(exc)) from None+ return value++def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> int:p = argparse.ArgumentParser(prog=prog, description=__doc__)p.add_argument("--model", required=True, help="source HF safetensors checkpoint dir")@@ -28,6 +38,9 @@ def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> ip.add_argument("--dtype", choices=sorted(_DTYPES), default="bfloat16")p.add_argument("--moe-backend", default="offload",help="offload (experts -> banks) or e.g. triton (experts stay dense)")+ p.add_argument("--quant-backend", type=_parse_quant_backend, default=None,+ help="kernel per quantized layer type, as for ft serve; the expert banks are packed for the "+ "chosen MoE kernel and the server picks that kernel back up from the FTW")p.add_argument("--shard-gib", type=float, default=8.0, help="max shard size in GiB")p.add_argument("--gpu", type=single_gpu_arg, default=None,help="GPU for the repack: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or "@@ -46,7 +59,7 @@ def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> it = time.perf_counter()index = convert_checkpoint(ns.model, ns.out, dtype=_DTYPES[ns.dtype],- moe_backend=ns.moe_backend, shard_limit=shard_limit, device=device,+ moe_backend=ns.moe_backend, quant_backend=ns.quant_backend, shard_limit=shard_limit, device=device,)dt = time.perf_counter() - tc = index["counts"]diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.pyindex 420faf2c..9ead434f 100644--- a/python/freetoken/checkpoint/convert.py+++ b/python/freetoken/checkpoint/convert.py@@ -119,9 +119,11 @@ class _ConvertSink:under one lock (disk-bound anyway)."""- def __init__(self, writer: FTWWriter, desc: str = "Converting expert banks") -> None:+ def __init__(self, writer: FTWWriter, desc: str = "Converting expert banks", *, names: dict[str, str] | None = None) -> None:self._writer = writerself._desc = desc+ # canonical role -> the bank name the file stores+ self._names = names or {}self._bar = Noneself._lock = threading.Lock()self._seen: set[int] = set()@@ -139,7 +141,7 @@ class _ConvertSink:nbytes = 0for bank_name, bank in banks.items():self._writer.add_tensor(- layer_bank_entry_name(bank_name, layer_id), bank.tensor, kind="experts_bank"+ layer_bank_entry_name(self._names.get(bank_name, bank_name), layer_id), bank.tensor, kind="experts_bank")nbytes += bank.nbytesbank.release()@@ -166,6 +168,7 @@ def convert_checkpoint(*,dtype: torch.dtype = torch.bfloat16,moe_backend: str = "offload",+ quant_backend: str | None = None,shard_limit: int = DEFAULT_SHARD_LIMIT,device: str | None = None,) -> dict:@@ -195,10 +198,17 @@ def convert_checkpoint(torch.zeros(1, device=dev) # init CUDA context (needed by nvfp4 backend pick / pinning)cfg = EngineConfig(model_path=model_path, tp_info=DistributedInfo(tp.rank, tp.size),- dtype=dtype, moe_backend=moe_backend)+ dtype=dtype, moe_strategy=moe_backend, quant_backend=quant_backend)mc = cfg.model_configoffload = moe_backend == "offload" and getattr(mc, "is_moe", False)include_moe_experts = not offload+ method = None+ if offload:+ from freetoken.engine.engine import offload_expert_method+ from freetoken.layers import set_rope_device++ set_rope_device(dev)+ method = offload_expert_method(cfg)from freetoken.utils.progress import byte_bar, count_bar@@ -220,14 +230,14 @@ def convert_checkpoint(quant_format = Nonenum_layers = Noneif offload:- # Streamable formats (bf16, ds_fp4, nvfp4 on the triton backend, gpt-oss mxfp4, q4_0,- # qwen3_5 fp8/bf16-dequant) write each layer to its own FTW entry as it completes (via- # the sink) instead of materializing the whole bank set first; the non-streamable ones- # (nvfp4 marlin/b12x -- repack mutates the whole bank set in place after load) ignore- # the sink. Which happened is per-provider (e.g. nvfp4's backend pick), so it's read- # back from ExpertBanks.streamed, not guessed here.- sink = _ConvertSink(writer)- banks = load_expert_banks(model_path, mc, device=dev, dtype=dtype, layer_sink=sink)+ # every method-packed format streams each layer to its own FTW entry as it completes (via the sink); the GGUF provider reports through ExpertBanks.streamed whether it engaged the sink or materialized the whole bank set first+ from freetoken.moe.legacy_format import legacy_bank_names, legacy_format_for++ names = legacy_bank_names(legacy_format_for(method.kind, method.kernel.name)) if method is not None else {}+ sink = _ConvertSink(writer, names=names)+ banks = load_expert_banks(+ model_path, mc, method=method, device=dev, dtype=dtype, layer_sink=sink+ )quant_format = banks.quant_formatif banks.streamed:sink.close()@@ -251,12 +261,13 @@ def convert_checkpoint(# has whole-tensor add_tensor, so the per-layer sources reassemble into one# flat tensor (a per-bank host RAM spike during conversion).items = []+ names = legacy_bank_names(quant_format)for name, per_layer in banks.sources.items():if num_layers is None:num_layers = len(per_layer)else:assert len(per_layer) == num_layers, (name, len(per_layer), num_layers)- items.append((name, torch.cat(per_layer, dim=0) if len(per_layer) > 1 else per_layer[0]))+ items.append((names.get(name, name), torch.cat(per_layer, dim=0) if len(per_layer) > 1 else per_layer[0]))for an in ("gate_up_alpha", "down_alpha"):if getattr(banks, an, None) is not None:items.append((an, getattr(banks, an)))diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.pyindex a365e3d5..207765b7 100644--- a/python/freetoken/checkpoint/ftw.py+++ b/python/freetoken/checkpoint/ftw.py@@ -106,6 +106,14 @@ def is_ftw_checkpoint(path: str) -> bool:return os.path.isfile(os.path.join(path, INDEX_NAME))+def ftw_quant_format(path: str) -> str | None:+ """The ``quant_format`` an FTW checkpoint's expert banks were packed for; None when ``path`` is not an FTW checkpoint or holds no banks."""+ if not is_ftw_checkpoint(path):+ return None+ with open(os.path.join(path, INDEX_NAME)) as f:+ return json.load(f).get("quant_format")++# ============================== writer ==============================class FTWWriter:"""Stream tensors into the FTW, rolling shard files at ``shard_limit``.@@ -384,7 +392,8 @@ def iter_ftw_weights(path: str, *, kinds=("weight",), workers: int = 8,reader.read_into(memoryview(buf), e, workers=workers, chunk=chunk)dt = _dtype_of(e["dtype"])t = torch.frombuffer(buf, dtype=dt, count=e["nbytes"] // _elsize(dt))- if not _put((e["name"], t.view(*e["shape"]) if e["shape"] else t, buf, e["nbytes"])):+ # a 0-d entry (a per-tensor scale) comes back 0-d, not [1]+ if not _put((e["name"], t.view(*e["shape"]) if e["shape"] else t.view(()), buf, e["nbytes"])):returnexcept BaseException as ex: # surface to consumererr.append(ex)@@ -590,8 +599,14 @@ def load_ftw_banks(views.append(raw.view(num_experts, *row_shape) if row_shape else raw.view(num_experts))sources[name] = views+ from freetoken.moe.legacy_format import canonical_role, kind_kernel_forfrom freetoken.moe.expert_banks import ExpertBanks+ # the file names the banks the legacy way; the quant_format tag names the (kind, kernel) they were packed for+ sources = {canonical_role(name): views for name, views in sources.items()}+ quant_format = reader.meta("quant_format")+ kind, kernel = kind_kernel_for(quant_format) if quant_format is not None else (None, None)+# a failed mlock leaves a LOCKED layer pageable; the log and labels report what the banks actually settled atapplied = list(residency)for banks in row_hb.values():@@ -628,8 +643,8 @@ def load_ftw_banks(# (not a separate kind); everything else under experts_bank is a weight source.alpha_kw = {n: alpha_hb[n].tensor for n in alpha_hb}return ExpertBanks(- reader.meta("quant_format"), sources, **alpha_kw,- layer_residency=applied,+ quant_format, sources, **alpha_kw,+ layer_residency=applied, kind=kind, kernel=kernel,)diff --git a/python/freetoken/core.py b/python/freetoken/core.pyindex ef0a539c..281ff41a 100644--- a/python/freetoken/core.py+++ b/python/freetoken/core.py@@ -11,7 +11,6 @@ if TYPE_CHECKING:from freetoken.attention.linear import FLAMetadatafrom freetoken.kvcache import BaseCacheHandle, BaseKVCachePoolfrom freetoken.kvcache.linear_state_pool import LinearStatePool- from freetoken.moe import BaseMoeBackendfrom freetoken.moe.offload_cache import OffloadMoeCache@@ -170,7 +169,6 @@ class Context:# NOTE: this table always treat page_size = 1page_table: torch.Tensor = field(init=False)attn_backend: BaseAttnBackend = field(init=False)- moe_backend: BaseMoeBackend = field(init=False)moe_offload_cache: OffloadMoeCache | None = Nonekv_cache: BaseKVCachePool = field(init=False)# Per-request recurrent state for GatedDeltaNet layers; set by the engine fordiff --git a/python/freetoken/engine/cache_budget.py b/python/freetoken/engine/cache_budget.pyindex ab7c0a9f..d522be49 100644--- a/python/freetoken/engine/cache_budget.py+++ b/python/freetoken/engine/cache_budget.py@@ -108,16 +108,18 @@ def resolve_moe_cache_auto(prefill_overlap: bool,kv_reserve_tokens: int,page_size: int,- quant_format: str,+ max_slots: int | None = None,) -> tuple[int, int, bool]:"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).+ ``max_slots`` is the expert kernel's addressable slot limit; the plan never exceeds it.+Applies memory_ratio to the persisted pre-model baseline exactly once, then defersthe MoE-vs-KV split to plan_cache_budget. The (1-memory_ratio) remainder is theCUDA-graph/activation headroom (not subtracted here)."""budget_bytes = net_cache_budget_bytes(memory_ratio, baseline_free, weights_bytes, fixed_cache_size)- max_slots = 992 if quant_format == "nvfp4_marlin" else total_experts+ max_slots = total_experts if max_slots is None else min(max_slots, total_experts)kv_reserve_pages = div_ceil(kv_reserve_tokens, page_size)return plan_cache_budget(budget_bytes=budget_bytes,diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.pyindex bcbe6bcf..28d9336a 100644--- a/python/freetoken/engine/config.py+++ b/python/freetoken/engine/config.py@@ -1,17 +1,20 @@from __future__ import annotations-from dataclasses import dataclass+from dataclasses import dataclass, field, replacefrom functools import cached_property-from typing import TYPE_CHECKING, List+from typing import TYPE_CHECKING, Any, Listimport torchfrom freetoken.distributed import DistributedInfofrom freetoken.models.register import _load_attr, get_model_spec-from freetoken.utils import cached_load_hf_config+from freetoken.utils import cached_load_hf_config, init_logger+from freetoken.utils.hf import optional_hf_fileif TYPE_CHECKING:from freetoken.models import ModelConfig+logger = init_logger(__name__)+@dataclass(frozen=True)class EngineConfig:@@ -20,9 +23,11 @@ class EngineConfig:dtype: torch.dtypemax_running_req: int = 4attention_backend: str = "auto"- moe_backend: str = "auto"- # NVFP4 routed-expert GEMM backend (--nvfp4-backend): auto|marlin|flashinfer|triton.- nvfp4_backend: str = "triton"+ moe_strategy: str = "auto"+ # old name of moe_strategy; __post_init__ folds it in+ moe_backend: str | None = field(default=None, repr=False)+ # --quant-backend: layer[.kind]=kernel entries, comma separated+ quant_backend: str | None = None# PLE table backend: "disk" (default) reads rows from the checkpoint files per fill, "pinned" preloads the table into page-locked host RAM.ple_backend: str = "disk"# Expert-bank host load (--expert-load): auto|serial|parallel. "auto" reads scattered@@ -41,16 +46,16 @@ class EngineConfig:# (cudaMemcpyBatchAsync); no-op unless moe_cache_size > 2 * num_experts.moe_prefill_hit_d2d: bool = Falsemoe_collect_stats: bool = False # capture decode miss-rate counters into the cuda graph- # CPU MoE backend (--moe-backend cpu): number of CPU worker threads computing+ # CPU MoE backend (--moe-strategy cpu): number of CPU worker threads computing# the decode experts. 0 = auto (physical cores). Ignored by other backends.moe_cpu_threads: int = 0- # Hybrid CPU/GPU decode (--moe-backend offload only): which MoE layers decode on+ # Hybrid CPU/GPU decode (--moe-strategy offload only): which MoE layers decode on# the CPU executor instead of the GPU offload/PCIe path. Spec is an explicit id# list ("3,7,11"), a count ("8" -> 8 layers evenly strided across depth), or a- # fraction ("0.5"). None/"" = all layers on GPU (plain offload). --moe-backend cpu+ # fraction ("0.5"). None/"" = all layers on GPU (plain offload). --moe-strategy cpu# already means all layers on CPU and ignores this.moe_cpu_layers: str | None = None- # Hybrid MoE backend (--moe-backend hybrid): max experts fetched over PCIe per+ # Hybrid MoE backend (--moe-strategy hybrid): max experts fetched over PCIe per# (layer, decode step); the rest of that step's misses are computed on the CPU.# -1 (default) = auto: fetch the benched pcie_bw/cpu_bw fraction of each step's# misses so the PCIe fetch and the CPU compute finish together (perfect overlap);@@ -83,6 +88,15 @@ class EngineConfig:# is final. Mutually exclusive with num_page_override.num_token_override: int | None = None+ def __post_init__(self):+ if self.moe_backend is None:+ return+ if self.moe_strategy != "auto":+ raise ValueError("moe_backend is the old name of moe_strategy; pass only moe_strategy")+ logger.warning("EngineConfig.moe_backend is deprecated; use moe_strategy")+ object.__setattr__(self, "moe_strategy", self.moe_backend)+ object.__setattr__(self, "moe_backend", None)+@cached_propertydef hf_config(self):return cached_load_hf_config(self.model_path)@@ -91,7 +105,8 @@ class EngineConfig:def model_config(self) -> ModelConfig:spec = get_model_spec(self.hf_config.architectures[0])parse_config = _load_attr(spec.module, spec.parse_config)- return parse_config(self.hf_config)+ model_config = parse_config(self.hf_config)+ return replace(model_config, quant=checkpoint_quant_config(self.model_path, self.hf_config, spec))@propertydef max_seq_len(self) -> int:@@ -106,3 +121,25 @@ class EngineConfig:@propertydef distributed_addr(self) -> str:return "tcp://127.0.0.1:2333"+++def checkpoint_quant_config(model_path: str, hf_config: Any, spec: Any):+ """The checkpoint's QuantConfig under the family's naming, or None for GGUF, whose native-quant ops the shared parser does not model yet."""+ from freetoken.layers.quantization import NameMap, QuantConfig++ if spec.parse_config == "parse_gguf_config":+ return None+ # NOTE: ModelOpt exports before 0.41 keep the quantization config only in hf_quant_config.json, and the weight download fetches nothing but the safetensors shards, so this sidecar is fetched on its own.+ hf_quant_config = None+ sidecar = optional_hf_file(model_path, "hf_quant_config.json")+ if sidecar is not None:+ import json++ with open(sidecar) as f:+ hf_quant_config = json.load(f)+ return QuantConfig.from_hf(+ hf_config,+ name_map=NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping),+ unquantized=spec.unquantized_modules,+ hf_quant_config=hf_quant_config,+ )diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex b5a6fa3b..006c1908 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -12,9 +12,12 @@ from freetoken.core import Batch, Context, Req, set_global_ctxfrom freetoken.distributed import destroy_distributed, enable_pynccl_distributed, set_tp_infofrom freetoken.gpu_select import gpu_identityfrom freetoken.layers import set_rope_device+from freetoken.layers.quantization import LayerKind, QuantBackend, finalize_quant, set_quant_backend+from freetoken.moe.offload_cache import iter_offload_moe_layersfrom freetoken.models import create_model, load_weight-from freetoken.moe import create_moe_backend, is_offload_moe_backend+from freetoken.moe import is_offload_moe_strategyfrom freetoken.moe.expert_banks import load_expert_banks+from freetoken.moe.host_banks import PinFailedfrom freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cachefrom freetoken.utils import align_ceil, init_logger, is_sm90_family, is_sm100_family, mem_GB, torch_dtype@@ -39,7 +42,7 @@ def _require_offload_cache_size(cache_size: int, num_experts: int) -> None:f"moe_cache_size={cache_size} is too small: need at least num_experts={num_experts} "f"slots. Pass --moe-cache-size/--moe-cache-rate, or use --moe-cache-auto "f"(the default for offload/hybrid backends when no cache-sizing flag is given; "- f"--moe-backend cpu always sizes its own fixed two-layer buffer and ignores "+ f"--moe-strategy cpu always sizes its own fixed two-layer buffer and ignores "f"cache-sizing flags).")@@ -250,7 +253,7 @@ def _make_dummy_weight_state_dict(if param.dtype in fp8_dtypes:# torch.randn is not implemented for fp8; fill via a uint8 view with small# codes (avoid NaN/inf fp8 encodings). Lets dummy-weight startup work for- # block-fp8 models (the dense fp8 linears are fp8 regardless of moe_backend).+ # block-fp8 models (the dense fp8 linears are fp8 regardless of moe_strategy).t = torch.empty(param.shape, dtype=param.dtype, device=device)t.view(torch.uint8).random_(0, 16)state_dict[key] = t@@ -294,6 +297,7 @@ class Engine:def __init__(self, config: EngineConfig):assert not torch.cuda.is_initialized()set_tp_info(rank=config.tp_info.rank, size=config.tp_info.size)+ set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))_ensure_expandable_segments() # before the first CUDA allocation belowfrom freetoken.gpu_select import bind_assigned_gpu@@ -323,6 +327,7 @@ class Engine:with torch.device("meta"), torch_dtype(config.dtype):self.model = create_model(config.model_config)self.model.load_state_dict(self._load_weight_state_dict(config))+ finalize_quant(self.model)post_weights_free = self._sync_get_memory()[0]self._weights_bytes = self._baseline_free - post_weights_free# Pool-budget baseline for the desktop cache sliders: free VRAM after the weights are@@ -339,7 +344,7 @@ class Engine:self._host_tables_bytes = 0if hasattr(self.model, "load_host_tables"):self._host_tables_bytes = int(self.model.load_host_tables(config) or 0)- if is_offload_moe_backend(config.moe_backend):+ if is_offload_moe_strategy(config.moe_strategy):self._init_offload_moe_cache(config)if hasattr(self.model, "prepare_for_runtime"):self.model.prepare_for_runtime()@@ -387,12 +392,10 @@ class Engine:# through them belongs to the attention backend, built later in init_capture_graph.self.kv_cache.attach_page_table(self.page_table)- # ======================= Attention & MoE backend initialization ========================+ # ======================= Attention backend initialization ========================self.ctx.attn_backend = self.attn_backend = create_attention_backend(config.attention_backend, config.model_config)- if config.model_config.is_moe:- self.ctx.moe_backend = self.moe_backend = create_moe_backend(config.moe_backend)# ======================= Sampler initialization ========================self.sampler = Sampler(self.device, config.model_config.vocab_size)@@ -470,12 +473,13 @@ class Engine:load_weight(config.model_path,self.device,- include_moe_experts=not is_offload_moe_backend(config.moe_backend),+ include_moe_experts=not is_offload_moe_strategy(config.moe_strategy),),device=self.device,)- def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks) -> tuple[int, int, bool]:++ def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None) -> tuple[int, int, bool]:"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).Pure glue over the Phase-1 budget policy; isolated here so it is unit-testable@@ -499,63 +503,38 @@ class Engine:prefill_overlap=config.moe_prefill_overlap,kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve),page_size=page_tokens,- quant_format=banks.quant_format,+ max_slots=method.slot_limit() if method is not None else None,)def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:- # A model may fully own cache construction via make_offload_moe_cache.- # Otherwise load_expert_banks gives the model module a setup hook first, then- # falls back to per-quant providers, and the engine wires the banks into cache.- cache_factory = getattr(self.model, "make_offload_moe_cache", None)- if cache_factory is not None and config.moe_cache_auto:- raise ValueError(- "--moe-cache-auto is not supported for models with a custom "- "make_offload_moe_cache; pass --moe-cache-size explicitly."- )- # decode_target picks the bank layout + the per-decode mechanism:- # "hybrid" -> GPU-cache + CPU-overflow co-compute, every layer (--moe-backend hybrid);- # "cpu" -> CPU executor for the cpu_layer_ids set (all layers under --moe-backend- # cpu, the --moe-cpu-layers subset under offload);- # "gpu" -> plain GPU offload.- # cpu/hybrid both read experts on the CPU, so banks load in the native (CPU-readable)- # layout; the GPU slot-cache GEMM reads those same native rows. decode_target also- # gates the CPU executor build below.- cpu_layer_ids = _resolve_cpu_layers(config, config.model_config.num_moe_layers)- if (- not cpu_layer_ids- and config.moe_cpu_layers is None- and config.moe_backend in ("offload", "hybrid")- and _pin_budget_bytes(self._host_tables_bytes) is not None- ):- cpu_layer_ids = _auto_cpu_layers(- config, config.model_config.num_moe_layers, reserved=self._host_tables_bytes- )- if config.moe_backend == "hybrid":- decode_target = "hybrid"- elif cpu_layer_ids:- decode_target = "cpu"- else:+ method = shared_offload_method(self.model)+ num_moe_layers = config.model_config.num_moe_layers+ cpu_layer_ids = _resolve_cpu_layers(config, num_moe_layers, reserved=self._host_tables_bytes, method=method)+ _check_pin_budget(config, reserved=self._host_tables_bytes, method=method)+ # the kernels were picked for model_config.decode_target; --moe-cpu-layers auto may still find that every bank fits the pin budget+ decode_target = config.model_config.decode_target+ if decode_target == "cpu" and not cpu_layer_ids:decode_target = "gpu"# split residency: where pinning is quota-capped (_pin_budget_bytes), pin only the GPU layers' banks and mlock the CPU layers'# uncapped hosts keep every bank pinned (CPU decode reads them the same; overlap prefill stays on)- # not applied to plain --moe-backend cpu; all-locked under a cap = --moe-backend offload --moe-cpu-layers 1.0+ # not applied to plain --moe-strategy cpu; all-locked under a cap = --moe-strategy offload --moe-cpu-layers 1.0split_residency = (bool(cpu_layer_ids)- and config.moe_backend in ("offload", "hybrid")+ and config.moe_strategy in ("offload", "hybrid")and _pin_budget_bytes(self._host_tables_bytes) is not None)- if config.moe_backend == "cpu" and not split_residency:+ if config.moe_strategy == "cpu" and not split_residency:# cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything insteadfrom freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytesbudget = _pin_budget_bytes(self._host_tables_bytes)bank_bytes = Noneif budget is not None:- bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)+ bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)if bank_bytes and bank_bytes > budget:split_residency = Truelogger.info_rank0(- f"--moe-backend cpu: banks {bank_bytes / 2**30:.2f} GiB exceed the "+ f"--moe-strategy cpu: banks {bank_bytes / 2**30:.2f} GiB exceed the "f"pin budget; OS-locking all layers instead of pinning")if split_residency and config.moe_prefill_overlap:@@ -565,25 +544,26 @@ class Engine:"(locked layers prefill via synchronous pageable copies)")object.__setattr__(config, "moe_prefill_overlap", False)- if cache_factory is None:- # Fast path: an FTW checkpoint loads its repacked banks directly.- # Slow path: load_expert_banks auto-picks parallel vs serial baseline by- # expert-tensor granularity. Both pin-after-fill.- # --expert-load: serial/parallel force the read; auto (None) lets load_expert_banks- # pick (parallel for scattered experts, with a low-RAM fallback to serial).- expert_parallel = {"serial": False, "parallel": True}.get(config.expert_load, None)- requested_residency = None- if split_residency:- from freetoken.moe.host_banks import HostResidency-- requested_residency = [- HostResidency.LOCKED.value if i in cpu_layer_ids- else HostResidency.PINNED.value- for i in range(config.model_config.num_moe_layers)- ]+ # Fast path: an FTW checkpoint loads its repacked banks directly.+ # Slow path: load_expert_banks auto-picks parallel vs serial baseline by+ # expert-tensor granularity. Both pin-after-fill.+ # --expert-load: serial/parallel force the read; auto (None) lets load_expert_banks+ # pick (parallel for scattered experts, with a low-RAM fallback to serial).+ expert_parallel = {"serial": False, "parallel": True}.get(config.expert_load, None)+ requested_residency = None+ if split_residency:+ from freetoken.moe.host_banks import HostResidency++ requested_residency = [+ HostResidency.LOCKED.value if i in cpu_layer_ids+ else HostResidency.PINNED.value+ for i in range(config.model_config.num_moe_layers)+ ]+ try:banks = load_expert_banks(config.model_path,config.model_config,+ method=method,device=self.device,dtype=self.dtype,dummy=config.use_dummy_weight,@@ -591,54 +571,60 @@ class Engine:decode_target=("cpu" if decode_target in ("cpu", "hybrid") else "gpu"),layer_residency=requested_residency,)- if config.moe_cache_auto:- size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks)- object.__setattr__(config, "moe_cache_size", size)- object.__setattr__(config, "moe_prefill_overlap", overlap)- if config.num_page_override is None:- # Honor the plan's KV half too: MoE slots and KV pages were solved- # against ONE budget (ratio x baseline - weights), so both must come- # from it. Re-solving pages later from a fresh free-memory reading- # double-counts everything allocated since the weights measurement- # (this expert cache, the CPU-executor GPU buffers, allocator- # slack) and goes negative whenever the expert fill is exact --- # a greedy fill leaves no headroom for the measurement delta.- object.__setattr__(config, "num_page_override", pages)- logger.info_rank0(- f"--moe-cache-auto resolved moe_cache_size={size} "- f"num_pages={pages} (prefill_overlap={overlap})"- )- _require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)- cache = OffloadMoeCache(- # Models with leading dense layers (GLM-4) only have experts on the MoE- # layers; num_moe_layers == num_layers when first_k_dense_replace == 0.- num_layers=config.model_config.num_moe_layers,- num_experts=config.model_config.num_experts,- cache_size=config.moe_cache_size,- device=self.device,- cache_policy=config.moe_cache_policy,- prefill_overlap=config.moe_prefill_overlap,- prefill_hit_d2d=config.moe_prefill_hit_d2d,- quant_format=banks.quant_format,- decode_target=decode_target,- hybrid_max_fetch=config.moe_hybrid_max_fetch,+ except PinFailed as exc:+ raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc+ if config.moe_cache_auto:+ size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method)+ object.__setattr__(config, "moe_cache_size", size)+ object.__setattr__(config, "moe_prefill_overlap", overlap)+ if config.num_page_override is None:+ # Honor the plan's KV half too: MoE slots and KV pages were solved+ # against ONE budget (ratio x baseline - weights), so both must come+ # from it. Re-solving pages later from a fresh free-memory reading+ # double-counts everything allocated since the weights measurement+ # (this expert cache, the CPU-executor GPU buffers, allocator+ # slack) and goes negative whenever the expert fill is exact --+ # a greedy fill leaves no headroom for the measurement delta.+ object.__setattr__(config, "num_page_override", pages)+ logger.info_rank0(+ f"--moe-cache-auto resolved moe_cache_size={size} "+ f"num_pages={pages} (prefill_overlap={overlap})")- # before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set- cache.cpu_layer_ids = cpu_layer_ids- cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)- cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)- else:- cache = cache_factory(config, self.device)- cache.decode_target = decode_target- cache.hybrid_max_fetch = config.moe_hybrid_max_fetch- cache.cpu_layer_ids = cpu_layer_ids+ _require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)+ layout = max_slots = None+ if method is not None:+ if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):+ raise ValueError(+ f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "+ f"binds {method.kind} / {method.kernel.name}; reconvert or select that kernel"+ )+ layout = method.layout()+ max_slots = method.slot_limit()+ cache = OffloadMoeCache(+ # Models with leading dense layers (GLM-4) only have experts on the MoE+ # layers; num_moe_layers == num_layers when first_k_dense_replace == 0.+ num_layers=config.model_config.num_moe_layers,+ num_experts=config.model_config.num_experts,+ cache_size=config.moe_cache_size,+ device=self.device,+ cache_policy=config.moe_cache_policy,+ prefill_overlap=config.moe_prefill_overlap,+ prefill_hit_d2d=config.moe_prefill_hit_d2d,+ quant_format=banks.quant_format,+ decode_target=decode_target,+ hybrid_max_fetch=config.moe_hybrid_max_fetch,+ layout=layout,+ max_slots=max_slots,+ )+ # before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set+ cache.cpu_layer_ids = cpu_layer_ids+ cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)+ cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)if decode_target == "hybrid":self._resolve_hybrid_fetch(config, cache)# Must be set before CUDA graph capture so the (device-side) accumulation ops are# captured and re-run on every decode replay.cache.collect_stats = config.moe_collect_stats- # attach_offload_moe_cache walks for OffloadMoELayers, or defers to a model's- # _iter_offload_moe_layers() hook when its MoE blocks are bespoke nn.Modules (DSV4).layers = attach_offload_moe_cache(self.model, cache)assert len(layers) == config.model_config.num_moe_layersif cache.decode_target in ("cpu", "hybrid"):@@ -698,7 +684,6 @@ class Engine:# Decode batches never exceed max_running_req, but CUDA-graph padding can# round a batch up to the largest captured size; cover both.max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, 1)- # gpt-oss mxfp4 carries clamped-swiglu scalars; other formats use the defaults.executor = CpuMoeExecutor(cache,top_k=sample.top_k,@@ -707,8 +692,10 @@ class Engine:num_threads=config.moe_cpu_threads,max_tokens=max_tokens,device=self.device,- swiglu_alpha=getattr(sample, "hidden_act_alpha", 1.702),- swiglu_limit=getattr(sample, "swiglu_limit", None),+ swiglu_alpha=float(sample.alpha),+ swiglu_limit=sample.limit,+ # FIXME: the None branch serves GGUF q4_0 banks, which have no quant method yet; drop it once GGUF joins the quant path+ fmt=sample.quant_method.cpu_format if sample.quant_method is not None else None,)cache.set_cpu_executor(executor)self.cpu_moe_executor = executor@@ -1124,20 +1111,31 @@ def _parse_cpu_layers_spec(spec: str, num_moe_layers: int) -> frozenset[int]:return frozenset(round(i * num_moe_layers / k) for i in range(k))-def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int]:+def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:"""MoE layer ids whose decode runs on the CPU executor.- ``--moe-backend cpu`` -> every layer. ``--moe-backend offload`` + ``--moe-cpu-layers``- -> the parsed subset (the rest stay on the GPU offload/PCIe path). Otherwise none.+ ``--moe-strategy cpu`` -> every layer. ``--moe-strategy offload`` + ``--moe-cpu-layers``+ -> the parsed subset, or the pin-budget pick for ``auto`` (the rest stay on the GPU offload/PCIe path). Otherwise none."""- if config.moe_backend == "cpu":+ if config.moe_strategy == "cpu":return frozenset(range(num_moe_layers))spec = config.moe_cpu_layers- if not spec or not is_offload_moe_backend(config.moe_backend):+ if not spec or not is_offload_moe_strategy(config.moe_strategy):return frozenset()+ if spec.strip() == "auto":+ return _auto_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)return _parse_cpu_layers_spec(spec, num_moe_layers)+def _decode_target(config: EngineConfig) -> str:+ """Where routed experts decode, from the flags alone: hybrid co-compute, the CPU executor for some or all layers, or the GPU slot cache."""+ if config.moe_strategy == "hybrid":+ return "hybrid"+ if config.moe_strategy == "cpu" or (config.moe_cpu_layers and is_offload_moe_strategy(config.moe_strategy)):+ return "cpu"+ return "gpu"++# expert activations the CPU MoE executor supports (csrc ActKind)_CPU_MOE_ACTS = ("silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",@@ -1179,25 +1177,50 @@ def _pin_budget_bytes(reserved: int = 0) -> int | None:return max(0, cap - reserved)-def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, reserved: int = 0) -> frozenset[int]:- """Pick CPU (locked) MoE layers automatically when the banks exceed the pin budget.-- Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""+def _bank_bytes(config: EngineConfig, method=None) -> int | None:from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes- bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)+ return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)+++def _pin_hint(reserved: int) -> str:+ if _pin_budget_bytes(reserved) is None:+ return "the expert banks need more page-locked host RAM than this host has; free host RAM or serve a smaller model"+ return (+ "pass --moe-cpu-layers auto to lock the layers over the pin budget for CPU decode, "+ "or --moe-cpu-layers <count|fraction|ids> to choose them yourself"+ )+++def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None) -> None:+ """Stop a plain offload boot whose banks exceed a known pin budget before any bank is read."""+ if config.moe_cpu_layers or config.moe_strategy not in ("offload", "hybrid"):+ return+ budget = _pin_budget_bytes(reserved)+ bank_bytes = _bank_bytes(config, method) if budget is not None else None+ if bank_bytes and bank_bytes > budget:+ raise ValueError(+ f"expert banks need {bank_bytes / 2**30:.1f} GiB of pinned host RAM but the pin budget is "+ f"{budget / 2**30:.1f} GiB (WSL caps CUDA pinning; FREETOKEN_PIN_BUDGET_GB overrides); {_pin_hint(reserved)}"+ )+++def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:+ """Pick CPU (locked) MoE layers for ``--moe-cpu-layers auto``: none while the banks fit the pin budget.++ Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""+ bank_bytes = _bank_bytes(config, method)if not bank_bytes:return frozenset()budget = _pin_budget_bytes(reserved)if budget is None or bank_bytes <= budget:return frozenset()if not _cpu_moe_executor_viable(config.model_config):- logger.info_rank0(+ raise ValueError(f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB exceed the "f"pin budget {budget / 2**30:.2f} GiB, but the CPU MoE executor cannot "- f"serve this model; keeping every layer pinned on the GPU offload path"+ f"serve this model (see --moe-strategy cpu requirements)")- return frozenset()n = min(num_moe_layers, math.ceil(num_moe_layers * (1 - budget / bank_bytes)))head = (n + 1) // 2ids = frozenset(range(head)) | frozenset(range(num_moe_layers - (n - head), num_moe_layers))@@ -1209,7 +1232,7 @@ def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, reserved: int =return ids-# MoE-only knobs and the value each resolves to on a dense model. moe_backend is handled+# MoE-only knobs and the value each resolves to on a dense model. moe_strategy is handled# separately (its dense value is 'fused', but 'auto' resolves there without a warning)._DENSE_MOE_SETTINGS = {"moe_cache_size": 0,@@ -1224,6 +1247,62 @@ _DENSE_MOE_SETTINGS = {}+def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> QuantBackend:+ """--quant-backend with an FTW checkpoint's packed expert kernel filled in where the flag leaves that table automatic.++ An entry that names another kernel is refused here, before any bank is read."""+ from freetoken.checkpoint.ftw import ftw_quant_format+ from freetoken.moe.legacy_format import kind_kernel_for++ fmt = ftw_quant_format(model_path) if model_path else None+ if fmt is None:+ return quant_backend+ try:+ kind, kernel = kind_kernel_for(fmt)+ except KeyError:+ return quant_backend+ requested = quant_backend.select(LayerKind.MOE, kind)+ if requested == kernel:+ return quant_backend+ if requested != "auto":+ raise ValueError(+ f"the FTW checkpoint's expert banks were packed for {kind} / {kernel} but --quant-backend asks for "+ f"{requested}; drop the entry, or reconvert with ft checkpoint --quant-backend moe.{kind}={requested}"+ )+ return QuantBackend(quant_backend.items + (((LayerKind.MOE, kind), kernel),))+++def shared_offload_method(model):+ """The expert method every offload MoE layer of ``model`` uses, or None for models whose MoE layers carry none (GGUF).++ The offload cache holds one bank layout, so the layers must agree on (kind, kernel)."""+ layers = [l for l in iter_offload_moe_layers(model) if getattr(l, "quant_method", None) is not None]+ if not layers:+ return None+ keys = {(layer.quant_method.kind, layer.quant_method.kernel.name) for layer in layers}+ if len(keys) != 1:+ found = sorted(f"{kind} / {kernel}" for kind, kernel in keys)+ raise ValueError(f"expert layers disagree on format / kernel: {found}; per-layer mixed expert formats are not supported")+ kind, kernel = keys.pop()+ logger.info_rank0(f"MoE experts: {kind} via {kernel}")+ return layers[0].quant_method+++def offload_expert_method(config: EngineConfig):+ """The offload expert method of ``config``'s model, from a meta-device build.++ For tools that load expert banks without an engine (the FTW converter); they pack for GPU decode."""+ from freetoken.models import create_model+ from freetoken.utils.torch_utils import torch_dtype++ set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))+ object.__setattr__(config.model_config, "moe_strategy", config.moe_strategy)+ object.__setattr__(config.model_config, "decode_target", "gpu")+ with torch.device("meta"), torch_dtype(config.dtype):+ model = create_model(config.model_config)+ return shared_offload_method(model)++def _adjust_config(config: EngineConfig):def override(attr: str, value: Any): # this is dangerous, use with cautionobject.__setattr__(config, attr, value)@@ -1248,9 +1327,9 @@ def _adjust_config(config: EngineConfig):for name, dense_value in _DENSE_MOE_SETTINGS.items()if getattr(config, name, dense_value) != dense_value]- if config.moe_backend not in ("auto", "fused"):- dropped.insert(0, f"moe_backend={config.moe_backend!r}")- override("moe_backend", "fused")+ if config.moe_strategy not in ("auto", "fused"):+ dropped.insert(0, f"moe_strategy={config.moe_strategy!r}")+ override("moe_strategy", "fused")for name, dense_value in _DENSE_MOE_SETTINGS.items():override(name, dense_value)if dropped:@@ -1351,12 +1430,12 @@ def _adjust_config(config: EngineConfig):if (is_moeand not _cpu_moe_act_ok- and (config.moe_backend in ("cpu", "hybrid") or config.moe_cpu_layers)+ and (config.moe_strategy in ("cpu", "hybrid") or config.moe_cpu_layers)):asked = (f"--moe-cpu-layers={config.moe_cpu_layers!r}"- if config.moe_backend not in ("cpu", "hybrid")- else f"--moe-backend {config.moe_backend!r}"+ if config.moe_strategy not in ("cpu", "hybrid")+ else f"--moe-strategy {config.moe_strategy!r}")raise ValueError(f"{asked}: the CPU MoE executor does not support this model's expert "@@ -1364,7 +1443,7 @@ def _adjust_config(config: EngineConfig):"and let every layer decode on the GPU offload path instead.")- if is_moe and config.moe_backend == "auto":+ if is_moe and config.moe_strategy == "auto":# A MoE model always defaults to the offload family: experts stream from pinned host# banks into an auto-sized GPU slot cache, which is the only default that serves a model# bigger than the GPU. The resident 'fused' path (bf16 / block-fp8 experts, the two@@ -1409,18 +1488,18 @@ def _adjust_config(config: EngineConfig):logger.info_rank0(f"benchbw profile recommends hybrid for {bench_fmt!r} experts on this GPU")- override("moe_backend", default_backend)- logger.info_rank0(f"Auto-selected MoE backend: {config.moe_backend}")+ override("moe_strategy", default_backend)+ logger.info_rank0(f"Auto-selected MoE strategy: {config.moe_strategy}")if (- is_offload_moe_backend(config.moe_backend)+ is_offload_moe_strategy(config.moe_strategy)and config.moe_cache_size <= 0and config.moe_cache_rate is Noneand not getattr(config, "moe_cache_auto", False)):# args.py's "no sizing flag -> default --moe-cache-auto" only fires when the# backend is already offload-family at *parse* time. A bare `ft serve <FTW MoE- # checkpoint>` (no --moe-backend, no cache flags) still has moe_backend=="auto" at+ # checkpoint>` (no --moe-strategy, no cache flags) still has moe_strategy=="auto" at# parse time -- the auto -> offload/cpu/hybrid resolution above is the first point# the concrete backend is known, so mirror the same default here: no sizing flag# was given, so let the scheduler resolve the cache size from free VRAM instead of@@ -1428,10 +1507,10 @@ def _adjust_config(config: EngineConfig):override("moe_cache_auto", True)logger.info_rank0("No MoE cache sizing flag given; defaulting to --moe-cache-auto for "- f"auto-selected backend {config.moe_backend!r}"+ f"auto-selected strategy {config.moe_strategy!r}")- if is_moe and config.moe_backend == "fused":+ if is_moe and config.moe_strategy == "fused":# An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The# sizing flags no longer redirect the backend, so ignore them here and say so -- the# geometry the user asked for is what runs. Report the flag actually passed: --moe-cache-@@ -1447,13 +1526,13 @@ def _adjust_config(config: EngineConfig):if inert:logger.warning_rank0(f"MoE backend 'fused' keeps its experts resident; ignoring {inert} "- "(use --moe-backend offload to serve experts from a slot cache)"+ "(use --moe-strategy offload to serve experts from a slot cache)")override("moe_cache_size", 0)override("moe_cache_rate", None)override("moe_cache_auto", False)- if is_moe and config.moe_backend == "cpu":+ if is_moe and config.moe_strategy == "cpu":# CPU-compute decode keeps experts in host RAM and computes them on the CPU;# the GPU only holds the two-layer prefill double buffer. So the slot cache is# fixed at exactly two expert layers (prefill overlap requires >= 2*num_experts)@@ -1471,23 +1550,27 @@ def _adjust_config(config: EngineConfig):if (is_moeand expert_quant not in ("none", "fp8_block")- and not is_offload_moe_backend(config.moe_backend)+ and not is_offload_moe_strategy(config.moe_strategy)):raise ValueError(- f"{expert_quant} experts require --moe-backend offload or cpu, "- f"got {config.moe_backend!r}"+ f"{expert_quant} experts require --moe-strategy offload or cpu, "+ f"got {config.moe_strategy!r}")- if is_moe and config.moe_cpu_layers and config.moe_backend not in ("offload", "hybrid"):+ if is_moe and config.moe_cpu_layers and config.moe_strategy not in ("offload", "hybrid"):# the layer split needs the offload host banks + slot cache; 'cpu' already runs every layer on CPU, 'fused' keeps experts resident on the GPU (no host banks)raise ValueError(- "--moe-cpu-layers requires --moe-backend offload or hybrid (got "- f"{config.moe_backend!r}); use --moe-backend cpu to run all layers on CPU"+ "--moe-cpu-layers requires --moe-strategy offload or hybrid (got "+ f"{config.moe_strategy!r}); use --moe-strategy cpu to run all layers on CPU")+ if is_moe and config.moe_cpu_layers and config.moe_cpu_layers.strip() != "auto":+ if not _parse_cpu_layers_spec(config.moe_cpu_layers, model_config.num_moe_layers):+ override("moe_cpu_layers", None)+if is_moe:- object.__setattr__(model_config, "moe_backend", config.moe_backend)- object.__setattr__(model_config, "nvfp4_backend", config.nvfp4_backend)+ object.__setattr__(model_config, "moe_strategy", config.moe_strategy)+ object.__setattr__(model_config, "decode_target", _decode_target(config))# Must stay LAST: page_size is only final here (_adjust_dsv4_config sets P=128, the# TRTLLM block sets 64). Also covers the programmatic LLM(...) path that bypasses parse_args.@@ -1518,7 +1601,7 @@ def _adjust_config(config: EngineConfig):)# The startup ServerArgs dump is the *requested* config, printed in the frontend process- # before any of the resolution above ran -- so "moe_backend='auto'" is all it can say. This+ # before any of the resolution above ran -- so "moe_strategy='auto'" is all it can say. This# is the one line that reports what actually runs, for every path (explicit backends never# hit an "Auto-selected ..." log at all).resolved = [@@ -1527,5 +1610,5 @@ def _adjust_config(config: EngineConfig):f"page_size={config.page_size}",]if is_moe:- resolved.insert(0, f"moe_backend={config.moe_backend!r}")+ resolved.insert(0, f"moe_strategy={config.moe_strategy!r}")logger.info_rank0(f"Resolved config: {', '.join(resolved)}")diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex 0ce249de..8d066d43 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -79,10 +79,10 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int"""H, I = hidden_size, moe_intermediate_sizeif fmt == "bf16":- # models/loader.py stream_moe_expert_sources: gate_up [E, 2I, H], down [E, H, I], bf16+ # bf16 expert banks: gate_up [E, 2I, H], down [E, H, I]return {"gate_up": 2 * I * H * 2, "down": H * I * 2}if fmt == "fp8_block":- # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block+ # block-fp8 expert banks: fp8 weights + bf16 128x128 block# scales, trailing scale dim 16B-padded (same helper as the loader)B = 128return {@@ -97,7 +97,7 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: intif fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"):# models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp16# per-row globals; marlin/b12x repacks are byte-identical with the globals- # folded into GPU-resident alphas (moe/nvfp4_backends.py), so no global banks.+ # folded into GPU-resident alphas (layers/quantization/moe/nvfp4.py), so no global banks.banks = {"gate_up_packed": 2 * I * (H // 2),"gate_up_scale": 2 * I * (H // 16),@@ -109,7 +109,7 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: intbanks["down_global"] = H * 2return banksif fmt == "mxfp4_triton":- # gpt_oss/weight.py _empty_mxfp4_triton_banks: transposed split-K blocks/scales + bf16 bias+ # gpt-oss mxfp4 expert banks: transposed split-K blocks/scales + bf16 biasreturn {"gate_up_blocks": (H // 2) * (2 * I),"gate_up_scales": (H // 32) * (2 * I),@@ -285,7 +285,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (kv_groups=(),top_k=8,moe_intermediate_size=2048,- expert_formats=_NVFP4_FORMATS,+ expert_formats=(*_NVFP4_FORMATS, "fp8_block"),aliases=("zai-org/GLM-5.3-Flash", "LibertAIDAI/GLM-5.3-Flash-NVFP4"),),AotModel(diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.pyindex fc42c49f..2a886cf7 100644--- a/python/freetoken/kernel/backend.py+++ b/python/freetoken/kernel/backend.py@@ -10,6 +10,8 @@ from __future__ import annotationsimport functoolsimport importlib.util+import torch+def _importable(name: str) -> bool:# find_spec normally returns None when a package is absent, but it can raise@@ -31,6 +33,20 @@ def is_sgl_kernel_installed() -> bool:return _importable("sgl_kernel")+def is_vllm_installed() -> bool:+ return _importable("vllm")+++def device_capability() -> tuple[int, int]:+ """Compute capability of the current device as (major, minor); (0, 0) without CUDA."""+ if not torch.cuda.is_available():+ return (0, 0)+ major, minor = torch.cuda.get_device_capability()+ return (int(major), int(minor))++@functools.cachedef driver_cuda_version() -> int | None:"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),diff --git a/python/freetoken/kernel/triton/fp8_block_linear.py b/python/freetoken/kernel/triton/fp8_block_linear.pyindex 0999c56d..b79aa5b2 100644--- a/python/freetoken/kernel/triton/fp8_block_linear.py+++ b/python/freetoken/kernel/triton/fp8_block_linear.py@@ -16,7 +16,6 @@ from __future__ import annotationsimport torchimport tritonimport triton.language as tl-from freetoken.layers import BaseOPfrom freetoken.kernel.triton.e4m3_compat import (e4m3_act_dtype,@@ -271,46 +270,8 @@ def dequant_block_fp8(weight: torch.Tensor, scale: torch.Tensor, block: int = _Breturn (weight.to(torch.float32) * s).to(torch.bfloat16)-# ======================================================================================-# BaseOP linear layers (TP=1, replicated).-# ======================================================================================-class Fp8BlockLinear(BaseOP):- """Replicated block-fp8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + bf16- ``weight_scale_inv`` ``[out//128, in//128]``. Activations are quantized dynamically;- the GEMM dequantizes per block, so activations keep bf16 precision."""-- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):- assert in_features % _BLOCK == 0 and out_features % _BLOCK == 0, (in_features, out_features)- self.in_features = in_features- self.out_features = out_features- self.weight = torch.empty(out_features, in_features, dtype=FP8)- self.weight_scale_inv = torch.empty(- out_features // _BLOCK, in_features // _BLOCK, dtype=torch.bfloat16- )- self.bias = torch.empty(out_features) if has_bias else None-- def forward(self, x: torch.Tensor) -> torch.Tensor:- return block_fp8_linear(x, self.weight, self.weight_scale_inv, self.bias)---class Fp8BlockColMerged(Fp8BlockLinear):- """Block-fp8 column-merged linear (drop-in for ``LinearColParallelMerged`` at TP=1).-- Holds one fp8 weight that is the concatenation of several projections along the output- dim; the caller splits the bf16 output by ``output_sizes`` exactly as before. Each part's- out dim must be a multiple of 128 so the concatenated ``weight_scale_inv`` blocks align."""-- def __init__(self, in_features: int, output_sizes: list[int], has_bias: bool = False):- for o in output_sizes:- assert o % _BLOCK == 0, (output_sizes, "each merged output size must be /128 for fp8")- self.output_sizes = list(output_sizes)- super().__init__(in_features, sum(output_sizes), has_bias)--__all__ = ["FP8",- "Fp8BlockLinear",- "Fp8BlockColMerged","block_fp8_linear","block_fp8_matmul","per_token_group_quant_fp8",diff --git a/python/freetoken/kernel/triton/fp8_blockscale_moe.py b/python/freetoken/kernel/triton/fp8_blockscale_moe.pyindex 8ba5626b..74760516 100644--- a/python/freetoken/kernel/triton/fp8_blockscale_moe.py+++ b/python/freetoken/kernel/triton/fp8_blockscale_moe.py@@ -98,14 +98,13 @@ def _decode_gemm(a, w, s, c, topk_weights, topk_ids, mul_routed_weight, a_row_isdef fused_experts_decode_fp8_blockscale(hidden_states, gate_up, gate_up_scale, down, down_scale,- topk_weights, topk_ids, activation="silu",+ topk_weights, topk_ids, activation="silu", act_alpha=1.0, act_limit=float("inf"),) -> torch.Tensor:"""Decode (bs-1) inline-dequant block-fp8 MoE. ``topk_ids`` index rows of the expertbanks (resident: expert id; offload: cache slot)."""from freetoken.kernel import moe_sum_reduce_triton- from freetoken.layers import silu_and_mul+ from freetoken.layers import gated_act_and_mul- assert activation == "silu"M, H = hidden_states.shapetop_k = topk_ids.shape[1]two_i = gate_up.shape[1]@@ -115,7 +114,7 @@ def fused_experts_decode_fp8_blockscale(ic1 = torch.empty((M, top_k, two_i), device=dev, dtype=dt)_decode_gemm(hidden_states, gate_up, gate_up_scale, ic1, topk_weights, topk_ids, False, False)ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)- silu_and_mul(ic1.view(-1, two_i), ic2)+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)_decode_gemm(ic2, down, down_scale, ic3, topk_weights, topk_ids, True, True)out = torch.empty_like(hidden_states)@@ -217,16 +216,15 @@ def _prefill_gemm(a_fp8, a_scale, w, s, c, tw, sorted_ids, expert_ids, ntpp, numdef fused_experts_fp8_blockscale(hidden_states, gate_up, gate_up_scale, down, down_scale,- topk_weights, topk_ids, num_experts, activation="silu",+ topk_weights, topk_ids, num_experts, activation="silu", act_alpha=1.0, act_limit=float("inf"),) -> torch.Tensor:"""Prefill inline-dequant block-fp8 MoE. ``topk_ids`` index expert rows in [0, num_experts)(materialized layer: position == expert id)."""from freetoken.kernel import moe_sum_reduce_triton- from freetoken.layers import silu_and_mul+ from freetoken.layers import gated_act_and_mulfrom freetoken.kernel.triton.fp8_block_linear import per_token_group_quant_fp8from freetoken.moe.fused import moe_align_block_size- assert activation == "silu"M, H = hidden_states.shapetop_k = topk_ids.shape[1]two_i = gate_up.shape[1]@@ -246,7 +244,7 @@ def fused_experts_fp8_blockscale(_prefill_gemm(a1_fp8, a1_scale, gate_up, gate_up_scale, ic1, tw, sorted_ids, expert_ids, ntpp,num_valid, top_k, False, cfg)ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)- silu_and_mul(ic1.view(-1, two_i), ic2)+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)a2_fp8, a2_scale = per_token_group_quant_fp8(ic2, 128)ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)_prefill_gemm(a2_fp8, a2_scale, down, down_scale, ic3, tw, sorted_ids, expert_ids, ntpp,diff --git a/python/freetoken/kernel/triton/fp8_pertensor_linear.py b/python/freetoken/kernel/triton/fp8_pertensor_linear.pyindex d1d4de5b..5561a86a 100644--- a/python/freetoken/kernel/triton/fp8_pertensor_linear.py+++ b/python/freetoken/kernel/triton/fp8_pertensor_linear.py@@ -20,14 +20,11 @@ accurate than the previous ``weight.to(bf16) * scale`` materialization, which itfrom __future__ import annotationsimport functools-import osimport reimport torchimport tritonimport triton.language as tl-from freetoken.layers import BaseOP-from freetoken.layers.base import _concat_prefixfrom freetoken.kernel.triton.e4m3_compat import (e4m3_kernel_view,@@ -39,10 +36,6 @@ from freetoken.kernel.triton.e4m3_compat import (FP8 = torch.float8_e4m3fn_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}-# Escape hatch: FREETOKEN_DEBUG_FP8_REF=1 swaps the triton kernels for a pure-torch dequant matmul-# (numeric reference / A-B debugging). Evaluated once; the kernels are the default.-_USE_REF = os.environ.get("FREETOKEN_DEBUG_FP8_REF") == "1"-# Row-wise _scaled_mm on sm_89 with torch < 2.12 launches its CUTLASS stream-K kernel off the# current stream (pytorch/pytorch#177651, fixed by pytorch/pytorch@252bb4a; #182/#72/#220), and@@ -55,11 +48,7 @@ def _torch_version() -> tuple[int, int]:@functools.cachedef rowwise_scaled_mm_ok() -> bool:"""Whether row-wise ``torch._scaled_mm`` may be issued from a side stream on this GPU.- Decided once per process, at load (never under graph capture). ``FREETOKEN_FP8_ROWWISE_MM=0/1``- forces the answer."""- forced = os.environ.get("FREETOKEN_FP8_ROWWISE_MM")- if forced in ("0", "1"):- return forced == "1"+ Decided once per process, at load (never under graph capture)."""if not torch.cuda.is_available():return Truefrom freetoken.gpu_select import assigned_visible_gpu@@ -347,10 +336,7 @@ def fp8_pertensor_linear(segments = scale_segments if scale_segments is not None else weight_scale_segments(weight_scale)if not _segments_w8a8_ok(segments):w8a8 = False # W8A16 below is exact for any per-row scale and never calls _scaled_mm- if _USE_REF: # numeric-reference fallback (debug / A-B)- w = weight.to(x.dtype) * weight_scale.to(x.dtype)[:, None]- out = (x.reshape(-1, K) @ w.t()).reshape(*lead, N)- elif w8a8:+ if w8a8:out = _scaled_mm(x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype,scale_segments=segments,@@ -367,67 +353,4 @@ def fp8_pertensor_linear(# ======================================================================================-# BaseOP linear layers (TP=1, replicated). Buffers: fp8 ``weight`` + fp32 ``weight_scale``.# ======================================================================================-class Fp8PerTensorLinear(BaseOP):- """Replicated per-tensor-FP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + per-row fp32- ``weight_scale`` ``[out]`` (a genuine per-tensor weight stores the same scalar in every- row; a fused projection stores each part's scalar across its own rows).-- ``input_scale`` (modelopt's calibrated per-tensor activation scale) is optional: when the- checkpoint ships it, batched decode runs W8A8 through cuBLASLt; when it does not (models- that quantize to fp8 at load, e.g. glm_moe_dsa), every batch size stays W8A16."""-- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):- self.in_features = in_features- self.out_features = out_features- self.weight = torch.empty(out_features, in_features, dtype=FP8)- self.weight_scale = torch.empty(out_features, dtype=torch.float32)- self.bias = torch.empty(out_features) if has_bias else None- # Set from the state dict when present. Left as None (not a tensor) so BaseOP's- # reflective state_dict/load_state_dict skip it entirely on checkpoints without one.- self.input_scale: torch.Tensor | None = None- self._uniform_scale = False- self._scale_segments: list[tuple[int, int]] | None = None-- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:- # Taken out before BaseOP's reflective pass (so it is not an "unexpected key") and- # put back after (so that pass does not try to pop it a second time on a reload).- input_scale = state_dict.pop(_concat_prefix(prefix, "input_scale"), None)- self.input_scale = None- super().load_state_dict(state_dict, prefix=prefix, _internal=_internal)- self.input_scale = input_scale- # Tensor-wise scaling needs one scalar for the whole weight; a fused projection is- # only piecewise-constant, so decide once here rather than syncing on every forward.- scale = self.weight_scale- self._uniform_scale = bool((scale == scale[0]).all().item())- # Segments for the per-part path; decide row-wise safety now, not under graph capture.- self._scale_segments = None if self._uniform_scale else weight_scale_segments(scale)- if self.input_scale is not None and not self._uniform_scale:- rowwise_scaled_mm_ok()-- def forward(self, x: torch.Tensor) -> torch.Tensor:- return fp8_pertensor_linear(- x, self.weight, self.weight_scale, self.bias,- self.input_scale, self._uniform_scale, scale_segments=self._scale_segments,- )---class Fp8PerTensorColMerged(Fp8PerTensorLinear):- """Column-merged per-tensor-FP8 linear (drop-in for ``LinearColParallelMerged`` at TP=1):- one fp8 weight concatenating several projections along the output dim; the caller splits- the bf16 output by ``output_sizes`` as before."""-- def __init__(self, in_features: int, output_sizes: list[int], has_bias: bool = False):- self.output_sizes = list(output_sizes)- super().__init__(in_features, sum(output_sizes), has_bias)---__all__ = [- "FP8",- "Fp8PerTensorLinear",- "Fp8PerTensorColMerged",- "fp8_pertensor_linear",- "rowwise_scaled_mm_ok",- "weight_scale_segments",-]diff --git a/python/freetoken/kernel/triton/mxfp8_linear.py b/python/freetoken/kernel/triton/mxfp8_linear.pyindex 98a7001c..ebe78217 100644--- a/python/freetoken/kernel/triton/mxfp8_linear.py+++ b/python/freetoken/kernel/triton/mxfp8_linear.py@@ -17,12 +17,10 @@ any other bf16 projection.from __future__ import annotations-import osimport torchimport tritonimport triton.language as tl-from freetoken.layers.base import BaseOPfrom freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view, e4m3_native_cx, e4m3_u8_to_f32@@ -30,17 +28,6 @@ FP8 = torch.float8_e4m3fnMXFP8_BLOCK = 32_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}-# Escape hatch: FREETOKEN_DEBUG_MXFP8_REF=1 swaps the kernels for a pure-torch-# dequant matmul (numeric reference / A-B debugging). Evaluated once, logged once.-_USE_REF = os.environ.get("FREETOKEN_DEBUG_MXFP8_REF") == "1"-if _USE_REF:- from freetoken.utils import init_logger-- init_logger(__name__).info(- "FREETOKEN_DEBUG_MXFP8_REF=1: MXFP8 linears serve the pure-torch fp32 "- "dequant reference (slow; debugging only)."- )-def mxfp8_dequant(weight: torch.Tensor, scale_codes: torch.Tensor,dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:@@ -250,10 +237,7 @@ def mxfp8_linear(N = weight.shape[0]M = x.numel() // Kassert K % MXFP8_BLOCK == 0 and scale_codes.shape == (N, K // MXFP8_BLOCK)- if _USE_REF: # numeric-reference fallback (debug / A-B)- w = mxfp8_dequant(weight, scale_codes, dtype=torch.float32)- out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, N)- elif M <= _GEMV_MAX_M:+ if M <= _GEMV_MAX_M:w8 = e4m3_kernel_view(weight)out = _gemv(x.reshape(M, K), w8, scale_codes, x.dtype).reshape(*lead, N)else:@@ -265,28 +249,4 @@ def mxfp8_linear(return out-# ======================================================================================-# BaseOP linear layers (TP=1, replicated). Buffers: fp8 ``weight`` + uint8-# ``weight_scale_inv`` (the checkpoint's e8m0 exponent codes, loaded verbatim).-# ======================================================================================-class Mxfp8Linear(BaseOP):- """Replicated MXFP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + uint8 e8m0- ``weight_scale_inv`` ``[out, in // 32]``. Fused projections concatenate several- checkpoint projections along the output dim; the scales are per-output-row so the- concatenation is exact."""-- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):- assert in_features % MXFP8_BLOCK == 0- self.in_features = in_features- self.out_features = out_features- self.weight = torch.empty(out_features, in_features, dtype=FP8)- self.weight_scale_inv = torch.empty(- out_features, in_features // MXFP8_BLOCK, dtype=torch.uint8- )- self.bias = torch.empty(out_features) if has_bias else None-- def forward(self, x: torch.Tensor) -> torch.Tensor:- return mxfp8_linear(x, self.weight, self.weight_scale_inv, self.bias)---__all__ = ["FP8", "MXFP8_BLOCK", "Mxfp8Linear", "mxfp8_linear", "mxfp8_dequant"]+__all__ = ["FP8", "MXFP8_BLOCK", "mxfp8_linear", "mxfp8_dequant"]diff --git a/python/freetoken/kernel/triton/nvfp4_linear.py b/python/freetoken/kernel/triton/nvfp4_linear.pyindex 9ed286f4..53abdedc 100644--- a/python/freetoken/kernel/triton/nvfp4_linear.py+++ b/python/freetoken/kernel/triton/nvfp4_linear.py@@ -35,13 +35,10 @@ CUDA-graph safe on the decode paths: fixed shapes, no host sync.from __future__ import annotations-import osimport torchimport tritonimport triton.language as tl-from freetoken.layers import BaseOP-from freetoken.layers.base import _concat_prefixfrom freetoken.kernel.triton.e4m3_compat import (e4m3_kernel_view,@@ -53,10 +50,6 @@ from freetoken.kernel.triton.e4m3_compat import (FP8 = torch.float8_e4m3fn_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}-# Escape hatch: FREETOKEN_DEBUG_DENSE_NVFP4_REF=1 swaps the triton kernels for a dequant_nvfp4 +-# torch matmul reference (numeric A-B debugging). Evaluated once; the kernels are the default.-_USE_REF = os.environ.get("FREETOKEN_DEBUG_DENSE_NVFP4_REF") == "1"-# Decode GEMV tiles (H100-tuned; the fp16-trick dequant is ALU-cheap so wider K tiles win).# Two sets: the transposed K-major resident layout (coalesced along N; narrower K tiles win)# vs the legacy row-major layout (coalesced along K).@@ -764,19 +757,6 @@ def _gemm(a: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tensor,return _gemm_scratch(a, packed_i32, scale, gscale, out_dtype, transposed)-def _ref(a: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor,- gscale: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor:- """Numeric reference: dequant the FP4 weight to bf16 (well-tested ``dequant_nvfp4``)- then a plain matmul. Used by FREETOKEN_DEBUG_DENSE_NVFP4_REF=1 for A-B validation."""- from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4-- slots = torch.zeros(1, dtype=torch.int32, device=a.device)- w = dequant_nvfp4(- packed.unsqueeze(0), scale.unsqueeze(0), gscale.unsqueeze(0), slots, dtype=torch.bfloat16- )[0] # [N, K] bf16- return (a.reshape(-1, a.shape[-1]) @ w.t()).to(out_dtype)--def _linear_impl(x: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tensor,gscale: torch.Tensor, bias: torch.Tensor | None, out_dtype: torch.dtype,@@ -802,10 +782,6 @@ def nvfp4_dense_linear() -> torch.Tensor:"""``y = x @ dequant(weight)^T`` on the checkpoint-native row-major layout.``weight`` [N, K//2] uint8, ``weight_scale`` [N, K//16] fp8-e4m3, ``weight_global`` [N] fp16."""- if _USE_REF:- *lead, K = x.shape- out = _ref(x, weight, weight_scale, weight_global, x.dtype).reshape(*lead, weight.shape[0])- return out + bias.to(out.dtype) if bias is not None else outreturn _linear_impl(x, weight.view(torch.int32), weight_scale, weight_global, bias, x.dtype, transposed=False)@@ -829,113 +805,14 @@ def nvfp4_dense_linear_t() -> torch.Tensor:"""``y = x @ dequant(W)^T`` on the K-major resident layout from:func:`nvfp4_transpose_resident` (``weight_t`` [K//8, N] int32, ``scale_t`` [K//16, N])."""- if _USE_REF:- out = _gemm_scratch(- x.reshape(-1, x.shape[-1]).contiguous(), weight_t.t(), scale_t.t(),- weight_global, x.dtype, transposed=True,- ).reshape(*x.shape[:-1], weight_t.shape[1])- return out + bias.to(out.dtype) if bias is not None else outreturn _linear_impl(x, weight_t.t(), scale_t.t(), weight_global, bias, x.dtype, transposed=True)-# ======================================================================================-# BaseOP linear layers (TP=1, replicated). Buffers: uint8 packed ``weight`` + fp8 block-# ``weight_scale`` + fp16 per-row ``weight_global``.-# ======================================================================================-class Nvfp4DenseLinear(BaseOP):- """Replicated NVFP4 dense linear (W4A16). Drop-in for ``LinearReplicated`` /- ``LinearRowParallel`` at TP=1 on the mixed-precision checkpoint's NVFP4 dense weights.-- Buffers are declared (and loaded) in the checkpoint's row-major layout; at load the- packed weight + block scales are repacked to K-major (:func:`nvfp4_transpose_resident`)- so the decode kernels' weight loads coalesce along N (~2x batched-decode throughput)."""-- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):- assert in_features % 16 == 0, f"NVFP4 in_features must be %16, got {in_features}"- self.in_features = in_features- self.out_features = out_features- self.weight = torch.empty(out_features, in_features // 2, dtype=torch.uint8)- self.weight_scale = torch.empty(out_features, in_features // 16, dtype=FP8)- self.weight_global = torch.empty(out_features, dtype=torch.float16)- self.bias = torch.empty(out_features) if has_bias else None- self._transposed = False-- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:- w = state_dict.pop(_concat_prefix(prefix, "weight"))- s = state_dict.pop(_concat_prefix(prefix, "weight_scale"))- assert w.shape == self.weight.shape and w.dtype == torch.uint8- assert s.shape == self.weight_scale.shape- self.weight, self.weight_scale = nvfp4_transpose_resident(w, s)- self.weight_global = state_dict.pop(_concat_prefix(prefix, "weight_global"))- if self.bias is not None:- self.bias = state_dict.pop(_concat_prefix(prefix, "bias"))- self._transposed = True- if not _internal and state_dict:- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")-- def forward(self, x: torch.Tensor) -> torch.Tensor:- if self._transposed:- return nvfp4_dense_linear_t(- x, self.weight, self.weight_scale, self.weight_global, self.bias- )- return nvfp4_dense_linear(x, self.weight, self.weight_scale, self.weight_global, self.bias)---class Nvfp4DenseColMerged(Nvfp4DenseLinear):- """Column-merged NVFP4 dense linear (drop-in for ``LinearColParallelMerged`` at TP=1):- one packed weight concatenating several projections on the output dim; each part keeps its- own per-row ``weight_global`` (and block scales), so the fused weight is exact. The caller- splits the output by ``output_sizes`` (e.g. shared-expert gate|up) as before."""-- def __init__(self, in_features: int, output_sizes: list[int], has_bias: bool = False):- self.output_sizes = list(output_sizes)- super().__init__(in_features, sum(output_sizes), has_bias)---class Nvfp4LMHead(BaseOP):- """NVFP4 (W4A16) LM head for the mixed checkpoint (TP=1, untied). Mirrors- ``ParallelLMHead.forward`` at TP=1: slice to the last token per sequence at prefill, then- the W4A16 GEMV/GEMM instead of a bf16 ``F.linear`` over the (here ~1 GB) bf16 weight."""-- def __init__(self, num_embeddings: int, embedding_dim: int):- assert embedding_dim % 16 == 0- self.num_embeddings = num_embeddings- self.embedding_dim = embedding_dim- self.weight = torch.empty(num_embeddings, embedding_dim // 2, dtype=torch.uint8)- self.weight_scale = torch.empty(num_embeddings, embedding_dim // 16, dtype=FP8)- self.weight_global = torch.empty(num_embeddings, dtype=torch.float16)- self._transposed = False-- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:- w = state_dict.pop(_concat_prefix(prefix, "weight"))- s = state_dict.pop(_concat_prefix(prefix, "weight_scale"))- assert w.shape == self.weight.shape and w.dtype == torch.uint8- self.weight, self.weight_scale = nvfp4_transpose_resident(w, s)- self.weight_global = state_dict.pop(_concat_prefix(prefix, "weight_global"))- self._transposed = True- if not _internal and state_dict:- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")-- def forward(self, x: torch.Tensor) -> torch.Tensor:- from freetoken.core import get_global_ctx-- batch = get_global_ctx().batch- if batch.is_prefill:- indices = batch.attn_metadata.get_last_indices(batch.size)- x = x[indices].contiguous()- if self._transposed:- return nvfp4_dense_linear_t(x, self.weight, self.weight_scale, self.weight_global)- return nvfp4_dense_linear(x, self.weight, self.weight_scale, self.weight_global)--__all__ = ["FP8","nvfp4_dense_linear","nvfp4_dense_linear_t","nvfp4_transpose_resident",- "Nvfp4DenseLinear",- "Nvfp4DenseColMerged",- "Nvfp4LMHead",]diff --git a/python/freetoken/layers/__init__.py b/python/freetoken/layers/__init__.pyindex 31c19bf9..579dc6e4 100644--- a/python/freetoken/layers/__init__.py+++ b/python/freetoken/layers/__init__.py@@ -1,4 +1,6 @@from .activation import (+ GATED_ACTIVATIONS,+ gated_act_and_mul,gelu_and_mul,gelu_tanh_and_mul,silu_and_mul,@@ -16,6 +18,7 @@ from .linear import ()from .moe import MoELayer, OffloadMoELayer, make_moe_layerfrom .norm import (+ GatedRMSNorm,GemmaPlusOneRMSNorm,GemmaPlusOneRMSNormFused,GemmaRMSNorm,@@ -30,6 +33,8 @@ __all__ = ["gelu_tanh_and_mul","swigluoai_and_mul","swiglu_clamp_and_mul",+ "gated_act_and_mul",+ "GATED_ACTIVATIONS","BaseOP","StateLessOP","OPList",@@ -41,6 +46,7 @@ __all__ = ["LinearQKVMerged","RMSNorm","RMSNormFused",+ "GatedRMSNorm","GemmaRMSNorm","GemmaPlusOneRMSNorm","GemmaPlusOneRMSNormFused",diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.pyindex ae1d9881..cd7a402a 100644--- a/python/freetoken/layers/activation.py+++ b/python/freetoken/layers/activation.py@@ -65,10 +65,30 @@ def swiglu_clamp_and_mul(return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit)+GATED_ACTIVATIONS = ("silu", "gelu", "gelu_tanh", "swigluoai", "swiglu_clamp")+++def gated_act_and_mul(activation: str, x, out, *, alpha: float = 1.0, limit: float = float("inf")):+ """The epilogue between the two expert GEMMs: ``activation`` over uninterleaved [gate; up] halves into ``out``."""+ if activation == "silu":+ return silu_and_mul(x, out)+ if activation == "gelu":+ return gelu_and_mul(x, out)+ if activation == "gelu_tanh":+ return gelu_tanh_and_mul(x, out)+ if activation == "swigluoai":+ return swigluoai_and_mul(x, out, alpha=alpha, limit=limit)+ if activation == "swiglu_clamp":+ return swiglu_clamp_and_mul(x, out, alpha=alpha, limit=limit)+ raise ValueError(f"no gated activation {activation!r}; known: {GATED_ACTIVATIONS}")++__all__ = ["silu_and_mul","gelu_and_mul","gelu_tanh_and_mul","swigluoai_and_mul","swiglu_clamp_and_mul",+ "gated_act_and_mul",+ "GATED_ACTIVATIONS",]diff --git a/python/freetoken/layers/embedding.py b/python/freetoken/layers/embedding.pyindex 76cd759b..a2221654 100644--- a/python/freetoken/layers/embedding.py+++ b/python/freetoken/layers/embedding.py@@ -9,6 +9,7 @@ from freetoken.distributed import DistributedCommunicator, get_tp_infofrom freetoken.utils import div_ceil, nvtx_annotatefrom .base import BaseOP+from .quantization import LayerKind, QuantConfig, quant_method_forclass VocabParallelEmbedding(BaseOP):@@ -58,6 +59,11 @@ class VocabParallelEmbedding(BaseOP):class ParallelLMHead(VocabParallelEmbedding):+ """The head is a linear layer over the vocab shard: its weights come from ``quant_method``+ unless they are tied to the input embedding."""++ quant_layer_kind = LayerKind.LINEAR+def __init__(self,num_embeddings: int,@@ -65,11 +71,27 @@ class ParallelLMHead(VocabParallelEmbedding):bias: bool = False,tie_word_embeddings: bool = False,tied_embedding: VocabParallelEmbedding | None = None,+ *,+ quant_config: QuantConfig | None = None,+ prefix: str = "",):super().__init__(num_embeddings, embedding_dim)- self.bias = torch.empty(self.num_embeddings_tp) if bias else None+ self.has_bias = bias+ self.prefix = prefixself.tied_embedding = tied_embeddingassert (tied_embedding is not None) == tie_word_embeddings+ self.in_features = embedding_dim+ self.out_features = self.num_embeddings_tp+ self.output_sizes = (self.num_embeddings_tp,)+ self.quant_method = None+ if tied_embedding is None:+ self.quant_method = quant_method_for(quant_config, self, prefix)+ self.quant_method.create_weights(self)+ self.bias = torch.empty(self.num_embeddings_tp) if bias else None++ def finalize(self) -> None:+ if self.quant_method is not None:+ self.quant_method.finalize(self)def load_state_dict(self,@@ -109,8 +131,10 @@ class ParallelLMHead(VocabParallelEmbedding):x = x[indices].contiguous()del indices- module = self.tied_embedding or self- logits = F.linear(x, module.weight, self.bias)+ if self.tied_embedding is not None:+ logits = F.linear(x, self.tied_embedding.weight, self.bias)+ else:+ logits = self.quant_method.apply(self, x)if self.tp_size == 1:return logitsinput_shape = logits.shapediff --git a/python/freetoken/layers/linear.py b/python/freetoken/layers/linear.pyindex f707e042..d8a3e861 100644--- a/python/freetoken/layers/linear.py+++ b/python/freetoken/layers/linear.py@@ -3,15 +3,20 @@ from __future__ import annotationsfrom typing import Listimport torch-import torch.nn.functional as Ffrom freetoken.distributed import DistributedCommunicator, get_tp_infofrom freetoken.utils import div_evenfrom .base import BaseOP+from .quantization import LayerKind, QuantConfig, quant_method_forclass _LinearTPImpl(BaseOP):- """Real implementation of a linear layer with tensor parallelism."""+ """Real implementation of a linear layer with tensor parallelism.++ The weights are declared and applied by ``quant_method``, picked from ``quant_config`` by+ the layer's ``prefix``; without a config the layer is plain bf16."""++ quant_layer_kind = LayerKind.LINEARdef __init__(self,@@ -20,16 +25,30 @@ class _LinearTPImpl(BaseOP):local_isize: int,local_osize: int,has_bias: bool,+ *,+ output_sizes: List[int] | None = None,+ quant_config: QuantConfig | None = None,+ prefix: str = "",):self.full_input_size = full_isizeself.full_output_size = full_osizeself.local_input_size = local_isizeself.local_output_size = local_osize- self.weight = torch.empty(local_osize, local_isize)+ self.has_bias = has_bias+ self.prefix = prefix+ # the TP-local shape the quant method declares weights for; fused projections list their segments+ self.in_features = local_isize+ self.out_features = local_osize+ self.output_sizes = tuple(output_sizes or (local_osize,))+ self.quant_method = quant_method_for(quant_config, self, prefix)+ self.quant_method.create_weights(self)self.bias = torch.empty(local_osize) if has_bias else None+ def finalize(self) -> None:+ self.quant_method.finalize(self)+def forward(self, x: torch.Tensor) -> torch.Tensor:- return F.linear(x, self.weight, self.bias)+ return self.quant_method.apply(self, x)class LinearReplicated(_LinearTPImpl):@@ -43,6 +62,9 @@ class LinearReplicated(_LinearTPImpl):input_size: int,output_size: int,has_bias: bool,+ *,+ quant_config: QuantConfig | None = None,+ prefix: str = "",):super().__init__(full_isize=input_size,@@ -50,6 +72,8 @@ class LinearReplicated(_LinearTPImpl):local_isize=input_size,local_osize=output_size,has_bias=has_bias,+ quant_config=quant_config,+ prefix=prefix,)@@ -59,13 +83,19 @@ class LinearColParallelMerged(_LinearTPImpl):input_size: int,output_sizes: List[int],has_bias: bool,+ *,+ quant_config: QuantConfig | None = None,+ prefix: str = "",):# check that all output sizes are divisible by tp_sizetp_info = get_tp_info()tp_output_sizes = [div_even(size, tp_info.size) for size in output_sizes]output_size = sum(output_sizes)tp_output_size = sum(tp_output_sizes)- super().__init__(input_size, output_size, input_size, tp_output_size, has_bias)+ super().__init__(+ input_size, output_size, input_size, tp_output_size, has_bias,+ output_sizes=tp_output_sizes, quant_config=quant_config, prefix=prefix,+ )class LinearQKVMerged(_LinearTPImpl):@@ -76,6 +106,9 @@ class LinearQKVMerged(_LinearTPImpl):num_qo_heads: int,num_kv_heads: int,has_bias: bool,+ *,+ quant_config: QuantConfig | None = None,+ prefix: str = "",):tp_info = get_tp_info()@@ -85,11 +118,23 @@ class LinearQKVMerged(_LinearTPImpl):full_osize = (num_qo_heads + 2 * num_kv_heads) * head_dimlocal_isize = hidden_sizelocal_osize = (local_num_qo + 2 * local_num_kv) * head_dim- super().__init__(full_isize, full_osize, local_isize, local_osize, has_bias)+ super().__init__(+ full_isize, full_osize, local_isize, local_osize, has_bias,+ output_sizes=[local_num_qo * head_dim, local_num_kv * head_dim, local_num_kv * head_dim],+ quant_config=quant_config, prefix=prefix,+ )class LinearOProj(_LinearTPImpl):- def __init__(self, input_size: int, output_size: int, has_bias: bool):+ def __init__(+ self,+ input_size: int,+ output_size: int,+ has_bias: bool,+ *,+ quant_config: QuantConfig | None = None,+ prefix: str = "",+ ):tp_info = get_tp_info()full_isize = input_sizefull_osize = output_size@@ -97,10 +142,13 @@ class LinearOProj(_LinearTPImpl):local_osize = output_sizeself._comm = DistributedCommunicator()self._tp_size = tp_info.size- super().__init__(full_isize, full_osize, local_isize, local_osize, has_bias)+ super().__init__(+ full_isize, full_osize, local_isize, local_osize, has_bias,+ quant_config=quant_config, prefix=prefix,+ )def forward(self, x: torch.Tensor) -> torch.Tensor:- y = F.linear(x, self.weight, self.bias)+ y = self.quant_method.apply(self, x)if self._tp_size > 1:y = self._comm.all_reduce(y)return y@@ -112,16 +160,22 @@ class LinearRowParallel(_LinearTPImpl):input_size: int,output_size: int,has_bias: bool,+ *,+ quant_config: QuantConfig | None = None,+ prefix: str = "",):tp_info = get_tp_info()local_input_size = div_even(input_size, tp_info.size)local_output_size = output_sizeself._comm = DistributedCommunicator()self._tp_size = tp_info.size- super().__init__(input_size, output_size, local_input_size, local_output_size, has_bias)+ super().__init__(+ input_size, output_size, local_input_size, local_output_size, has_bias,+ quant_config=quant_config, prefix=prefix,+ )def forward(self, x: torch.Tensor) -> torch.Tensor:- y = F.linear(x, self.weight, self.bias)+ y = self.quant_method.apply(self, x)if self._tp_size > 1:y = self._comm.all_reduce(y)return ydiff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.pyindex d68d8ded..42ad822b 100644--- a/python/freetoken/layers/moe.py+++ b/python/freetoken/layers/moe.py@@ -4,20 +4,20 @@ from typing import TYPE_CHECKING, Tupleimport torchfrom freetoken.core import get_global_ctxfrom freetoken.distributed import DistributedCommunicator, get_tp_info-from freetoken.moe import is_offload_moe_backend-from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl, fused_topk+from freetoken.moe import is_offload_moe_strategy+from freetoken.moe.fused import fused_topkfrom freetoken.moe.offload_cache import OffloadMoeCache-from freetoken.utils import div_even+from .base import BaseOP+from .quantization import ExpertView, LayerKind, QuantConfig, quant_method_forif TYPE_CHECKING:from freetoken.models.config import ModelConfig# Router decision (topk_weights[float32], topk_ids[int32]) for models whose router-# is computed outside the MoE layer. Such models call ``routed_forward`` (offload) or-# ``_run_experts`` (dense) with a precomputed routing instead of going through the-# generic softmax+top-k path.+# is computed outside the MoE layer. Such models call ``routed_forward`` with a+# precomputed routing instead of going through the generic softmax+top-k path.TopK = Tuple[torch.Tensor, torch.Tensor]# Hybrid decode overlaps the CPU overflow GEMV behind the GPU PCIe fetch + GEMM by@@ -27,6 +27,15 @@ _HYBRID_OVERLAP = os.getenv("FREETOKEN_HYBRID_OVERLAP", "1") != "0"class MoELayer(BaseOP):+ """Resident routed experts.++ The expert format comes from ``quant_method`` (declared by ``create_weights``, run by+ ``apply``); without a ``quant_config`` the experts are plain bf16. The gated activation is+ ``act(clamp(g, limit) * alpha) * (clamp(u) + beta)`` with ``interleaved`` gate|up rows+ for gpt-oss."""++ quant_layer_kind = LayerKind.MOE+def __init__(self,num_experts: int,@@ -37,7 +46,17 @@ class MoELayer(BaseOP):activation: str = "silu",apply_router_weight_on_input: bool = False,allocate_experts: bool = True,- weight_format: str = "bf16",+ *,+ alpha: float = 1.0,+ beta: float = 0.0,+ limit: float | None = None,+ interleaved: bool = False,+ has_bias: bool = False,+ layer_id: int | None = None,+ strategy: str = "resident",+ decode_target: str = "gpu",+ quant_config: QuantConfig | None = None,+ prefix: str = "",):super().__init__()@@ -48,101 +67,47 @@ class MoELayer(BaseOP):self._comm = DistributedCommunicator()tp_info = get_tp_info()+ self.tp_rank = tp_info.rankself.tp_size = tp_size = tp_info.sizeself.renormalize = renormalizeself.activation = activationself.apply_router_weight_on_input = apply_router_weight_on_input- self.weight_format = weight_format- intermediate_size_per_partition = div_even(intermediate_size, tp_size)- if allocate_experts:- self._alloc_resident_experts(intermediate_size_per_partition)-- def _alloc_resident_experts(self, intermediate_size_per_partition: int) -> None:- """Allocate the resident (in-GPU) expert weights for ``self.weight_format``.-- The resident sibling of the offload bank schemas: each format owns its- tensor layout here and its kernel branch in ``_resident_gemm``.- """- if self.weight_format == "fp8_block":- # Stacked block-fp8 experts + bf16 per-128x128-block inverse scales.- # Full (unpartitioned) intermediate size: this layout is TP=1-only.- from freetoken.kernel.triton.fp8_block_linear import FP8-- blk = 128- n, i, h = self.num_experts, self.intermediate_size, self.hidden_size- self.gate_up_proj = torch.empty(n, 2 * i, h, dtype=FP8)- self.gate_up_scale_inv = torch.empty(- n, 2 * i // blk, h // blk, dtype=torch.bfloat16- )- self.down_proj = torch.empty(n, h, i, dtype=FP8)- self.down_scale_inv = torch.empty(n, h // blk, i // blk, dtype=torch.bfloat16)- return- assert self.weight_format == "bf16", (- f"no resident expert allocation for weight_format {self.weight_format!r}"- )- self.gate_up_proj = torch.empty(- self.num_experts,- 2 * intermediate_size_per_partition,- self.hidden_size,- )- self.down_proj = torch.empty(- self.num_experts,- self.hidden_size,- intermediate_size_per_partition,- )+ self.alpha = alpha+ self.beta = beta+ self.limit = limit+ self.interleaved = interleaved+ self.has_bias = has_bias+ self.layer_id = layer_id+ self.strategy = strategy+ self.decode_target = decode_target+ self.prefix = prefix+ # offload layers without a quant config stay on the format-tag banks (GGUF q4_0)+ self.quant_method = None+ if quant_config is not None or allocate_experts:+ self.quant_method = quant_method_for(quant_config, self, prefix)+ if allocate_experts:+ self.quant_method.create_weights(self)++ def finalize(self) -> None:+ if self.quant_method is not None:+ self.quant_method.finalize(self)def _maybe_all_reduce(self, hidden_states: torch.Tensor) -> torch.Tensor:if self.tp_size > 1:return self._comm.all_reduce(hidden_states)return hidden_states- def _run_experts(- self,- hidden_states: torch.Tensor,- topk_weights: torch.Tensor,- topk_ids: torch.Tensor,- ) -> torch.Tensor:- """Dense (in-GPU) expert compute for a precomputed routing decision."""- return fused_experts_impl(- hidden_states,- self.gate_up_proj,- self.down_proj,- topk_weights,- topk_ids,- self.activation,- apply_router_weight_on_input=self.apply_router_weight_on_input,- )-def _resident_gemm(self,hidden_states: torch.Tensor,topk_weights: torch.Tensor,topk_ids: torch.Tensor,) -> torch.Tensor:- """Kernel dispatch on ``self.weight_format`` -- the resident mirror of- ``OffloadMoELayer._expert_gemm``'s ``cache.quant_format`` dispatch."""- if self.weight_format == "fp8_block":- # Prefill dequantizes the layer's experts to bf16 and runs the bf16- # grouped GEMM; decode dequantizes only the routed rows.- from freetoken.moe.fused_fp8_block import (- fused_experts_decode_fp8_block,- fused_experts_fp8_block,- )-- if get_global_ctx().batch.is_prefill:- return fused_experts_fp8_block(- hidden_states, self.gate_up_proj, self.gate_up_scale_inv,- self.down_proj, self.down_scale_inv,- topk_weights, topk_ids, self.num_experts,- )- return fused_experts_decode_fp8_block(- hidden_states, self.gate_up_proj, self.gate_up_scale_inv,- self.down_proj, self.down_scale_inv, topk_weights, topk_ids,- )- assert self.weight_format == "bf16", (- f"no resident expert kernel for weight_format {self.weight_format!r}"+ assert self.quant_method is not None+ return self.quant_method.apply(+ hidden_states, topk_weights, topk_ids, self.quant_method.resident_view(self),+ layer=self, is_prefill=get_global_ctx().batch.is_prefill,)- return self._run_experts(hidden_states, topk_weights, topk_ids)def routed_forward(self,@@ -168,30 +133,13 @@ class MoELayer(BaseOP):hidden_states: torch.Tensor,router_logits: torch.Tensor | None = None,):- if self.weight_format != "bf16":- # Quantized resident experts: generic softmax router + format kernel.- # The bf16 path below stays on ctx.moe_backend byte-for-byte.- topk_weights, topk_ids = fused_topk(- hidden_states=hidden_states,- gating_output=router_logits,- topk=self.top_k,- renormalize=self.renormalize,- )- return self._maybe_all_reduce(- self._resident_gemm(hidden_states, topk_weights, topk_ids)- )- ctx = get_global_ctx()- final_hidden_states = ctx.moe_backend.forward(+ topk_weights, topk_ids = fused_topk(hidden_states=hidden_states,- w1=self.gate_up_proj,- w2=self.down_proj,gating_output=router_logits,topk=self.top_k,renormalize=self.renormalize,- activation=self.activation,- apply_router_weight_on_input=self.apply_router_weight_on_input,)- return self._maybe_all_reduce(final_hidden_states)+ return self._maybe_all_reduce(self._resident_gemm(hidden_states, topk_weights, topk_ids))class OffloadMoELayer(MoELayer):@@ -205,6 +153,16 @@ class OffloadMoELayer(MoELayer):renormalize: bool = True,activation: str = "silu",apply_router_weight_on_input: bool = False,+ *,+ alpha: float = 1.0,+ beta: float = 0.0,+ limit: float | None = None,+ interleaved: bool = False,+ has_bias: bool = False,+ strategy: str = "offload",+ decode_target: str = "gpu",+ quant_config: QuantConfig | None = None,+ prefix: str = "",):super().__init__(num_experts=num_experts,@@ -215,8 +173,17 @@ class OffloadMoELayer(MoELayer):activation=activation,apply_router_weight_on_input=apply_router_weight_on_input,allocate_experts=False,+ alpha=alpha,+ beta=beta,+ limit=limit,+ interleaved=interleaved,+ has_bias=has_bias,+ layer_id=layer_id,+ strategy=strategy,+ decode_target=decode_target,+ quant_config=quant_config,+ prefix=prefix,)- self.layer_id = layer_idself.offload_cache: OffloadMoeCache | None = Nonedef forward(@@ -423,9 +390,8 @@ class OffloadMoELayer(MoELayer):return cache.wait_prefill_layer(self.layer_id)# ------------------------------------------------------------------- # Kernel dispatch -- pure routing on the cache's quant format. ``views``- # are the bank tensors the movement step produced (in bank registration- # order) and ``topk_ids`` already index their rows.+ # Kernel dispatch: ``views`` are the bank tensors the movement step produced (in bank registration order) and ``topk_ids`` already index their rows.+ # GGUF q4_0 experts still dispatch on the cache's format tag until they get a method.# ------------------------------------------------------------------def _expert_gemm(@@ -440,88 +406,17 @@ class OffloadMoELayer(MoELayer):alphas: tuple[torch.Tensor, torch.Tensor] | None,is_prefill: bool,) -> torch.Tensor:- fmt = cache.quant_format- if fmt in ("nvfp4_marlin", "nvfp4_b12x"):- # Borrowed W4A16 fused MoE -- Marlin (vLLM, sm_80-99) or b12x- # (flashinfer, sm_120) over their pre-tiled banks; one kernel serves- # prefill and decode, with the movement-matched per-row global scales.- from freetoken.moe.nvfp4_backends import b12x_fused_experts, marlin_fused_experts-- assert alphas is not None- gate_up_packed, gate_up_scale, down_packed, down_scale = views- fused = marlin_fused_experts if fmt == "nvfp4_marlin" else b12x_fused_experts- return fused(- hidden_states,- gate_up_packed,- gate_up_scale,- alphas[0],- down_packed,- down_scale,- alphas[1],- topk_weights,- topk_ids,- self.activation,- self.apply_router_weight_on_input,- )- if fmt == "nvfp4":- # FreeToken's Triton inline-dequant kernels over the native ModelOpt- # rows: the FP4 banks are read directly in the GEMM, no BF16 copy of- # the experts is ever materialized. The swigluoai scalars (MiniMax-M3)- # live on the layer via make_moe_layer's extra_attrs (gpt-oss precedent)- # and are ignored by the plain *_and_mul activations.- act_alpha = getattr(self, "hidden_act_alpha", 1.702)- act_limit = getattr(self, "swiglu_limit", None)- # None == "no clamp" everywhere else in the repo (mxfp4 maps it to +inf).- act_limit = float("inf") if act_limit is None else act_limit- if is_prefill:- from freetoken.moe.fused_nvfp4 import fused_experts_nvfp4-- return fused_experts_nvfp4(- hidden_states,- *views,- topk_weights,- topk_ids,- n,- self.activation,- self.apply_router_weight_on_input,- act_alpha,- act_limit,- )- # Marlin-style int32 wide-load GEMV (arithmetic dequant, no HW cvt).- # Bit-identical to the byte-at-a-time path; lifts gate/up BW ~43%->51%- # (I=512), ~41%->53% (I=768), 65%->72% (I=1536). CUDA-graph safe (fixed- # shapes, no host sync).- from freetoken.moe.fused_nvfp4 import fused_experts_decode_nvfp4_marlin-- return fused_experts_decode_nvfp4_marlin(- hidden_states,- *views,- topk_weights,- topk_ids,- self.activation,- self.apply_router_weight_on_input,- act_alpha,- act_limit,- )- if fmt == "fp8_block":- # Block-fp8 experts: fused inline-dequant grouped GEMM reads the routed fp8 rows- # directly (fp8 banks halve host/cache bytes; no bf16 materialization).- from freetoken.moe.fused_fp8_block import (- fused_experts_decode_fp8_block,- fused_experts_fp8_block,- )+ if self.quant_method is not None:+ from freetoken.moe.legacy_format import canonical_role # legacy_format imports this package- gate_up, gate_up_scale, down, down_scale = views- if is_prefill:- return fused_experts_fp8_block(- hidden_states, gate_up, gate_up_scale, down, down_scale,- topk_weights, topk_ids, n, self.activation,- self.apply_router_weight_on_input,- )- return fused_experts_decode_fp8_block(- hidden_states, gate_up, gate_up_scale, down, down_scale,- topk_weights, topk_ids, self.activation, self.apply_router_weight_on_input,+ view = ExpertView(+ {canonical_role(name): t for name, t in zip(cache.bank_schema, views)},+ slots=None if n is not None else topk_ids, n=n, alphas=alphas,+ )+ return self.quant_method.apply(+ hidden_states, topk_weights, topk_ids, view, layer=self, is_prefill=is_prefill)+ fmt = cache.quant_formatif fmt == "q4_0":# Native GGUF Q4_0 experts: dequant-in-kernel grouped GEMV (MMVQ) over the# streamed packed banks; topk_ids already index the cache slots / layer.@@ -531,57 +426,7 @@ class OffloadMoELayer(MoELayer):return fused_experts_gguf_q4_0(hidden_states, gate_up, down, topk_weights, topk_ids, self.activation)- if fmt == "mxfp4_triton":- # gpt-oss MXFP4 experts (biased, clamped swiglu): transposed split-K GEMV- # decode + grouped `_t` prefill. The swiglu scalars live on the layer- # (set at construction), not in the base signature.- from freetoken.moe.fused_mxfp4 import (- run_mxfp4_prefill_experts_t,- run_mxfp4_splitk_decode_experts,- )-- gu_blocks, gu_scales, gu_bias, dn_blocks, dn_scales, dn_bias = views- run = run_mxfp4_prefill_experts_t if is_prefill else run_mxfp4_splitk_decode_experts- return run(- hidden_states, topk_weights, topk_ids,- gu_blocks, gu_scales, gu_bias, dn_blocks, dn_scales, dn_bias,- top_k=self.top_k,- hidden_act_alpha=self.hidden_act_alpha,- swiglu_limit=self.swiglu_limit,- )- if fmt == "ds_fp4":- # DeepSeek-V4 FP4 experts: grouped inline-dequant GEMM for streaming- # prefill chunks (n = bank rows to sort over); per-route dequant GEMV- # for decode and the sparse small-chunk slot path (n is None there,- # and sorting over the full slot cache would drown in padding).- gate_up_packed, gate_up_scale, down_packed, down_scale = views- if is_prefill and n is not None:- from freetoken.moe.fused_ds_fp4 import routed_experts_fp4_prefill-- return routed_experts_fp4_prefill(- hidden_states, topk_ids, topk_weights,- gate_up_packed, gate_up_scale, down_packed, down_scale,- self.swiglu_limit, n,- )- from freetoken.moe.fused_ds_fp4 import routed_experts_fp4-- return routed_experts_fp4(- hidden_states, topk_ids, topk_weights,- gate_up_packed, gate_up_scale, down_packed, down_scale,- self.swiglu_limit,- )- assert fmt == "bf16", f"unknown quant_format {fmt!r}"- gate_up, down = views- impl = fused_experts_impl if is_prefill else fused_experts_decode_impl- return impl(- hidden_states,- gate_up,- down,- topk_weights,- topk_ids,- self.activation,- self.apply_router_weight_on_input,- )+ raise AssertionError(f"offload experts without a quant method only serve q4_0 banks, got {fmt!r}")def make_moe_layer(@@ -589,7 +434,6 @@ def make_moe_layer(*,layer_id: int | None = None,activation: str = "silu",- weight_format: str = "bf16",renormalize: bool | None = None,apply_router_weight_on_input: bool = False,num_experts: int | None = None,@@ -598,21 +442,23 @@ def make_moe_layer(intermediate_size: int | None = None,resident_cls: type[MoELayer] | None = None,offload_cls: "type[OffloadMoELayer] | None" = None,- extra_attrs: dict | None = None,+ alpha: float = 1.0,+ beta: float = 0.0,+ limit: float | None = None,+ interleaved: bool = False,+ has_bias: bool = False,+ quant_config: QuantConfig | None = None,+ prefix: str = "",) -> MoELayer:- """Build the experts layer for ``config.moe_backend`` -- the one construction+ """Build the experts layer for ``config.moe_strategy`` -- the one constructionseam between a model and the MoE strategy.- Picks ``OffloadMoELayer`` for the offload family (offload/cpu/hybrid, which- ignore ``weight_format``: the quant format comes from the offload cache) and+ Picks ``OffloadMoELayer`` for the offload family (offload/cpu/hybrid) and``MoELayer`` otherwise. Geometry defaults come from ``config``; pass overrides- for models whose fields deviate. ``extra_attrs`` become instance attributes --- the seam for per-format scalars the base signature does not carry (e.g.- ``hidden_act_alpha``/``swiglu_limit``, read back via ``getattr`` by the engine- and format kernels). ``resident_cls``/``offload_cls`` keep model-specific+ for models whose fields deviate. ``resident_cls``/``offload_cls`` keep model-specificsubclasses constructible through the same seam."""- offload = is_offload_moe_backend(config.moe_backend)+ offload = is_offload_moe_strategy(config.moe_strategy)layer_cls = (offload_cls or OffloadMoELayer) if offload else (resident_cls or MoELayer)kwargs = dict(num_experts=num_experts if num_experts is not None else config.num_experts,@@ -624,13 +470,17 @@ def make_moe_layer(renormalize=renormalize if renormalize is not None else config.norm_topk_prob,activation=activation,apply_router_weight_on_input=apply_router_weight_on_input,+ alpha=alpha,+ beta=beta,+ limit=limit,+ interleaved=interleaved,+ has_bias=has_bias,+ quant_config=quant_config,+ prefix=prefix,)if offload:assert layer_id is not None, "offload MoE backends need the layer_id"kwargs["layer_id"] = layer_id- else:- kwargs["weight_format"] = weight_format- layer = layer_cls(**kwargs)- for name, value in (extra_attrs or {}).items():- setattr(layer, name, value)- return layer+ kwargs["strategy"] = config.moe_strategy+ kwargs["decode_target"] = config.decode_target+ return layer_cls(**kwargs)diff --git a/python/freetoken/layers/norm.py b/python/freetoken/layers/norm.pyindex df248136..5eca789b 100644--- a/python/freetoken/layers/norm.py+++ b/python/freetoken/layers/norm.py@@ -168,3 +168,25 @@ class RMSNormFused(BaseOP):return self.rmsnorm(x, self.weight, self.eps), xself.fused_add_rmsnorm(x, residual, self.weight, self.eps)return x, residual+++_GATE_ACTIVATIONS = ("silu", "swish", "sigmoid")+++class GatedRMSNorm(BaseOP):+ """``rmsnorm(x) * activation(z)`` in one fla kernel: the GDN / KDA output norm."""++ def __init__(self, size: int, eps: float, activation: str = "silu") -> None:+ # rms_norm_gated drops the gate entirely (no error) for a name it does not know+ assert activation in _GATE_ACTIVATIONS, f"unsupported gate activation {activation!r}"+ self.weight = torch.empty(size)+ self.eps = eps+ self.activation = activation++ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.fla import rms_norm_gated++ return rms_norm_gated(+ x=x, weight=self.weight, bias=None, z=z, eps=self.eps,+ is_rms_norm=True, norm_before_gate=True, activation=self.activation,+ )diff --git a/python/freetoken/layers/quantization/__init__.py b/python/freetoken/layers/quantization/__init__.pynew file mode 100644index 00000000..99e3d71f--- /dev/null+++ b/python/freetoken/layers/quantization/__init__.py@@ -0,0 +1,50 @@+from .scheme import GroupShape, QuantKind, QuantScheme, WeightDesc+from .registry import LayerKind, dialects, method_class, methods_for, register_dialect, register_method+from .method import KernelSelectionError, QuantMethod, finalize_quant, select_kernel+from .linear import (+ Fp8BlockLinearMethod,+ Fp8TensorLinearMethod,+ LinearConfig,+ LinearKernel,+ LinearMethod,+ Mxfp8LinearMethod,+ Nvfp4LinearMethod,+ UnquantizedLinearMethod,+)+from .moe import (+ BankSpec,+ ExpertView,+ Fp8BlockMoEMethod,+ MoEConfig,+ MoEKernel,+ MoEMethod,+ Mxfp4MoEMethod,+ Mxfp8MoEMethod,+ Nvfp4MoEMethod,+ UnquantizedMoEMethod,+)+from .quant_backend import QuantBackend, get_quant_backend, set_quant_backend+from .names import NameMap+from .configs import (+ CompressedTensorsConfig,+ Fp8BlockConfig,+ ModelOptConfig,+ Mxfp4Config,+ NoQuantConfig,+ QuantConfig,+ quant_method_for,+ quantization_config_of,+)++__all__ = [+ "QuantKind", "GroupShape", "QuantScheme", "WeightDesc",+ "LayerKind", "dialects", "method_class", "methods_for", "register_dialect", "register_method",+ "KernelSelectionError", "select_kernel", "QuantMethod",+ "LinearConfig", "LinearKernel", "LinearMethod",+ "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",+ "BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",+ "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",+ "QuantBackend", "set_quant_backend", "get_quant_backend", "NameMap",+ "QuantConfig", "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",+ "quant_method_for", "quantization_config_of", "finalize_quant",+]diff --git a/python/freetoken/layers/quantization/configs/__init__.py b/python/freetoken/layers/quantization/configs/__init__.pynew file mode 100644index 00000000..6905b60d--- /dev/null+++ b/python/freetoken/layers/quantization/configs/__init__.py@@ -0,0 +1,22 @@+"""One QuantConfig per checkpoint dialect; importing the package registers them all."""++from .base import NoQuantConfig, QuantConfig, quantization_config_of+from . import compressed_tensors, fp8, modelopt, mxfp4+from .compressed_tensors import CompressedTensorsConfig+from .fp8 import Fp8BlockConfig+from .modelopt import ModelOptConfig+from .mxfp4 import Mxfp4Config++_NO_QUANT = NoQuantConfig()+++def quant_method_for(quant_config, layer, prefix: str):+ """The layer's Method from its config; a layer built without one is plain bf16."""+ return (quant_config if quant_config is not None else _NO_QUANT).get_quant_method(layer, prefix)+++__all__ = [+ "QuantConfig", "quantization_config_of", "quant_method_for",+ "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",+ "compressed_tensors", "fp8", "modelopt", "mxfp4",+]diff --git a/python/freetoken/layers/quantization/configs/base.py b/python/freetoken/layers/quantization/configs/base.pynew file mode 100644index 00000000..d786a333--- /dev/null+++ b/python/freetoken/layers/quantization/configs/base.py@@ -0,0 +1,110 @@+"""QuantConfig: reads one quantization_config dialect and answers what scheme each module has."""++from __future__ import annotations++from abc import ABC, abstractmethod+from typing import Any, ClassVar++from ..linear import LinearConfig+from ..moe import MoEConfig+from ..names import Matcher, NameMap, name_set+from ..quant_backend import get_quant_backend+from ..registry import LayerKind, dialects, method_class, register_dialect+from ..scheme import QuantKind, QuantScheme++LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig}+++def cfg_get(cfg: Any, key: str, default: Any = None) -> Any:+ if cfg is None:+ return default+ if isinstance(cfg, dict):+ return cfg.get(key, default)+ return getattr(cfg, key, default)+++def quantization_config_of(hf_config: Any) -> dict[str, Any] | None:+ q = cfg_get(hf_config, "quantization_config")+ if q is None:+ q = cfg_get(cfg_get(hf_config, "text_config"), "quantization_config")+ if q is None:+ return None+ if isinstance(q, dict):+ return q+ if hasattr(q, "to_dict"):+ return dict(q.to_dict())+ return dict(vars(q))+++class QuantConfig(ABC):+ dialect: ClassVar[str]++ def __init__(self, name_map: NameMap | None = None, unquantized: tuple[str, ...] = ()):+ self.name_map = name_map or NameMap()+ # modules the family serves in bf16 regardless of the dialect (glob patterns on checkpoint names)+ self.unquantized: Matcher = name_set(tuple(unquantized))+ self._schemes: dict[str, QuantScheme | None] = {}++ @classmethod+ def claims(cls, q: dict[str, Any]) -> bool:+ """Whether this dialect reads the given ``quantization_config``."""+ return str(q.get("quant_method") or "").lower() == cls.dialect++ @abstractmethod+ def scheme_for_name(self, name: str) -> QuantScheme | None:+ """Scheme of one module by its checkpoint name; None means unquantized."""++ def scheme_for(self, prefix: str) -> QuantScheme | None:+ if prefix in self._schemes:+ return self._schemes[prefix]+ names = self.name_map.to_checkpoint(prefix)+ schemes = {None if self.unquantized(n) else self.scheme_for_name(n) for n in names}+ if len(schemes) != 1:+ raise ValueError(f"fused module {prefix!r} mixes quantization schemes across {names}: {schemes}")+ scheme = schemes.pop()+ self._schemes[prefix] = scheme+ return scheme++ def get_quant_method(self, layer: Any, prefix: str):+ scheme = self.scheme_for(prefix)+ layer_kind = layer.quant_layer_kind+ kind = scheme.kind if scheme else QuantKind.NONE+ cls = method_class(kind, layer_kind)+ cfg = LAYER_CONFIGS[layer_kind].from_layer(layer, scheme)+ return cls(cfg, get_quant_backend().select(layer_kind, kind))++ @staticmethod+ def from_hf(+ hf_config: Any,+ *,+ name_map: NameMap | None = None,+ unquantized: tuple[str, ...] = (),+ hf_quant_config: dict[str, Any] | None = None,+ ) -> "QuantConfig":+ """``hf_quant_config`` is the parsed ``hf_quant_config.json`` of ModelOpt exports that keep+ no ``quantization_config`` in config.json (modelopt < 0.41); ``unquantized`` lists the modules+ the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors)."""+ q = quantization_config_of(hf_config)+ if q is None and hf_quant_config and isinstance(hf_quant_config.get("quantization"), dict):+ q = dict(hf_quant_config["quantization"])+ q.setdefault("quant_method", "modelopt")+ if q is None:+ return NoQuantConfig(name_map, unquantized)+ for cls in dialects():+ if cls.claims(q):+ return cls(q, hf_config, name_map=name_map, unquantized=unquantized)+ raise NotImplementedError(f"quantization method {q.get('quant_method')!r} is not supported")+++@register_dialect+class NoQuantConfig(QuantConfig):+ """No ``quantization_config``: every module is bf16."""++ dialect = "none"++ @classmethod+ def claims(cls, q: dict[str, Any]) -> bool:+ return False++ def scheme_for_name(self, name: str) -> QuantScheme | None:+ return Nonediff --git a/python/freetoken/layers/quantization/configs/compressed_tensors.py b/python/freetoken/layers/quantization/configs/compressed_tensors.pynew file mode 100644index 00000000..d2b31dba--- /dev/null+++ b/python/freetoken/layers/quantization/configs/compressed_tensors.py@@ -0,0 +1,64 @@+from __future__ import annotations++from typing import Any, ClassVar++from ..names import Matcher, ct_set+from ..registry import register_dialect+from ..scheme import QuantScheme+from ..scheme import fp8_block_scheme, fp8_tensor_scheme, nvfp4_scheme+from .base import QuantConfig+++@register_dialect+class CompressedTensorsConfig(QuantConfig):+ """llm-compressor exports: ordered ``config_groups`` with ``targets`` (class names, names or ``re:``) minus ``ignore``."""++ dialect = "compressed-tensors"++ SCHEMES: ClassVar[dict[str, QuantScheme]] = {+ "NVFP4": nvfp4_scheme(input_scale=False),+ "NVFP4_LOCAL": nvfp4_scheme(input_scale=True),+ "FP8_TENSOR_STATIC": fp8_tensor_scheme("bf16", input_scale=True),+ "FP8_TENSOR_DYNAMIC": fp8_tensor_scheme("bf16"),+ "FP8_CHANNEL": fp8_tensor_scheme("bf16", per_row=True),+ "FP8_BLOCK": fp8_block_scheme("float"),+ }++ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):+ super().__init__(name_map, unquantized)+ self.ignore = ct_set(tuple(q.get("ignore") or ()), class_names=False)+ groups = q.get("config_groups") or {}+ self.groups: list[tuple[Matcher, QuantScheme]] = [+ (ct_set(tuple(spec.get("targets") or ()), class_names=True), self._scheme_of(spec)) for spec in groups.values()+ ]++ def scheme_for_name(self, name: str) -> QuantScheme | None:+ if self.ignore(name):+ return None+ for targets, scheme in self.groups:+ if targets(name):+ return scheme+ return None++ @classmethod+ def _scheme_of(cls, spec: dict[str, Any]) -> QuantScheme:+ w = spec.get("weights") or {}+ act = spec.get("input_activations")+ bits = int(w.get("num_bits") or 0)+ wtype = str(w.get("type") or "").lower()+ strategy = str(w.get("strategy") or "").lower()+ group = int(w.get("group_size") or 0)+ if bits == 4 and wtype == "float":+ if strategy == "tensor_group" and group == 16:+ return cls.SCHEMES["NVFP4_LOCAL" if act and act.get("dynamic") == "local" else "NVFP4"]+ raise NotImplementedError(f"compressed-tensors 4-bit float scheme (strategy={strategy!r}, group_size={group}) is not supported; only NVFP4 is")+ if bits == 8 and wtype == "float":+ if strategy == "tensor":+ return cls.SCHEMES["FP8_TENSOR_DYNAMIC" if act is None or act.get("dynamic") else "FP8_TENSOR_STATIC"]+ if strategy == "channel":+ return cls.SCHEMES["FP8_CHANNEL"]+ if strategy == "block":+ block = tuple(int(x) for x in (w.get("block_structure") or ()))+ if block == (128, 128):+ return cls.SCHEMES["FP8_BLOCK"]+ raise NotImplementedError(f"compressed-tensors weight scheme {w} is not supported")diff --git a/python/freetoken/layers/quantization/configs/fp8.py b/python/freetoken/layers/quantization/configs/fp8.pynew file mode 100644index 00000000..11c8fbe2--- /dev/null+++ b/python/freetoken/layers/quantization/configs/fp8.py@@ -0,0 +1,46 @@+from __future__ import annotations++from typing import Any, ClassVar++from ..names import is_routed_expert, name_set, substr_set+from ..registry import register_dialect+from ..scheme import QuantScheme+from ..scheme import FP8_BLOCK, fp8_block_scheme, fp8_tensor_scheme, mxfp4_scheme+from .base import QuantConfig, cfg_get+++@register_dialect+class Fp8BlockConfig(QuantConfig):+ """HF ``quant_method: fp8`` (DeepSeek-V3 style 128x128 block scales) plus the DeepSeek-V4 e8m0 / fp4-expert variant."""++ dialect = "fp8"++ SCHEMES: ClassVar[dict[str, QuantScheme]] = {+ "BLOCK": fp8_block_scheme("float"),+ "BLOCK_E8M0": fp8_block_scheme("e8m0"),+ # HF ``modules_to_convert``: a table (Qwen3.8-Flash-Next PLE) stored e4m3 with one scalar scale+ "TABLE": fp8_tensor_scheme("float"),+ "EXPERT_MXFP4": mxfp4_scheme(),+ }++ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):+ super().__init__(name_map, unquantized)+ block = tuple(int(x) for x in (q.get("weight_block_size") or ()))+ if q.get("weight_per_tensor") or block != (FP8_BLOCK, FP8_BLOCK):+ raise NotImplementedError(f"fp8 checkpoint with weight_block_size={block} per_tensor={q.get('weight_per_tensor')} is not supported; only 128x128 blocks are")+ # transformers skips lm_head when the checkpoint gives no list+ not_convert = tuple(q.get("modules_to_not_convert") or ("lm_head",))+ self.not_convert = name_set(not_convert)+ self.not_convert_substr = substr_set(not_convert)+ self.convert_tables = name_set(tuple(q.get("modules_to_convert") or ()))+ self.e8m0 = str(q.get("scale_fmt") or "").lower() == "ue8m0"+ self.expert_fp4 = str(cfg_get(hf_config, "expert_dtype") or "").lower() == "fp4"++ def scheme_for_name(self, name: str) -> QuantScheme | None:+ if self.convert_tables(name):+ return self.SCHEMES["TABLE"]+ if self.not_convert(name) or self.not_convert_substr(name):+ return None+ if self.expert_fp4 and is_routed_expert(name):+ return self.SCHEMES["EXPERT_MXFP4"]+ return self.SCHEMES["BLOCK_E8M0" if self.e8m0 else "BLOCK"]diff --git a/python/freetoken/layers/quantization/configs/modelopt.py b/python/freetoken/layers/quantization/configs/modelopt.pynew file mode 100644index 00000000..813b0667--- /dev/null+++ b/python/freetoken/layers/quantization/configs/modelopt.py@@ -0,0 +1,66 @@+from __future__ import annotations++from typing import Any, ClassVar++from ..names import ancestors, name_set+from ..registry import register_dialect+from ..scheme import QuantScheme+from ..scheme import fp8_block_scheme, fp8_tensor_scheme, mxfp8_scheme, nvfp4_scheme+from .base import QuantConfig+++@register_dialect+class ModelOptConfig(QuantConfig):+ """NVIDIA ModelOpt exports: one ``quant_algo`` for every Linear minus ``ignore``, or+ ``MIXED_PRECISION`` with a per-module ``quantized_layers`` allow-list."""++ dialect = "modelopt"++ SCHEMES: ClassVar[dict[str, QuantScheme]] = {+ "NVFP4": nvfp4_scheme(input_scale=True),+ "NVFP4_NO_INPUT": nvfp4_scheme(input_scale=False),+ "FP8": fp8_tensor_scheme("fp32", input_scale=True),+ "FP8_PER_CHANNEL_PER_TOKEN": fp8_tensor_scheme("fp32", per_row=True),+ "FP8_PB_WO": fp8_block_scheme("fp32"),+ "MXFP8": mxfp8_scheme(),+ }++ @classmethod+ def claims(cls, q: dict[str, Any]) -> bool:+ method = str(q.get("quant_method") or "").lower()+ return method == "modelopt" or (not method and bool(q.get("quant_algo")))++ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):+ super().__init__(name_map, unquantized)+ self.algo = str(q.get("quant_algo") or "").upper()+ self.ignore = name_set(tuple(q.get("ignore") or q.get("exclude_modules") or ()))+ layers = q.get("quantized_layers") or {}+ self.quantized_layers = {k: str((v or {}).get("quant_algo") or "").upper() for k, v in layers.items()} if isinstance(layers, dict) else {}+ self.with_input_scale = bool(q.get("with_input_scale", True))+ if self.algo == "MIXED_PRECISION" and not self.quantized_layers:+ raise NotImplementedError("ModelOpt MIXED_PRECISION without quantized_layers in quantization_config")+ if self.algo != "MIXED_PRECISION":+ self._scheme_of(self.algo)++ def scheme_for_name(self, name: str) -> QuantScheme | None:+ if self.ignore(name):+ return None+ algo = self._module_algo(name)+ return None if algo is None else self._scheme_of(algo)++ def _module_algo(self, name: str) -> str | None:+ if self.algo != "MIXED_PRECISION":+ return self.algo+ for a in ancestors(name):+ algo = self.quantized_layers.get(a)+ if algo is not None:+ return algo+ return None++ def _scheme_of(self, algo: str) -> QuantScheme:+ if algo in ("NVFP4", "W4A16_NVFP4"):+ return self.SCHEMES["NVFP4" if self.with_input_scale else "NVFP4_NO_INPUT"]+ try:+ return self.SCHEMES[algo]+ except KeyError:+ raise NotImplementedError(f"ModelOpt quant_algo {algo!r} is not supported") from Nonediff --git a/python/freetoken/layers/quantization/configs/mxfp4.py b/python/freetoken/layers/quantization/configs/mxfp4.pynew file mode 100644index 00000000..e9458f03--- /dev/null+++ b/python/freetoken/layers/quantization/configs/mxfp4.py@@ -0,0 +1,27 @@+from __future__ import annotations++from typing import Any, ClassVar++from ..names import name_set+from ..registry import register_dialect+from ..scheme import QuantScheme+from ..scheme import mxfp4_scheme+from .base import QuantConfig+++@register_dialect+class Mxfp4Config(QuantConfig):+ """gpt-oss: the stacked ``experts`` containers are MXFP4 with biases, everything else bf16."""++ dialect = "mxfp4"++ SCHEME: ClassVar[QuantScheme] = mxfp4_scheme()++ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):+ super().__init__(name_map, unquantized)+ self.not_convert = name_set(tuple(q.get("modules_to_not_convert") or ()))++ def scheme_for_name(self, name: str) -> QuantScheme | None:+ if not name.endswith(".experts") or self.not_convert(name):+ return None+ return self.SCHEMEdiff --git a/python/freetoken/layers/quantization/linear/__init__.py b/python/freetoken/layers/quantization/linear/__init__.pynew file mode 100644index 00000000..8236abe0--- /dev/null+++ b/python/freetoken/layers/quantization/linear/__init__.py@@ -0,0 +1,13 @@+from .base import LinearConfig, LinearKernel, LinearMethod+from . import fp8_block, fp8_tensor, mxfp8, nvfp4, unquantized+from .fp8_block import Fp8BlockLinearMethod+from .fp8_tensor import Fp8TensorLinearMethod+from .mxfp8 import Mxfp8LinearMethod+from .nvfp4 import Nvfp4LinearMethod+from .unquantized import UnquantizedLinearMethod++__all__ = [+ "LinearConfig", "LinearKernel", "LinearMethod",+ "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",+ "fp8_block", "fp8_tensor", "mxfp8", "nvfp4", "unquantized",+]diff --git a/python/freetoken/layers/quantization/linear/base.py b/python/freetoken/layers/quantization/linear/base.pynew file mode 100644index 00000000..e8bbeff4--- /dev/null+++ b/python/freetoken/layers/quantization/linear/base.py@@ -0,0 +1,64 @@+"""The linear side: per-layer config, the kernel ABC and the Method base."""++from __future__ import annotations++from abc import ABC, abstractmethod+from dataclasses import dataclass+from typing import Any, ClassVar++import torch++from ..method import QuantMethod+from ..scheme import QuantScheme+++@dataclass(frozen=True)+class LinearConfig:+ """What a linear kernel may look at to accept or decline a layer: the TP-local shape and the weight scheme."""++ in_features: int+ out_features: int+ # fused segments along N; a plain linear is one segment+ output_sizes: tuple[int, ...] = ()+ scheme: QuantScheme | None = None++ def __post_init__(self) -> None:+ if not self.output_sizes:+ object.__setattr__(self, "output_sizes", (self.out_features,))+ if sum(self.output_sizes) != self.out_features:+ raise ValueError(f"output_sizes {self.output_sizes} do not sum to {self.out_features}")++ @classmethod+ def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "LinearConfig":+ return cls(layer.in_features, layer.out_features, tuple(layer.output_sizes), scheme)+++class LinearKernel(ABC):+ """One backend for one linear kind: says whether it can run a layer, then finalizes and applies it."""++ name: ClassVar[str]++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ return None++ def worth_it(self, cfg: LinearConfig) -> bool:+ return True++ def finalize(self, layer: Any) -> None:+ pass++ @abstractmethod+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor: ...+++class LinearMethod(QuantMethod):+ """Declares a linear layer's weights for its kind; finalize and apply go to the kernel."""++ @abstractmethod+ def create_weights(self, layer: Any) -> None: ...++ def finalize(self, layer: Any) -> None:+ self.kernel.finalize(layer)++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ return self.kernel.apply(layer, x)diff --git a/python/freetoken/layers/quantization/linear/fp8_block.py b/python/freetoken/layers/quantization/linear/fp8_block.pynew file mode 100644index 00000000..3b78e977--- /dev/null+++ b/python/freetoken/layers/quantization/linear/fp8_block.py@@ -0,0 +1,60 @@+"""fp8 e4m3 weight with 128x128 block scales (DeepSeek-V3 style)."""++from __future__ import annotations++from typing import Any++import torch++from ..registry import LayerKind, register_method+from ..scheme import FP8_BLOCK as BLOCK, QuantKind+from .base import LinearConfig, LinearKernel, LinearMethod++FP8 = torch.float8_e4m3fn+E8M0 = torch.float8_e8m0fnu+++def _e8m0(cfg: LinearConfig) -> bool:+ return cfg.scheme is not None and cfg.scheme.weight.scale == "e8m0"+++class Dsv4Fp8BlockLinearKernel(LinearKernel):+ """DeepSeek-V4's reference path: activations quantized to fp8 with power-of-two block scales, e8m0 weight scales read as codes."""++ name = "dsv4"++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ return None if _e8m0(cfg) else "serves e8m0 block scales only"++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.dsv4.fp8_linear import block_fp8_linear++ return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)+++class TritonFp8BlockLinearKernel(LinearKernel):+ """W8A16 GEMV at M=1, dynamic 1x128 W8A8 GEMM above."""++ name = "triton"++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ return "reads float block scales; e8m0 codes go to the dsv4 kernel" if _e8m0(cfg) else None++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.fp8_block_linear import block_fp8_linear++ return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)+++@register_method(QuantKind.FP8_BLOCK, LayerKind.LINEAR)+class Fp8BlockLinearMethod(LinearMethod):+ candidates = (Dsv4Fp8BlockLinearKernel, TritonFp8BlockLinearKernel)++ def create_weights(self, layer: Any) -> None:+ g = self.cfg+ if g.in_features % BLOCK or any(o % BLOCK for o in g.output_sizes):+ raise ValueError(f"block-fp8 needs in/out sizes divisible by {BLOCK}, got K={g.in_features} N={g.output_sizes}")+ layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)+ # e8m0 codes stay codes for the dsv4 kernel; float scales are bf16 as the readers push them today+ scale_dtype = E8M0 if _e8m0(g) else torch.bfloat16+ layer.weight_scale_inv = torch.empty(g.out_features // BLOCK, g.in_features // BLOCK, dtype=scale_dtype)diff --git a/python/freetoken/layers/quantization/linear/fp8_tensor.py b/python/freetoken/layers/quantization/linear/fp8_tensor.pynew file mode 100644index 00000000..9461e044--- /dev/null+++ b/python/freetoken/layers/quantization/linear/fp8_tensor.py@@ -0,0 +1,84 @@+"""fp8 e4m3 weight with one fp32 scale per output row (per-tensor scales are broadcast)."""++from __future__ import annotations++from typing import Any++import torch++from freetoken.kernel import backend++from ..registry import LayerKind, register_method+from ..scheme import QuantKind+from .base import LinearConfig, LinearKernel, LinearMethod++FP8 = torch.float8_e4m3fn+++class TorchFp8TensorLinearKernel(LinearKernel):+ """Static W8A8 through torch._scaled_mm; needs the checkpoint's input_scale and fp8 tensor cores."""++ name = "torch"++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ if cfg.scheme is None or not cfg.scheme.has("input_scale"):+ return "the checkpoint has no static activation scale"+ cc = backend.device_capability()+ if cc < (8, 9):+ return f"fp8 tensor cores need sm_89+, got sm_{cc[0]}{cc[1]}"+ from freetoken.kernel.triton.e4m3_compat import e4m3_native++ if not e4m3_native():+ return "e4m3 emulation mode has no fp8 GEMM"+ return None++ def finalize(self, layer: Any) -> None:+ from freetoken.kernel.triton.fp8_pertensor_linear import rowwise_scaled_mm_ok, weight_scale_segments++ # a fused projection carries one scalar per part; decide the GEMM shape once, not under graph capture+ scale = layer.weight_scale+ uniform = bool((scale == scale[0]).all().item())+ layer._fp8_uniform_scale = uniform+ layer._fp8_scale_segments = None if uniform else weight_scale_segments(scale)+ if not uniform:+ rowwise_scaled_mm_ok()++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear++ return fp8_pertensor_linear(+ x, layer.weight, layer.weight_scale, layer.bias,+ layer.input_scale, layer._fp8_uniform_scale, scale_segments=layer._fp8_scale_segments,+ )+++class TritonFp8TensorLinearKernel(LinearKernel):+ """W8A16: split-K GEMV at M=1, GEMM above."""++ name = "triton"++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear++ return fp8_pertensor_linear(x, layer.weight, layer.weight_scale, layer.bias)+++class EmulationFp8TensorLinearKernel(LinearKernel):+ name = "emulation"++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ *lead, K = x.shape+ w = layer.weight.to(x.dtype) * layer.weight_scale.to(x.dtype)[:, None]+ out = (x.reshape(-1, K) @ w.t()).reshape(*lead, w.shape[0])+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out+++@register_method(QuantKind.FP8_TENSOR, LayerKind.LINEAR)+class Fp8TensorLinearMethod(LinearMethod):+ candidates = (TorchFp8TensorLinearKernel, TritonFp8TensorLinearKernel, EmulationFp8TensorLinearKernel)++ def create_weights(self, layer: Any) -> None:+ g = self.cfg+ layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)+ layer.weight_scale = torch.empty(g.out_features, dtype=torch.float32)+ layer.input_scale = torch.empty((), dtype=torch.float32) if self.scheme.has("input_scale") else Nonediff --git a/python/freetoken/layers/quantization/linear/mxfp8.py b/python/freetoken/layers/quantization/linear/mxfp8.pynew file mode 100644index 00000000..3102727f--- /dev/null+++ b/python/freetoken/layers/quantization/linear/mxfp8.py@@ -0,0 +1,48 @@+"""fp8 e4m3 weight with e8m0 scales per 32 input elements (OCP MXFP8)."""++from __future__ import annotations++from typing import Any++import torch++from ..registry import LayerKind, register_method+from ..scheme import MX_GROUP as BLOCK, QuantKind+from .base import LinearKernel, LinearMethod++FP8 = torch.float8_e4m3fn+++class TritonMxfp8LinearKernel(LinearKernel):+ """W8A16: split-K GEMV up to M=256, bf16 dequant + cuBLAS above."""++ name = "triton"++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_linear++ return mxfp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)+++class EmulationMxfp8LinearKernel(LinearKernel):+ name = "emulation"++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant++ *lead, K = x.shape+ w = mxfp8_dequant(layer.weight, layer.weight_scale_inv, dtype=torch.float32)+ out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, w.shape[0])+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out+++@register_method(QuantKind.MXFP8, LayerKind.LINEAR)+class Mxfp8LinearMethod(LinearMethod):+ candidates = (TritonMxfp8LinearKernel, EmulationMxfp8LinearKernel)++ def create_weights(self, layer: Any) -> None:+ g = self.cfg+ if g.in_features % BLOCK:+ raise ValueError(f"mxfp8 needs in_features divisible by {BLOCK}, got {g.in_features}")+ layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)+ layer.weight_scale_inv = torch.empty(g.out_features, g.in_features // BLOCK, dtype=torch.uint8)diff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.pynew file mode 100644index 00000000..6644acdb--- /dev/null+++ b/python/freetoken/layers/quantization/linear/nvfp4.py@@ -0,0 +1,123 @@+"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp16 global per output row."""++from __future__ import annotations++from types import SimpleNamespace+from typing import Any++import torch++from freetoken.kernel import backend++from ..registry import LayerKind, register_method+from ..scheme import NVFP4_GROUP as GROUP, QuantKind+from .base import LinearConfig, LinearKernel, LinearMethod++FP8 = torch.float8_e4m3fn+++class TritonNvfp4LinearKernel(LinearKernel):+ """W4A16 on the K-major resident layout."""++ name = "triton"++ def finalize(self, layer: Any) -> None:+ from freetoken.kernel.triton.nvfp4_linear import nvfp4_transpose_resident++ layer.weight, layer.weight_scale = nvfp4_transpose_resident(layer.weight, layer.weight_scale)++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.nvfp4_linear import nvfp4_dense_linear_t++ return nvfp4_dense_linear_t(x, layer.weight, layer.weight_scale, layer.weight_global, layer.bias)+++class MarlinNvfp4LinearKernel(LinearKernel):+ """vLLM's fused FP4 Marlin GEMM (W4A16); needs one global scale for the whole weight."""++ name = "marlin"++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ if not backend.is_vllm_installed():+ return "vLLM is not installed"+ if len(cfg.output_sizes) > 1:+ return "fused projection carries one global scale per part"+ try:+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401+ apply_fp4_marlin_linear,+ is_fp4_marlin_supported,+ prepare_fp4_layer_for_marlin,+ )+ except Exception as exc:+ return f"vLLM Marlin FP4 symbols are unusable ({exc!r})"+ if not is_fp4_marlin_supported():+ return "vLLM reports FP4 Marlin unsupported on this device"+ return None++ def finalize(self, layer: Any) -> None:+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import prepare_fp4_layer_for_marlin++ # Marlin wants the per-tensor scalar; weight_global is that scalar broadcast to [N]+ packed = SimpleNamespace(+ weight=layer.weight,+ weight_scale=layer.weight_scale,+ weight_scale_2=layer.weight_global.reshape(-1)[0].to(torch.bfloat16),+ output_size_per_partition=layer.out_features,+ input_size_per_partition=layer.in_features,+ params_dtype=torch.bfloat16,+ )+ prepare_fp4_layer_for_marlin(packed)+ layer._marlin_weight = packed.weight+ layer._marlin_scale = packed.weight_scale+ layer._marlin_global = packed.weight_scale_2+ layer._marlin_workspace = packed.workspace+ dev = layer.weight.device+ layer.weight = torch.empty(0, dtype=torch.uint8, device=dev)+ layer.weight_scale = torch.empty(0, dtype=FP8, device=dev)+ layer.weight_global = torch.empty(0, dtype=torch.float16, device=dev)++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import apply_fp4_marlin_linear++ out = apply_fp4_marlin_linear(+ input=x,+ weight=layer._marlin_weight,+ weight_scale=layer._marlin_scale,+ weight_scale_2=layer._marlin_global,+ workspace=layer._marlin_workspace,+ size_n=layer.out_features,+ size_k=layer.in_features,+ )+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out+++class EmulationNvfp4LinearKernel(LinearKernel):+ """Dequantize to bf16 with dequant_nvfp4, then a plain matmul; keeps the row-major layout."""++ name = "emulation"++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4++ *lead, K = x.shape+ slots = torch.zeros(1, dtype=torch.int32, device=x.device)+ w = dequant_nvfp4(+ layer.weight.unsqueeze(0), layer.weight_scale.unsqueeze(0), layer.weight_global.unsqueeze(0),+ slots, dtype=torch.bfloat16,+ )[0]+ out = (x.reshape(-1, K) @ w.t()).to(x.dtype).reshape(*lead, w.shape[0])+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out+++@register_method(QuantKind.NVFP4, LayerKind.LINEAR)+class Nvfp4LinearMethod(LinearMethod):+ candidates = (TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel)++ def create_weights(self, layer: Any) -> None:+ g = self.cfg+ if g.in_features % GROUP:+ raise ValueError(f"nvfp4 needs in_features divisible by {GROUP}, got {g.in_features}")+ layer.weight = torch.empty(g.out_features, g.in_features // 2, dtype=torch.uint8)+ layer.weight_scale = torch.empty(g.out_features, g.in_features // GROUP, dtype=FP8)+ layer.weight_global = torch.empty(g.out_features, dtype=torch.float16)+ # input_scale stays undeclared: the W4A16 kernels never read it and today's readers drop itdiff --git a/python/freetoken/layers/quantization/linear/unquantized.py b/python/freetoken/layers/quantization/linear/unquantized.pynew file mode 100644index 00000000..6cd8de8a--- /dev/null+++ b/python/freetoken/layers/quantization/linear/unquantized.py@@ -0,0 +1,33 @@+"""bf16 Linear: one kernel (torch), no scheme."""++from __future__ import annotations++from typing import Any++import torch+import torch.nn.functional as F++from ..registry import LayerKind, register_method+from ..scheme import QuantKind+from .base import LinearKernel, LinearMethod+++class TorchLinearKernel(LinearKernel):+ name = "torch"++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ # an fp32 activation stream (DeepSeek-V4's compressors) upcasts the bf16 weight on the fly, as the reference does+ w, b = layer.weight, layer.bias+ if w.dtype != x.dtype:+ w = w.to(x.dtype)+ b = b.to(x.dtype) if b is not None else None+ return F.linear(x, w, b)+++@register_method(QuantKind.NONE, LayerKind.LINEAR)+class UnquantizedLinearMethod(LinearMethod):+ candidates = (TorchLinearKernel,)++ def create_weights(self, layer: Any) -> None:+ g = self.cfg+ layer.weight = torch.empty(g.out_features, g.in_features)diff --git a/python/freetoken/layers/quantization/method.py b/python/freetoken/layers/quantization/method.pynew file mode 100644index 00000000..2a079539--- /dev/null+++ b/python/freetoken/layers/quantization/method.py@@ -0,0 +1,89 @@+"""QuantMethod: one (kind, layer kind) pair bound to one kernel object; the selector and the post-load hook."""++from __future__ import annotations++from abc import ABC+from typing import Any, ClassVar, Sequence++from freetoken.utils import init_logger++from .registry import LayerKind+from .scheme import QuantKind++logger = init_logger(__name__)+++class KernelSelectionError(RuntimeError):+ pass+++def select_kernel(candidates: Sequence[type], requested: str, cfg: Any):+ """Pick one kernel from an ordered table: the requested name if usable, else the first usable one that is worth it."""+ if not candidates:+ raise KernelSelectionError("empty kernel table")+ names = [c.name for c in candidates]+ if requested != "auto" and requested in names:+ kernel = candidates[names.index(requested)]()+ reason = kernel.unusable_reason(cfg)+ if reason:+ raise KernelSelectionError(f"kernel {requested!r} was requested but cannot run here: {reason}")+ return kernel+ skipped: list[str] = []+ fallback = None+ for cls in candidates:+ kernel = cls()+ reason = kernel.unusable_reason(cfg)+ if reason:+ skipped.append(f"{kernel.name}: {reason}")+ continue+ if kernel.worth_it(cfg):+ if skipped:+ logger.info("kernel %s selected; skipped %s", kernel.name, "; ".join(skipped))+ return kernel+ if fallback is None:+ fallback = kernel+ if fallback is not None:+ return fallback+ raise KernelSelectionError("no usable kernel in table; " + "; ".join(skipped))+++class QuantMethod(ABC):+ kind: ClassVar[QuantKind]+ layer_kind: ClassVar[LayerKind]+ candidates: ClassVar[tuple[type, ...]] = ()++ def __init__(self, cfg: Any, requested: str = "auto"):+ self.cfg = cfg+ self.requested = requested+ self.kernel = select_kernel(self.candidates, requested, cfg)++ @property+ def scheme(self):+ return self.cfg.scheme++def finalize_quant(root: Any) -> int:+ """Call ``quant_method.finalize(layer)`` on every layer under ``root``; returns the count."""+ from freetoken.layers.base import BaseOP++ seen: set[int] = set()+ count = 0++ def walk(op: Any) -> None:+ nonlocal count+ if id(op) in seen:+ return+ seen.add(id(op))+ method = getattr(op, "quant_method", None)+ if method is not None:+ method.finalize(op)+ count += 1+ for value in vars(op).values():+ if isinstance(value, BaseOP):+ walk(value)+ elif isinstance(value, (list, tuple)):+ for item in value:+ if isinstance(item, BaseOP):+ walk(item)++ walk(root)+ return countdiff --git a/python/freetoken/layers/quantization/moe/__init__.py b/python/freetoken/layers/quantization/moe/__init__.pynew file mode 100644index 00000000..a5f2dc66--- /dev/null+++ b/python/freetoken/layers/quantization/moe/__init__.py@@ -0,0 +1,13 @@+from .base import BankSpec, ExpertView, MoEConfig, MoEKernel, MoEMethod+from . import fp8_block, mxfp4, mxfp8, nvfp4, unquantized+from .fp8_block import Fp8BlockMoEMethod+from .mxfp4 import Mxfp4MoEMethod+from .mxfp8 import Mxfp8MoEMethod+from .nvfp4 import Nvfp4MoEMethod+from .unquantized import UnquantizedMoEMethod++__all__ = [+ "BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",+ "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",+ "fp8_block", "mxfp4", "mxfp8", "nvfp4", "unquantized",+]diff --git a/python/freetoken/layers/quantization/moe/base.py b/python/freetoken/layers/quantization/moe/base.pynew file mode 100644index 00000000..89f13ed0--- /dev/null+++ b/python/freetoken/layers/quantization/moe/base.py@@ -0,0 +1,201 @@+"""The MoE side: per-layer config, bank descriptions, the kernel ABC and the Method base."""++from __future__ import annotations++from abc import ABC, abstractmethod+from dataclasses import dataclass+from typing import Any, ClassVar++import torch++from ..method import QuantMethod+from ..scheme import QuantScheme+++@dataclass(frozen=True)+class MoEConfig:+ """Everything an expert kernel may look at to accept or decline a layer and to size its banks."""++ num_experts: int+ hidden: int+ intermediate: int+ top_k: int+ tp_rank: int = 0+ tp_size: int = 1+ scheme: QuantScheme | None = None+ activation: str = "silu"+ alpha: float = 1.0+ beta: float = 0.0+ limit: float | None = None+ interleaved: bool = False+ has_bias: bool = False+ apply_router_weight_on_input: bool = False+ strategy: str = "resident"+ decode_target: str = "gpu"++ @classmethod+ def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "MoEConfig":+ return cls(+ num_experts=layer.num_experts,+ hidden=layer.hidden_size,+ intermediate=layer.intermediate_size,+ top_k=layer.top_k,+ tp_rank=layer.tp_rank,+ tp_size=layer.tp_size,+ scheme=scheme,+ activation=layer.activation,+ alpha=float(layer.alpha),+ beta=float(layer.beta),+ limit=layer.limit,+ interleaved=bool(layer.interleaved),+ has_bias=bool(layer.has_bias),+ apply_router_weight_on_input=bool(layer.apply_router_weight_on_input),+ strategy=layer.strategy,+ decode_target=layer.decode_target,+ )++ @property+ def local_intermediate(self) -> int:+ return self.intermediate // self.tp_size++ @property+ def plain_silu(self) -> bool:+ return (self.activation, self.alpha, self.beta, self.limit) == ("silu", 1.0, 0.0, None)+++@dataclass(frozen=True)+class BankSpec:+ """One bank of an expert layout, per expert (the E dim is prepended by the allocator)."""++ shape: tuple[int, ...]+ dtype: torch.dtype+ # GPU-resident per-expert vector (marlin / b12x alphas) instead of a host bank+ resident: bool = False+++@dataclass+class ExpertView:+ """The weights an apply() call reads: layout roles -> [E, ...] (resident, materialized layer) or [S, ...] (slot cache)."""++ tensors: dict[str, torch.Tensor]+ slots: torch.Tensor | None = None+ n: int | None = None+ alphas: tuple[torch.Tensor, torch.Tensor] | None = None+++def fused_piece(pieces: dict[str, torch.Tensor], role: str) -> torch.Tensor:+ """A gate_up-family piece, concatenating separate gate / up pieces along the row axis."""+ if role in pieces:+ return pieces[role]+ suffix = role[len("gate_up"):]+ return torch.cat([pieces["gate" + suffix], pieces["up" + suffix]], dim=1)+++def global_rows(piece: torch.Tensor, rows: int) -> torch.Tensor:+ """Per-expert global scale as one fp16 value per output row (a scalar per expert is broadcast)."""+ flat = piece.reshape(piece.shape[0], -1).to(torch.float16)+ if flat.shape[1] == rows:+ return flat+ assert flat.shape[1] == 1, (flat.shape, rows)+ return flat.expand(-1, rows)+++def fused_global(pieces: dict[str, torch.Tensor], half_rows: int) -> torch.Tensor:+ if "gate_up_global" in pieces:+ return global_rows(pieces["gate_up_global"], 2 * half_rows)+ return torch.cat([global_rows(pieces["gate_global"], half_rows), global_rows(pieces["up_global"], half_rows)], dim=1)+++def limit_or_inf(layer: Any) -> float:+ limit = getattr(layer, "limit", None)+ return float("inf") if limit is None else float(limit)+++def gated_epilogue_reason(cfg: "MoEConfig") -> str | None:+ """Why ``gated_act_and_mul`` cannot run ``cfg``'s activation quadruple, or None; the plain kinds have no alpha / limit and nothing takes beta or interleaved rows."""+ from freetoken.layers import GATED_ACTIVATIONS++ if cfg.activation not in GATED_ACTIVATIONS:+ return f"no {cfg.activation!r} epilogue"+ if cfg.interleaved:+ return "the epilogue reads uninterleaved [gate; up] halves"+ if cfg.beta != 0.0:+ return f"the epilogue has no beta (got {cfg.beta})"+ if cfg.activation in ("silu", "gelu", "gelu_tanh") and (cfg.alpha != 1.0 or cfg.limit is not None):+ return f"{cfg.activation} takes no alpha / limit (got alpha={cfg.alpha}, limit={cfg.limit})"+ return None+++def is_resident(layer: Any) -> bool:+ return layer.quant_method.cfg.strategy == "resident"+++class MoEKernel(ABC):+ """One backend for one expert kind: bank layout, packing from checkpoint pieces, and the fused forward."""++ name: ClassVar[str]+ cpu_format: ClassVar[str | None] = None+ max_slots: ClassVar[int | None] = None++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ return None++ def worth_it(self, cfg: MoEConfig) -> bool:+ return True++ def slot_limit(self, cfg: MoEConfig) -> int | None:+ """Most GPU cache slots the kernel can address for ``cfg``; None for no limit."""+ return self.max_slots++ def _common_reject(self, cfg: MoEConfig, *, resident_ok: bool, tp_ok: bool, cpu_ok: bool, plain_silu_only: bool) -> str | None:+ if not resident_ok and cfg.strategy == "resident":+ return "not served resident; use --moe-strategy offload or cpu"+ if not tp_ok and cfg.tp_size > 1:+ return "TP > 1 is not supported for this expert format"+ if not cpu_ok and cfg.decode_target != "gpu":+ return "has no CPU executor format; decode must run on the GPU"+ if cfg.has_bias:+ return "kernel has no bias epilogue"+ if plain_silu_only and not cfg.plain_silu:+ return f"kernel is plain-silu only, experts use ({cfg.activation}, alpha={cfg.alpha}, beta={cfg.beta}, limit={cfg.limit})"+ return None++ @abstractmethod+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]: ...++ @abstractmethod+ def pack(self, pieces: dict[str, torch.Tensor], cfg: MoEConfig, out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:+ """Write one batch of expert pieces into ``out`` rows; returns the GPU-resident per-expert values, if any."""++ @abstractmethod+ def apply(self, layer: Any, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, is_prefill: bool) -> torch.Tensor: ...+++class MoEMethod(QuantMethod):+ """Declares resident experts for its kind; layout, pack and apply go to the kernel."""++ def layout(self) -> dict[str, BankSpec]:+ return self.kernel.layout(self.cfg)++ def pack(self, pieces: dict[str, torch.Tensor], out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:+ return self.kernel.pack(pieces, self.cfg, out)++ def slot_limit(self) -> int | None:+ return self.kernel.slot_limit(self.cfg)++ def apply(self, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, layer: Any, is_prefill: bool) -> torch.Tensor:+ return self.kernel.apply(layer, x, topk_weights, topk_ids, view, is_prefill=is_prefill)++ @property+ def cpu_format(self) -> str | None:+ return self.kernel.cpu_format++ @abstractmethod+ def create_weights(self, layer: Any) -> None: ...++ def finalize(self, layer: Any) -> None:+ pass++ @abstractmethod+ def resident_view(self, layer: Any) -> ExpertView:+ """Layout roles -> the resident layer's tensors, for apply()."""diff --git a/python/freetoken/layers/quantization/moe/fp8_block.py b/python/freetoken/layers/quantization/moe/fp8_block.pynew file mode 100644index 00000000..6c0167b7--- /dev/null+++ b/python/freetoken/layers/quantization/moe/fp8_block.py@@ -0,0 +1,80 @@+"""fp8 e4m3 experts with 128x128 block scales."""++from __future__ import annotations++import torch++from ..registry import LayerKind, register_method+from ..scheme import FP8_BLOCK as BLOCK, QuantKind+from .base import BankSpec, ExpertView, fused_piece, gated_epilogue_reason, limit_or_inf, MoEConfig, MoEKernel, MoEMethod++FP8 = torch.float8_e4m3fn+++def _pad_scale(rows: int, cols: int) -> int:+ from freetoken.kernel.aot_models import fp8_block_scale_pad++ return fp8_block_scale_pad(rows, cols)+++class TritonFp8BlockMoEKernel(MoEKernel):+ name = "triton"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ reason = self._common_reject(cfg, resident_ok=True, tp_ok=False, cpu_ok=False, plain_silu_only=False)+ if reason:+ return reason+ reason = gated_epilogue_reason(cfg)+ if reason:+ return f"fp8 block MoE kernel: {reason}"+ if cfg.apply_router_weight_on_input:+ return "fp8 block MoE kernel cannot apply the router weight on the input"+ return None++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ i, h, b = cfg.intermediate, cfg.hidden, BLOCK+ return {+ "gate_up": BankSpec((2 * i, h), FP8),+ "gate_up_scale": BankSpec((2 * i // b, _pad_scale(2 * i // b, h // b)), torch.bfloat16),+ "down": BankSpec((h, i), FP8),+ "down_scale": BankSpec((h // b, _pad_scale(h // b, i // b)), torch.bfloat16),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ out["gate_up"].copy_(fused_piece(pieces, "gate_up"))+ out["down"].copy_(pieces["down"])+ # the scale banks carry 16-byte padding on the last dim+ gs = fused_piece(pieces, "gate_up_scale")+ out["gate_up_scale"][:, :, : gs.shape[2]].copy_(gs)+ ds = pieces["down_scale"]+ out["down_scale"][:, :, : ds.shape[2]].copy_(ds)+ return {}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ from freetoken.moe.fused_fp8_block import fused_experts_decode_fp8_block, fused_experts_fp8_block++ t = view.tensors+ alpha, limit = float(layer.alpha), limit_or_inf(layer)+ if is_prefill:+ n = view.n if view.n is not None else layer.num_experts+ return fused_experts_fp8_block(x, t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"], topk_weights, topk_ids, n, layer.activation, layer.apply_router_weight_on_input, alpha, limit)+ return fused_experts_decode_fp8_block(x, t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, alpha, limit)+++@register_method(QuantKind.FP8_BLOCK, LayerKind.MOE)+class Fp8BlockMoEMethod(MoEMethod):+ candidates = (TritonFp8BlockMoEKernel,)++ def create_weights(self, layer) -> None:+ g = self.cfg+ e, i, h, b = g.num_experts, g.intermediate, g.hidden, BLOCK+ layer.gate_up_proj = torch.empty(e, 2 * i, h, dtype=FP8)+ layer.gate_up_scale_inv = torch.empty(e, 2 * i // b, h // b, dtype=torch.bfloat16)+ layer.down_proj = torch.empty(e, h, i, dtype=FP8)+ layer.down_scale_inv = torch.empty(e, h // b, i // b, dtype=torch.bfloat16)++ def resident_view(self, layer) -> ExpertView:+ return ExpertView({+ "gate_up": layer.gate_up_proj, "gate_up_scale": layer.gate_up_scale_inv,+ "down": layer.down_proj, "down_scale": layer.down_scale_inv,+ })diff --git a/python/freetoken/layers/quantization/moe/mxfp4.py b/python/freetoken/layers/quantization/moe/mxfp4.pynew file mode 100644index 00000000..aa020ee3--- /dev/null+++ b/python/freetoken/layers/quantization/moe/mxfp4.py@@ -0,0 +1,142 @@+"""MXFP4 experts: the standard OCP form (DeepSeek-V4) and the gpt-oss variant with biases."""++from __future__ import annotations++import torch++from ..registry import LayerKind, register_method+from ..scheme import MX_GROUP as GROUP, QuantKind+from .base import BankSpec, ExpertView, fused_piece, is_resident, limit_or_inf, MoEConfig, MoEKernel, MoEMethod++E8M0 = torch.float8_e8m0fnu+++class TritonMxfp4MoEKernel(MoEKernel):+ """Standard OCP MXFP4 experts (DeepSeek-V4 ds_fp4): e2m1 pairs + e8m0 scales, no bias."""++ name = "triton"+ cpu_format = "ds_fp4"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ if cfg.interleaved:+ return "standard MXFP4 kernel reads the concatenated gate|up row order"+ if (cfg.alpha, cfg.beta) != (1.0, 0.0):+ return "standard MXFP4 kernel has no alpha / beta in its swiglu"+ return self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=True, plain_silu_only=False)++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ i, h = cfg.intermediate, cfg.hidden+ return {+ "gate_up": BankSpec((2 * i, h // 2), torch.uint8),+ "gate_up_scale": BankSpec((2 * i, h // GROUP), E8M0),+ "down": BankSpec((h, i // 2), torch.uint8),+ "down_scale": BankSpec((h, i // GROUP), E8M0),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ out["gate_up"].copy_(fused_piece(pieces, "gate_up").view(torch.uint8))+ out["gate_up_scale"].copy_(fused_piece(pieces, "gate_up_scale").view(E8M0))+ out["down"].copy_(pieces["down"].view(torch.uint8))+ out["down_scale"].copy_(pieces["down_scale"].view(E8M0))+ return {}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ t = view.tensors+ banks = (t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"])+ limit = limit_or_inf(layer)+ if is_prefill and view.n is not None:+ from freetoken.moe.fused_ds_fp4 import routed_experts_fp4_prefill++ return routed_experts_fp4_prefill(x, topk_ids, topk_weights, *banks, limit, view.n)+ from freetoken.moe.fused_ds_fp4 import routed_experts_fp4++ return routed_experts_fp4(x, topk_ids, topk_weights, *banks, limit)+++class TritonGptossMxfp4MoEKernel(MoEKernel):+ """gpt-oss MXFP4: bias, alpha / limit swiglu, interleaved gate|up rows, transposed split-K banks."""++ name = "triton_gptoss"+ cpu_format = "mxfp4_triton"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ if not cfg.has_bias:+ return "gpt-oss kernel needs the expert biases"+ return None++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ i, h = cfg.local_intermediate, cfg.hidden+ return {+ "gate_up": BankSpec((h // 2, 2 * i), torch.uint8),+ "gate_up_scale": BankSpec((h // GROUP, 2 * i), torch.uint8),+ "gate_up_bias": BankSpec((2 * i,), torch.bfloat16),+ "down": BankSpec((i // 2, h), torch.uint8),+ "down_scale": BankSpec((i // GROUP, h), torch.uint8),+ "down_bias": BankSpec((h,), torch.bfloat16),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ # HF [e, N, K//32, 16] blocks / [e, N, K//32] scales -> split-K layout with N innermost+ def blocks_t(t):+ e, n = t.shape[0], t.shape[1]+ return t.reshape(e, n, -1).permute(0, 2, 1)++ out["gate_up"].copy_(blocks_t(fused_piece(pieces, "gate_up")))+ out["gate_up_scale"].copy_(fused_piece(pieces, "gate_up_scale").permute(0, 2, 1))+ out["gate_up_bias"].copy_(fused_piece(pieces, "gate_up_bias"))+ out["down"].copy_(blocks_t(pieces["down"]))+ out["down_scale"].copy_(pieces["down_scale"].permute(0, 2, 1))+ out["down_bias"].copy_(pieces["down_bias"])+ return {}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ from freetoken.moe.fused_mxfp4 import MXFP4_DECODE_MAX_TOKENS, run_mxfp4_prefill_experts_t, run_mxfp4_splitk_decode_experts++ t = view.tensors+ # the resident layer picks the split-K decode kernel by token count, not by phase+ decode = x.shape[0] <= MXFP4_DECODE_MAX_TOKENS if is_resident(layer) else not is_prefill+ run = run_mxfp4_splitk_decode_experts if decode else run_mxfp4_prefill_experts_t+ return run(+ x, topk_weights, topk_ids,+ t["gate_up"], t["gate_up_scale"], t["gate_up_bias"], t["down"], t["down_scale"], t["down_bias"],+ top_k=layer.top_k, hidden_act_alpha=float(layer.alpha), swiglu_limit=layer.limit,+ )+++@register_method(QuantKind.MXFP4, LayerKind.MOE)+class Mxfp4MoEMethod(MoEMethod):+ candidates = (TritonMxfp4MoEKernel, TritonGptossMxfp4MoEKernel)++ def create_weights(self, layer) -> None:+ if not self.cfg.has_bias:+ raise NotImplementedError("standard MXFP4 experts are served from the offload cache, not resident")+ g = self.cfg+ e, i, h = g.num_experts, g.local_intermediate, g.hidden+ if h % GROUP:+ raise ValueError(f"MXFP4 hidden size must be divisible by {GROUP}")+ layer.gate_up_proj_blocks = torch.empty(e, 2 * i, h // GROUP, 16, dtype=torch.uint8)+ layer.gate_up_proj_scales = torch.empty(e, 2 * i, h // GROUP, dtype=torch.uint8)+ layer.gate_up_proj_bias = torch.empty(e, 2 * i, dtype=torch.bfloat16)+ layer.down_proj_blocks = torch.empty(e, h, i // GROUP, 16, dtype=torch.uint8)+ layer.down_proj_scales = torch.empty(e, h, i // GROUP, dtype=torch.uint8)+ layer.down_proj_bias = torch.empty(e, h, dtype=torch.bfloat16)++ def finalize(self, layer) -> None:+ if getattr(layer, "gate_up_proj_blocks", None) is None:+ return+ from freetoken.moe.fused_mxfp4 import _transpose_mxfp4_for_decode++ # one transposed copy serves prefill and decode; the HF blocks are freed so 120B fits+ layer._gu_blocks_t, layer._gu_scales_t = _transpose_mxfp4_for_decode(layer.gate_up_proj_blocks, layer.gate_up_proj_scales)+ layer._dn_blocks_t, layer._dn_scales_t = _transpose_mxfp4_for_decode(layer.down_proj_blocks, layer.down_proj_scales)+ layer.gate_up_proj_blocks = None+ layer.gate_up_proj_scales = None+ layer.down_proj_blocks = None+ layer.down_proj_scales = None+ torch.cuda.empty_cache()++ def resident_view(self, layer) -> ExpertView:+ return ExpertView({+ "gate_up": layer._gu_blocks_t, "gate_up_scale": layer._gu_scales_t, "gate_up_bias": layer.gate_up_proj_bias,+ "down": layer._dn_blocks_t, "down_scale": layer._dn_scales_t, "down_bias": layer.down_proj_bias,+ })diff --git a/python/freetoken/layers/quantization/moe/mxfp8.py b/python/freetoken/layers/quantization/moe/mxfp8.pynew file mode 100644index 00000000..f91d6dba--- /dev/null+++ b/python/freetoken/layers/quantization/moe/mxfp8.py@@ -0,0 +1,21 @@+"""MXFP8 experts: registered so the table is complete, not served yet."""++from __future__ import annotations++from ..registry import LayerKind, register_method+from ..scheme import QuantKind+from .base import ExpertView, MoEMethod+++@register_method(QuantKind.MXFP8, LayerKind.MOE)+class Mxfp8MoEMethod(MoEMethod):+ candidates = ()++ def __init__(self, *args, **kwargs):+ raise NotImplementedError("mxfp8 experts are not served yet")++ def create_weights(self, layer) -> None:+ raise NotImplementedError++ def resident_view(self, layer) -> ExpertView:+ raise NotImplementedErrordiff --git a/python/freetoken/layers/quantization/moe/nvfp4.py b/python/freetoken/layers/quantization/moe/nvfp4.pynew file mode 100644index 00000000..bb341cdf--- /dev/null+++ b/python/freetoken/layers/quantization/moe/nvfp4.py@@ -0,0 +1,575 @@+"""NVFP4 experts: served from the offload cache through the Triton inline-dequant kernels, vLLM's Marlin or flashinfer's b12x.++FreeToken stores MoE experts as ModelOpt NVFP4 (packed e2m1 codes + fp8-e4m3 per-16+block scales + per-tensor global scale) in pinned host banks and gathers the routed+experts into a GPU slot cache. Rules every kernel here follows:++* A kernel owns ``{load-time pack, forward}`` as a unit. Prefill and decode consume+ the *same* bank layout -- there is never a second copy of the experts.+* Every pack keeps the expert dimension outermost with per-expert blocks contiguous+ and byte-identical in size to the native layout, so the banks are repacked *in+ place* and the offload cache's slot gather (``copy_missing``) works unchanged.+* The per-tensor global scales never enter the offload banks: they are tiny+ ``[L*E]`` vectors kept resident on the GPU and gathered per forward call+ (device-side, CUDA-graph safe).+"""++from __future__ import annotations++import math++import torch++from freetoken.kernel import backend+from freetoken.utils import init_logger++from ..registry import LayerKind, register_method+from ..scheme import NVFP4_GROUP as GROUP, QuantKind+from .base import BankSpec, ExpertView, fused_global, fused_piece, gated_epilogue_reason, global_rows, limit_or_inf, MoEConfig, MoEKernel, MoEMethod++logger = init_logger(__name__)++FP8 = torch.float8_e4m3fn+MARLIN_MAX_SLOTS = 992+B12X_MIN_INTERMEDIATE = 1024+++class TritonNvfp4MoEKernel(MoEKernel):+ """FreeToken's inline-dequant kernels over the native ModelOpt rows."""++ name = "triton"+ cpu_format = "nvfp4"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ reason = self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=True, plain_silu_only=False)+ if reason:+ return reason+ reason = gated_epilogue_reason(cfg)+ return f"triton nvfp4 MoE kernel: {reason}" if reason else None++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ i, h = cfg.intermediate, cfg.hidden+ return {+ "gate_up": BankSpec((2 * i, h // 2), torch.uint8),+ "gate_up_scale": BankSpec((2 * i, h // GROUP), FP8),+ "gate_up_global": BankSpec((2 * i,), torch.float16),+ "down": BankSpec((h, i // 2), torch.uint8),+ "down_scale": BankSpec((h, i // GROUP), FP8),+ "down_global": BankSpec((h,), torch.float16),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ out["gate_up"].copy_(fused_piece(pieces, "gate_up"))+ out["gate_up_scale"].copy_(fused_piece(pieces, "gate_up_scale"))+ out["gate_up_global"].copy_(fused_global(pieces, cfg.intermediate))+ out["down"].copy_(pieces["down"])+ out["down_scale"].copy_(pieces["down_scale"])+ out["down_global"].copy_(global_rows(pieces["down_global"], cfg.hidden))+ return {}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ from freetoken.moe.fused_nvfp4 import fused_experts_decode_nvfp4_marlin, fused_experts_nvfp4++ t = view.tensors+ banks = (t["gate_up"], t["gate_up_scale"], t["gate_up_global"], t["down"], t["down_scale"], t["down_global"])+ alpha, limit = float(layer.alpha), limit_or_inf(layer)+ if is_prefill:+ return fused_experts_nvfp4(x, *banks, topk_weights, topk_ids, view.n, layer.activation, layer.apply_router_weight_on_input, alpha, limit)+ return fused_experts_decode_nvfp4_marlin(x, *banks, topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, alpha, limit)+++# ---------------------------------------------------------------------------+# Marlin: vLLM's fused Marlin MoE (W4A16, dequant in kernel) over Marlin-tiled banks.+# The Marlin kernels come from vLLM (not sgl-kernel) deliberately: vLLM ships the+# NVFP4 instantiations in its AOT wheel together with the matching host-side layout+# transforms, so the pair stays consistent by construction. sgl-kernel's AOT wheel+# excludes the NVFP4 MoE Marlin kernels to cut wheel size (its python transforms pair+# with sglang's *JIT* kernel tree, whose scale encoding has since diverged).+#+# The small layout-transform helpers are ported from sglang+# (``srt/layers/quantization/marlin_utils{,_fp4}.py``, Apache-2.0); the kernels+# themselves are imported, not vendored.+# ---------------------------------------------------------------------------+++def _marlin_symbols_ok() -> bool:+ """Probe the exact vLLM symbols the pack/forward paths below use.++ ``find_spec`` only proves the top-level package exists; an incompatible donor+ version would otherwise crash mid-init, after minutes of bank loading. Probing+ at selection time degrades to triton before any bank is touched.+ """+ try:+ from vllm import _custom_ops # noqa: F401+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( # noqa: F401+ fused_marlin_moe,+ )+ from vllm.model_executor.layers.quantization.utils.marlin_utils import ( # noqa: F401+ marlin_permute_scales,+ )+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401+ nvfp4_marlin_process_global_scale,+ nvfp4_marlin_process_scales,+ )+ from vllm.scalar_type import scalar_types # noqa: F401+ except Exception as exc:+ logger.warning(+ f"NVFP4 marlin backend is installed but unusable ({exc!r}); "+ "falling back to the Triton inline-dequant backend"+ )+ return False+ return True+++# Marlin pack: mirrors vLLM's prepare_moe_fp4_layer_for_marlin so the host prep always matches the AOT op.+def _marlin_pack_proj(+ packed: torch.Tensor,+ scale: torch.Tensor,+ row_global: torch.Tensor,+ *,+ size_k: int,+ size_n: int,+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """Pack one expert projection (on GPU) into the Marlin layout.++ ``packed`` ``[N, K//2]`` uint8 e2m1 pairs, ``scale`` ``[N, K//16]`` e4m3,+ ``row_global`` ``[N]`` fp16 per-row global (rows of a merged gate_up may carry+ w1's and w3's different globals). Returns ``(qweight [K//16, 2N] int32,+ scales [K//16, N] e4m3-coded, global scalar bf16)``.++ Marlin takes one global scalar per projection, so when the row globals differ+ the ratio is folded into the fp16 block scales before they are re-encoded+ (exact when all rows share one global, <=1 scale ulp otherwise).+ """+ from vllm import _custom_ops as ops+ from vllm.model_executor.layers.quantization.utils.marlin_utils import (+ marlin_permute_scales,+ )+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (+ nvfp4_marlin_process_global_scale,+ nvfp4_marlin_process_scales,+ )++ assert size_n % 64 == 0, f"Marlin requires N % 64 == 0, got {size_n}"+ qweight = ops.gptq_marlin_repack(+ b_q_weight=packed.view(torch.int32).T.contiguous(),+ perm=torch.empty(0, dtype=torch.int, device=packed.device),+ size_k=size_k,+ size_n=size_n,+ num_bits=4,+ )++ g = row_global.to(torch.float32)+ g_max = g.max()+ s = scale.to(torch.bfloat16)+ if not torch.all(g == g_max):+ s = (s.float() * (g / g_max).unsqueeze(1)).to(torch.bfloat16)+ s = marlin_permute_scales(s.T, size_k=size_k, size_n=size_n, group_size=16)+ s = nvfp4_marlin_process_scales(s)+ if isinstance(s, tuple): # newer vLLM returns (scales, scale_factor)+ s, factor = s+ g_max = g_max / factor+ g_out = nvfp4_marlin_process_global_scale(g_max.to(torch.bfloat16).reshape(1))+ return qweight, s, g_out++[email protected]_grad()+def marlin_fused_experts(+ hidden_states: torch.Tensor,+ gate_up_q: torch.Tensor,+ gate_up_s: torch.Tensor,+ gate_up_alpha: torch.Tensor,+ down_q: torch.Tensor,+ down_s: torch.Tensor,+ down_alpha: torch.Tensor,+ topk_weights: torch.Tensor,+ topk_ids: torch.Tensor,+ activation: str,+ apply_router_weight_on_input: bool,+) -> torch.Tensor:+ """Marlin W4A16 fused MoE (two grouped GEMMs + activation + reduce).++ Two calling regimes share this entry: decode passes the full ``[S]`` slot cache+ with ``topk_ids`` rewritten to slot ids; full-layer prefill passes banks whose+ position == expert id (the materialized ``[:E]`` slot view or the overlap double+ buffer views), so the raw routing ids arrive unmapped. The ``*_alpha`` vectors+ are the matching per-row global scales in both regimes.+ vLLM's implementation is device-side only (no host syncs), so the decode call is+ CUDA-graph capturable.+ """+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe+ from vllm.scalar_type import scalar_types++ assert activation == "silu", "Marlin NVFP4 backend supports gated silu only"+ return fused_marlin_moe(+ hidden_states,+ gate_up_q,+ down_q,+ None, # bias1+ None, # bias2+ gate_up_s,+ down_s,+ gating_output=None,+ topk_weights=topk_weights,+ topk_ids=topk_ids,+ quant_type_id=scalar_types.float4_e2m1f.id,+ apply_router_weight_on_input=apply_router_weight_on_input,+ global_num_experts=gate_up_q.size(0),+ activation=activation,+ global_scale1=gate_up_alpha,+ global_scale2=down_alpha,+ )+++class MarlinNvfp4MoEKernel(MoEKernel):+ """vLLM's fused Marlin MoE over pre-tiled banks; the globals fold into GPU-resident alphas."""++ name = "marlin"+ max_slots = MARLIN_MAX_SLOTS++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ if not backend.is_vllm_installed():+ return "vLLM is not installed"+ reason = self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=False, plain_silu_only=True)+ if reason:+ return reason+ if not _marlin_symbols_ok():+ return "vLLM Marlin donor symbols are unusable"+ return None++ def worth_it(self, cfg: MoEConfig) -> bool:+ return (8, 0) <= backend.device_capability() < (10, 0)++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ i, h = cfg.intermediate, cfg.hidden+ return {+ "gate_up": BankSpec((h // GROUP, 4 * i), torch.int32),+ "gate_up_scale": BankSpec((h // GROUP, 2 * i), FP8),+ "down": BankSpec((i // GROUP, 2 * h), torch.int32),+ "down_scale": BankSpec((i // GROUP, h), FP8),+ "gate_up_alpha": BankSpec((), torch.bfloat16, resident=True),+ "down_alpha": BankSpec((), torch.bfloat16, resident=True),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ i, h = cfg.intermediate, cfg.hidden+ device = torch.device("cuda")+ gu, gus, gug = fused_piece(pieces, "gate_up"), fused_piece(pieces, "gate_up_scale"), fused_global(pieces, i)+ dn, dns, dng = pieces["down"], pieces["down_scale"], global_rows(pieces["down_global"], h)+ e = gu.shape[0]+ gate_up_alpha = torch.empty(e, dtype=torch.bfloat16, device=device)+ down_alpha = torch.empty(e, dtype=torch.bfloat16, device=device)+ for k in range(e):+ qw, sc, al = _marlin_pack_proj(gu[k].to(device), gus[k].to(device), gug[k].to(device), size_k=h, size_n=2 * i)+ out["gate_up"][k].copy_(qw)+ out["gate_up_scale"][k].copy_(sc.view(FP8))+ gate_up_alpha[k] = al[0]+ qw, sc, al = _marlin_pack_proj(dn[k].to(device), dns[k].to(device), dng[k].to(device), size_k=i, size_n=h)+ out["down"][k].copy_(qw)+ out["down_scale"][k].copy_(sc.view(FP8))+ down_alpha[k] = al[0]+ torch.cuda.synchronize(device)+ return {"gate_up_alpha": gate_up_alpha, "down_alpha": down_alpha}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ t = view.tensors+ assert view.alphas is not None+ return marlin_fused_experts(x, t["gate_up"], t["gate_up_scale"], view.alphas[0], t["down"], t["down_scale"], view.alphas[1], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input)+++# ---------------------------------------------------------------------------+# b12x: flashinfer's SM12x CuTe-DSL fused MoE (W4A16) over b12x-packed banks.+# The pack (``prepare_w4a16_packed_weights``) runs on any CUDA build, but the fused-MoE+# kernel needs sm_120/121 AND a CUDA>=13 *driver* (it JIT-compiles PTX at runtime). The+# kernel object gates on that, so the forward is only reached on hardware that can run it.+#+# Layout contract (verified against flashinfer 0.6.12 prepare_w4a16_packed_weights):+# * prepare() takes the block scales already *swizzled* (it calls unswizzle_expert_+# scales internally), so the native row-major fp8 scales are run through+# nvfp4_block_scale_interleave first.+# * prepare() reorders the w13 rows with reorder_w13_to_gate_up == cat([second, first]),+# i.e. it expects the merged proj as [up, gate] and emits [gate, up]. FreeToken stores+# [gate, up], so the halves are swapped before prepare() to come out [gate, up] again+# (the kernel computes silu(first) * second == silu(gate) * up).+# * modelopt globals are passed straight through (no 1/g inversion).+# The fused forward reconstructs the W4A16PackedWeights from the banks and uses the+# prepared-weights launch (_launch_sm120_w4a16_moe) directly, so weights are prepared+# exactly once at load time (b12x_fused_moe would re-prepare + ptr-cache every call).+# ---------------------------------------------------------------------------+++def _b12x_symbols_ok() -> bool:+ """Probe the exact flashinfer symbols the pack/forward paths below use.++ ``find_spec`` only proves the top-level package exists; an incompatible donor+ version would otherwise crash mid-init, after minutes of bank loading. Probing+ at selection time degrades to triton before any bank is touched.+ """+ try:+ # Probe exactly what the pack/forward use: the row-major->swizzled scale+ # transform, the W4A16 prepare, and the prepared-weights launch (which also+ # transitively imports the CuTe-DSL kernel + cutlass).+ from flashinfer import nvfp4_block_scale_interleave # noqa: F401+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import ( # noqa: F401+ _launch_sm120_w4a16_moe,+ )+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import ( # noqa: F401+ W4A16PackedWeights,+ _make_workspace,+ prepare_w4a16_packed_weights,+ )+ except Exception as exc:+ logger.warning(+ f"NVFP4 b12x backend is installed but unusable ({exc!r}); "+ "falling back to the Triton inline-dequant backend"+ )+ return False+ return True+++def _flashinfer_cuda_major() -> int | None:+ """flashinfer's own toolkit-CUDA major; None if it can't be determined. Only a+ fallback proxy when the driver version is unavailable -- the b12x kernel is JIT+ PTX-compiled through the driver, so the toolkit major is *not* what gates it."""+ try:+ from flashinfer.jit.cpp_ext import get_cuda_version++ return int(get_cuda_version().major)+ except Exception:+ return None+++def _b12x_unusable_reason(cc: tuple[int, int]) -> str | None:+ """None if the flashinfer b12x decode/prefill kernel can actually run on this device,+ else a human-readable reason. The b12x *pack* works anywhere, but the SM12x CuTe-DSL+ kernel itself requires sm_120+ and a CUDA>=13 *driver* (it JIT-compiles PTX through the+ driver), so selection must check the runtime here -- not just that flashinfer imports --+ or the model loads in the b12x layout and then crashes on the first decode."""+ import importlib.util++ if cc < (12, 0):+ return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"+ if importlib.util.find_spec("flashinfer") is None:+ return "flashinfer is not installed"+ from freetoken.kernel.backend import driver_cuda_version++ drv = driver_cuda_version()+ if drv is not None:+ if drv < 13000:+ return (+ "b12x fused MoE requires a CUDA>=13 driver "+ f"(driver supports CUDA {drv // 1000}.{(drv % 1000) // 10})"+ )+ else:+ # Driver version undetermined: fall back to flashinfer's toolkit major as a+ # conservative proxy rather than risk loading the b12x layout and crashing.+ major = _flashinfer_cuda_major()+ if major is not None and major < 13:+ return (+ "b12x fused MoE requires CUDA>=13 (driver undetermined; "+ f"flashinfer toolkit is CUDA {major}.x)"+ )+ if not _b12x_symbols_ok():+ return "flashinfer b12x donor symbols are unusable"+ return None+++def _b12x_swizzle_block_scales(scale: torch.Tensor) -> torch.Tensor:+ """Row-major per-16 fp8 block scales ``[n, N, K//16]`` -> flashinfer's expert-leading+ *swizzled* storage, which is what :func:`prepare_w4a16_packed_weights` expects on input+ (it unswizzles internally). The 128x4 interleave is per-expert; ``N`` is a multiple of+ 128 and ``K//16`` a multiple of 4 for the supported models, so the shape is preserved."""+ from flashinfer import nvfp4_block_scale_interleave++ pieces = [+ nvfp4_block_scale_interleave(scale[e].view(torch.uint8).contiguous())+ for e in range(scale.shape[0])+ ]+ return torch.stack(pieces, dim=0).view(torch.float8_e4m3fn)+++# Per-device scratch (sized sms*4+2) that run_w4a16_moe reads off the prepared object;+# tiny and shape-independent, so one per device is reused across all layers/decodes.+_B12X_WORKSPACE: dict[torch.device, torch.Tensor] = {}++[email protected]_grad()+def _b12x_small_workspace(device: torch.device) -> torch.Tensor:+ ws = _B12X_WORKSPACE.get(device)+ if ws is None:+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (+ _make_workspace,+ )++ ws = _make_workspace(device, max_blocks_per_sm=4)+ _B12X_WORKSPACE[device] = ws+ return ws+++def b12x_fused_experts(+ hidden_states: torch.Tensor,+ gate_up_q: torch.Tensor,+ gate_up_s: torch.Tensor,+ gate_up_alpha: torch.Tensor,+ down_q: torch.Tensor,+ down_s: torch.Tensor,+ down_alpha: torch.Tensor,+ topk_weights: torch.Tensor,+ topk_ids: torch.Tensor,+ activation: str,+ apply_router_weight_on_input: bool,+) -> torch.Tensor:+ """SM12x W4A16 fused MoE over cache slots (same calling convention as+ :func:`marlin_fused_experts`). Requires sm_120/121 + a CUDA>=13 driver; reached only+ when the b12x kernel object has confirmed the runtime supports it.++ The banks already hold flashinfer's prepared (tiled) layout from+ the b12x kernel object's pack, so this wraps them back into a+ ``W4A16PackedWeights`` and calls the prepared-weights launch directly -- the public+ ``b12x_fused_moe`` would re-prepare the raw modelopt weights (and ptr-cache them) on+ every call, which both costs a prepare per step and breaks for the offload cache,+ whose slot contents move between calls.++ CUDA-graph note: the launch resolves a (shape-keyed, module-cached) scratch workspace+ on first use and flashinfer raises if that happens *during* capture, so the decode+ path must be warmed once eagerly before graph capture (FreeToken already does)."""+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import (+ _launch_sm120_w4a16_moe,+ )+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (+ W4A16PackedWeights,+ )++ assert activation == "silu", "b12x backend supports gated silu only"+ assert not apply_router_weight_on_input+ num_experts = gate_up_q.size(0)+ hidden_size = hidden_states.size(-1)+ # down bank is the prepared w2 == [E, K_tiles, ...] with K_tiles == intermediate//16.+ intermediate_size = down_q.size(1) * 16+ prepared = W4A16PackedWeights(+ w13=gate_up_q,+ w13_scale=gate_up_s,+ w13_global_scale=gate_up_alpha,+ w2=down_q,+ w2_scale=down_s,+ w2_global_scale=down_alpha,+ workspace=_b12x_small_workspace(hidden_states.device),+ hidden_size=hidden_size,+ intermediate_size=intermediate_size,+ num_experts=num_experts,+ is_gated=True,+ params_dtype=hidden_states.dtype,+ source_format="modelopt",+ )+ out = torch.empty(+ hidden_states.size(0), hidden_size, dtype=hidden_states.dtype, device=hidden_states.device+ )+ _launch_sm120_w4a16_moe(+ a=hidden_states,+ topk_ids=topk_ids,+ topk_weights=topk_weights,+ w1_weight=gate_up_q,+ w1_weight_sf=gate_up_s,+ w1_alpha=gate_up_alpha,+ w2_weight=down_q,+ w2_weight_sf=down_s,+ w2_alpha=down_alpha,+ num_experts=num_experts,+ top_k=topk_ids.size(1),+ num_local_experts=num_experts,+ scatter_output=out,+ activation="silu",+ source_format="modelopt",+ _prepared_weights=prepared,+ )+ return out+++class B12xNvfp4MoEKernel(MoEKernel):+ """flashinfer's SM12x CuTe-DSL W4A16 MoE; the packed blocks reuse the native bank bytes."""++ name = "b12x"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ cc = backend.device_capability()+ if cc < (12, 0):+ return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"+ if not backend.is_flashinfer_installed():+ return "flashinfer is not installed"+ reason = self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=False, plain_silu_only=True)+ if reason:+ return reason+ return _b12x_unusable_reason(cc)++ def worth_it(self, cfg: MoEConfig) -> bool:+ # NOTE: never auto-selected. flashinfer's cute launcher indexes each bank with int32 element offsets, so the GPU slot cache is capped at (2^31 - 1) / elements-per-slot (about 1000 slots for GLM-5.3-Flash's 12960 experts); until flashinfer lifts that, b12x stays behind triton in the table; the selection does not weigh the cap, cache-auto clamps to slot_limit() and OffloadMoeCache refuses a larger cache.+ return cfg.intermediate >= B12X_MIN_INTERMEDIATE++ def slot_limit(self, cfg: MoEConfig) -> int | None:+ # the cute launcher indexes each bank with int32 element offsets+ per_slot = max(math.prod(spec.shape) for spec in self.layout(cfg).values() if not spec.resident)+ return (2**31 - 1) // per_slot++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ # flashinfer's prepared tiles: the Marlin shapes, byte-identical to the native rows+ i, h = cfg.intermediate, cfg.hidden+ return {+ "gate_up": BankSpec((h // GROUP, 4 * i), torch.int32),+ "gate_up_scale": BankSpec((h // GROUP, 2 * i), FP8),+ "down": BankSpec((i // GROUP, 2 * h), torch.int32),+ "down_scale": BankSpec((i // GROUP, h), FP8),+ "gate_up_alpha": BankSpec((), torch.float32, resident=True),+ "down_alpha": BankSpec((), torch.float32, resident=True),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import prepare_w4a16_packed_weights++ i, h = cfg.intermediate, cfg.hidden+ device = torch.device("cuda")+ gu = fused_piece(pieces, "gate_up").to(device)+ gug = fused_global(pieces, i).to(device).float()+ g_max = gug.max(dim=1, keepdim=True).values+ gus = fused_piece(pieces, "gate_up_scale").to(device).to(torch.float16)+ ratio = gug / g_max+ if not torch.all(ratio == 1.0):+ gus = (gus.float() * ratio.unsqueeze(-1)).to(torch.float16)+ dng = global_rows(pieces["down_global"], h).to(device)+ if not torch.all(dng == dng[:, :1]):+ raise ValueError("b12x pack requires a row-constant per-expert down_proj global scale")+ # b12x wants up|gate row order+ gu = torch.cat([gu[:, i:], gu[:, :i]], dim=1).contiguous()+ gus = torch.cat([gus[:, i:], gus[:, :i]], dim=1).contiguous()+ prepared = prepare_w4a16_packed_weights(+ gu,+ _b12x_swizzle_block_scales(gus.to(FP8)),+ g_max.squeeze(1),+ pieces["down"].to(device),+ _b12x_swizzle_block_scales(pieces["down_scale"].to(device)),+ dng[:, 0].float(),+ activation="silu",+ params_dtype=torch.bfloat16,+ source_format="modelopt",+ )+ e = gu.shape[0]+ for role, t in (("gate_up", prepared.w13), ("gate_up_scale", prepared.w13_scale), ("down", prepared.w2), ("down_scale", prepared.w2_scale)):+ bank = out[role]+ assert t.shape[1:] == bank.shape[1:] and t.dtype == bank.dtype, (role, t.shape, t.dtype, bank.shape, bank.dtype)+ bank.copy_(t[:e])+ return {"gate_up_alpha": prepared.w13_global_scale[:e].float(), "down_alpha": prepared.w2_global_scale[:e].float()}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ t = view.tensors+ assert view.alphas is not None+ return b12x_fused_experts(x, t["gate_up"], t["gate_up_scale"], view.alphas[0], t["down"], t["down_scale"], view.alphas[1], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input)+++@register_method(QuantKind.NVFP4, LayerKind.MOE)+class Nvfp4MoEMethod(MoEMethod):+ candidates = (TritonNvfp4MoEKernel, MarlinNvfp4MoEKernel, B12xNvfp4MoEKernel)++ def create_weights(self, layer) -> None:+ raise NotImplementedError("NVFP4 experts are served from the offload cache, not resident")++ def resident_view(self, layer) -> ExpertView:+ raise NotImplementedError("NVFP4 experts are not resident")diff --git a/python/freetoken/layers/quantization/moe/unquantized.py b/python/freetoken/layers/quantization/moe/unquantized.pynew file mode 100644index 00000000..ef435686--- /dev/null+++ b/python/freetoken/layers/quantization/moe/unquantized.py@@ -0,0 +1,48 @@+"""bf16 experts: the fused Triton MoE over gate_up / down banks."""++from __future__ import annotations++import torch++from ..registry import LayerKind, register_method+from ..scheme import QuantKind+from .base import BankSpec, ExpertView, fused_piece, gated_epilogue_reason, is_resident, limit_or_inf, MoEConfig, MoEKernel, MoEMethod+++class FusedMoEKernel(MoEKernel):+ name = "fused"+ cpu_format = "bf16"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ reason = gated_epilogue_reason(cfg)+ return f"bf16 fused MoE: {reason}" if reason else None++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ i = cfg.local_intermediate+ return {"gate_up": BankSpec((2 * i, cfg.hidden), torch.bfloat16), "down": BankSpec((cfg.hidden, i), torch.bfloat16)}++ def pack(self, pieces, cfg: MoEConfig, out):+ out["gate_up"].copy_(fused_piece(pieces, "gate_up"))+ out["down"].copy_(pieces["down"])+ return {}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl++ # the resident layer runs the in-place prefill kernel for both phases+ impl = fused_experts_impl if is_prefill or is_resident(layer) else fused_experts_decode_impl+ return impl(x, view.tensors["gate_up"], view.tensors["down"], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, float(layer.alpha), limit_or_inf(layer))+++@register_method(QuantKind.NONE, LayerKind.MOE)+class UnquantizedMoEMethod(MoEMethod):+ candidates = (FusedMoEKernel,)++ def create_weights(self, layer) -> None:+ g = self.cfg+ i = g.local_intermediate+ layer.gate_up_proj = torch.empty(g.num_experts, 2 * i, g.hidden)+ layer.down_proj = torch.empty(g.num_experts, g.hidden, i)++ def resident_view(self, layer) -> ExpertView:+ return ExpertView({"gate_up": layer.gate_up_proj, "down": layer.down_proj})diff --git a/python/freetoken/layers/quantization/names.py b/python/freetoken/layers/quantization/names.pynew file mode 100644index 00000000..c7a79743--- /dev/null+++ b/python/freetoken/layers/quantization/names.py@@ -0,0 +1,92 @@+"""Module-name matching for the dialect configs and the attribute -> checkpoint name map."""++from __future__ import annotations++import fnmatch+import re+from dataclasses import dataclass+from typing import Callable, Iterator++Matcher = Callable[[str], bool]+++def ancestors(name: str) -> Iterator[str]:+ while name:+ yield name+ name, _, _ = name.rpartition(".")+++_GLOB_CHARS = re.compile(r"[*?\[]")+++def name_set(patterns: tuple[str, ...]) -> Matcher:+ """Membership of a module or any parent in a list of literal names and glob patterns."""+ literal = frozenset(p for p in patterns if not _GLOB_CHARS.search(p))+ globs = [p for p in patterns if _GLOB_CHARS.search(p)]+ glob = re.compile("|".join(fnmatch.translate(p) for p in globs)) if globs else None++ def hit(name: str) -> bool:+ return any(a in literal or (glob is not None and glob.match(a)) for a in ancestors(name))++ return hit+++def substr_set(patterns: tuple[str, ...]) -> Matcher:+ """transformers' fp8 integration skips a module when ``entry + "."`` occurs in ``name + "."``."""+ rx = re.compile("|".join(re.escape(p) + r"\." for p in patterns)) if patterns else None+ return lambda name: rx is not None and rx.search(name + ".") is not None+++def ct_set(patterns: tuple[str, ...], *, class_names: bool) -> Matcher:+ """compressed-tensors ``targets`` / ``ignore``: module names, ``re:`` regexes, or the class name Linear."""+ names = name_set(tuple(p for p in patterns if not p.startswith("re:") and p != "Linear"))+ regexes = [p[3:] for p in patterns if p.startswith("re:")]+ rx = re.compile("|".join(f"(?:{r})" for r in regexes)) if regexes else None+ any_linear = class_names and "Linear" in patterns+ return lambda name: any_linear or names(name) or (rx is not None and rx.match(name) is not None)+++_ROUTED_EXPERT = re.compile(r"\.experts\.\d+(\.|$)")+++def is_routed_expert(name: str) -> bool:+ return bool(_ROUTED_EXPERT.search(name)) or name.endswith(".experts")+++@dataclass(frozen=True)+class NameMap:+ """FreeToken attribute path -> checkpoint module name(s), applied to every scheme_for query.++ ``roots`` rename a leading path (``model.layers`` -> ``model.language_model.layers``);+ ``segments`` rename an inner path (``feed_forward.shared_mlp`` -> ``mlp``);+ ``packed`` expands a fused leaf into its source leaves (``qkv_proj`` -> q/k/v_proj).+ """++ roots: tuple[tuple[str, str], ...] = ()+ segments: tuple[tuple[str, str], ...] = ()+ packed: tuple[tuple[str, tuple[str, ...]], ...] = ()++ def to_checkpoint(self, prefix: str) -> tuple[str, ...]:+ for attr_root, ckpt_root in sorted(self.roots, key=lambda r: -len(r[0])):+ if prefix == attr_root or prefix.startswith(attr_root + "."):+ prefix = ckpt_root + prefix[len(attr_root):]+ break+ for attr_seg, ckpt_seg in self.segments:+ prefix = _replace_segment(prefix, attr_seg, ckpt_seg)+ head, _, leaf = prefix.rpartition(".")+ sources = dict(self.packed).get(leaf)+ if not sources:+ return (prefix,)+ return tuple(f"{head}.{s}" if head else s for s in sources)+++def _replace_segment(prefix: str, attr_seg: str, ckpt_seg: str) -> str:+ parts, seg, out, i = prefix.split("."), attr_seg.split("."), [], 0+ while i < len(parts):+ if parts[i : i + len(seg)] == seg:+ out.extend(ckpt_seg.split(".") if ckpt_seg else [])+ i += len(seg)+ else:+ out.append(parts[i])+ i += 1+ return ".".join(out)diff --git a/python/freetoken/layers/quantization/quant_backend.py b/python/freetoken/layers/quantization/quant_backend.pynew file mode 100644index 00000000..1eebada6--- /dev/null+++ b/python/freetoken/layers/quantization/quant_backend.py@@ -0,0 +1,80 @@+"""The --quant-backend value: the kernel name wanted for each table, ``layer=name`` for every kind of a layer type, ``layer.kind=name`` for one."""++from __future__ import annotations++from dataclasses import dataclass++from .registry import LayerKind, method_class, methods_for+from .scheme import QuantKind++Table = tuple[LayerKind, "QuantKind | None"]+++def _kernel_names(layer_kind: LayerKind, kind: QuantKind | None) -> set[str]:+ if kind is not None:+ try:+ return {c.name for c in method_class(kind, layer_kind).candidates}+ except NotImplementedError:+ return set()+ return {c.name for cls in methods_for(layer_kind).values() for c in cls.candidates}+++def _parse_table(key: str) -> Table | None:+ layer, _, kind = key.partition(".")+ try:+ return LayerKind(layer.strip()), QuantKind(kind.strip()) if kind else None+ except ValueError:+ return None+++def _table_text(table: Table) -> str:+ layer, kind = table+ return f"{layer}.{kind}" if kind is not None else str(layer)+++@dataclass(frozen=True)+class QuantBackend:+ items: tuple[tuple[Table, str], ...] = ()++ @classmethod+ def parse(cls, text: str | None) -> "QuantBackend":+ items: dict[Table, str] = {}+ for raw in (text or "").split(","):+ item = raw.strip()+ if not item:+ continue+ key, sep, name = item.partition("=")+ table = _parse_table(key)+ name = name.strip().lower()+ if not sep or not name or table is None:+ layers = [l.value for l in LayerKind]+ raise ValueError(f"bad --quant-backend item {item!r}; expected layer[.kind]=name with layer in {layers}")+ known = _kernel_names(*table)+ if name not in known:+ raise ValueError(f"--quant-backend {key}: no kernel {name!r}; known: {sorted(known)}")+ items[table] = name+ return cls(tuple(sorted(items.items(), key=lambda kv: _table_text(kv[0]))))++ def select(self, layer_kind: LayerKind, kind: QuantKind) -> str:+ """The kernel name for one table: its own entry, else the layer-wide entry when the table lists that kernel, else ``"auto"``."""+ entries = dict(self.items)+ exact = entries.get((layer_kind, kind))+ if exact:+ return exact+ broad = entries.get((layer_kind, None))+ if broad and broad in _kernel_names(layer_kind, kind):+ return broad+ return "auto"+++_QUANT_BACKEND = QuantBackend()+++def set_quant_backend(backend: QuantBackend) -> None:+ """Install the kernel requests every layer built afterwards consults; the engine sets them once before create_model."""+ global _QUANT_BACKEND+ _QUANT_BACKEND = backend+++def get_quant_backend() -> QuantBackend:+ return _QUANT_BACKENDdiff --git a/python/freetoken/layers/quantization/registry.py b/python/freetoken/layers/quantization/registry.pynew file mode 100644index 00000000..2d13cf50--- /dev/null+++ b/python/freetoken/layers/quantization/registry.py@@ -0,0 +1,59 @@+"""Two registries: checkpoint dialects, and (kind, layer kind) -> Method."""++from __future__ import annotations++from enum import Enum+from typing import Any++from .scheme import QuantKind+++class LayerKind(Enum):+ LINEAR = "linear"+ MOE = "moe"++ def __str__(self) -> str:+ return self.value+++_DIALECTS: dict[str, type] = {}+_METHODS: dict[tuple[QuantKind, LayerKind], type] = {}+++def register_dialect(cls: type) -> type:+ name = cls.dialect+ if name in _DIALECTS:+ raise ValueError(f"quant dialect {name!r} already registered: {_DIALECTS[name].__name__}")+ _DIALECTS[name] = cls+ return cls+++def dialects() -> tuple[type, ...]:+ return tuple(_DIALECTS.values())+++def register_method(kind: QuantKind, layer_kind: LayerKind):+ if not isinstance(kind, QuantKind) or not isinstance(layer_kind, LayerKind):+ raise TypeError(f"cannot register quant method for {layer_kind!r}.{kind!r}")++ def deco(cls: type) -> type:+ key = (kind, layer_kind)+ if key in _METHODS:+ raise ValueError(f"quant method for {layer_kind}.{kind} already registered: {_METHODS[key].__name__}")+ cls.kind = kind+ cls.layer_kind = layer_kind+ _METHODS[key] = cls+ return cls++ return deco+++def method_class(kind: QuantKind, layer_kind: LayerKind) -> type[Any]:+ try:+ return _METHODS[(kind, layer_kind)]+ except KeyError:+ raise NotImplementedError(f"no quant method for {layer_kind}.{kind}") from None+++def methods_for(layer_kind: LayerKind) -> dict[QuantKind, type]:+ return {kind: cls for (kind, lk), cls in _METHODS.items() if lk == layer_kind}diff --git a/python/freetoken/layers/quantization/scheme.py b/python/freetoken/layers/quantization/scheme.pynew file mode 100644index 00000000..668eb6fb--- /dev/null+++ b/python/freetoken/layers/quantization/scheme.py@@ -0,0 +1,97 @@+"""QuantScheme: the storage description and role set of one quantized weight, plus the per-kind builders."""++from __future__ import annotations++from dataclasses import dataclass+from enum import Enum+from typing import Iterable++import torch++# -1 selects the whole dimension: (-1, -1) one scale per tensor, (1, -1) one per output row, (1, 16) one per 16 input elements, (128, 128) a 2-D block+GroupShape = tuple[int, int]++class QuantKind(Enum):+ """The quantization kinds; one Method per (kind, layer kind)."""++ NONE = "none"+ FP8_TENSOR = "fp8_tensor"+ FP8_BLOCK = "fp8_block"+ MXFP8 = "mxfp8"+ NVFP4 = "nvfp4"+ MXFP4 = "mxfp4"++ def __str__(self) -> str:+ return self.value+++FP8_BLOCK = 128+NVFP4_GROUP = 16+MX_GROUP = 32+++@dataclass(frozen=True)+class WeightDesc:+ elem: str+ group: GroupShape | None+ scale: str | None+++++@dataclass(frozen=True, eq=False)+class QuantScheme:+ """``roles`` are the canonical tensor names the kind's Method declares; which checkpoint tensors feed them is the dialect Config's business."""++ kind: QuantKind+ weight: WeightDesc+ roles: frozenset[str]++ def __init__(self, kind: QuantKind, weight: WeightDesc, roles: Iterable[str]):+ if not isinstance(kind, QuantKind):+ raise TypeError(f"kind must be a QuantKind, got {kind!r}")+ object.__setattr__(self, "kind", kind)+ object.__setattr__(self, "weight", weight)+ object.__setattr__(self, "roles", frozenset(roles))++ def _key(self):+ return (self.kind, self.weight, self.roles)++ def __eq__(self, other: object) -> bool:+ return isinstance(other, QuantScheme) and self._key() == other._key()++ def __hash__(self) -> int:+ return hash(self._key())++ def has(self, role: str) -> bool:+ return role in self.roles++ def __repr__(self) -> str:+ return f"QuantScheme({self.kind}, {self.weight}, roles={sorted(self.roles)})"+++# ---------------------------------------------------------------------------+# Builders: one per kind; the dialect configs pick the variant their checkpoints carry+# ---------------------------------------------------------------------------+++def fp8_tensor_scheme(scale: str, *, per_row: bool = False, input_scale: bool = False) -> QuantScheme:+ roles = {"weight", "weight_scale"} | ({"input_scale"} if input_scale else set())+ return QuantScheme(QuantKind.FP8_TENSOR, WeightDesc("e4m3", (1, -1) if per_row else (-1, -1), scale), roles)+++def fp8_block_scheme(scale: str) -> QuantScheme:+ return QuantScheme(QuantKind.FP8_BLOCK, WeightDesc("e4m3", (FP8_BLOCK, FP8_BLOCK), scale), {"weight", "weight_scale_inv"})+++def mxfp8_scheme() -> QuantScheme:+ return QuantScheme(QuantKind.MXFP8, WeightDesc("e4m3", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale_inv"})+++def nvfp4_scheme(*, input_scale: bool) -> QuantScheme:+ roles = {"weight", "weight_scale", "weight_global"} | ({"input_scale"} if input_scale else set())+ return QuantScheme(QuantKind.NVFP4, WeightDesc("e2m1", (1, NVFP4_GROUP), "e4m3"), roles)+++def mxfp4_scheme() -> QuantScheme:+ return QuantScheme(QuantKind.MXFP4, WeightDesc("e2m1", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale"})diff --git a/python/freetoken/models/__init__.py b/python/freetoken/models/__init__.pyindex 2167d089..532b1371 100644--- a/python/freetoken/models/__init__.py+++ b/python/freetoken/models/__init__.py@@ -11,7 +11,7 @@ from .config import (SWAAttentionGroupConfig,)from .register import get_model_class-from .weight import load_moe_expert_sources, load_weight+from .weight import load_weightdef create_model(model_config: ModelConfig) -> BaseLLMModel:@@ -22,7 +22,6 @@ __all__ = ["BaseLLMModel","create_model","load_weight",- "load_moe_expert_sources","AttentionGroupConfig","BaseAttentionGroupConfig","DSV4AttentionGroupConfig",diff --git a/python/freetoken/models/blocks.py b/python/freetoken/models/blocks.pyindex e811ca80..0fa5fe4d 100644--- a/python/freetoken/models/blocks.py+++ b/python/freetoken/models/blocks.py@@ -18,6 +18,7 @@ if TYPE_CHECKING:import torchfrom freetoken.core import Batch+ from freetoken.layers.quantization import QuantConfigfrom .config import ModelConfig@@ -33,11 +34,13 @@ class BaseLLMModel(ABC, BaseOP):class GatedMLP(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, quant_config: QuantConfig | None = None, prefix: str = ""):self.gate_up_proj = LinearColParallelMerged(config.hidden_size,[config.intermediate_size, config.intermediate_size],has_bias=False,+ quant_config=quant_config,+ prefix=f"{prefix}.gate_up_proj",)fn_map = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}@@ -49,6 +52,8 @@ class GatedMLP(BaseOP):config.intermediate_size,config.hidden_size,has_bias=False,+ quant_config=quant_config,+ prefix=f"{prefix}.down_proj",)@nvtx_annotate("MLP")diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex 1bce039c..07a16df5 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -260,7 +260,11 @@ class ModelConfig:norm_topk_prob: boolmodel_type: strarchitectures: list[str]- moe_backend: str = "fused"+ moe_strategy: str = "fused"+ # where routed experts decode (gpu / cpu / hybrid); set by the engine from the flags, it gates which expert kernels can serve+ decode_target: str = "gpu"+ # The QuantConfig the engine builds from the checkpoint; layers ask it for their method.+ quant: Any | None = None# ----- optional, model-specific extensions (default keeps other models intact) -----moe_enabled: bool = False# Weight quantization of the MoE experts only. "none" keeps the default BF16@@ -268,26 +272,13 @@ class ModelConfig:# "fp8_block" is DeepSeek-V3-style 128x128 block-fp8 (weight fp8-e4m3 +# weight_scale_inv per block), also applied to the dense projections.expert_quant: str = "none"- # NVFP4 routed-expert GEMM backend (--nvfp4-backend); injected from EngineConfig.- nvfp4_backend: str = "triton"# Block size (out, in) for block-wise weight quantization (fp8_block: (128, 128)).weight_block_size: tuple[int, int] | None = None- # Weight quantization of the *dense* attention / GatedDeltaNet projections (separate- # from the routed experts above). "fp8_pertensor" keeps them fp8-e4m3 + a per-output-row- # scale and runs a W8A16 kernel (modelopt MIXED_PRECISION); "none" leaves them bf16- # (dequant-at-load for any other dense quant, e.g. NVFP4 shared_expert/lm_head).+ # the checkpoint's quant kind for the dense attention / GatedDeltaNet projections, detected by the family's parse_config for its readerattn_quant: str = "none"- # Weight quantization of the *dense* NVFP4 MLP projections -- the shared expert, and dense- # (non-MoE) MLP layers -- which NVFP4 checkpoints store as packed FP4 like the routed- # experts. "nvfp4" keeps them packed and runs the W4A16 dense kernels (quartering their- # decode weight traffic); "none" dequantizes them to bf16 at load. Set independently of the- # routed experts and lm_head: e.g. pure-NVFP4 Qwen3.5 has bf16 attn + bf16 lm_head but FP4- # shared experts, so this is "nvfp4" while attn_quant / lm_head_quant are "none".+ # the checkpoint's quant kind for the dense MLP projections (shared expert, dense layers), detected the same waydense_quant: str = "none"- # Weight quantization of the lm_head. "nvfp4" keeps the (untied) FP4 head native (W4A16) --- # the bf16 dequant of this ~1 GB matrix was the single largest decode kernel; "none" leaves- # it bf16. Separate from dense_quant because only some NVFP4 checkpoints quantize lm_head- # (modelopt MIXED_PRECISION does; pure NVFP4 leaves it bf16).+ # the checkpoint's quant kind for the lm_head, detected the same way (only some NVFP4 exports quantize it)lm_head_quant: str = "none"shared_expert_intermediate_size: int = 0use_qk_norm: bool = Falsediff --git a/python/freetoken/models/deepseek_v4/__init__.py b/python/freetoken/models/deepseek_v4/__init__.pyindex b3cb41db..37f5076b 100644--- a/python/freetoken/models/deepseek_v4/__init__.py+++ b/python/freetoken/models/deepseek_v4/__init__.py@@ -16,7 +16,7 @@ addressed by page tables, and sparse attention is a physical-slot gather (seefrom .args import DeepseekV4Args, load_argsfrom .config import parse_configfrom .model import DeepseekV4ForCausalLM-from .weight import iter_weights, load_dsfp4_expert_sources+from .weight import iter_expert_pieces, iter_weights__all__ = ["DeepseekV4Args",@@ -24,5 +24,5 @@ __all__ = ["parse_config","DeepseekV4ForCausalLM","iter_weights",- "load_dsfp4_expert_sources",+ "iter_expert_pieces",]diff --git a/python/freetoken/models/deepseek_v4/attention.py b/python/freetoken/models/deepseek_v4/attention.pyindex 357585ee..cb0dd315 100644--- a/python/freetoken/models/deepseek_v4/attention.py+++ b/python/freetoken/models/deepseek_v4/attention.py@@ -5,19 +5,19 @@ from __future__ import annotationsimport torchimport torch.nn.functional as F-from torch import nnfrom freetoken.core import get_global_ctxfrom freetoken.kernel.triton.dsv4.fp8_linear import act_quant_fp8_inplacefrom freetoken.kernel.triton.dsv4.norm import rms_norm+from freetoken.layers import BaseOP, LinearColParallelMerged, LinearReplicated, LinearRowParallel, RMSNormfrom .args import DeepseekV4Argsfrom .compress import Compressor, Indexer-from .layers import Linear, RMSNorm, get_compress_topk_idxs, get_window_topk_idxs+from .layers import get_compress_topk_idxs, get_window_topk_idxsfrom .ops import apply_rotary_emb, apply_rotary_emb_decode, get_freqs_cis-class Attention(nn.Module):+class Attention(BaseOP):"""Multi-head Latent Attention with sliding window + optional KV compression.KV lives in the paged DSV4 pools (window / compressed / indexer), resolved per token via@@ -26,8 +26,7 @@ class Attention(nn.Module):``window_pool[L]`` / ``cmp_pool[L]`` inside the kernel -- no per-forward staging slab. Bothprefill and decode use the same paged kernel."""- def __init__(self, layer_id: int, args: DeepseekV4Args):- super().__init__()+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, quant_config=None, prefix: str = ""):self.layer_id = layer_idself.dim = args.dimself.n_heads = args.n_heads@@ -40,22 +39,23 @@ class Attention(nn.Module):self.compress_ratio = args.compress_ratios[layer_id]self.eps = args.norm_eps- self.attn_sink = nn.Parameter(torch.empty(self.n_heads, dtype=torch.float32), requires_grad=False)- self.wq_a = Linear(self.dim, self.q_lora_rank, kind="fp8")+ self.attn_sink = torch.empty(self.n_heads, dtype=torch.float32)+ # the latent projections are replicated; wq_b shards over heads, wo_b over the output groups+ self.wq_a = LinearReplicated(self.dim, self.q_lora_rank, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wq_a")self.q_norm = RMSNorm(self.q_lora_rank, self.eps)- self.wq_b = Linear(self.q_lora_rank, self.n_heads * self.head_dim, kind="fp8")- self.wkv = Linear(self.dim, self.head_dim, kind="fp8")+ self.wq_b = LinearColParallelMerged(self.q_lora_rank, [self.n_heads * self.head_dim], has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wq_b")+ self.wkv = LinearReplicated(self.dim, self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wkv")self.kv_norm = RMSNorm(self.head_dim, self.eps)- # wo_a: dequantized to bf16 (reference runs a bf16 grouped-output einsum).+ # wo_a is one [o_lora_rank, K] matrix per output group, stacked on N and applied as a bmm; the reference dequantizes it to bf16 and so does the reader. Under TP it shards on N by group like wo_b shards on K.wo_a_rows = self.n_groups * args.o_lora_rankwo_a_k = self.n_heads * self.head_dim // self.n_groups- self.wo_a = nn.Parameter(torch.empty(wo_a_rows, wo_a_k, dtype=torch.bfloat16), requires_grad=False)- self.wo_b = Linear(self.n_groups * args.o_lora_rank, self.dim, kind="fp8")+ self.wo_a = torch.empty(wo_a_rows, wo_a_k, dtype=torch.bfloat16)+ self.wo_b = LinearRowParallel(self.n_groups * args.o_lora_rank, self.dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wo_b")self.softmax_scale = self.head_dim ** -0.5if self.compress_ratio:- self.compressor = Compressor(args, self.compress_ratio, self.head_dim)- self.indexer = Indexer(args, self.compress_ratio) if self.compress_ratio == 4 else None+ self.compressor = Compressor(args, self.compress_ratio, self.head_dim, quant_config=quant_config, prefix=f"{prefix}.compressor")+ self.indexer = Indexer(args, self.compress_ratio, quant_config=quant_config, prefix=f"{prefix}.indexer") if self.compress_ratio == 4 else Noneelse:self.compressor = Noneself.indexer = None@@ -69,7 +69,7 @@ class Attention(nn.Module):rope_theta, args.rope_factor, args.beta_fast, args.beta_slow,)# Bound on first forward (freqs/arange only; pool buffers are read off the LIVE pool).- self.freqs_cis: torch.Tensor | None = None+ self._freqs_cis: torch.Tensor | None = None# KV addressing lives in the attention backend (ctx.attn_backend), pool buffers on the live# pool (ctx.kv_cache) -- both read per access, so the model holds NO state a runtime rebuild@@ -85,11 +85,11 @@ class Attention(nn.Module):L = self.layer_idwin = self.window_sizeself.P = pool.P- self.freqs_cis = get_freqs_cis(*self._freqs_params, device)+ self._freqs_cis = get_freqs_cis(*self._freqs_params, device)if self.compress_ratio:- self.compressor.bind_paged(pool, L, self.freqs_cis, device, tier="attn")+ self.compressor.bind_paged(pool, L, self._freqs_cis, device, tier="attn")if self.indexer is not None:- self.indexer.bind_paged(pool, L, self.freqs_cis, device)+ self.indexer.bind_paged(pool, L, self._freqs_cis, device)def reset(self) -> None:if self.compressor is not None:@@ -102,7 +102,7 @@ class Attention(nn.Module):o = o.reshape(bsz, seqlen, self.n_groups, -1)wo_a = self.wo_a.view(self.n_groups, self.o_lora_rank, -1)o = torch.einsum("bsgd,grd->bsgr", o, wo_a).flatten(2)- return self.wo_b(o)+ return self.wo_b.forward(o)def _prefill_segment(self, x_seg, qr_seg, kv_seg, ti: int, start_pos: int, n: int):"""One request's prefill work inside a (possibly ragged) batch: persist its window KV,@@ -144,17 +144,17 @@ class Attention(nn.Module):if start_pos == 0:self.reset() # re-seed the compressor/indexer carry from scratchblocks = (- self.indexer(x_seg, qr_seg, 0, 0, slots, ti) if self.indexer is not None+ self.indexer.forward(x_seg, qr_seg, 0, 0, slots, ti) if self.indexer is not Noneelse get_compress_topk_idxs(ratio, 1, n, 0, 0).to(device))- self.compressor(x_seg, 0, slots, ti=ti)+ self.compressor.forward(x_seg, 0, slots, ti=ti)else:blocks = (self.indexer.extend(x_seg, qr_seg, start_pos, 0, slots, tail_ws, ti)if self.indexer is not Noneelse self._compress_topk_extend(n, start_pos, end, 0, device, 1))- self.compressor(x_seg, start_pos, slots, tail_window_slot=tail_ws, ti=ti)+ self.compressor.forward(x_seg, start_pos, slots, tail_window_slot=tail_ws, ti=ti)return win_global, self.attn.blocks_to_global(blocks, ratio, ti=ti)def forward_ragged(self, x, segments, flat_positions):@@ -178,17 +178,17 @@ class Attention(nn.Module):_, T, _ = x.size()if len(segments) == 1:# single contiguous segment: a free slice view instead of a per-layer gather- freqs = self.freqs_cis[segments[0][3]:segments[0][3] + T]+ freqs = self._freqs_cis[segments[0][3]:segments[0][3] + T]else:- freqs = self.freqs_cis.index_select(0, flat_positions) # [T, rd//2] per-token rope+ freqs = self._freqs_cis.index_select(0, flat_positions) # [T, rd//2] per-token rope- qr = q = self.q_norm(self.wq_a(x))- q = self.wq_b(q).unflatten(-1, (self.n_heads, self.head_dim))+ qr = q = self.q_norm.forward(self.wq_a.forward(x))+ q = self.wq_b.forward(q).unflatten(-1, (self.n_heads, self.head_dim))q = rms_norm(q, None, self.eps)apply_rotary_emb(q[..., -rd:], freqs)- kv = self.wkv(x)- kv = self.kv_norm(kv)+ kv = self.wkv.forward(x)+ kv = self.kv_norm.forward(kv)apply_rotary_emb(kv[..., -rd:], freqs)act_quant_fp8_inplace(kv[..., :-rd], 64)@@ -273,15 +273,15 @@ class Attention(nn.Module):if wctx is None:wctx = get_global_ctx().batch.attn_metadata.window_ctx(pos, rows)window_slots, prev_window_slots, window_slots_topk = wctx- freqs_t = self.freqs_cis.index_select(0, pos) # [B, rd_pairs] (per-layer rope)+ freqs_t = self._freqs_cis.index_select(0, pos) # [B, rd_pairs] (per-layer rope)- qr = q = self.q_norm(self.wq_a(x))- q = self.wq_b(q).unflatten(-1, (self.n_heads, self.head_dim))+ qr = q = self.q_norm.forward(self.wq_a.forward(x))+ q = self.wq_b.forward(q).unflatten(-1, (self.n_heads, self.head_dim))q = rms_norm(q, None, self.eps)apply_rotary_emb_decode(q[..., -rd:], freqs_t) # per-row position freqs- kv = self.wkv(x)- kv = self.kv_norm(kv)+ kv = self.wkv.forward(x)+ kv = self.kv_norm.forward(kv)apply_rotary_emb_decode(kv[..., -rd:], freqs_t)act_quant_fp8_inplace(kv[..., :-rd], 64)# Persistent window write: each row's new token to its own window slot (swa_ratio=1).diff --git a/python/freetoken/models/deepseek_v4/compress.py b/python/freetoken/models/deepseek_v4/compress.pyindex 4d578f36..3295976f 100644--- a/python/freetoken/models/deepseek_v4/compress.py+++ b/python/freetoken/models/deepseek_v4/compress.py@@ -3,25 +3,23 @@from __future__ import annotationsimport torch-from torch import nnfrom freetoken.core import get_global_ctxfrom freetoken.kernel.triton.dsv4.bf16_linear import bf16_linear_fp32from freetoken.kernel.triton.dsv4.compress import gated_poolfrom freetoken.kernel.triton.dsv4.fp8_linear import act_quant_fp8_inplace, fp4_act_quant_inplacefrom freetoken.kernel.triton.dsv4.hadamard import hadamard_transform+from freetoken.layers import BaseOP, LinearReplicated, RMSNormfrom .args import DeepseekV4Args-from .layers import Linear, RMSNormfrom .ops import apply_rotary_emb, apply_rotary_emb_decode-class Compressor(nn.Module):+class Compressor(BaseOP):"""Learned gated pooling of KV over ``compress_ratio`` tokens (CSA overlap whenratio==4). Stateful across decode steps. KV / state buffers are bound from the pool."""- def __init__(self, args: DeepseekV4Args, compress_ratio: int, head_dim: int, rotate: bool = False):- super().__init__()+ def __init__(self, args: DeepseekV4Args, compress_ratio: int, head_dim: int, rotate: bool = False, *, quant_config=None, prefix: str = ""):self.dim = args.dimself.head_dim = head_dimself.rope_head_dim = args.rope_head_dim@@ -31,10 +29,10 @@ class Compressor(nn.Module):self.max_batch_size = args.max_batch_sizecoff = 1 + self.overlap- self.ape = nn.Parameter(torch.empty(compress_ratio, coff * self.head_dim, dtype=torch.float32), requires_grad=False)+ self.ape = torch.empty(compress_ratio, coff * self.head_dim, dtype=torch.float32)# checkpoint stores these bf16; matmul upcasts on-chip (halves footprint + read).- self.wkv = Linear(self.dim, coff * self.head_dim, kind="bf16")- self.wgate = Linear(self.dim, coff * self.head_dim, kind="bf16")+ self.wkv = LinearReplicated(self.dim, coff * self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wkv")+ self.wgate = LinearReplicated(self.dim, coff * self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wgate")self.norm = RMSNorm(self.head_dim, args.norm_eps)# Paged-KV binding (set on first forward). All addressing -- the compressed-KV pool view,# the per-window-page compress-state ring, the decode snapshot -- is reached through the@@ -46,7 +44,7 @@ class Compressor(nn.Module):self.coff = coffself.item_size = coff * self.head_dim # the ring row's kv|score split pointself.P: int = 128 # window-page size (set in bind)- self.freqs_cis: torch.Tensor | None = None+ self._freqs_cis: torch.Tensor | None = None# Rolling state register held in fp32 (the carry); seeded by prefill, advanced by# decode. At bs=1 it mirrors the page's ring block; persisted to the ring on write.self._kv_state: torch.Tensor | None = None@@ -70,7 +68,7 @@ class Compressor(nn.Module):self.layer_id = layer_idself.tier = tierself.P = pool.P- self.freqs_cis = freqs_cis+ self._freqs_cis = freqs_cisself._device = devicecoff = self.coffself._kv_state = torch.zeros(@@ -140,8 +138,8 @@ class Compressor(nn.Module):return self.extend(x, start_pos, window_slots, int(tail_window_slot), ti)dtype = x.dtypex = x.float()- kv = self.wkv(x)- score = self.wgate(x)+ kv = self.wkv.forward(x)+ score = self.wgate.forward(x)kv_full, score_full = kv, score # pre-split (for the page-boundary carry write-through)ks, ss = self._kv_state, self._score_stateshould_compress = seqlen >= ratio@@ -171,8 +169,8 @@ class Compressor(nn.Module):self._write_through_carry(int(window_slots[-1].item()))if not should_compress:return None- kv = self.norm(kv.to(dtype))- freqs_cis = self.freqs_cis[:cutoff:ratio]+ kv = self.norm.forward(kv.to(dtype))+ freqs_cis = self._freqs_cis[:cutoff:ratio]apply_rotary_emb(kv[..., -rd:], freqs_cis)if self.rotate:kv = hadamard_transform(kv)@@ -210,8 +208,8 @@ class Compressor(nn.Module):self._seed_carry_from_ring(tail_window_slot)x = x.float()- kv = self.wkv(x)- score = self.wgate(x)+ kv = self.wkv.forward(x)+ score = self.wgate.forward(x)kv_full, score_full = kv, scoreks, ss = self._kv_state, self._score_stateshould_compress = (start_pos + seqlen) // ratio > start_pos // ratio@@ -268,13 +266,13 @@ class Compressor(nn.Module):self._write_through_carry(int(window_slots[-1].item()))if not should_compress:return None- reduced = self.norm(reduced.to(dtype))+ reduced = self.norm.forward(reduced.to(dtype))# freqs per compressed block: position b*ratio for absolute block b. start_pos is ratio-# aligned, so the new FULL blocks span [start_pos, start_pos+cutoff) at stride ratio. Use# the block-aligned `cutoff` end (NOT start_pos+seqlen): a stepped slice over an unaligned# span ceils to one extra element when seqlen % ratio != 0 (mirrors the from-scratch# prefill's freqs_cis[:cutoff:ratio]).- freqs_cis = self.freqs_cis[start_pos:start_pos + cutoff:ratio]+ freqs_cis = self._freqs_cis[start_pos:start_pos + cutoff:ratio]assert freqs_cis.size(0) == reduced.size(1), f"{freqs_cis.shape=} {reduced.shape=}"apply_rotary_emb(reduced[..., -rd:], freqs_cis)if self.rotate:@@ -342,8 +340,8 @@ class Compressor(nn.Module):self.layer_id, self.tier, window_slots, self.ring_size, torch.cat([ks, ss], dim=-1))- compressed = self.norm(compressed)- freqs_t = self.freqs_cis.index_select(0, (pos + 1 - ratio).clamp_min(0)) # [B, rd//2]+ compressed = self.norm.forward(compressed)+ freqs_t = self._freqs_cis.index_select(0, (pos + 1 - ratio).clamp_min(0)) # [B, rd//2]apply_rotary_emb_decode(compressed[..., -rd:], freqs_t)if self.rotate:compressed = hadamard_transform(compressed)@@ -363,27 +361,26 @@ class Compressor(nn.Module):self.attn.scatter_compressed(self.layer_id, self.tier, cmp_dst, compressed.view(B, -1))-class Indexer(nn.Module):+class Indexer(BaseOP):"""Lightning Indexer: scores compressed KV and returns top-k positions to attend."""- def __init__(self, args: DeepseekV4Args, compress_ratio: int):- super().__init__()+ def __init__(self, args: DeepseekV4Args, compress_ratio: int, *, quant_config=None, prefix: str = ""):self.dim = args.dimself.n_heads = args.index_n_headsself.head_dim = args.index_head_dimself.rope_head_dim = args.rope_head_dimself.index_topk = args.index_topkself.q_lora_rank = args.q_lora_rank- self.wq_b = Linear(self.q_lora_rank, self.n_heads * self.head_dim, kind="fp8")- self.weights_proj = Linear(self.dim, self.n_heads, kind="bf16")+ self.wq_b = LinearReplicated(self.q_lora_rank, self.n_heads * self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wq_b")+ self.weights_proj = LinearReplicated(self.dim, self.n_heads, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.weights_proj")self.softmax_scale = self.head_dim ** -0.5self.compress_ratio = compress_ratio- self.compressor = Compressor(args, compress_ratio, self.head_dim, rotate=True)+ self.compressor = Compressor(args, compress_ratio, self.head_dim, rotate=True, quant_config=quant_config, prefix=f"{prefix}.compressor")# The indexer's own compressor writes its compressed keys into the idx tier; scoring# gathers them by block index. Both go through the LIVE backend (self.attn -># ctx.attn_backend) per access, so a runtime rebuild needs no model unbind.self.layer_id: int | None = None- self.freqs_cis: torch.Tensor | None = None+ self._freqs_cis: torch.Tensor | None = None@propertydef attn(self):@@ -398,7 +395,7 @@ class Indexer(nn.Module):# arithmetic rows (full_loc // ratio); its OWN compress-state ring is pool.indexer_state_ring[L]# (distinct from the attention compressor's, so the two never collide on per-page state slots).self.layer_id = layer_id- self.freqs_cis = freqs_cis+ self._freqs_cis = freqs_cisself._device = deviceself.compressor.bind_paged(pool, layer_id, freqs_cis, device, tier="idx")@@ -407,17 +404,17 @@ class Indexer(nn.Module):def forward(self, x: torch.Tensor, qr: torch.Tensor, start_pos: int, offset: int, window_slots: torch.Tensor, ti: int = 0):bsz, seqlen, _ = x.size()- freqs_cis = self.freqs_cis[start_pos:start_pos + seqlen]+ freqs_cis = self._freqs_cis[start_pos:start_pos + seqlen]ratio = self.compress_ratiord = self.rope_head_dimend_pos = start_pos + seqlen- q = self.wq_b(qr)+ q = self.wq_b.forward(qr)q = q.unflatten(-1, (self.n_heads, self.head_dim))apply_rotary_emb(q[..., -rd:], freqs_cis)q = hadamard_transform(q)fp4_act_quant_inplace(q, 32)- self.compressor(x, start_pos, window_slots, ti=ti) # scatters indexer keys to idx_pool- weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads ** -0.5)+ self.compressor.forward(x, start_pos, window_slots, ti=ti) # scatters indexer keys to idx_pool+ weights = self.weights_proj.forward(x) * (self.softmax_scale * self.n_heads ** -0.5)keys = self.attn.indexer_keys(ti, end_pos // ratio, ratio, self.layer_id, bsz)scores = self.attn.indexer_prefill_logits(q, keys, weights)return self.attn.indexer_select_prefill(@@ -431,14 +428,14 @@ class Indexer(nn.Module):blocks [0, end//ratio) with a causal mask, returns top-k block indices offset by ``offset``."""bsz, seqlen, _ = x.size()end = start_pos + seqlen- freqs_cis = self.freqs_cis[start_pos:end]+ freqs_cis = self._freqs_cis[start_pos:end]ratio, rd = self.compress_ratio, self.rope_head_dim- q = self.wq_b(qr).unflatten(-1, (self.n_heads, self.head_dim))+ q = self.wq_b.forward(qr).unflatten(-1, (self.n_heads, self.head_dim))apply_rotary_emb(q[..., -rd:], freqs_cis)q = hadamard_transform(q)fp4_act_quant_inplace(q, 32)- self.compressor(x, start_pos, window_slots, tail_window_slot=tail_window_slot, ti=ti) # writes idx_pool- weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads ** -0.5)+ self.compressor.forward(x, start_pos, window_slots, tail_window_slot=tail_window_slot, ti=ti) # writes idx_pool+ weights = self.weights_proj.forward(x) * (self.softmax_scale * self.n_heads ** -0.5)keys = self.attn.indexer_keys(ti, end // ratio, ratio, self.layer_id, bsz)scores = self.attn.indexer_prefill_logits(q, keys, weights)return self.attn.indexer_select_prefill(@@ -460,13 +457,13 @@ class Indexer(nn.Module):fixed staging width (== max valid count in eager, a static capture width under graph)."""B = x.size(0)ratio, rd = self.compress_ratio, self.rope_head_dim- q = self.wq_b(qr).unflatten(-1, (self.n_heads, self.head_dim))- apply_rotary_emb_decode(q[..., -rd:], self.freqs_cis.index_select(0, pos)) # per-row freqs+ q = self.wq_b.forward(qr).unflatten(-1, (self.n_heads, self.head_dim))+ apply_rotary_emb_decode(q[..., -rd:], self._freqs_cis.index_select(0, pos)) # per-row freqsq = hadamard_transform(q)fp4_act_quant_inplace(q, 32)# Advance the indexer's own compressor (writes this token's idx key to idx_pool per row).self.compressor.decode_step(x, pos, prev_window_slots, window_slots, rows)- weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads ** -0.5)+ weights = self.weights_proj.forward(x) * (self.softmax_scale * self.n_heads ** -0.5)valid = (pos + 1) // ratio # [B] per-row valid block count# Head-reduced scores in one pass: the kernel gathers each block's key off the full-loc# SNAPSHOT (not the live map -> overlap-safe replay) and reads its column bound fromdiff --git a/python/freetoken/models/deepseek_v4/layers.py b/python/freetoken/models/deepseek_v4/layers.pyindex c3a4e35f..83bc7689 100644--- a/python/freetoken/models/deepseek_v4/layers.py+++ b/python/freetoken/models/deepseek_v4/layers.py@@ -1,59 +1,9 @@-"""Shared DSV4 primitives: weight-only linears, RMSNorm, and the static window /-compressed top-k index builders."""+"""Static window / compressed top-k index builders."""from __future__ import annotationsimport torchimport torch.nn.functional as F-from torch import nn--from freetoken.kernel.triton.dsv4.fp8_linear import block_fp8_linear-from freetoken.kernel.triton.dsv4.norm import rms_norm--FP8 = torch.float8_e4m3fn---class Linear(nn.Module):- """BF16 / FP32 / block-scaled-FP8 linear (weight-only).-- ``kind="fp8"``: weight float8_e4m3fn + 128x128 e8m0 ``scale`` (dequant in the- Triton GEMM). ``kind="fp32"``/``"bf16"``: plain ``F.linear``.- """-- def __init__(self, in_features, out_features, bias=False, kind="fp8"):- super().__init__()- self.in_features = in_features- self.out_features = out_features- self.kind = kind- if kind == "fp8":- self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=FP8), requires_grad=False)- self.scale = nn.Parameter(- torch.empty(out_features // 128, in_features // 128, dtype=torch.float8_e8m0fnu),- requires_grad=False,- )- else:- dtype = torch.float32 if kind == "fp32" else torch.bfloat16- self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype), requires_grad=False)- self.register_parameter("scale", None)- if bias:- self.bias = nn.Parameter(torch.empty(out_features), requires_grad=False)- else:- self.register_parameter("bias", None)-- def forward(self, x: torch.Tensor) -> torch.Tensor:- if self.kind == "fp8":- return block_fp8_linear(x, self.weight, self.scale, self.bias)- return F.linear(x, self.weight.to(x.dtype), self.bias)---class RMSNorm(nn.Module):- def __init__(self, dim: int, eps: float = 1e-6):- super().__init__()- self.eps = eps- self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32), requires_grad=False)-- def forward(self, x: torch.Tensor):- return rms_norm(x, self.weight, self.eps)def get_window_topk_idxs(window_size, bsz, seqlen, start_pos):diff --git a/python/freetoken/models/deepseek_v4/model.py b/python/freetoken/models/deepseek_v4/model.pyindex c00d20cc..a2d4387a 100644--- a/python/freetoken/models/deepseek_v4/model.py+++ b/python/freetoken/models/deepseek_v4/model.py@@ -12,8 +12,9 @@ sqrtsoftplus / hash MoE), wired onto FreeToken's shared paged engine:- The model is a registered :class:`BaseLLMModel` built by ``create_model``; it is drivenby ``ctx.batch`` (positions / input_ids), captured by the engine ``GraphRunner`` fordecode, and freed by dropping page-table indices (no separate runner / python cursor).- - Routed FP4 experts are served from :class:`OffloadMoeCache` (on-demand) -- the- framework's core acceleration.+ - Routed MXFP4 experts are served from :class:`OffloadMoeCache` (on-demand) -- the+ framework's core acceleration; their format and the dense fp8 block linears come from+ ``config.quant``.Heavy ops are FreeToken Triton kernels. Precision matches the reference (FP8/FP4activation quant + Hadamard rotation re-introduced; see ``ops.py`` / the dsv4 kernels).@@ -23,11 +24,11 @@ from __future__ import annotationsimport torchimport torch.nn.functional as F-from torch import nnfrom freetoken.core import get_global_ctxfrom freetoken.kernel.triton.dsv4.hc import hc_post_combine, hc_pre_combinefrom freetoken.kernel.triton.dsv4.sinkhorn import hc_split_sinkhorn+from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNorm, VocabParallelEmbeddingfrom freetoken.models.blocks import BaseLLMModelfrom .args import DeepseekV4Args@@ -37,26 +38,19 @@ from .moe import MoE# Re-exports: keep every class/helper previously defined here importable from .model# (external import stability; the moved definitions live in their own modules).from .compress import Compressor, Indexer # noqa: F401-from .layers import ( # noqa: F401- FP8,- Linear,- RMSNorm,- get_compress_topk_idxs,- get_window_topk_idxs,-)+from .layers import get_compress_topk_idxs, get_window_topk_idxs # noqa: F401from .moe import Expert, Gate # noqa: F401-class Block(nn.Module):+class Block(BaseOP):"""Decoder block with manifold-constrained Hyper-Connections (4 residual streams)."""- def __init__(self, layer_id: int, args: DeepseekV4Args):- super().__init__()+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, strategy: str = "offload", decode_target: str = "gpu", quant_config=None, prefix: str = ""):self.layer_id = layer_idself.norm_eps = args.norm_epsself.dim = args.dim- self.attn = Attention(layer_id, args)- self.ffn = MoE(layer_id, args)+ self.attn = Attention(layer_id, args, quant_config=quant_config, prefix=f"{prefix}.attn")+ self.ffn = MoE(layer_id, args, strategy=strategy, decode_target=decode_target, quant_config=quant_config, prefix=f"{prefix}.ffn")self.attn_norm = RMSNorm(args.dim, self.norm_eps)self.ffn_norm = RMSNorm(args.dim, self.norm_eps)self.hc_mult = hc_mult = args.hc_mult@@ -64,12 +58,12 @@ class Block(nn.Module):self.hc_eps = args.hc_epsmix_hc = (2 + hc_mult) * hc_multhc_dim = hc_mult * args.dim- self.hc_attn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim, dtype=torch.float32), requires_grad=False)- self.hc_ffn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim, dtype=torch.float32), requires_grad=False)- self.hc_attn_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32), requires_grad=False)- self.hc_ffn_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32), requires_grad=False)- self.hc_attn_scale = nn.Parameter(torch.empty(3, dtype=torch.float32), requires_grad=False)- self.hc_ffn_scale = nn.Parameter(torch.empty(3, dtype=torch.float32), requires_grad=False)+ self.hc_attn_fn = torch.empty(mix_hc, hc_dim, dtype=torch.float32)+ self.hc_ffn_fn = torch.empty(mix_hc, hc_dim, dtype=torch.float32)+ self.hc_attn_base = torch.empty(mix_hc, dtype=torch.float32)+ self.hc_ffn_base = torch.empty(mix_hc, dtype=torch.float32)+ self.hc_attn_scale = torch.empty(3, dtype=torch.float32)+ self.hc_ffn_scale = torch.empty(3, dtype=torch.float32)def hc_pre(self, x, hc_fn, hc_scale, hc_base):shape, dtype = x.size(), x.dtype@@ -101,51 +95,49 @@ class Block(nn.Module):# metadata; ``flat_positions`` [T] = per-token ABSOLUTE position (batch.positions).residual = xx, post, comb = self.hc_pre(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)- x = self.attn_norm(x)+ x = self.attn_norm.forward(x)x = self.attn.forward_ragged(x, segments, flat_positions)x = self.hc_post(x, residual, post, comb)residual = xx, post, comb = self.hc_pre(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)- x = self.ffn_norm(x)- x = self.ffn(x, input_ids)+ x = self.ffn_norm.forward(x)+ x = self.ffn.forward(x, input_ids)x = self.hc_post(x, residual, post, comb)return xdef decode_step(self, x, pos, rows, cmp_stage_cap, input_ids, wctx=None):residual = xx, post, comb = self.hc_pre(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)- x = self.attn_norm(x)+ x = self.attn_norm.forward(x)x = self.attn.decode_step(x, pos, rows, cmp_stage_cap, wctx)x = self.hc_post(x, residual, post, comb)residual = xx, post, comb = self.hc_pre(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)- x = self.ffn_norm(x)- x = self.ffn(x, input_ids)+ x = self.ffn_norm.forward(x)+ x = self.ffn.forward(x, input_ids)x = self.hc_post(x, residual, post, comb)return x-class Transformer(nn.Module):- def __init__(self, args: DeepseekV4Args):- super().__init__()+class Transformer(BaseOP):+ def __init__(self, args: DeepseekV4Args, quant_config=None, *, strategy: str = "offload", decode_target: str = "gpu", prefix: str = ""):self.args = argsself.norm_eps = args.norm_epsself.hc_eps = args.hc_epsself.hc_mult = hc_mult = args.hc_mult- self.embed = nn.Embedding(args.vocab_size, args.dim)- self.embed.weight.requires_grad_(False)- self.layers = nn.ModuleList([Block(i, args) for i in range(args.n_layers)])+ self.embed = VocabParallelEmbedding(args.vocab_size, args.dim)+ self.layers = OPList([Block(i, args, strategy=strategy, decode_target=decode_target, quant_config=quant_config, prefix=f"{prefix}.layers.{i}") for i in range(args.n_layers)])self.norm = RMSNorm(args.dim, self.norm_eps)- self.head = nn.Parameter(torch.empty(args.vocab_size, args.dim, dtype=torch.bfloat16), requires_grad=False)+ self.head = ParallelLMHead(args.vocab_size, args.dim, quant_config=quant_config, prefix=f"{prefix}.head")hc_dim = hc_mult * args.dim- self.hc_head_fn = nn.Parameter(torch.empty(hc_mult, hc_dim, dtype=torch.float32), requires_grad=False)- self.hc_head_base = nn.Parameter(torch.empty(hc_mult, dtype=torch.float32), requires_grad=False)- self.hc_head_scale = nn.Parameter(torch.empty(1, dtype=torch.float32), requires_grad=False)+ self.hc_head_fn = torch.empty(hc_mult, hc_dim, dtype=torch.float32)+ self.hc_head_base = torch.empty(hc_mult, dtype=torch.float32)+ self.hc_head_scale = torch.empty(1, dtype=torch.float32)def bind(self, pool, device: torch.device) -> None:- for layer in self.layers:+ for layer in self.layers.op_list:layer.attn.bind(pool, device)def hc_head(self, x):@@ -160,7 +152,6 @@ class Transformer(nn.Module):def prefill_batched(self, input_ids: torch.Tensor, segments, flat_positions: torch.Tensor,- last_indices: torch.Tensor,) -> torch.Tensor:# Ragged batched prefill (bs >= 1). ``input_ids`` is [1, T] -- the requests' NEW tokens# concatenated (cu_seqlens, no padding); each request starts at its own cached_len@@ -170,15 +161,15 @@ class Transformer(nn.Module):# allocate_paged, so the per-request attention never reads another request's KV/carry.# ``segments`` [(offset, extend_len, table_idx, start_pos)] comes off the attention# metadata; ``flat_positions`` [T] is the scheduler-staged batch.positions (per-token- # ABSOLUTE position); ``last_indices`` [B] is the flattened index of each request's- # final token -> its next-token logits row.- h = self.embed(input_ids)+ # ABSOLUTE position); the head picks each request's final token off the attention+ # metadata -> its next-token logits row.+ h = self.embed.forward(input_ids.view(-1)).view(1, -1, self.args.dim)h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)- for layer in self.layers:+ for layer in self.layers.op_list:h = layer.prefill_batched(h, input_ids, segments, flat_positions)h = self.hc_head(h)- h = self.norm(h)- return F.linear(h[0, last_indices], self.head) # [B, vocab]+ h = self.norm.forward(h)+ return self.head.forward(h[0]) # [B, vocab]def decode(self, input_ids: torch.Tensor, pos: torch.Tensor, cmp_stage_cap: int@@ -194,7 +185,7 @@ class Transformer(nn.Module):# in-flight replay (it mutates only the live map).B = input_ids.size(0)rows = torch.arange(B, device=input_ids.device)- h = self.embed(input_ids)+ h = self.embed.forward(input_ids.view(-1)).view(B, 1, self.args.dim)h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)# Hoist the layer-invariant per-step decode tensors (shared window-ring global slots):# resolved ONCE off the attention metadata (recomputed per call, so a capture records@@ -202,32 +193,30 @@ class Transformer(nn.Module):# into every layer. They read only the shared snapshot / positions, so they are identical# across layers.wctx = get_global_ctx().batch.attn_metadata.window_ctx(pos, rows)- for layer in self.layers:+ for layer in self.layers.op_list:h = layer.decode_step(h, pos, rows, cmp_stage_cap, input_ids, wctx)h = self.hc_head(h)- h = self.norm(h)- return F.linear(h[:, -1], self.head)+ h = self.norm.forward(h)+ return self.head.forward(h[:, -1])class DeepseekV4ForCausalLM(BaseLLMModel):"""Engine adapter: a registered :class:`BaseLLMModel` wrapping the DSV4 transformer.- Built on the engine ``meta`` device then loaded via :meth:`load_state_dict` (which- casts each incoming tensor to its parameter dtype and assigns it). KV pools, rope- constants and compressor state are bound on the first forward.+ KV pools, rope constants and compressor state are bound on the first forward."""def __init__(self, config):self._config = configself._args: DeepseekV4Args = config.dsv4_args- self._transformer = Transformer(self._args)+ self.model = Transformer(self._args, config.quant, strategy=config.moe_strategy, decode_target=config.decode_target, prefix="model")self._bound = Falsedef _ensure_bound(self) -> None:if self._bound:returnpool = get_global_ctx().kv_cache- self._transformer.bind(pool, pool.device)+ self.model.bind(pool, pool.device)self._bound = Truedef mark_for_rebind(self) -> None:@@ -237,30 +226,6 @@ class DeepseekV4ForCausalLM(BaseLLMModel):block count, freqs) depends on the new pool's geometry, so re-derive it via _ensure_bound."""self._bound = False- def _iter_offload_moe_layers(self):- # Hook for moe.offload_cache.iter_offload_moe_layers: DSV4's MoE block (block.ffn)- # is a bespoke nn.Module (not OffloadMoELayer), so yield its inner routed-expert- # layer instead, whose .offload_cache the generic attach sets.- for block in self._transformer.layers:- yield block.ffn.experts-- def state_dict(self, *, prefix: str = "", result=None):- result = {} if result is None else result- for name, p in self._transformer.named_parameters():- result[f"{prefix}.{name}" if prefix else name] = p- return result-- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:- casted = {}- for name, p in self._transformer.named_parameters():- key = f"{prefix}.{name}" if prefix else name- if key not in state_dict:- raise RuntimeError(f"Missing weight for DeepSeek-V4 parameter: {key}")- casted[name] = state_dict.pop(key).to(p.dtype)- if state_dict and not _internal:- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")- self._transformer.load_state_dict(casted, assign=True, strict=False)-def forward(self) -> torch.Tensor:self._ensure_bound()batch = get_global_ctx().batch@@ -273,9 +238,8 @@ class DeepseekV4ForCausalLM(BaseLLMModel):# FROM THE RING. Per-token ops (embed / HC / norm / MoE) run batched over the# concatenated tokens; attention runs per segment so the carry / slot maps never# cross requests.- return self._transformer.prefill_batched(+ return self.model.prefill_batched(input_ids.view(1, -1), md.segments, batch.positions.long(),- md.last_indices.long(),)# DECODE (bs>=1): per-row position (GPU int tensor -> no host syncs / graph safe). The# compressed staging cap is the max position any row reaches (eager); a static max_seq-1@@ -290,7 +254,7 @@ class DeepseekV4ForCausalLM(BaseLLMModel):cmp_stage_cap = md.stage_width - 1else:cmp_stage_cap = int(pos.max().item())- return self._transformer.decode(input_ids.view(B, 1), pos, cmp_stage_cap)+ return self.model.decode(input_ids.view(B, 1), pos, cmp_stage_cap)__all__ = ["Transformer", "DeepseekV4ForCausalLM"]diff --git a/python/freetoken/models/deepseek_v4/moe.py b/python/freetoken/models/deepseek_v4/moe.pyindex d7e555a9..6b624567 100644--- a/python/freetoken/models/deepseek_v4/moe.py+++ b/python/freetoken/models/deepseek_v4/moe.py@@ -1,37 +1,32 @@-"""DSV4 MoE: sqrtsoftplus/hash router, shared SwiGLU expert, offloaded FP4 routed+"""DSV4 MoE: sqrtsoftplus/hash router, shared SwiGLU expert, offloaded MXFP4 routedexperts (GPU slot-cache / cpu / hybrid decode paths)."""from __future__ import annotationsimport torchimport torch.nn.functional as F-from torch import nnfrom freetoken.kernel.triton.dsv4.bf16_linear import bf16_linear_fp32from freetoken.kernel.triton.dsv4.swiglu import fused_swiglu-from freetoken.layers import OffloadMoELayer+from freetoken.layers import BaseOP, LinearColParallelMerged, LinearRowParallel, OffloadMoELayerfrom .args import DeepseekV4Args-from .layers import Linear-class Gate(nn.Module):+class Gate(BaseOP):"""MoE router: sqrtsoftplus scoring + hash routing (first ``n_hash_layers``)."""def __init__(self, layer_id: int, args: DeepseekV4Args):- super().__init__()self.topk = args.n_activated_expertsself.score_func = args.score_funcself.route_scale = args.route_scaleself.hash = layer_id < args.n_hash_layers- self.weight = nn.Parameter(torch.empty(args.n_routed_experts, args.dim, dtype=torch.bfloat16), requires_grad=False)+ self.weight = torch.empty(args.n_routed_experts, args.dim, dtype=torch.bfloat16)if self.hash:- self.tid2eid = nn.Parameter(- torch.empty(args.vocab_size, args.n_activated_experts, dtype=torch.int64), requires_grad=False- )- self.register_parameter("bias", None)+ self.tid2eid = torch.empty(args.vocab_size, args.n_activated_experts, dtype=torch.int64)+ self.bias = Noneelse:- self.bias = nn.Parameter(torch.empty(args.n_routed_experts, dtype=torch.float32), requires_grad=False)+ self.bias = torch.empty(args.n_routed_experts, dtype=torch.float32)def forward(self, x: torch.Tensor, input_ids: torch.Tensor):scores = bf16_linear_fp32(x, self.weight)@@ -55,28 +50,27 @@ class Gate(nn.Module):return weights, indices-class Expert(nn.Module):+class Expert(BaseOP):"""Dense SwiGLU expert (the shared expert; routed experts are offloaded FP4)."""- def __init__(self, dim: int, inter_dim: int, swiglu_limit: float):- super().__init__()- self.w1 = Linear(dim, inter_dim, kind="fp8")- self.w2 = Linear(inter_dim, dim, kind="fp8")- self.w3 = Linear(dim, inter_dim, kind="fp8")+ def __init__(self, dim: int, inter_dim: int, swiglu_limit: float, *, quant_config=None, prefix: str = ""):+ self.w1 = LinearColParallelMerged(dim, [inter_dim], has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w1")+ self.w2 = LinearRowParallel(inter_dim, dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w2")+ self.w3 = LinearColParallelMerged(dim, [inter_dim], has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w3")self.swiglu_limit = swiglu_limitdef forward(self, x: torch.Tensor) -> torch.Tensor:- h = fused_swiglu(self.w1(x), self.w3(x), self.swiglu_limit, x.dtype)- return self.w2(h)+ h = fused_swiglu(self.w1.forward(x), self.w3.forward(x), self.swiglu_limit, x.dtype)+ return self.w2.forward(h)class DSV4OffloadMoELayer(OffloadMoELayer):- """Routed FP4 experts on the shared offload cache: the base whole-layer+ """Routed MXFP4 experts on the shared offload cache: the base whole-layerstreaming prefill (grouped inline-dequant GEMM for dense chunks, GEMVbelow the route crossover) and slot-cache / cpu / hybrid decode paths(per-route dequant GEMV)."""- def __init__(self, layer_id: int, args: DeepseekV4Args):+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, strategy: str = "offload", decode_target: str = "gpu", quant_config=None, prefix: str = ""):super().__init__(layer_id=layer_id,num_experts=args.n_routed_experts,@@ -85,8 +79,12 @@ class DSV4OffloadMoELayer(OffloadMoELayer):intermediate_size=args.moe_inter_dim,renormalize=True,activation="silu",+ limit=args.swiglu_limit,+ strategy=strategy,+ decode_target=decode_target,+ quant_config=quant_config,+ prefix=prefix,)- self.swiglu_limit = args.swiglu_limitdef _prefill_routed(self,@@ -124,24 +122,23 @@ class DSV4OffloadMoELayer(OffloadMoELayer):)-class MoE(nn.Module):- """Sparse MoE: hash/score router -> offloaded FP4 routed experts + shared expert."""+class MoE(BaseOP):+ """Sparse MoE: hash/score router -> offloaded MXFP4 routed experts + shared expert."""- def __init__(self, layer_id: int, args: DeepseekV4Args):- super().__init__()+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, strategy: str = "offload", decode_target: str = "gpu", quant_config=None, prefix: str = ""):self.dim = args.dimself.gate = Gate(layer_id, args)- self.shared_experts = Expert(args.dim, args.moe_inter_dim, args.swiglu_limit)- self.experts = DSV4OffloadMoELayer(layer_id, args)+ self.shared_experts = Expert(args.dim, args.moe_inter_dim, args.swiglu_limit, quant_config=quant_config, prefix=f"{prefix}.shared_experts")+ self.experts = DSV4OffloadMoELayer(layer_id, args, strategy=strategy, decode_target=decode_target, quant_config=quant_config, prefix=f"{prefix}.experts")def forward(self, x: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:shape = x.size()x = x.view(-1, self.dim)- weights, indices = self.gate(x, input_ids.flatten())+ weights, indices = self.gate.forward(x, input_ids.flatten())# Shared expert enqueued before routed_forward: hybrid decode blocks on the# CPU pool inside routed_forward, so this GEMM must already be on the stream# to overlap the CPU overflow compute.- shared = self.shared_experts(x)+ shared = self.shared_experts.forward(x)# routed_forward may mutate the ids in place (offload decode slot remap);# indices.to(int32) always copies (int64 source), so no clone needed here.routed = self.experts.routed_forward(diff --git a/python/freetoken/models/deepseek_v4/weight.py b/python/freetoken/models/deepseek_v4/weight.pyindex f5a3fd17..4728f7f9 100644--- a/python/freetoken/models/deepseek_v4/weight.py+++ b/python/freetoken/models/deepseek_v4/weight.py@@ -1,24 +1,24 @@"""Weight loading for DeepSeek-V4-Flash (engine path).- - :func:`iter_weights` streams resident (non-expert) tensors keyed to engine param- names; model's ``load_state_dict`` casts each. ``wo_a`` dequantized to bf16 to match+ - :func:`iter_weights` streams resident (non-expert) tensors keyed by the model's+ attribute paths (``model.`` + the checkpoint name). ``wo_a`` dequantized to bf16 to matchthe reference bf16 einsum.- - :func:`load_dsfp4_expert_sources` packs routed FP4 experts into pinned CPU banks for- the offload cache. DeepSeek FP4: e8m0 per-32 block scale, no global scale.+ - :func:`iter_expert_pieces` streams the routed MXFP4 experts (e2m1 pairs + e8m0 per-32+ scales, no global) as per-expert pieces for the expert quant method's banks."""from __future__ import annotations-import collectionsimport jsonimport osimport re-from typing import Iteratorimport safetensorsimport torchfrom tqdm import tqdm+from freetoken.layers.quantization import QuantKind+from freetoken.distributed import get_tp_infofrom freetoken.models.loader import drop_page_cachefrom .args import DeepseekV4Args, load_args@@ -81,14 +81,14 @@ def iter_weights():"""Stream resident (non-expert) weights as ``(name, tensor)`` keyed to engine params.- Routed FP4 experts come from the offload cache, so ``include_moe_experts`` must be- False (DeepSeek-V4 only runs ``--moe-backend offload``). Tensors yielded in checkpoint+ Routed MXFP4 experts come from the offload cache, so ``include_moe_experts`` must be+ False (DeepSeek-V4 only runs ``--moe-strategy offload``). Tensors yielded in checkpointdtype (fp8 + e8m0 preserved); ``wo_a`` dequantized to bf16 to match the reference einsum."""if include_moe_experts:raise ValueError("DeepSeek-V4 routed experts are served from the offload cache; "- "run with --moe-backend offload (include_moe_experts must be False)."+ "run with --moe-strategy offload (include_moe_experts must be False).")if not include_non_moe:return@@ -99,248 +99,117 @@ def iter_weights(def get(name: str) -> torch.Tensor:return reader.get(name)- def linear(prefix: str):- yield f"{prefix}.weight", get(f"{prefix}.weight")- if reader.has(f"{prefix}.scale"):- yield f"{prefix}.scale", get(f"{prefix}.scale")+ def linear(src: str, dst: str):+ yield f"{dst}.weight", get(f"{src}.weight")+ # fp8 linears declare the e8m0 block scale under the quant method's role name+ if reader.has(f"{src}.scale"):+ yield f"{dst}.weight_scale_inv", get(f"{src}.scale")try:- yield "embed.weight", get("embed.weight")- yield "norm.weight", get("norm.weight")- yield "head", get("head.weight")+ yield "model.embed.weight", get("embed.weight")+ yield "model.norm.weight", get("norm.weight")+ yield "model.head.weight", get("head.weight")for nm in ("hc_head_fn", "hc_head_base", "hc_head_scale"):- yield nm, get(nm)+ yield f"model.{nm}", get(nm)for L in range(args.n_layers):a = f"layers.{L}.attn"- yield from linear(f"{a}.wq_a")- yield f"{a}.q_norm.weight", get(f"{a}.q_norm.weight")- yield from linear(f"{a}.wq_b")- yield from linear(f"{a}.wkv")- yield f"{a}.kv_norm.weight", get(f"{a}.kv_norm.weight")+ m = f"model.{a}"+ yield from linear(f"{a}.wq_a", f"{m}.wq_a")+ yield f"{m}.q_norm.weight", get(f"{a}.q_norm.weight")+ yield from linear(f"{a}.wq_b", f"{m}.wq_b")+ yield from linear(f"{a}.wkv", f"{m}.wkv")+ yield f"{m}.kv_norm.weight", get(f"{a}.kv_norm.weight")# wo_a: FP8 in the checkpoint, dequantized to bf16 (reference bf16 einsum).- yield f"{a}.wo_a", _dequant_fp8_block(+ yield f"{m}.wo_a", _dequant_fp8_block(get(f"{a}.wo_a.weight"), get(f"{a}.wo_a.scale"))- yield from linear(f"{a}.wo_b")- yield f"{a}.attn_sink", get(f"{a}.attn_sink")+ yield from linear(f"{a}.wo_b", f"{m}.wo_b")+ yield f"{m}.attn_sink", get(f"{a}.attn_sink")ratio = args.compress_ratios[L]if ratio:c = f"{a}.compressor"- yield f"{c}.ape", get(f"{c}.ape")- yield f"{c}.wkv.weight", get(f"{c}.wkv.weight")- yield f"{c}.wgate.weight", get(f"{c}.wgate.weight")- yield f"{c}.norm.weight", get(f"{c}.norm.weight")+ for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):+ yield f"model.{c}.{nm}", get(f"{c}.{nm}")if ratio == 4:idx = f"{a}.indexer"- yield from linear(f"{idx}.wq_b")- yield f"{idx}.weights_proj.weight", get(f"{idx}.weights_proj.weight")+ yield from linear(f"{idx}.wq_b", f"model.{idx}.wq_b")+ yield f"model.{idx}.weights_proj.weight", get(f"{idx}.weights_proj.weight")ic = f"{idx}.compressor"- yield f"{ic}.ape", get(f"{ic}.ape")- yield f"{ic}.wkv.weight", get(f"{ic}.wkv.weight")- yield f"{ic}.wgate.weight", get(f"{ic}.wgate.weight")- yield f"{ic}.norm.weight", get(f"{ic}.norm.weight")+ for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):+ yield f"model.{ic}.{nm}", get(f"{ic}.{nm}")- yield f"layers.{L}.attn_norm.weight", get(f"layers.{L}.attn_norm.weight")- yield f"layers.{L}.ffn_norm.weight", get(f"layers.{L}.ffn_norm.weight")+ yield f"model.layers.{L}.attn_norm.weight", get(f"layers.{L}.attn_norm.weight")+ yield f"model.layers.{L}.ffn_norm.weight", get(f"layers.{L}.ffn_norm.weight")g = f"layers.{L}.ffn.gate"- yield f"{g}.weight", get(f"{g}.weight")+ yield f"model.{g}.weight", get(f"{g}.weight")if L < args.n_hash_layers:- yield f"{g}.tid2eid", get(f"{g}.tid2eid")+ yield f"model.{g}.tid2eid", get(f"{g}.tid2eid")else:- yield f"{g}.bias", get(f"{g}.bias")+ yield f"model.{g}.bias", get(f"{g}.bias")for proj in ("w1", "w2", "w3"):- yield from linear(f"layers.{L}.ffn.shared_experts.{proj}")+ src = f"layers.{L}.ffn.shared_experts.{proj}"+ yield from linear(src, f"model.{src}")for nm in ("hc_attn_fn", "hc_ffn_fn", "hc_attn_base","hc_ffn_base", "hc_attn_scale", "hc_ffn_scale",):- yield f"layers.{L}.{nm}", get(f"layers.{L}.{nm}")+ yield f"model.layers.{L}.{nm}", get(f"layers.{L}.{nm}")finally:reader.close()# ---------------------------------------------------------------------------------------# Routed FP4 expert pinned banks.+# Routed MXFP4 expert pieces.# --------------------------------------------------------------------------------------_EXPERT_RE = re.compile(r"^layers\.(?P<layer>\d+)\.ffn\.experts\.(?P<expert>\d+)\."r"(?P<proj>w1|w2|w3)\.(?P<kind>weight|scale)$")+_PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}+_KIND_SUFFIX = {"weight": "", "scale": "_scale"}-def load_dsfp4_expert_sources(- model_path: str, args: DeepseekV4Args, *, layer_sink=None-) -> dict[str, list[torch.Tensor]]:- """Build pinned CPU DeepSeek-FP4 banks for the routed experts.-- 4 banks, one tensor per layer (independent allocations): ``gate_up_packed/scale``- ``[E, 2I, H//2]`` uint8 / ``[..., H//32]`` e8m0, ``down_packed/scale`` ``[E, H, I//2]``- / ``[..., I//32]``.-- ``layer_sink=None`` (serving): pin each layer as its writes complete, via an- internally-owned :class:`PinPipeline`. ``layer_sink`` given (converter): the- completion tracker fires into it instead -- nothing here is pinned, and the sink- may release banks it has written out, so the returned tensors are only valid- until then (the caller owns that tradeoff).- """- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks-- folder = model_path- weight_map = _weight_map(folder)- L, E = args.n_layers, args.n_routed_experts- H, I = args.dim, args.moe_inter_dim-- for shard in sorted(set(weight_map.values())):- drop_page_cache(os.path.join(folder, shard))-- shards: dict[str, list[tuple[str, re.Match]]] = collections.defaultdict(list)- for name, shard in weight_map.items():- m = _EXPERT_RE.match(name)- if m is None:- continue- if int(m.group("layer")) >= L: # skip the MTP layer (index L)- continue- shards[shard].append((name, m))-- e8m0 = torch.float8_e8m0fnu- specs = { # alloc UNPINNED, fill, then pin-after-fill (skips slow cudaHostAlloc zero-fill)- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),- "gate_up_scale": ((E, 2 * I, H // 32), e8m0),- "down_packed": ((E, H, I // 2), torch.uint8),- "down_scale": ((E, H, I // 32), e8m0),- }- hb = alloc_layer_banks(specs, L)- banks = {name: [b.tensor for b in hb[name]] for name in specs}-- def _load(sink) -> int:- tracker = LayerCompletionTracker(E * 6, hb, sink) # {w1,w2,w3} x {weight,scale} x experts- placed = 0- for shard in tqdm(sorted(shards), desc="Loading DSV4 FP4 experts"):- path = os.path.join(folder, shard)- with safetensors.safe_open(path, framework="pt", device="cpu") as f:- for name, m in shards[shard]:- layer = _place_dsfp4(banks, name, f.get_tensor(name), I)- tracker.note(layer)- placed += 1- drop_page_cache(path)- return placed-- if layer_sink is not None:- placed = _load(layer_sink)- else:- with PinPipeline() as pins:- placed = _load(pins)-- expected = L * E * 6 # {w1,w2,w3} x {weight, scale}- assert placed == expected, f"loaded {placed} expert tensors, expected {expected}"- return banks---def dummy_dsfp4_expert_sources(args: DeepseekV4Args) -> dict[str, list[torch.Tensor]]:- """Fabricate the 4 ds_fp4 banks for --dummy-weight (no checkpoint on disk)."""- from freetoken.moe.host_banks import alloc_layer_banks, pin_banks-- L, E = args.n_layers, args.n_routed_experts- H, I = args.dim, args.moe_inter_dim- e8m0 = torch.float8_e8m0fnu- specs = {- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),- "gate_up_scale": ((E, 2 * I, H // 32), e8m0),- "down_packed": ((E, H, I // 2), torch.uint8),- "down_scale": ((E, H, I // 32), e8m0),- }- hb = alloc_layer_banks(specs, L)- banks = {name: [b.tensor for b in hb[name]] for name in specs}- for t in banks["gate_up_packed"]: # packed e2m1; scales stay 0 (valid e8m0)- t.random_(0, 256)- for t in banks["down_packed"]:- t.random_(0, 256)- pin_banks(hb)- return banks---def is_expert_tensor(name: str) -> bool:- """Predicate for the common parallel reader: is this a routed-expert tensor?"""- return _EXPERT_RE.match(name) is not None---def _place_dsfp4(banks: dict, name: str, t: torch.Tensor, I: int) -> int:- """Copy one expert tensor into its layer/expert slot (shared by serial + parallel- readers): w1->gate_up[:,:I], w3->gate_up[:,I:], w2->down. Returns the layer index."""- m = _EXPERT_RE.match(name)- layer, expert = int(m.group("layer")), int(m.group("expert"))- proj, kind = m.group("proj"), m.group("kind")- if kind == "weight":- t = t.view(torch.uint8)- if proj == "w1":- banks["gate_up_packed"][layer][expert, :I] = t- elif proj == "w3":- banks["gate_up_packed"][layer][expert, I:] = t- else: # w2 -> down- banks["down_packed"][layer][expert] = t- else: # scale (e8m0)- if proj == "w1":- banks["gate_up_scale"][layer][expert, :I] = t- elif proj == "w3":- banks["gate_up_scale"][layer][expert, I:] = t- else:- banks["down_scale"][layer][expert] = t- return layer---def load_dsfp4_expert_sources_parallel(- model_path: str, args: DeepseekV4Args, *, workers: int = 8, chunk: int = 8 << 20,- layer_sink=None,-) -> dict[str, list[torch.Tensor]]:- """parallel path: same banks as load_dsfp4_expert_sources, filled from the common- chunked multi-threaded O_DIRECT reader instead of serial per-shard safe_open.- ``layer_sink``: see :func:`load_dsfp4_expert_sources`."""+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):+ """Routed experts, one piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0+ ``_scale`` companions (``w1`` / ``w3`` / ``w2``). The MTP layer's experts are skipped."""+ if kind is not QuantKind.MXFP4:+ return None+ if get_tp_info().size > 1:+ raise NotImplementedError("DeepSeek-V4 expert banks support TP=1 only")from freetoken.models.weight import iter_expert_tensors_parallel- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks+ from freetoken.moe.expert_pieces import per_expert_pieces+ args = load_args(model_path, max_batch_size=1)L, E = args.n_layers, args.n_routed_experts- H, I = args.dim, args.moe_inter_dim- e8m0 = torch.float8_e8m0fnu- specs = {- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),- "gate_up_scale": ((E, 2 * I, H // 32), e8m0),- "down_packed": ((E, H, I // 2), torch.uint8),- "down_scale": ((E, H, I // 32), e8m0),- }- hb = alloc_layer_banks(specs, L) # lazy host banks (unpinned)- banks = {name: [b.tensor for b in hb[name]] for name in specs}-- def _is_expert(name: str) -> bool:- m = _EXPERT_RE.match(name)- return m is not None and int(m.group("layer")) < L # skip the MTP layer (index L)-- def _load(sink) -> int:- tracker = LayerCompletionTracker(E * 6, hb, sink)- placed = 0- for name, t in iter_expert_tensors_parallel(model_path, _is_expert, workers=workers, chunk=chunk):- layer = _place_dsfp4(banks, name, t, I)- tracker.note(layer)- placed += 1- return placed-- if layer_sink is not None:- placed = _load(layer_sink)- else:- with PinPipeline() as pins:- placed = _load(pins)-- expected = L * E * 6- assert placed == expected, f"loaded {placed} expert tensors, expected {expected}"- return banks---__all__ = [- "iter_weights",- "load_dsfp4_expert_sources",- "load_dsfp4_expert_sources_parallel",- "is_expert_tensor",-]++ def locate(raw_name: str):+ m = _EXPERT_RE.match(raw_name)+ if m is None or int(m["layer"]) >= L:+ return None+ return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]++ if parallel:+ tensors = iter_expert_tensors_parallel(model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk)+ return per_expert_pieces(tensors, locate, tensors_per_expert=6)++ def _serial():+ reader = _ShardReader(model_path, _weight_map(model_path), torch.device("cpu"))+ try:+ for li in tqdm(range(L), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):+ for e in range(E):+ base = f"layers.{li}.ffn.experts.{e}"+ for proj in ("w1", "w3", "w2"):+ for kind_ in ("weight", "scale"):+ name = f"{base}.{proj}.{kind_}"+ yield name, reader.get(name)+ finally:+ reader.close()++ return per_expert_pieces(_serial(), locate, tensors_per_expert=6)+++__all__ = ["iter_weights", "iter_expert_pieces"]diff --git a/python/freetoken/models/gemma4/__init__.py b/python/freetoken/models/gemma4/__init__.pyindex 37a8dcb3..a6b285f2 100644--- a/python/freetoken/models/gemma4/__init__.py+++ b/python/freetoken/models/gemma4/__init__.py@@ -10,13 +10,13 @@ from .model import Gemma4ForCausalLMfrom .moe import Gemma4Routerfrom .vision import Gemma4MultimodalEmbedder, Gemma4VisionModelfrom .weight import (+ nvfp4_expert_spec,iter_weights,iter_weights_parallel,- load_nvfp4_expert_sources,- load_nvfp4_expert_sources_parallel,)__all__ = [+ "nvfp4_expert_spec","Gemma4Attention","Gemma4ForCausalLM","Gemma4MultimodalEmbedder",@@ -25,8 +25,6 @@ __all__ = ["parse_config","iter_weights","iter_weights_parallel",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel","parse_gguf_config","iter_gguf_weights","load_q4_0_expert_sources",diff --git a/python/freetoken/models/gemma4/attention.py b/python/freetoken/models/gemma4/attention.pyindex 9103bf34..4f85255f 100644--- a/python/freetoken/models/gemma4/attention.py+++ b/python/freetoken/models/gemma4/attention.py@@ -17,7 +17,7 @@ if TYPE_CHECKING:class Gemma4Attention(BaseOP):"""Gemma 4 attention for one full-context or SWA layer."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self.layer_id = layer_idgroup = config.attention_group_for_layer(layer_id)self.is_swa = isinstance(group, SWAAttentionGroupConfig)@@ -37,8 +37,13 @@ class Gemma4Attention(BaseOP):self.num_qo_heads,self.num_kv_heads,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",+ )+ self.o_proj = LinearReplicated(+ self.q_dim, config.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.o_proj",)- self.o_proj = LinearReplicated(self.q_dim, config.hidden_size, has_bias=False)self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)self.v_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps, with_scale=False)diff --git a/python/freetoken/models/gemma4/model.py b/python/freetoken/models/gemma4/model.pyindex 66fcf412..83e49c4d 100644--- a/python/freetoken/models/gemma4/model.py+++ b/python/freetoken/models/gemma4/model.py@@ -28,11 +28,13 @@ class Gemma4DecoderLayer(BaseOP):per-layer ``layer_scalar``. The feed-forward is the dual (shared MLP || routed MoE)branch for MoE checkpoints, or a single dense MLP branch for dense checkpoints."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self._layer_id = layer_id- self.self_attn = Gemma4Attention(config, layer_id)+ self.self_attn = Gemma4Attention(config, layer_id, prefix=f"{prefix}.self_attn")self.feed_forward = (- Gemma4MLP(config, layer_id) if config.is_moe else Gemma4DenseMLP(config)+ Gemma4MLP(config, layer_id, prefix=f"{prefix}.feed_forward")+ if config.is_moe+ else Gemma4DenseMLP(config, prefix=f"{prefix}.feed_forward"))eps = config.rms_norm_eps@@ -53,14 +55,17 @@ class Gemma4DecoderLayer(BaseOP):class Gemma4Model(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,embed_scale=config.embedding_scale,)self.layers = OPList(- [Gemma4DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ Gemma4DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)self._image_token_id = config.image_token_id@@ -101,6 +106,8 @@ class Gemma4ForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)self._final_logit_softcapping = config.final_logit_softcappingif config.is_multimodal:diff --git a/python/freetoken/models/gemma4/moe.py b/python/freetoken/models/gemma4/moe.pyindex 0185561b..f5c8ca22 100644--- a/python/freetoken/models/gemma4/moe.py+++ b/python/freetoken/models/gemma4/moe.py@@ -39,12 +39,14 @@ class Gemma4Router(BaseOP):class Gemma4MLP(BaseOP):"""Gemma 4 feed-forward sandwich: shared MLP plus routed MoE branch."""- def __init__(self, config: ModelConfig, layer_id: int):- self.shared_mlp = GatedMLP(config)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.shared_mlp = GatedMLP(config, quant_config=config.quant, prefix=f"{prefix}.shared_mlp")self.experts = make_moe_layer(config,layer_id=layer_id,activation="gelu_tanh",+ quant_config=config.quant,+ prefix=f"{prefix}.experts",)self.router = Gemma4Router(config)@@ -80,29 +82,6 @@ class Gemma4MLP(BaseOP):)-class _Nvfp4GatedMLP(BaseOP):- """W4A16 NVFP4 gated MLP (fused gate_up + down, gelu-tanh gated). Drop-in for the bf16- GatedMLP inside Gemma4DenseMLP on modelopt-NVFP4 dense checkpoints; same attribute names- (gate_up_proj / down_proj) so the loader targets it identically."""-- def __init__(self, config: ModelConfig):- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear- from freetoken.layers import gelu_tanh_and_mul-- self.gate_up_proj = Nvfp4DenseColMerged(- config.hidden_size,- [config.intermediate_size, config.intermediate_size],- has_bias=False,- )- self.down_proj = Nvfp4DenseLinear(- config.intermediate_size, config.hidden_size, has_bias=False- )- self._act = gelu_tanh_and_mul-- def forward(self, x: torch.Tensor) -> torch.Tensor:- return self.down_proj.forward(self._act(self.gate_up_proj.forward(x)))--class Gemma4DenseMLP(BaseOP):"""Gemma 4 dense feed-forward: a single gated MLP branch (no routed experts),RMS-normed then residual-added, scaled by the per-layer ``layer_scalar``.@@ -111,15 +90,10 @@ class Gemma4DenseMLP(BaseOP):``out = (x + post_feedforward_layernorm(shared_mlp(pre_ff))) * layer_scalar``.Attribute names match the MoE Gemma4MLP so weight.py's rename machinery(``mlp.* -> feed_forward.shared_mlp.*``, bare ``layer_scalar`` /- ``post_feedforward_layernorm.``) resolves the dense checkpoint unchanged. The gated- MLP is bf16, or W4A16 NVFP4 when the checkpoint quantizes the dense MLP."""+ ``post_feedforward_layernorm.``) resolves the dense checkpoint unchanged."""- def __init__(self, config: ModelConfig):- self.shared_mlp = (- _Nvfp4GatedMLP(config)- if getattr(config, "dense_quant", "none") == "nvfp4"- else GatedMLP(config)- )+ def __init__(self, config: ModelConfig, *, prefix: str = ""):+ self.shared_mlp = GatedMLP(config, quant_config=config.quant, prefix=f"{prefix}.shared_mlp")self.post_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.pyindex 43c2355d..71f5531b 100644--- a/python/freetoken/models/gemma4/weight.py+++ b/python/freetoken/models/gemma4/weight.py@@ -14,7 +14,6 @@ from freetoken.models.loader import ()from freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,- load_nvfp4_expert_source_banks,)from freetoken.utils import cached_load_hf_configfrom tqdm import tqdm@@ -178,7 +177,7 @@ def iter_weights(if name is None:continue- # Per-expert NVFP4 tensors go to the offload cache (load_nvfp4_expert_sources),+ # Per-expert NVFP4 tensors go to the offload cache (expert pieces),# not this dense pass; fused bf16/q4_0 experts lack ".experts.<int>." so are unaffected.if _NVFP4_EXPERT_RE.search(raw_name):continue@@ -255,7 +254,7 @@ def iter_weights_parallel((``feed_forward.experts.{gate_up_proj,down_proj}``), so no merge needed; same keyrename as iter_weights, read via the common chunked O_DIRECT reader."""assert include_moe_experts and not include_non_moe, (- "gemma4 parallel reader is experts-only (used by load_moe_expert_sources)"+ "gemma4 parallel reader is experts-only (used by the expert piece reader)")from freetoken.models.weight import iter_expert_tensors_parallel@@ -275,41 +274,12 @@ def iter_weights_parallel(yield _expert_name(raw_name), tensor-def load_nvfp4_expert_sources(- model_path: str, config, *, layer_sink=None-) -> dict[str, list[torch.Tensor]]:- """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""- return load_nvfp4_expert_source_banks(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- layer_sink=layer_sink,- )---def load_nvfp4_expert_sources_parallel(- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None-):- """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel-- return load_nvfp4_expert_source_banks_parallel(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- workers=workers,- chunk=chunk,- layer_sink=layer_sink,- )+def nvfp4_expert_spec(model_path: str, config):+ return _NVFP4_SOURCE_SPEC__all__ = [+ "nvfp4_expert_spec","iter_weights","iter_weights_parallel",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel",]diff --git a/python/freetoken/models/glm4_moe/__init__.py b/python/freetoken/models/glm4_moe/__init__.pyindex a07134c3..b8748b1d 100644--- a/python/freetoken/models/glm4_moe/__init__.py+++ b/python/freetoken/models/glm4_moe/__init__.py@@ -1,11 +1,10 @@from .config import parse_configfrom .model import Glm4MoeForCausalLM-from .weight import iter_weights, load_nvfp4_expert_sources, load_nvfp4_expert_sources_parallel+from .weight import iter_weights, nvfp4_expert_spec__all__ = [+ "nvfp4_expert_spec","Glm4MoeForCausalLM","parse_config","iter_weights",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel",]diff --git a/python/freetoken/models/glm4_moe/mlp.py b/python/freetoken/models/glm4_moe/mlp.pyindex f8fb8bef..13fb6ee1 100644--- a/python/freetoken/models/glm4_moe/mlp.py+++ b/python/freetoken/models/glm4_moe/mlp.py@@ -3,29 +3,26 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGimport torch.nn.functional as F-from freetoken.layers import BaseOP+from freetoken.layers import BaseOP, LinearReplicatedfrom freetoken.utils import nvtx_annotate-from .nvfp4_linear import LinearNVFP4-if TYPE_CHECKING:import torch+ from freetoken.layers.quantization import QuantConfig+class GlmGatedMLP(BaseOP):- """SwiGLU MLP whose projections keep the checkpoint's native NVFP4 weights.-- Used for both the leading dense layers (``intermediate_size``) and each MoE layer's- always-on shared expert (``moe_intermediate_size``). GLM-4 ships these in NVFP4; we keep- them NVFP4 and dequantize in the forward (:class:`LinearNVFP4`) -- identical math to the- routed experts (faithful to the checkpoint) and the smallest footprint, which matters- because GLM activates 89*8 experts per token. Activations stay bf16.- """-- def __init__(self, hidden_size: int, intermediate_size: int):- self.gate_proj = LinearNVFP4(hidden_size, intermediate_size)- self.up_proj = LinearNVFP4(hidden_size, intermediate_size)- self.down_proj = LinearNVFP4(intermediate_size, hidden_size)+ """SwiGLU MLP for the leading dense layers (``intermediate_size``) and each MoE layer's+ always-on shared expert (``moe_intermediate_size``); the projections take whatever the+ checkpoint stores for their prefix (NVFP4 in the GLM-4 releases)."""++ def __init__(+ self, hidden_size: int, intermediate_size: int, *, quant_config: QuantConfig | None = None, prefix: str = ""+ ):+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")@nvtx_annotate("MLP")def forward(self, x: torch.Tensor) -> torch.Tensor:diff --git a/python/freetoken/models/glm4_moe/model.py b/python/freetoken/models/glm4_moe/model.pyindex 17f4f0aa..b5c647e1 100644--- a/python/freetoken/models/glm4_moe/model.py+++ b/python/freetoken/models/glm4_moe/model.py@@ -18,12 +18,14 @@ if TYPE_CHECKING:class Glm4MoeDecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self.self_attn = Glm4MoeAttention(config, layer_id)if layer_id >= config.first_k_dense_replace:- self.mlp: BaseOP = Glm4MoeSparseBlock(config, layer_id)+ self.mlp: BaseOP = Glm4MoeSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")else:- self.mlp = GlmGatedMLP(config.hidden_size, config.intermediate_size)+ self.mlp = GlmGatedMLP(+ config.hidden_size, config.intermediate_size, quant_config=config.quant, prefix=f"{prefix}.mlp"+ )self.input_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)self.post_attention_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps@@ -42,13 +44,16 @@ class Glm4MoeDecoderLayer(BaseOP):class Glm4MoeModel(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = EmbeddingDF11(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [Glm4MoeDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ Glm4MoeDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)@@ -68,6 +73,8 @@ class Glm4MoeForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/glm4_moe/moe.py b/python/freetoken/models/glm4_moe/moe.pyindex 757af0f5..e1b0f18b 100644--- a/python/freetoken/models/glm4_moe/moe.py+++ b/python/freetoken/models/glm4_moe/moe.py@@ -23,7 +23,7 @@ class Glm4MoeSparseBlock(BaseOP):``routed_scaling_factor``. The shared expert is applied to the same input and added."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self.top_k = config.num_experts_per_tokself.num_experts = config.num_expertsself.norm_topk_prob = config.norm_topk_prob@@ -44,10 +44,14 @@ class Glm4MoeSparseBlock(BaseOP):config,layer_id=layer_id - config.first_k_dense_replace,renormalize=config.norm_topk_prob,+ quant_config=config.quant,+ prefix=f"{prefix}.experts",)self.shared_experts = GlmGatedMLP(config.hidden_size,config.moe_intermediate_size * max(1, config.n_shared_experts),+ quant_config=config.quant,+ prefix=f"{prefix}.shared_experts",)def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:diff --git a/python/freetoken/models/glm4_moe/nvfp4_linear.py b/python/freetoken/models/glm4_moe/nvfp4_linear.pydeleted file mode 100644index 1eff394b..00000000--- a/python/freetoken/models/glm4_moe/nvfp4_linear.py+++ /dev/null@@ -1,114 +0,0 @@-from __future__ import annotations--import os--import torch-from freetoken.layers import BaseOP-from freetoken.layers.base import _concat_prefix-from freetoken.kernel.triton.nvfp4_linear import (- nvfp4_dense_linear_t,- nvfp4_transpose_resident,-)--# Resident NVFP4 linears default to vLLM's fused FP4 Marlin GEMM (W4A16) on Marlin-capable GPUs-# (cc in [8.0, 10.0), which lack native FP4 tensor cores): ~2-7x faster per-linear at bs=1 and it-# frees the bf16 dequant scratch, with numerically-faithful output (validated under the model's-# sampling regime). When Marlin is unavailable (e.g. sm_120, or vLLM not installed) the-# forward runs FreeToken's own shared-# W4A16 kernel (freetoken.kernel.triton.nvfp4_linear) -- reading the FP4 weight directly (fp32-# accumulation, no per-forward bf16 dequant + scratch), which is faster *and* strictly more-# accurate than the old dequant fallback. Set GLM_NVFP4_NOMARLIN=1 to force the shared kernel.-# (Marlin's ~bf16-level difference can perturb *greedy* decoding, but GLM's recommended decoding-# is sampling, where it is correctness-equivalent.)-_DISABLE_MARLIN = os.environ.get("GLM_NVFP4_NOMARLIN", "0") == "1"---class LinearNVFP4(BaseOP):- """Replicated (TP=1) linear that keeps the checkpoint's native NVFP4 weight as-is.-- Used for GLM-4's always-resident dense MLPs and shared experts (W4A16: bf16 activation,- FP4 weight). By default the weight is repacked to Marlin layout at load and the forward is a- single fused dequant+GEMM (vLLM Marlin). With GLM_NVFP4_NOMARLIN=1 (or on non-Marlin GPUs /- when vLLM is absent) it runs the shared FreeToken W4A16 kernel directly on the packed FP4- weight (decode int32 GEMV / prefill GEMM, fp32 accumulation) -- no bf16 dequant scratch.-- Keeping NVFP4 (4 bits) is faithful to the checkpoint (identical math to the routed experts)- and the smallest option -- re-quantizing to fp8 would add ~2.6% error *and* use twice the- memory, and the budget is tight once attention is lossless DF11.- """-- def __init__(self, in_features: int, out_features: int):- assert in_features % 16 == 0- self.in_features = in_features- self.out_features = out_features- self.weight = torch.empty(out_features, in_features // 2, dtype=torch.uint8)- self.weight_scale = torch.empty(out_features, in_features // 16, dtype=torch.float8_e4m3fn)- self.weight_global = torch.empty(out_features, dtype=torch.float16)- self._marlin = False-- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:- weight = state_dict.pop(_concat_prefix(prefix, "weight"))- weight_scale = state_dict.pop(_concat_prefix(prefix, "weight_scale"))- weight_global = state_dict.pop(_concat_prefix(prefix, "weight_global"))-- if not _DISABLE_MARLIN:- try:- from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (- apply_fp4_marlin_linear,- is_fp4_marlin_supported,- prepare_fp4_layer_for_marlin,- )-- marlin_supported = is_fp4_marlin_supported()- except Exception:- marlin_supported = False-- if marlin_supported:- from types import SimpleNamespace-- # weight_global is the per-tensor weight_scale_2 broadcast to [out]; Marlin- # wants the scalar.- layer = SimpleNamespace(- weight=weight,- weight_scale=weight_scale,- weight_scale_2=weight_global.reshape(-1)[0].to(torch.bfloat16),- output_size_per_partition=self.out_features,- input_size_per_partition=self.in_features,- params_dtype=torch.bfloat16,- )- prepare_fp4_layer_for_marlin(layer) # repacks weight + processes scales in-place- self._mweight = layer.weight- self._mscale = layer.weight_scale- self._mglobal = layer.weight_scale_2- self._mworkspace = layer.workspace- self._apply = apply_fp4_marlin_linear- self._marlin = True- dev = weight.device- self.weight = torch.empty(0, dtype=torch.uint8, device=dev)- self.weight_scale = torch.empty(0, dtype=torch.float8_e4m3fn, device=dev)- self.weight_global = torch.empty(0, dtype=torch.float16, device=dev)-- if not self._marlin:- # K-major resident repack: coalesced weight loads in the shared W4A16 kernels.- self.weight, self.weight_scale = nvfp4_transpose_resident(weight, weight_scale)- self.weight_global = weight_global-- if not _internal and state_dict:- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")-- def forward(self, x: torch.Tensor) -> torch.Tensor:- if self._marlin:- return self._apply(- input=x,- weight=self._mweight,- weight_scale=self._mscale,- weight_scale_2=self._mglobal,- workspace=self._mworkspace,- size_n=self.out_features,- size_k=self.in_features,- )- # Shared FreeToken W4A16 kernel: read the packed FP4 weight directly (no bf16 scratch).- return nvfp4_dense_linear_t(x, self.weight, self.weight_scale, self.weight_global)---__all__ = ["LinearNVFP4"]diff --git a/python/freetoken/models/glm4_moe/weight.py b/python/freetoken/models/glm4_moe/weight.pyindex 7efc6e6b..5d4f9fce 100644--- a/python/freetoken/models/glm4_moe/weight.py+++ b/python/freetoken/models/glm4_moe/weight.py@@ -11,7 +11,6 @@ from freetoken.distributed import get_tp_infofrom freetoken.models.loader import drop_page_cachefrom freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,- load_nvfp4_expert_source_banks,)from freetoken.utils import cached_load_hf_config, download_hf_weightfrom tqdm import tqdm@@ -79,8 +78,8 @@ def _iter_nvfp4_resident("""Yield native NVFP4 buffers for an always-resident Linear.``weight`` (packed fp4) + ``weight_scale`` (fp8 block scale) verbatim; per-tensor- ``weight_scale_2`` broadcast to per-row fp16 ``weight_global`` as LinearNVFP4 /- dequant_nvfp4 expect. lossless vs checkpoint; same dequant math as routed experts.+ ``weight_scale_2`` broadcast to per-row fp16 ``weight_global`` as the NVFP4 linear+ method / dequant_nvfp4 expect. lossless vs checkpoint; same dequant math as routed experts."""packed = reader.get(f"{src_prefix}.weight") # [OUT, IN//2] uint8scale = reader.get(f"{src_prefix}.weight_scale") # [OUT, IN//16] fp8-e4m3@@ -122,7 +121,7 @@ def iter_weights("""assert not include_moe_experts, ("GLM-4 MoE stores experts as NVFP4 and only supports the offload backend; experts "- "are loaded into the offload cache via load_nvfp4_expert_sources()."+ "are loaded into the offload cache from their NVFP4 pieces.")assert include_non_moeconfig = parse_config(cached_load_hf_config(model_path))@@ -191,39 +190,8 @@ def _iter_resident_weights(reader, config, primary) -> Iterator[tuple[str, torch# --------------------------------------------------------------------------------------# Routed expert host banks (NVFP4) for the offload cache.# ---------------------------------------------------------------------------------------def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None) -> dict[str, torch.Tensor]:- """Build the pinned CPU NVFP4 banks for GLM-4's routed experts.-- experts exist only for layers [first_k_dense_replace, num_layers) and pack by MoE layer- index (layer - first_k_dense_replace) so banks have no holes for leading dense layers.- MTP layer excluded.- """- return load_nvfp4_expert_source_banks(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- layer_sink=layer_sink,- )---def load_nvfp4_expert_sources_parallel(- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None-):- """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel-- return load_nvfp4_expert_source_banks_parallel(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- workers=workers,- chunk=chunk,- layer_sink=layer_sink,- )+def nvfp4_expert_spec(model_path: str, config):+ return _NVFP4_SOURCE_SPEC-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]+__all__ = ["iter_weights", "nvfp4_expert_spec"]diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.pyindex c1c05140..7e949097 100644--- a/python/freetoken/models/glm5_next/__init__.py+++ b/python/freetoken/models/glm5_next/__init__.py@@ -1,10 +1,11 @@from .config import parse_configfrom .model import Glm5NextForCausalLM-from .weight import iter_weights, load_nvfp4_expert_sources+from .weight import iter_expert_pieces, iter_weights, nvfp4_expert_spec__all__ = [+ "nvfp4_expert_spec","Glm5NextForCausalLM","parse_config","iter_weights",- "load_nvfp4_expert_sources",+ "iter_expert_pieces",]diff --git a/python/freetoken/models/glm5_next/attention.py b/python/freetoken/models/glm5_next/attention.pyindex 1ab6c340..0814a687 100644--- a/python/freetoken/models/glm5_next/attention.py+++ b/python/freetoken/models/glm5_next/attention.py@@ -31,10 +31,11 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGimport torch+from freetoken.layers.quantization import QuantKindfrom freetoken.core import get_global_ctxfrom freetoken.layers import BaseOP, LinearReplicated, RMSNorm# Shared with GLM-5.2 (weight.py imports privately from the same package).-from freetoken.models.glm_moe_dsa.attention import _IdxLayerNorm, _make_proj+from freetoken.models.glm_moe_dsa.attention import _IdxLayerNormfrom freetoken.utils import nvtx_annotateif TYPE_CHECKING:@@ -46,17 +47,17 @@ class Glm5NextIndexer(BaseOP):checkpoint's NoPE geometry leaves ``qk_rope_head_dim == 0`` so position entersonly via the compression APE).- Kept bf16 in every quant mode: small (~17 MB/layer) and the top-k boundary is- precision-sensitive (same reasoning as glm_moe_dsa's indexer).+ ``wq_b`` follows the checkpoint's quant config; ``wk`` and ``weights_proj`` stay bf16 (same reasoning as glm_moe_dsa's indexer)."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.glm5_argsself.n_heads = args.index_n_headsself.head_dim = args.index_head_dimself.kpool = args.index_kpoolself.wq_b = LinearReplicated(- args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False+ args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.wq_b",)self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False)self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6)@@ -91,10 +92,10 @@ class Glm5NextIndexer(BaseOP):class Glm5NextAttention(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.glm5_argsself.layer_id = layer_id- self.indexer = Glm5NextIndexer(config, layer_id)+ self.indexer = Glm5NextIndexer(config, layer_id, prefix=f"{prefix}.indexer")self.num_heads = args.num_headsself.qk_nope_head_dim = args.qk_nope_head_dimself.qk_rope_head_dim = args.qk_rope_head_dim # 0 (NoPE)@@ -106,25 +107,32 @@ class Glm5NextAttention(BaseOP):"would need the glm_moe_dsa rope plumbing back")- quant = config.attn_quant- self.q_a_proj = _make_proj(quant, args.hidden_size, args.q_lora_rank)+ self.q_a_proj = LinearReplicated(+ args.hidden_size, args.q_lora_rank, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.q_a_proj",+ )self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps)- self.q_b_proj = _make_proj(- quant, args.q_lora_rank, self.num_heads * self.qk_head_dim+ self.q_b_proj = LinearReplicated(+ args.q_lora_rank, self.num_heads * self.qk_head_dim, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.q_b_proj",)# NoPE: kv_a projects to bare ckv (no +qk_rope_head_dim rows).- self.kv_a_proj_with_mqa = _make_proj(- quant, args.hidden_size, self.kv_lora_rank+ self.kv_a_proj_with_mqa = LinearReplicated(+ args.hidden_size, self.kv_lora_rank, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.kv_a_proj_with_mqa",)self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps)- # kv_b stays bf16 in every mode (bmm absorption operand, not a Linear).+ # the MLA absorption reads kv_b_proj.weight as bmm operands, so only an unquantized scheme is servedself.kv_b_proj = LinearReplicated(self.kv_lora_rank,self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.kv_b_proj",)- self.o_proj = _make_proj(- quant, self.num_heads * self.v_head_dim, args.hidden_size+ assert self.kv_b_proj.quant_method.kind is QuantKind.NONE, f"{prefix}.kv_b_proj: a quantized kv_b_proj needs a dequantized copy for the MLA absorption"+ self.o_proj = LinearReplicated(+ self.num_heads * self.v_head_dim, args.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.o_proj",)self._w_uk: torch.Tensor | None = Noneself._w_uv: torch.Tensor | None = Nonediff --git a/python/freetoken/models/glm5_next/config.py b/python/freetoken/models/glm5_next/config.pyindex 294ef579..1cc6d798 100644--- a/python/freetoken/models/glm5_next/config.py+++ b/python/freetoken/models/glm5_next/config.py@@ -17,15 +17,7 @@ scaling 2.5) with 288 routed experts and a clamped SwiGLU(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in``glm5_args`` (``Glm5NextArgs``).-Resident-weight quantization (attn/dense/lm_head fp8-at-load, the GLM-5.2-bandwidth trick) is OFF by default: the NVFP4 exports deliberately quantize only-the routed experts (attention / shared expert / lm_head ship bf16 behind the-quantization_config ignore list), and a serving engine must not override the-checkpoint author's precision decision silently -- the checkpoint is served-as-is, like vLLM/sglang. FREETOKEN_GLM5_ATTN_FP8=1 / FREETOKEN_GLM5_MLP_FP8=1-opt into the W8A16 requantization (measured decode 36 -> 45.5 tok/s on the-hybrid reference setup; the win needs CUDA graphs -- launch-bound eager decode-gets slower).+The checkpoint is served in the precision its quantization_config declares."""from __future__ import annotations@@ -43,12 +35,6 @@ from freetoken.models.config import (from .args import load_args-# Load-time W8A16 fp8 for resident weights: default OFF, env opt-in (see module-# docstring for the rationale and measured numbers). attn covers the KDA-# in_proj_qkv/o_proj + DSA projections (the precision-sensitive b|f_a|g_a gate-# slice stays bf16, see kda.py); mlp covers dense MLPs, shared expert, lm_head.-_ATTN_FP8 = os.getenv("FREETOKEN_GLM5_ATTN_FP8", "0") != "0"-_MLP_FP8 = os.getenv("FREETOKEN_GLM5_MLP_FP8", "0") != "0"def _dsa_on(args, dsa_layer_ids) -> bool:@@ -142,6 +128,11 @@ def parse_config(hf_config: Any) -> ModelConfig:t == "sparse" for t in mlp_types[first_dense:]), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}"+ from freetoken.models.qwen3_5_moe.config import _fp8_block_quant # shared until the legacy quant fields go++ expert_quant, weight_block_size = _fp8_block_quant(hf_config)+ if expert_quant == "none":+ expert_quant = detect_expert_quant(hf_config)return ModelConfig(num_layers=num_layers,num_qo_heads=args.num_heads,@@ -177,7 +168,8 @@ def parse_config(hf_config: Any) -> ModelConfig:hf_config, "architectures", ["Glm5NextForConditionalGeneration"]),moe_enabled=True,- expert_quant=detect_expert_quant(hf_config),+ expert_quant=expert_quant,+ weight_block_size=weight_block_size,first_k_dense_replace=first_dense,n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0),routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),@@ -186,9 +178,6 @@ def parse_config(hf_config: Any) -> ModelConfig:attn_sm_scale=args.qk_head_dim**-0.5,has_attn_bias=bool(getattr(text, "attention_bias", False)),swiglu_limit=args.swiglu_limit,- attn_quant="fp8_pertensor" if _ATTN_FP8 else "none",- dense_quant="fp8_pertensor" if _MLP_FP8 else "none",- lm_head_quant="fp8_pertensor" if _MLP_FP8 else "none",vision_config=None, # text-only milestone; model.visual.* weights are droppedimage_token_id=getattr(hf_config, "image_token_id", None),glm5_args=args,diff --git a/python/freetoken/models/glm5_next/kda.py b/python/freetoken/models/glm5_next/kda.pyindex 1d44e3a4..ba722ae2 100644--- a/python/freetoken/models/glm5_next/kda.py+++ b/python/freetoken/models/glm5_next/kda.py@@ -28,7 +28,7 @@ from typing import TYPE_CHECKINGimport torchfrom freetoken.core import get_global_ctxfrom freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen-from freetoken.layers import BaseOP, LinearColParallelMerged, LinearReplicated+from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicatedfrom freetoken.utils import nvtx_annotateif TYPE_CHECKING:@@ -43,29 +43,13 @@ class _DepthwiseConv1d(BaseOP):self.weight = torch.empty(conv_dim, 1, kernel)-class _GatedRMSNormSigmoid(BaseOP):- """RMSNorm(x) * sigmoid(z), fused (KDA's o_norm; GDN's variant gates with silu)."""-- def __init__(self, dim: int, eps: float):- self.weight = torch.empty(dim)- self.eps = eps-- def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:- from freetoken.kernel.fla import rms_norm_gated-- return rms_norm_gated(- x=x, weight=self.weight, bias=None, z=z, eps=self.eps,- is_rms_norm=True, norm_before_gate=True, activation="sigmoid",- )--class Glm5NextKDA(BaseOP):"""KDA op; state is held in ``ctx.linear_state_pool`` keyed by the request'slinear slot (``FLAMetadata.cache_indices``). Parameter names follow thecheckpoint modulo two load-time fusions (see weight.py): ``in_proj`` isq|k|v|b|f_a|g_a concatenated, ``conv1d`` is q|k|v conv concatenated."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.glm5_argsself.layer_id = layer_idself.num_heads = args.linear_num_heads@@ -77,41 +61,22 @@ class Glm5NextKDA(BaseOP):self.scale = self.head_dim**-0.5p, h, d = self.proj_size, self.num_heads, self.head_dim- # q|k|v dominate the resident read (3 * 8192 x 4096 = 201 MB/layer bf16 --- # the single biggest dense-weight stream in the model); under the fp8- # resident mode they split into their own W8A16 GEMM while the small,- # precision-sensitive gate projections (b|f_a|g_a) stay bf16 (the GDN- # qkvz/ba split precedent). BF16 mode keeps the single fused GEMM.- self._fp8 = config.attn_quant == "fp8_pertensor"- self._bfg_split = [h, d, d]- if self._fp8:- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged-- self.in_proj_qkv = Fp8PerTensorColMerged(- args.hidden_size, [p, p, p], has_bias=False- )- self.in_proj_bfg = LinearColParallelMerged(- args.hidden_size, self._bfg_split, has_bias=False- )- else:- self._in_proj_split = [p, p, p, h, d, d]- self.in_proj = LinearColParallelMerged(- args.hidden_size, self._in_proj_split, has_bias=False- )+ # one fused input GEMM over q|k|v|b|f_a|g_a+ self._in_proj_split = [p, p, p, h, d, d]+ self.in_proj = LinearColParallelMerged(+ args.hidden_size, self._in_proj_split, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.in_proj",+ )# Low-rank gate up-projections (128 -> 8192): forget gate and output gate.- self.f_b_proj = LinearReplicated(d, p, has_bias=False)- self.g_b_proj = LinearReplicated(d, p, has_bias=False)+ self.f_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.f_b_proj")+ self.g_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.g_b_proj")self.conv1d = _DepthwiseConv1d(self.conv_dim, self.conv_kernel_size)# Gate params stay fp32 (exp/sigmoid precision; the kernels read fp32).# models/weight.py exempts *.A_log / *.dt_bias from the model-dtype downcast.self.A_log = torch.empty(h, dtype=torch.float32)self.dt_bias = torch.empty(p, dtype=torch.float32)- self.o_norm = _GatedRMSNormSigmoid(d, eps=args.norm_eps)- # o_proj follows the resident quant mode (rationale: the split comment- # at _fp8 above); f_b/g_b stay bf16 alongside the gate slice.- from .attention import _make_proj-- self.o_proj = _make_proj(config.attn_quant, p, args.hidden_size)+ self.o_norm = GatedRMSNorm(d, eps=args.norm_eps, activation="sigmoid")+ self.o_proj = LinearReplicated(p, args.hidden_size, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.o_proj")def _conv_weight(self) -> torch.Tensor:return self.conv1d.weight.squeeze(1) # [conv_dim, kernel]@@ -143,16 +108,10 @@ class Glm5NextKDA(BaseOP):fla = build_fla_metadata(batch, hidden_states.device)batch.fla_metadata = fla- if self._fp8:- conv_in = self.in_proj_qkv.forward(hidden_states)- b, f_a, g_a = torch.split(- self.in_proj_bfg.forward(hidden_states), self._bfg_split, dim=-1- )- else:- proj = self.in_proj.forward(hidden_states)- conv_in, b, f_a, g_a = torch.split(- proj, [self.conv_dim, h, d, d], dim=-1- )+ proj = self.in_proj.forward(hidden_states)+ conv_in, b, f_a, g_a = torch.split(+ proj, [self.conv_dim, h, d, d], dim=-1+ )g1 = self.f_b_proj.forward(f_a) # raw forget-gate logits [T, H*D]g2 = self.g_b_proj.forward(g_a) # output-gate logits [T, H*D]li = pool.local_index(self.layer_id)diff --git a/python/freetoken/models/glm5_next/mlp.py b/python/freetoken/models/glm5_next/mlp.pyindex 1ee508bf..5ec3c66a 100644--- a/python/freetoken/models/glm5_next/mlp.py+++ b/python/freetoken/models/glm5_next/mlp.py@@ -1,7 +1,6 @@"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers and shared experts.-Same shape as glm_moe_dsa's GlmDsaGatedMLP (bf16 in the NVFP4 checkpoint;-optional W8A16 fp8-at-load via ``ModelConfig.dense_quant``), but the activation+Same shape as glm_moe_dsa's GlmDsaGatedMLP (every projection built from the QuantConfig), but the activationis the GLM-5.3 clamped SwiGLU (``swiglu_limit``):``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``."""@@ -13,7 +12,7 @@ import torchfrom freetoken.layers import BaseOP, swiglu_clamp_and_mulfrom freetoken.utils import nvtx_annotate-from .attention import _make_proj+from freetoken.layers import LinearReplicatedclass Glm5NextGatedMLP(BaseOP):@@ -21,12 +20,14 @@ class Glm5NextGatedMLP(BaseOP):self,hidden_size: int,intermediate_size: int,- quant: str = "none",swiglu_limit: float | None = None,+ *,+ quant_config=None,+ prefix: str = "",):- self.gate_proj = _make_proj(quant, hidden_size, intermediate_size)- self.up_proj = _make_proj(quant, hidden_size, intermediate_size)- self.down_proj = _make_proj(quant, intermediate_size, hidden_size)+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")self.swiglu_limit = swiglu_limit@nvtx_annotate("MLP")diff --git a/python/freetoken/models/glm5_next/model.py b/python/freetoken/models/glm5_next/model.pyindex 4c011dd6..ae287215 100644--- a/python/freetoken/models/glm5_next/model.py+++ b/python/freetoken/models/glm5_next/model.py@@ -41,28 +41,8 @@ if TYPE_CHECKING:from freetoken.models.config import ModelConfig-class Glm5Fp8LMHead(ParallelLMHead):- """W8A16 lm_head (fp8-e4m3 weight + per-row scale, quantized at load); the- full-vocab decode GEMV reads the whole head every step -- fp8 halves it.- Same contract as glm_moe_dsa's GlmFp8LMHead."""-- def __init__(self, num_embeddings: int, embedding_dim: int):- super().__init__(num_embeddings, embedding_dim, tie_word_embeddings=False)- self.weight = torch.empty(num_embeddings, embedding_dim, dtype=torch.float8_e4m3fn)- self.weight_scale = torch.empty(num_embeddings, dtype=torch.float32)-- def forward(self, x: torch.Tensor) -> torch.Tensor:- from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear-- batch = get_global_ctx().batch- if batch.is_prefill:- indices = batch.attn_metadata.get_last_indices(batch.size)- x = x[indices].contiguous()- return fp8_pertensor_linear(x, self.weight, self.weight_scale)--class Glm5NextDecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.glm5_argsself._layer_id = layer_idself._is_last = layer_id == config.num_layers - 1@@ -74,15 +54,15 @@ class Glm5NextDecoderLayer(BaseOP):self._sinkhorn = args.mhc_sinkhorn_iterationsif args.is_kda_layer(layer_id):- self.self_attn: BaseOP = Glm5NextKDA(config, layer_id)+ self.self_attn: BaseOP = Glm5NextKDA(config, layer_id, prefix=f"{prefix}.self_attn")else:- self.self_attn = Glm5NextAttention(config, layer_id)+ self.self_attn = Glm5NextAttention(config, layer_id, prefix=f"{prefix}.self_attn")if layer_id >= config.first_k_dense_replace:- self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id)+ self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")else:self.mlp = Glm5NextGatedMLP(- config.hidden_size, config.intermediate_size,- quant=config.dense_quant, swiglu_limit=config.swiglu_limit,+ config.hidden_size, config.intermediate_size, swiglu_limit=config.swiglu_limit,+ quant_config=config.quant, prefix=f"{prefix}.mlp",)self.input_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)self.post_attention_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)@@ -156,13 +136,13 @@ class Glm5NextDecoderLayer(BaseOP):class Glm5NextModel(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [Glm5NextDecoderLayer(config, i) for i in range(config.num_layers)]+ [Glm5NextDecoderLayer(config, i, prefix=f"{prefix}.layers.{i}") for i in range(config.num_layers)])self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps)@@ -178,17 +158,14 @@ class Glm5NextForCausalLM(BaseLLMModel):def __init__(self, config: ModelConfig):self._config = configself.model = Glm5NextModel(config)- if config.lm_head_quant == "fp8_pertensor" and not config.tie_word_embeddings:- self.lm_head: BaseOP = Glm5Fp8LMHead(- num_embeddings=config.vocab_size, embedding_dim=config.hidden_size- )- else:- self.lm_head = ParallelLMHead(- num_embeddings=config.vocab_size,- embedding_dim=config.hidden_size,- tie_word_embeddings=config.tie_word_embeddings,- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,- )+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",+ )def prepare_for_runtime(self) -> None:"""Post-load, pre-KV-sizing hook: materialize the DSA layers' bmm-readydiff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.pyindex 038ffc0d..1cf27ca6 100644--- a/python/freetoken/models/glm5_next/moe.py+++ b/python/freetoken/models/glm5_next/moe.py@@ -4,7 +4,7 @@ Routing is identical to glm_moe_dsa (sigmoid scores + selection-only``e_score_correction_bias``, optional group-limited top-k, gather unbiasedscores, renormalize, scale by ``routed_scaling_factor``); the deltas are theexpert count (288, top-8) and the clamped-SwiGLU activation (``swiglu_limit`` =-10), which rides ``make_moe_layer``'s ``extra_attrs`` into the offload kernels+10), carried as the experts layer's ``limit`` into the offload kernels(triton NVFP4 in-GPU, generic-epilogue CPU GEMV). The marlin/b12x borrowedkernels hard-code silu; the backend selector already falls back to triton fornon-silu experts.@@ -27,7 +27,7 @@ TopK = Tuple[torch.Tensor, torch.Tensor]class Glm5NextSparseBlock(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self.top_k = config.num_experts_per_tokself.num_experts = config.num_expertsself.norm_topk_prob = config.norm_topk_prob@@ -44,16 +44,16 @@ class Glm5NextSparseBlock(BaseOP):layer_id=layer_id - config.first_k_dense_replace,activation="swiglu_clamp" if config.swiglu_limit is not None else "silu",renormalize=config.norm_topk_prob,- extra_attrs={- "swiglu_limit": config.swiglu_limit,- "hidden_act_alpha": 1.0, # plain sigmoid inside the clamped swiglu- },+ limit=config.swiglu_limit,+ quant_config=config.quant,+ prefix=f"{prefix}.experts",)self.shared_experts = Glm5NextGatedMLP(config.hidden_size,config.moe_intermediate_size * max(1, config.n_shared_experts),- quant=config.dense_quant,swiglu_limit=config.swiglu_limit,+ quant_config=config.quant,+ prefix=f"{prefix}.shared_experts",)def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.pyindex fda85225..03e2a7a5 100644--- a/python/freetoken/models/glm5_next/weight.py+++ b/python/freetoken/models/glm5_next/weight.py@@ -1,13 +1,14 @@"""Weight loading for GLM-5.3-Flash (``glm5_next``).-Supported checkpoints: NVFP4 exports of GLM-5.3-Flash in the multimodal-wrapper-layout (``model.language_model.*``) -- ModelOpt tensor kinds (LibertAIDAI) or-compressed-tensors kinds (RedHatAI), selected by ``quantization_config``. Not-supported: bf16-expert originals (zai-org), text-only key layouts, TP > 1.+Supported checkpoints, all in the multimodal-wrapper layout (``model.language_model.*``):+NVFP4 exports (ModelOpt tensor kinds from LibertAIDAI, compressed-tensors kinds from+RedHatAI, selected by ``quantization_config``) and the zai-org block-fp8 release. Not+supported: text-only key layouts, TP > 1, resident routed experts.-Routed experts go to the offload cache via ``load_nvfp4_expert_sources``;-everything else loads bf16 with keys renamed ``model.language_model.X`` ->-``model.X``. ``model.visual.*`` and the trailing MTP layer are never read.+Routed experts go to the offload cache from their NVFP4 or block-fp8 pieces; every+other projection loads as stored (bf16, or fp8 codes with their block scales) with keys+renamed ``model.language_model.X`` -> ``model.X``. ``model.visual.*`` and the trailing+MTP layer are never read.Load-time fusions (must mirror the module split orders):@@ -15,9 +16,7 @@ Load-time fusions (must mirror the module split orders):* KDA ``conv1d`` = q|k|v depthwise conv weights concatenated on the channel axisfp32-kept tensors: ``A_log`` / ``dt_bias``, the mHC ``hc_*`` tensors, the indexer-APE, and the router ``e_score_correction_bias``. Optional W8A16 fp8-at-load-follows ``ModelConfig.attn_quant`` / ``dense_quant`` / ``lm_head_quant``-(defaults and env opt-ins: see config.py).+APE, and the router ``e_score_correction_bias``."""from __future__ import annotations@@ -29,11 +28,11 @@ from typing import Iteratorimport torchfrom freetoken.distributed import get_tp_info-from freetoken.models.glm_moe_dsa.weight import _ShardReader, _quant_fp8_per_row+from freetoken.layers.quantization import QuantKind+from freetoken.models.glm_moe_dsa.weight import _ShardReaderfrom freetoken.models.loader import drop_page_cachefrom freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,- load_nvfp4_expert_source_banks,)from freetoken.utils import cached_load_hf_config, download_hf_weightfrom tqdm import tqdm@@ -94,73 +93,54 @@ def _select_expert_source_spec(model_path: str) -> Nvfp4ExpertSourceSpec:# KDA in_proj fusion order; MUST match Glm5NextKDA._in_proj_split._KDA_IN_PROJ = ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")--def load_nvfp4_expert_sources(model_path: str, config, layer_sink=None):- return load_nvfp4_expert_source_banks(- model_path,- config,- _select_expert_source_spec(model_path),- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- layer_sink=layer_sink,- )+# zai-org fp8 release: fp8 codes + fp32 128x128 block scales per expert projection+_FP8_EXPERT_RE = re.compile(+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."+ r"(?P<proj>gate|up|down)_proj\.(?P<kind>weight|weight_scale_inv)$"+)-def _maybe_fp8(key: str, w: torch.Tensor, fp8: bool):- if fp8:- q, scale = _quant_fp8_per_row(w)- yield f"{key}.weight", q- yield f"{key}.weight_scale", scale+def _proj(reader, src: str, dst: str) -> Iterator[tuple[str, torch.Tensor]]:+ """One projection as the checkpoint stores it: bf16, or fp8 codes with their block scales."""+ w = reader.get(f"{src}.weight")+ if w.dtype == torch.float8_e4m3fn:+ yield f"{dst}.weight", w+ yield f"{dst}.weight_scale_inv", reader.get(f"{src}.weight_scale_inv")else:- yield f"{key}.weight", w.to(torch.bfloat16)+ yield f"{dst}.weight", w.to(torch.bfloat16)-def _iter_kda_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:+def nvfp4_expert_spec(model_path: str, config) -> Nvfp4ExpertSourceSpec:+ return _select_expert_source_spec(model_path)+++def _iter_kda_layer(reader, layer: int) -> Iterator[tuple[str, torch.Tensor]]:src = f"{_CKPT}.layers.{layer}.self_attn"dst = f"{_MODEL}.layers.{layer}.self_attn"- if attn_fp8:- # fp8 resident: q|k|v (the 201 MB/layer read) as one W8A16 GEMM with- # per-row scales; the small gate projections b|f_a|g_a stay bf16.- qkv = torch.cat(- [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("q_proj", "k_proj", "v_proj")],- dim=0,- )- q, scale = _quant_fp8_per_row(qkv)- yield f"{dst}.in_proj_qkv.weight", q- yield f"{dst}.in_proj_qkv.weight_scale", scale- bfg = torch.cat(- [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("b_proj", "f_a_proj", "g_a_proj")],- dim=0,- )- yield f"{dst}.in_proj_bfg.weight", bfg- else:- # One fused input GEMM: q|k|v|b|f_a|g_a (output-axis concat).- fused = torch.cat(- [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in _KDA_IN_PROJ], dim=0- )- yield f"{dst}.in_proj.weight", fused+ # One fused input GEMM: q|k|v|b|f_a|g_a (output-axis concat).+ parts = [reader.get(f"{src}.{p}.weight") for p in _KDA_IN_PROJ]+ if any(p.dtype == torch.float8_e4m3fn for p in parts):+ raise NotImplementedError("fp8 KDA input projections are not fused by this reader")+ yield f"{dst}.in_proj.weight", torch.cat([p.to(torch.bfloat16) for p in parts], dim=0)# One merged depthwise conv over the q|k|v stream (channel-axis concat).conv = torch.cat([reader.get(f"{src}.{p}_conv1d.weight").to(torch.bfloat16) for p in ("q", "k", "v")],dim=0,)yield f"{dst}.conv1d.weight", conv- for p in ("f_b_proj", "g_b_proj"):- yield f"{dst}.{p}.weight", reader.get(f"{src}.{p}.weight").to(torch.bfloat16)- yield from _maybe_fp8(f"{dst}.o_proj", reader.get(f"{src}.o_proj.weight"), attn_fp8)+ for p in ("f_b_proj", "g_b_proj", "o_proj"):+ yield from _proj(reader, f"{src}.{p}", f"{dst}.{p}")# Gate params stay fp32 (the recurrent kernels read them as fp32).yield f"{dst}.A_log", reader.get(f"{src}.A_log").to(torch.float32)yield f"{dst}.dt_bias", reader.get(f"{src}.dt_bias").to(torch.float32)yield f"{dst}.o_norm.weight", reader.get(f"{src}.o_norm.weight").to(torch.bfloat16)-def _iter_dsa_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:+def _iter_dsa_layer(reader, layer: int) -> Iterator[tuple[str, torch.Tensor]]:src = f"{_CKPT}.layers.{layer}.self_attn"dst = f"{_MODEL}.layers.{layer}.self_attn"- fp8_projs = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "o_proj") if attn_fp8 else ()for proj in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"):- w = reader.get(f"{src}.{proj}.weight")- yield from _maybe_fp8(f"{dst}.{proj}", w, proj in fp8_projs)+ yield from _proj(reader, f"{src}.{proj}", f"{dst}.{proj}")for norm in ("q_a_layernorm", "kv_a_layernorm"):yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to(torch.bfloat16)# kpool indexer (every DSA layer owns one). Kept bf16; the APE is fp32.@@ -185,8 +165,7 @@ def iter_weights(include_non_moe: bool,) -> Iterator[tuple[str, torch.Tensor]]:assert not include_moe_experts, (- "GLM-5.3 stores routed experts as NVFP4 and only supports the offload backend; "- "experts are loaded into the offload cache via load_nvfp4_expert_sources()."+ "GLM-5.3 routed experts only serve from the offload cache; they are loaded from their expert pieces.")assert include_non_moeif get_tp_info().size > 1:@@ -201,17 +180,6 @@ def iter_weights(weight_map = json.load(f)["weight_map"]reader = _ShardReader(folder, weight_map, device)primary = get_tp_info().is_primary()- attn_fp8 = config.attn_quant == "fp8_pertensor"- mlp_fp8 = config.dense_quant == "fp8_pertensor"- head_fp8 = config.lm_head_quant == "fp8_pertensor"- if primary:- from freetoken.utils import init_logger-- init_logger(__name__).info(- f"GLM-5.3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "- f"lm_head={config.lm_head_quant} (FREETOKEN_GLM5_ATTN_FP8/FREETOKEN_GLM5_MLP_FP8; "- "an FTW conversion records these choices implicitly -- serve with the same flags)"- )try:for layer in tqdm(range(config.num_layers),@@ -221,9 +189,9 @@ def iter_weights(src = f"{_CKPT}.layers.{layer}"dst = f"{_MODEL}.layers.{layer}"if args.is_kda_layer(layer):- yield from _iter_kda_layer(reader, layer, attn_fp8)+ yield from _iter_kda_layer(reader, layer)else:- yield from _iter_dsa_layer(reader, layer, attn_fp8)+ yield from _iter_dsa_layer(reader, layer)# mHC mixing tensors, fp32 on every layer.for hc in ("hc_attn_fn", "hc_attn_base", "hc_attn_scale",@@ -237,9 +205,7 @@ def iter_weights(if layer < config.first_k_dense_replace:for proj in ("gate_proj", "up_proj", "down_proj"):- yield from _maybe_fp8(- f"{dst}.mlp.{proj}", reader.get(f"{src}.mlp.{proj}.weight"), mlp_fp8- )+ yield from _proj(reader, f"{src}.mlp.{proj}", f"{dst}.mlp.{proj}")else:yield f"{dst}.mlp.gate.weight", reader.get(f"{src}.mlp.gate.weight").to(torch.bfloat16@@ -251,25 +217,63 @@ def iter_weights(reader.get(f"{src}.mlp.gate.e_score_correction_bias").to(torch.float32),)for proj in ("gate_proj", "up_proj", "down_proj"):- yield from _maybe_fp8(- f"{dst}.mlp.shared_experts.{proj}",- reader.get(f"{src}.mlp.shared_experts.{proj}.weight"),- mlp_fp8,- )+ yield from _proj(reader, f"{src}.mlp.shared_experts.{proj}", f"{dst}.mlp.shared_experts.{proj}")yield f"{_MODEL}.embed_tokens.weight", reader.get(f"{_CKPT}.embed_tokens.weight").to(torch.bfloat16)yield f"{_MODEL}.norm.weight", reader.get(f"{_CKPT}.norm.weight").to(torch.bfloat16)- head = reader.get("lm_head.weight")- if head_fp8 and not config.tie_word_embeddings:- q, scale = _quant_fp8_per_row(head)- yield "lm_head.weight", q- yield "lm_head.weight_scale", scale- else:- yield "lm_head.weight", head.to(torch.bfloat16)+ yield "lm_head.weight", reader.get("lm_head.weight").to(torch.bfloat16)finally:reader.close()-__all__ = ["iter_weights", "load_nvfp4_expert_sources"]+def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):+ """Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their ``_scale`` companions; other kinds use the generic readers."""+ if kind is not QuantKind.FP8_BLOCK:+ return None+ if get_tp_info().size > 1:+ raise NotImplementedError("glm5_next fp8 expert banks support TP=1 only")+ from freetoken.models.weight import experts_scattered, iter_expert_tensors_parallel+ from freetoken.moe.expert_pieces import per_expert_pieces++ suffix = {"weight": "", "weight_scale_inv": "_scale"}++ def locate(raw_name: str):+ m = _FP8_EXPERT_RE.match(raw_name)+ if m is None:+ return None+ bank = _layer_to_bank(int(m["layer"]), config)+ if bank is None:+ return None+ return bank, int(m["expert"]), m["proj"] + suffix[m["kind"]]++ if parallel is None:+ parallel = experts_scattered(model_path)+ if parallel:+ tensors = iter_expert_tensors_parallel(+ model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk+ )+ return per_expert_pieces(tensors, locate, tensors_per_expert=6)++ def _serial():+ folder = download_hf_weight(model_path)+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:+ weight_map = json.load(f)["weight_map"]+ reader = _ShardReader(folder, weight_map, torch.device("cpu"))+ try:+ layers = range(config.first_k_dense_replace, config.num_layers)+ for layer in tqdm(layers, desc="Loading GLM-5.3 fp8 experts (serial)", disable=not get_tp_info().is_primary()):+ for e in range(config.num_experts):+ base = f"{_CKPT}.layers.{layer}.mlp.experts.{e}"+ for proj in ("gate", "up", "down"):+ for kind_name in suffix:+ name = f"{base}.{proj}_proj.{kind_name}"+ yield name, reader.get(name)+ finally:+ reader.close()++ return per_expert_pieces(_serial(), locate, tensors_per_expert=6)+++__all__ = ["iter_weights", "iter_expert_pieces", "nvfp4_expert_spec"]diff --git a/python/freetoken/models/glm_moe_dsa/__init__.py b/python/freetoken/models/glm_moe_dsa/__init__.pyindex 1f2c189a..a0c05545 100644--- a/python/freetoken/models/glm_moe_dsa/__init__.py+++ b/python/freetoken/models/glm_moe_dsa/__init__.py@@ -1,15 +1,13 @@from .config import parse_configfrom .model import GlmMoeDsaForCausalLMfrom .weight import (+ nvfp4_expert_spec,iter_weights,- load_nvfp4_expert_sources,- load_nvfp4_expert_sources_parallel,)__all__ = [+ "nvfp4_expert_spec","GlmMoeDsaForCausalLM","parse_config","iter_weights",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel",]diff --git a/python/freetoken/models/glm_moe_dsa/attention.py b/python/freetoken/models/glm_moe_dsa/attention.pyindex e25d961f..4381de5b 100644--- a/python/freetoken/models/glm_moe_dsa/attention.py+++ b/python/freetoken/models/glm_moe_dsa/attention.py@@ -27,6 +27,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGimport torch+from freetoken.layers.quantization import QuantKindfrom freetoken.core import get_global_ctxfrom freetoken.layers import BaseOP, LinearReplicated, RMSNormfrom freetoken.layers.rotary import get_rope@@ -36,16 +37,6 @@ if TYPE_CHECKING:from freetoken.models.config import ModelConfig-def _make_proj(quant: str, in_features: int, out_features: int) -> BaseOP:- """A resident projection in the model's resolved quant mode: ``"fp8_pertensor"``- (W8A16, per-row scale, quantized at load -- see weight.py) or bf16."""- if quant == "fp8_pertensor":- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear-- return Fp8PerTensorLinear(in_features, out_features, has_bias=False)- return LinearReplicated(in_features, out_features, has_bias=False)--class _IdxLayerNorm(BaseOP):"""LayerNorm with bias (the indexer's ``k_norm``; RMSNorm everywhere else)."""@@ -63,17 +54,18 @@ class _IdxLayerNorm(BaseOP):class GlmDsaIndexer(BaseOP):"""DSA lightning indexer ("full" layers only; "shared" layers carry no weights).- Kept bf16 in every quant mode: the projections are small (~17 MB/layer) and the- top-k boundary is precision-sensitive. The HF reference itself is the bf16- equivalent of DeepSeek's fp8 scoring kernel, so bf16 scoring is faithful.+ ``wq_b`` follows the checkpoint's quant config; ``wk`` and ``weights_proj`` stay bf16 as in vLLM (the top-k boundary is precision-sensitive, and they are ~17 MB/layer)."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.glm_dsa_argsself.n_heads = args.index_n_headsself.head_dim = args.index_head_dimself.softmax_scale = args.index_head_dim**-0.5- self.wq_b = LinearReplicated(args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False)+ self.wq_b = LinearReplicated(+ args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.wq_b",+ )self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False)self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6)self.weights_proj = LinearReplicated(args.hidden_size, self.n_heads, has_bias=False)@@ -106,14 +98,14 @@ class GlmDsaIndexer(BaseOP):class GlmMoeDsaAttention(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.glm_dsa_argsself.layer_id = layer_id# IndexShare: "full" layers own an indexer; "shared" layers reuse the most# recent full layer's top-k selection (resolved in the backend).idx_types = args.indexer_typesself.indexer = (- GlmDsaIndexer(config, layer_id)+ GlmDsaIndexer(config, layer_id, prefix=f"{prefix}.indexer")if idx_types and idx_types[layer_id] == "full"else None)@@ -137,22 +129,32 @@ class GlmMoeDsaAttention(BaseOP):is_neox=not args.rope_interleave, # GLM-5.2: interleaved (config-driven))- quant = config.attn_quant- self.q_a_proj = _make_proj(quant, args.hidden_size, args.q_lora_rank)+ self.q_a_proj = LinearReplicated(+ args.hidden_size, args.q_lora_rank, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.q_a_proj",+ )self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps)- self.q_b_proj = _make_proj(quant, args.q_lora_rank, self.num_heads * self.qk_head_dim)- self.kv_a_proj_with_mqa = _make_proj(- quant, args.hidden_size, self.kv_lora_rank + self.qk_rope_head_dim+ self.q_b_proj = LinearReplicated(+ args.q_lora_rank, self.num_heads * self.qk_head_dim, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.q_b_proj",+ )+ self.kv_a_proj_with_mqa = LinearReplicated(+ args.hidden_size, self.kv_lora_rank + self.qk_rope_head_dim, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.kv_a_proj_with_mqa",)self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps)- # kv_b stays bf16 in every mode: it is consumed as bmm operands by the MLA- # absorption (below), not through a Linear forward.+ # the MLA absorption reads kv_b_proj.weight as bmm operands, so only an unquantized scheme is servedself.kv_b_proj = LinearReplicated(self.kv_lora_rank,self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.kv_b_proj",+ )+ assert self.kv_b_proj.quant_method.kind is QuantKind.NONE, f"{prefix}.kv_b_proj: a quantized kv_b_proj needs a dequantized copy for the MLA absorption"+ self.o_proj = LinearReplicated(+ self.num_heads * self.v_head_dim, args.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.o_proj",)- self.o_proj = _make_proj(quant, self.num_heads * self.v_head_dim, args.hidden_size)# Contiguous per-head kv_b split, cached on first forward (see _kv_b). Absorbing# kv_b into Q/O runs as a per-head bf16 bmm on these instead of re-slicing +# re-upcasting the bf16 weight per token (bf16 einsum with tensor-core fp32diff --git a/python/freetoken/models/glm_moe_dsa/config.py b/python/freetoken/models/glm_moe_dsa/config.pyindex c0ba25c5..615bdc7f 100644--- a/python/freetoken/models/glm_moe_dsa/config.py+++ b/python/freetoken/models/glm_moe_dsa/config.py@@ -9,11 +9,7 @@ kernel, identity selection for the dense regimes).MoE routing knobs mirror the glm4_moe path (sigmoid ``noaux_tc`` router, sharedexpert, routed scaling). The DSA indexer geometry rides in ``glm_dsa_args``.-Resident-weight quantization is resolved HERE (from the FREETOKEN_GLM_*_FP8 env-switches, default on) into the standard ``ModelConfig`` fields -- ``attn_quant`` /-``dense_quant`` / ``lm_head_quant`` = ``"fp8_pertensor"`` or ``"none"`` -- and every-consumer (the module constructors and the weight loader) reads those fields, so the-resolved config is the single record of what the served weights actually are.+Every resident projection is built from the QuantConfig, so the served precision is whatever the checkpoint's quantization_config declares."""from __future__ import annotations@@ -30,13 +26,6 @@ from freetoken.models.config import (from .args import load_args-# W8A16 fp8 (per-row scale, quantized at load) for the resident weights; decode is-# weight-bandwidth bound and the freed VRAM densifies the expert cache. =0 restores-# checkpoint-faithful bf16. NB: these resolve into ModelConfig at parse time and change-# what iter_weights yields -- an FTW checkpoint converted under one setting must be-# served under the same one (see weight.py).-_ATTN_FP8 = os.getenv("FREETOKEN_GLM_ATTN_FP8", "1") != "0"-_MLP_FP8 = os.getenv("FREETOKEN_GLM_MLP_FP8", "1") != "0"def _dsa_on(args, num_layers: int) -> bool:@@ -124,12 +113,6 @@ def parse_config(hf_config: Any) -> ModelConfig:topk_group=int(getattr(hf_config, "topk_group", 1)),attn_sm_scale=args.qk_head_dim**-0.5,has_attn_bias=bool(getattr(hf_config, "attention_bias", False)),- # Resident-weight quant modes (see the module docstring): recorded here so the- # resolved config describes the served weights; the constructors and- # iter_weights read these fields, never the env directly.- attn_quant="fp8_pertensor" if _ATTN_FP8 else "none",- dense_quant="fp8_pertensor" if _MLP_FP8 else "none",- lm_head_quant="fp8_pertensor" if _MLP_FP8 else "none",glm_dsa_args=args,)diff --git a/python/freetoken/models/glm_moe_dsa/mlp.py b/python/freetoken/models/glm_moe_dsa/mlp.pyindex be9430c5..32c50ea7 100644--- a/python/freetoken/models/glm_moe_dsa/mlp.py+++ b/python/freetoken/models/glm_moe_dsa/mlp.py@@ -1,11 +1,6 @@"""SwiGLU MLP for GLM-5.2's leading dense layers and per-layer shared experts.-The checkpoint ships these bf16 (NVIDIA's NVFP4 recipe leaves them unquantized; only-the routed experts are FP4). Serving default is W8A16 fp8 with per-row scales,-quantized at load (``ModelConfig.dense_quant == "fp8_pertensor"``, from-FREETOKEN_GLM_MLP_FP8): decode reads every shared expert each token, so this halves-~5.6 GiB/token of weight traffic and frees the same VRAM for expert-cache slots.-``FREETOKEN_GLM_MLP_FP8=0`` restores the checkpoint-faithful bf16 weights.+Every projection is built from the QuantConfig, so it serves whatever precision the checkpoint stores (bf16 under NVIDIA's NVFP4 recipe, which only quantizes the routed experts)."""from __future__ import annotations@@ -16,17 +11,17 @@ import torch.nn.functional as Ffrom freetoken.layers import BaseOPfrom freetoken.utils import nvtx_annotate-from .attention import _make_proj+from freetoken.layers import LinearReplicatedif TYPE_CHECKING:import torchclass GlmDsaGatedMLP(BaseOP):- def __init__(self, hidden_size: int, intermediate_size: int, quant: str = "none"):- self.gate_proj = _make_proj(quant, hidden_size, intermediate_size)- self.up_proj = _make_proj(quant, hidden_size, intermediate_size)- self.down_proj = _make_proj(quant, intermediate_size, hidden_size)+ def __init__(self, hidden_size: int, intermediate_size: int, *, quant_config=None, prefix: str = ""):+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")@nvtx_annotate("MLP")def forward(self, x: torch.Tensor) -> torch.Tensor:diff --git a/python/freetoken/models/glm_moe_dsa/model.py b/python/freetoken/models/glm_moe_dsa/model.pyindex 57644932..b6e21a37 100644--- a/python/freetoken/models/glm_moe_dsa/model.py+++ b/python/freetoken/models/glm_moe_dsa/model.py@@ -16,38 +16,15 @@ if TYPE_CHECKING:from freetoken.models.config import ModelConfig-class GlmFp8LMHead(ParallelLMHead):- """W8A16 lm_head (fp8-e4m3 weight + per-row scale, quantized at load).-- The full-vocab logits GEMV reads the whole ~1.9 GiB bf16 head every decode step;- fp8 halves that. Selected by ``ModelConfig.lm_head_quant == "fp8_pertensor"``- (weight.py quantizes at load off the same field). GLM-5.2 does not tie embeddings,- so the fp8 weight is head-only.- """-- def __init__(self, num_embeddings: int, embedding_dim: int):- super().__init__(num_embeddings, embedding_dim, tie_word_embeddings=False)- self.weight = torch.empty(num_embeddings, embedding_dim, dtype=torch.float8_e4m3fn)- self.weight_scale = torch.empty(num_embeddings, dtype=torch.float32)-- def forward(self, x: torch.Tensor) -> torch.Tensor:- from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear-- batch = get_global_ctx().batch- if batch.is_prefill:- indices = batch.attn_metadata.get_last_indices(batch.size)- x = x[indices].contiguous()- return fp8_pertensor_linear(x, self.weight, self.weight_scale)--class GlmMoeDsaDecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = GlmMoeDsaAttention(config, layer_id)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = GlmMoeDsaAttention(config, layer_id, prefix=f"{prefix}.self_attn")if layer_id >= config.first_k_dense_replace:- self.mlp: BaseOP = GlmMoeDsaSparseBlock(config, layer_id)+ self.mlp: BaseOP = GlmMoeDsaSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")else:self.mlp = GlmDsaGatedMLP(- config.hidden_size, config.intermediate_size, quant=config.dense_quant+ config.hidden_size, config.intermediate_size,+ quant_config=config.quant, prefix=f"{prefix}.mlp",)self.input_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)self.post_attention_layernorm = RMSNormFused(@@ -67,13 +44,16 @@ class GlmMoeDsaDecoderLayer(BaseOP):class GlmMoeDsaModel(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [GlmMoeDsaDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ GlmMoeDsaDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)@@ -88,18 +68,14 @@ class GlmMoeDsaModel(BaseOP):class GlmMoeDsaForCausalLM(BaseLLMModel):def __init__(self, config: ModelConfig):self.model = GlmMoeDsaModel(config)- if config.lm_head_quant == "fp8_pertensor" and not config.tie_word_embeddings:- self.lm_head: BaseOP = GlmFp8LMHead(- num_embeddings=config.vocab_size,- embedding_dim=config.hidden_size,- )- else:- self.lm_head = ParallelLMHead(- num_embeddings=config.vocab_size,- embedding_dim=config.hidden_size,- tie_word_embeddings=config.tie_word_embeddings,- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,- )+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",+ )super().__init__()def prepare_for_runtime(self) -> None:diff --git a/python/freetoken/models/glm_moe_dsa/moe.py b/python/freetoken/models/glm_moe_dsa/moe.pyindex cec4f0e3..0e17c9be 100644--- a/python/freetoken/models/glm_moe_dsa/moe.py+++ b/python/freetoken/models/glm_moe_dsa/moe.py@@ -25,7 +25,7 @@ TopK = Tuple[torch.Tensor, torch.Tensor]class GlmMoeDsaSparseBlock(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self.top_k = config.num_experts_per_tokself.num_experts = config.num_expertsself.norm_topk_prob = config.norm_topk_prob@@ -44,11 +44,14 @@ class GlmMoeDsaSparseBlock(BaseOP):config,layer_id=layer_id - config.first_k_dense_replace,renormalize=config.norm_topk_prob,+ quant_config=config.quant,+ prefix=f"{prefix}.experts",)self.shared_experts = GlmDsaGatedMLP(config.hidden_size,config.moe_intermediate_size * max(1, config.n_shared_experts),- quant=config.dense_quant,+ quant_config=config.quant,+ prefix=f"{prefix}.shared_experts",)def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:diff --git a/python/freetoken/models/glm_moe_dsa/weight.py b/python/freetoken/models/glm_moe_dsa/weight.pyindex df22053f..900d71af 100644--- a/python/freetoken/models/glm_moe_dsa/weight.py+++ b/python/freetoken/models/glm_moe_dsa/weight.py@@ -1,21 +1,11 @@"""Weight loading for GLM-5.2 (``glm_moe_dsa``).-Resident (non routed-expert) weights are bf16 in the checkpoint. What this loader-yields follows the quant modes RESOLVED IN ``parse_config`` (``ModelConfig.attn_quant``-/ ``dense_quant`` / ``lm_head_quant``, from the FREETOKEN_GLM_*_FP8 switches, default-on): in the default fp8 mode the big projections are requantized at load to W8A16-fp8-e4m3 with per-output-row scales (an extra ``*.weight_scale`` tensor per-projection); with the switches off everything streams through verbatim as bf16. The+Resident (non routed-expert) weights stream through verbatim in the checkpoint's precision (bf16). Therouter selection bias is remapped ``mlp.gate.e_score_correction_bias ->mlp.e_score_correction_bias``; the DSA indexer tensors load bf16 on "full" indexerlayers (serving runs faithful DSA top-k sparse attention; see attention.py); only thetrailing MTP layer is skipped. Routed experts are NVFP4and go to the offload cache via the shared glm4_moe loader (identical key layout).--FTW caveat: an FTW checkpoint stores whatever iter_weights yielded at CONVERSION time,-and the model is built from the env at SERVE time -- the two must agree (a mismatch-fails loudly in load_state_dict on the ``*.weight_scale`` keys). The active modes are-logged at load so conversion logs record the choice."""from __future__ import annotations@@ -28,8 +18,7 @@ import safetensorsimport torchfrom freetoken.distributed import get_tp_infofrom freetoken.models.glm4_moe.weight import (- load_nvfp4_expert_sources,- load_nvfp4_expert_sources_parallel,+ nvfp4_expert_spec,)from freetoken.models.loader import drop_page_cachefrom freetoken.utils import cached_load_hf_config, download_hf_weight@@ -38,17 +27,6 @@ from tqdm import tqdmfrom .config import parse_config# fp8-e4m3 dynamic range for the per-row W8A16 quantization of the big MLA projections.-_FP8_MAX = 448.0---def _quant_fp8_per_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:- """Per-output-row fp8-e4m3 quantization: ``w ~= weight_fp8 * scale[:, None]``."""- wf = w.float()- scale = (wf.abs().amax(dim=1) / _FP8_MAX).clamp(min=1e-12)- q = (wf / scale[:, None]).clamp(-_FP8_MAX, _FP8_MAX).to(torch.float8_e4m3fn)- return q, scale.to(torch.float32)--class _ShardReader:def __init__(self, folder: str, weight_map: dict, device: torch.device):self._folder = folder@@ -88,7 +66,7 @@ def iter_weights() -> Iterator[tuple[str, torch.Tensor]]:assert not include_moe_experts, ("GLM-5.2 stores routed experts as NVFP4 and only supports the offload backend; "- "experts are loaded into the offload cache via load_nvfp4_expert_sources()."+ "experts are loaded into the offload cache from their NVFP4 pieces.")assert include_non_moeconfig = parse_config(cached_load_hf_config(model_path))@@ -98,17 +76,6 @@ def iter_weights(reader = _ShardReader(folder, weight_map, device)primary = get_tp_info().is_primary()dense = config.first_k_dense_replace- attn_fp8 = config.attn_quant == "fp8_pertensor"- mlp_fp8 = config.dense_quant == "fp8_pertensor"- head_fp8 = config.lm_head_quant == "fp8_pertensor"- if primary:- from freetoken.utils import init_logger-- init_logger(__name__).info(- f"GLM-5.2 resident quant: attn={config.attn_quant} dense={config.dense_quant} "- f"lm_head={config.lm_head_quant} (FREETOKEN_GLM_ATTN_FP8/FREETOKEN_GLM_MLP_FP8; "- "an FTW conversion records these choices implicitly -- serve with the same flags)"- )try:for layer in tqdm(range(config.num_layers),@@ -116,17 +83,8 @@ def iter_weights(disable=not primary,):a = f"model.layers.{layer}.self_attn"- fp8_projs = (- ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "o_proj") if attn_fp8 else ()- )for proj in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"):- w = reader.get(f"{a}.{proj}.weight")- if proj in fp8_projs:- q, scale = _quant_fp8_per_row(w)- yield f"{a}.{proj}.weight", q- yield f"{a}.{proj}.weight_scale", scale- else:- yield f"{a}.{proj}.weight", w+ yield f"{a}.{proj}.weight", reader.get(f"{a}.{proj}.weight")for norm in ("q_a_layernorm", "kv_a_layernorm"):yield f"{a}.{norm}.weight", reader.get(f"{a}.{norm}.weight")# DSA lightning indexer ("full" layers only; "shared" layers reuse their@@ -145,18 +103,9 @@ def iter_weights(m = f"model.layers.{layer}.mlp"- def _mlp_weight(key: str):- w = reader.get(f"{key}.weight")- if mlp_fp8:- q, scale = _quant_fp8_per_row(w)- yield f"{key}.weight", q- yield f"{key}.weight_scale", scale- else:- yield f"{key}.weight", w-if layer < dense:for proj in ("gate_proj", "up_proj", "down_proj"):- yield from _mlp_weight(f"{m}.{proj}")+ yield f"{m}.{proj}.weight", reader.get(f"{m}.{proj}.weight")else:yield f"{m}.gate.weight", reader.get(f"{m}.gate.weight")yield (@@ -164,19 +113,13 @@ def iter_weights(reader.get(f"{m}.gate.e_score_correction_bias").to(torch.bfloat16),)for proj in ("gate_proj", "up_proj", "down_proj"):- yield from _mlp_weight(f"{m}.shared_experts.{proj}")+ yield f"{m}.shared_experts.{proj}.weight", reader.get(f"{m}.shared_experts.{proj}.weight")yield "model.embed_tokens.weight", reader.get("model.embed_tokens.weight")yield "model.norm.weight", reader.get("model.norm.weight")- head = reader.get("lm_head.weight")- if head_fp8 and not config.tie_word_embeddings:- q, scale = _quant_fp8_per_row(head)- yield "lm_head.weight", q- yield "lm_head.weight_scale", scale- else:- yield "lm_head.weight", head+ yield "lm_head.weight", reader.get("lm_head.weight")finally:reader.close()-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]+__all__ = ["iter_weights", "nvfp4_expert_spec"]diff --git a/python/freetoken/models/gpt_oss/__init__.py b/python/freetoken/models/gpt_oss/__init__.pyindex 92637487..45ef7d37 100644--- a/python/freetoken/models/gpt_oss/__init__.py+++ b/python/freetoken/models/gpt_oss/__init__.py@@ -1,5 +1,5 @@from .config import parse_configfrom .model import GptOssForCausalLM-from .weight import iter_weights, setup_offload_expert_banks+from .weight import iter_expert_pieces, iter_weights-__all__ = ["GptOssForCausalLM", "parse_config", "iter_weights", "setup_offload_expert_banks"]+__all__ = ["GptOssForCausalLM", "parse_config", "iter_weights", "iter_expert_pieces"]diff --git a/python/freetoken/models/gpt_oss/attention.py b/python/freetoken/models/gpt_oss/attention.pyindex 8a67a465..259e54d9 100644--- a/python/freetoken/models/gpt_oss/attention.py+++ b/python/freetoken/models/gpt_oss/attention.py@@ -17,7 +17,7 @@ if TYPE_CHECKING:class GptOssAttention(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):import torchself.layer_id = layer_id@@ -42,12 +42,16 @@ class GptOssAttention(BaseOP):num_qo_heads=config.num_qo_heads,num_kv_heads=config.num_kv_heads,has_bias=config.has_attn_bias,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",)self.sinks = torch.empty(self.num_qo_heads)self.o_proj = LinearOProj(input_size=config.num_qo_heads * self.head_dim,output_size=config.hidden_size,has_bias=config.has_attn_bias,+ quant_config=config.quant,+ prefix=f"{prefix}.o_proj",)rotary_config = group.rotary_configdiff --git a/python/freetoken/models/gpt_oss/model.py b/python/freetoken/models/gpt_oss/model.pyindex c0caa3a8..8398fb8b 100644--- a/python/freetoken/models/gpt_oss/model.py+++ b/python/freetoken/models/gpt_oss/model.py@@ -22,9 +22,9 @@ if TYPE_CHECKING:class GptOssDecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = GptOssAttention(config, layer_id)- self.mlp = GptOssMLP(config, layer_id)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = GptOssAttention(config, layer_id, prefix=f"{prefix}.self_attn")+ self.mlp = GptOssMLP(config, layer_id, prefix=f"{prefix}.mlp")self.input_layernorm = RMSNormFused(size=config.hidden_size,eps=config.rms_norm_eps,@@ -47,13 +47,16 @@ class GptOssDecoderLayer(BaseOP):class GptOssModel(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [GptOssDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ GptOssDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size,@@ -80,6 +83,8 @@ class GptOssForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)self.config = configsuper().__init__()diff --git a/python/freetoken/models/gpt_oss/moe.py b/python/freetoken/models/gpt_oss/moe.pyindex 8e4bd0c6..3b58946b 100644--- a/python/freetoken/models/gpt_oss/moe.py+++ b/python/freetoken/models/gpt_oss/moe.py@@ -3,197 +3,39 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGimport torch-import freetoken.layers.moe as moe_layers-from freetoken.layers import BaseOP, LinearReplicated, MoELayer, OffloadMoELayer-from freetoken.moe import is_offload_moe_backend-from freetoken.moe.fused_mxfp4 import (- MXFP4_DECODE_MAX_TOKENS,- _transpose_mxfp4_for_decode,- run_mxfp4_prefill_experts_t,- run_mxfp4_splitk_decode_experts,-)+from freetoken.layers import BaseOP, LinearReplicated, MoELayer, OffloadMoELayer, make_moe_layerfrom freetoken.utils import nvtx_annotate-from .weight import local_mxfp4_intermediate_range-if TYPE_CHECKING:from freetoken.models.config import ModelConfig-class GptOssMxfp4TritonMoELayer(MoELayer):- def __init__(self, config: ModelConfig):- tp_info = moe_layers.get_tp_info()- _, _, local_intermediate = local_mxfp4_intermediate_range(- config.moe_intermediate_size,- rank=tp_info.rank,- world_size=tp_info.size,- )- if config.hidden_size % 32 != 0:- raise ValueError("GPT-OSS MXFP4 hidden size must be divisible by 32")-- super().__init__(- num_experts=config.num_experts,- top_k=config.num_experts_per_tok,- hidden_size=config.hidden_size,- intermediate_size=config.moe_intermediate_size,- renormalize=config.norm_topk_prob,- activation="gpt_oss_swiglu",- allocate_experts=False,- # Descriptive only: this subclass owns its allocation and forward, but the- # field must not claim bf16 for mxfp4 tensors.- weight_format="mxfp4_triton",- )- self.local_intermediate_size = local_intermediate- self.hidden_act_alpha = config.hidden_act_alpha- self.swiglu_limit = config.swiglu_limit+def _route(router_logits: torch.Tensor, top_k: int) -> tuple[torch.Tensor, torch.Tensor]:+ from freetoken.kernel import gpt_oss_fused_routing- hidden_blocks = config.hidden_size // 32- intermediate_blocks = local_intermediate // 32- self.gate_up_proj_blocks = torch.empty(- config.num_experts,- 2 * local_intermediate,- hidden_blocks,- 16,- dtype=torch.uint8,- )- self.gate_up_proj_scales = torch.empty(- config.num_experts,- 2 * local_intermediate,- hidden_blocks,- dtype=torch.uint8,- )- self.gate_up_proj_bias = torch.empty(- config.num_experts,- 2 * local_intermediate,- dtype=torch.bfloat16,- )- self.down_proj_blocks = torch.empty(- config.num_experts,- config.hidden_size,- intermediate_blocks,- 16,- dtype=torch.uint8,- )- self.down_proj_scales = torch.empty(- config.num_experts,- config.hidden_size,- intermediate_blocks,- dtype=torch.uint8,- )- self.down_proj_bias = torch.empty(- config.num_experts,- config.hidden_size,- dtype=torch.bfloat16,- )- # Transposed split-K decode weights, built lazily in _ensure_decode_weights.- self._gu_blocks_t = None- self._gu_scales_t = None- self._dn_blocks_t = None- self._dn_scales_t = None-- def _topk(self, router_logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:- from freetoken.kernel import gpt_oss_fused_routing-- return gpt_oss_fused_routing(router_logits, self.top_k)-- def _ensure_decode_weights(self) -> None:- """Build the transposed split-K/prefill weights once and free the HF blocks/- scales (single transposed layout ~61GB so 120B fits). Both prefill and decode- read the transposed weights. Must run before CUDA graph capture."""- if self._gu_blocks_t is not None:- return- self._gu_blocks_t, self._gu_scales_t = _transpose_mxfp4_for_decode(- self.gate_up_proj_blocks, self.gate_up_proj_scales- )- self._dn_blocks_t, self._dn_scales_t = _transpose_mxfp4_for_decode(- self.down_proj_blocks, self.down_proj_scales- )- self.gate_up_proj_blocks = None- self.gate_up_proj_scales = None- self.down_proj_blocks = None- self.down_proj_scales = None- torch.cuda.empty_cache()+ return gpt_oss_fused_routing(router_logits.contiguous(), top_k)- def prepare_for_runtime(self) -> None:- self._ensure_decode_weights()-- def forward(- self,- hidden_states: torch.Tensor,- router_logits: torch.Tensor,- ) -> torch.Tensor:- if not hidden_states.is_cuda:- raise RuntimeError("GPT-OSS MXFP4 MoE requires the Triton CUDA kernel")- if not hidden_states.is_contiguous():- hidden_states = hidden_states.contiguous()- if not router_logits.is_contiguous():- router_logits = router_logits.contiguous()-- topk_weights, topk_ids = self._topk(router_logits)- self._ensure_decode_weights()- if hidden_states.shape[0] <= MXFP4_DECODE_MAX_TOKENS:- output = run_mxfp4_splitk_decode_experts(- hidden_states, topk_weights, topk_ids,- self._gu_blocks_t, self._gu_scales_t, self.gate_up_proj_bias,- self._dn_blocks_t, self._dn_scales_t, self.down_proj_bias,- top_k=self.top_k,- hidden_act_alpha=self.hidden_act_alpha,- swiglu_limit=self.swiglu_limit,- )- else:- output = run_mxfp4_prefill_experts_t(- hidden_states, topk_weights, topk_ids,- self._gu_blocks_t, self._gu_scales_t, self.gate_up_proj_bias,- self._dn_blocks_t, self._dn_scales_t, self.down_proj_bias,- top_k=self.top_k,- hidden_act_alpha=self.hidden_act_alpha,- swiglu_limit=self.swiglu_limit,- )- return self._maybe_all_reduce(output)+class GptOssMoELayer(MoELayer):+ """Resident experts routed by the fused softmax-over-top-k kernel."""-class GptOssMxfp4OffloadMoELayer(OffloadMoELayer):- def __init__(self, config: ModelConfig, layer_id: int):- tp_info = moe_layers.get_tp_info()- _, _, local_intermediate = local_mxfp4_intermediate_range(- config.moe_intermediate_size,- rank=tp_info.rank,- world_size=tp_info.size,- )- super().__init__(- layer_id=layer_id,- num_experts=config.num_experts,- top_k=config.num_experts_per_tok,- hidden_size=config.hidden_size,- intermediate_size=config.moe_intermediate_size,- renormalize=config.norm_topk_prob,- activation="gpt_oss_swiglu",- )- self.local_intermediate_size = local_intermediate- self.hidden_act_alpha = config.hidden_act_alpha- self.swiglu_limit = config.swiglu_limit+ def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor | None = None):+ assert router_logits is not None+ topk_weights, topk_ids = _route(router_logits, self.top_k)+ return self.routed_forward(hidden_states.contiguous(), topk_weights, topk_ids)- def _topk(self, router_logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:- from freetoken.kernel import gpt_oss_fused_routing- return gpt_oss_fused_routing(router_logits, self.top_k)+class GptOssOffloadMoELayer(OffloadMoELayer):+ """Offloaded experts routed by the fused softmax-over-top-k kernel."""- def forward(- self,- hidden_states: torch.Tensor,- router_logits: torch.Tensor | None = None,- ) -> torch.Tensor:+ def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor | None = None):assert router_logits is not None- if not router_logits.is_contiguous():- router_logits = router_logits.contiguous()- topk_weights, topk_ids = self._topk(router_logits)- # routed_forward dispatches prefill/decode movement and all-reduces once;- # topk_ids is cloned because decode rewrites it in place into slot ids.- return self.routed_forward(hidden_states, topk_weights, topk_ids.clone())+ topk_weights, topk_ids = _route(router_logits, self.top_k)+ return self.routed_forward(hidden_states, topk_weights, topk_ids)class GptOssMLP(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int | None = None):+ def __init__(self, config: ModelConfig, layer_id: int | None = None, *, prefix: str = ""):self.router = LinearReplicated(config.hidden_size,config.num_experts,@@ -204,11 +46,19 @@ class GptOssMLP(BaseOP):f"gpt-oss supports only mxfp4 expert weights, got "f"moe_weight_format={config.moe_weight_format!r}")- if is_offload_moe_backend(config.moe_backend):- assert layer_id is not None- self.experts = GptOssMxfp4OffloadMoELayer(config, layer_id)- else:- self.experts = GptOssMxfp4TritonMoELayer(config)+ self.experts = make_moe_layer(+ config,+ layer_id=layer_id,+ activation="gpt_oss_swiglu",+ alpha=config.hidden_act_alpha,+ limit=config.swiglu_limit,+ interleaved=True,+ has_bias=True,+ resident_cls=GptOssMoELayer,+ offload_cls=GptOssOffloadMoELayer,+ quant_config=config.quant,+ prefix=f"{prefix}.experts",+ )self._layer_id = layer_id@nvtx_annotate("MoE")@@ -226,6 +76,6 @@ class GptOssMLP(BaseOP):__all__ = ["GptOssMLP",- "GptOssMxfp4OffloadMoELayer",- "GptOssMxfp4TritonMoELayer",+ "GptOssMoELayer",+ "GptOssOffloadMoELayer",]diff --git a/python/freetoken/models/gpt_oss/weight.py b/python/freetoken/models/gpt_oss/weight.pyindex 38f16d8a..686f0246 100644--- a/python/freetoken/models/gpt_oss/weight.py+++ b/python/freetoken/models/gpt_oss/weight.py@@ -4,6 +4,7 @@ from typing import Iteratorimport safetensorsimport torch+from freetoken.layers.quantization import QuantKindfrom freetoken.distributed import get_tp_infofrom freetoken.models.loader import (MergeRule,@@ -253,71 +254,6 @@ def _expert_layer_and_name(key: str) -> tuple[int, str] | None:return layer_id, parts[5]-def _empty_mxfp4_triton_banks(- config,- *,- dtype: torch.dtype,- tp_info=None,-):- """returns ``(banks, host_banks)``, unpinned. ``banks[name]`` is one ``[E, ...]``- tensor per layer (independent allocations). host banks are lazy anon mmaps (already- zero, so no ``zero_`` pass); caller fills then pins (pin-after-fill skips cudaHostAlloc's- slow commit) -- per-layer via ``PinPipeline`` for the real streaming loaders, or- ``pin_banks`` for the dummy path. unwritten padding stays zero."""- if tp_info is None:- tp_info = get_tp_info()- _, _, local_intermediate = local_mxfp4_intermediate_range(- config.moe_intermediate_size,- rank=tp_info.rank,- world_size=tp_info.size,- )- num_layers, E = config.num_layers, config.num_experts- hidden_blocks = config.hidden_size // 32- intermediate_blocks = local_intermediate // 32- specs = {- "gate_up_blocks": ((E, config.hidden_size // 2, 2 * local_intermediate), torch.uint8),- "gate_up_scales": ((E, hidden_blocks, 2 * local_intermediate), torch.uint8),- "gate_up_bias": ((E, 2 * local_intermediate), dtype),- "down_blocks": ((E, local_intermediate // 2, config.hidden_size), torch.uint8),- "down_scales": ((E, intermediate_blocks, config.hidden_size), torch.uint8),- "down_bias": ((E, config.hidden_size), dtype),- }- if torch.cuda.is_available():- from freetoken.moe.host_banks import alloc_layer_banks-- hb = alloc_layer_banks(specs, num_layers) # lazy anon mmap -> zero-initialized, unpinned- return {name: [b.tensor for b in hb[name]] for name in specs}, hb- return {- name: [torch.zeros(shape, dtype=dt) for _ in range(num_layers)]- for name, (shape, dt) in specs.items()- }, {}---def _make_dummy_mxfp4_triton_banks(- config,- *,- dtype: torch.dtype,- tp_info=None,-) -> dict[str, list[torch.Tensor]]:- from freetoken.moe.host_banks import pin_banks-- banks, _hb = _empty_mxfp4_triton_banks(config, dtype=dtype, tp_info=tp_info)- for t in banks["gate_up_blocks"]:- t.random_(0, 256)- for t in banks["gate_up_scales"]:- t.fill_(127)- for t in banks["gate_up_bias"]:- t.normal_()- for t in banks["down_blocks"]:- t.random_(0, 256)- for t in banks["down_scales"]:- t.fill_(127)- for t in banks["down_bias"]:- t.normal_()- pin_banks(_hb) # pin-after-fill- return banks--def _read_safetensor_slice(f,name: str,@@ -328,286 +264,87 @@ def _read_safetensor_slice(return f.get_slice(name)[slices]-def _copy_transposed_mxfp4_blocks(dst: torch.Tensor, raw: torch.Tensor) -> None:- # HF [E, N, K//32, 16] -> split-K [E, K//2, N].- if raw.numel() == 0:- return- source = raw.reshape(raw.shape[0], raw.shape[1], raw.shape[2] * raw.shape[3]).permute(- 0,- 2,- 1,- )- dst[:, : source.shape[1], : source.shape[2]].copy_(source)---def _copy_transposed_mxfp4_scales(dst: torch.Tensor, raw: torch.Tensor) -> None:- # HF [E, N, K//32] -> split-K [E, K//32, N].- if raw.numel() == 0:- return- source = raw.permute(0, 2, 1)- dst[:, : source.shape[1], : source.shape[2]].copy_(source)+_EXPERT_SOURCES = (+ "gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias",+ "down_proj_blocks", "down_proj_scales", "down_proj_bias",+)-def _copy_prefix(dst: torch.Tensor, raw: torch.Tensor) -> None:- dst[:, : raw.shape[1]].copy_(raw)+def _source_slices(config, tp_info) -> dict[str, tuple[str, tuple[slice, ...]]]:+ """HF expert tensor -> (piece role, this rank's slice of the intermediate axis)."""+ tp_start, tp_end, _ = local_mxfp4_intermediate_range(+ config.moe_intermediate_size, rank=tp_info.rank, world_size=tp_info.size+ )+ tp_end = max(tp_start, tp_end)+ rows = slice(2 * tp_start, 2 * tp_end)+ cols = slice(tp_start // 32, tp_end // 32)+ return {+ "gate_up_proj_blocks": ("gate_up", (slice(None), rows, slice(None), slice(None))),+ "gate_up_proj_scales": ("gate_up_scale", (slice(None), rows, slice(None))),+ "gate_up_proj_bias": ("gate_up_bias", (slice(None), rows)),+ "down_proj_blocks": ("down", (slice(None), slice(None), cols, slice(None))),+ "down_proj_scales": ("down_scale", (slice(None), slice(None), cols)),+ "down_proj_bias": ("down_bias", (slice(None), slice(None))),+ }-def load_mxfp4_triton_banks_streaming(- model_path: str,- model_config=None,- *,- dtype: torch.dtype,- tp_info=None,- layer_sink=None,-) -> dict[str, list[torch.Tensor]]:- """``layer_sink=None`` (serving): pin each layer as its writes complete, via an- internally-owned :class:`PinPipeline` (or no pinning at all in the CPU-only fallback,- where ``_hb`` is empty). ``layer_sink`` given (converter): the completion tracker- fires into it instead -- nothing here is pinned, and the sink may release banks it- has written out, so the returned tensors are only valid until then (the caller owns- that tradeoff)."""- config = model_config if model_config is not None else parse_config(cached_load_hf_config(model_path))+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20):+ """gpt-oss experts as HF ships them, one piece per layer: stacked ``[E, ...]`` mxfp4+ ``_blocks`` / ``_scales`` / ``_bias`` tensors sliced to this rank's intermediate range."""+ if kind is not QuantKind.MXFP4:+ return Noneif config.moe_weight_format != "mxfp4":raise ValueError("GPT-OSS offload requires MXFP4 expert weights")+ tp_info = get_tp_info()+ slices = _source_slices(config, tp_info)+ num_layers, num_experts = config.num_layers, config.num_experts- if tp_info is None:- tp_info = get_tp_info()- tp_start, tp_end, local_intermediate = local_mxfp4_intermediate_range(- config.moe_intermediate_size,- rank=tp_info.rank,- world_size=tp_info.size,- )- tp_end = max(tp_start, tp_end)- banks, _hb = _empty_mxfp4_triton_banks(config, dtype=dtype, tp_info=tp_info)- seen: set[tuple[int, str]] = set()- all_expert_sources = (- "gate_up_proj_blocks",- "gate_up_proj_scales",- "gate_up_proj_bias",- "down_proj_blocks",- "down_proj_scales",- "down_proj_bias",- )- copy_plan = {- "gate_up_proj_blocks": (- "gate_up_blocks",- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None), slice(None)),- _copy_transposed_mxfp4_blocks,- ),- "gate_up_proj_scales": (- "gate_up_scales",- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None)),- _copy_transposed_mxfp4_scales,- ),- "gate_up_proj_bias": (- "gate_up_bias",- (slice(None), slice(2 * tp_start, 2 * tp_end)),- _copy_prefix,- ),- "down_proj_blocks": (- "down_blocks",- (- slice(None),- slice(None),- slice(tp_start // 32, tp_end // 32),- slice(None),- ),- _copy_transposed_mxfp4_blocks,- ),- "down_proj_scales": (- "down_scales",- (slice(None), slice(None), slice(tp_start // 32, tp_end // 32)),- _copy_transposed_mxfp4_scales,- ),- }+ def _is_expert(name: str) -> bool:+ info = _expert_layer_and_name(name)+ return info is not None and 0 <= info[0] < num_layers- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline+ def _tensors():+ if parallel:+ from freetoken.models.weight import iter_expert_tensors_parallel- def _load(sink) -> None:- # _hb is empty in the CPU-only fallback (no banks to pin/stream): no-op tracker.- tracker = LayerCompletionTracker(len(all_expert_sources), _hb, sink) if _hb else None+ for name, whole in iter_expert_tensors_parallel(model_path, _is_expert, workers=workers, chunk=chunk):+ _, source = _expert_layer_and_name(name)+ yield name, whole[slices[source][1]]+ returnfor file in iter_root_safetensor_files_from_index(model_path):with safetensors.safe_open(file, framework="pt", device="cpu") as f:for name in f.keys():info = _expert_layer_and_name(name)if info is None:continue- layer_id, source_name = info- if layer_id < 0 or layer_id >= config.num_layers:+ layer_id, source = info+ if not 0 <= layer_id < num_layers:raise ValueError(f"Unexpected GPT-OSS expert layer in checkpoint: {name}")- seen.add((layer_id, source_name))-- if source_name == "down_proj_bias":- if tp_info.rank == 0:- raw = f.get_tensor(name)- banks["down_bias"][layer_id].copy_(raw)- if tracker is not None:- tracker.note(layer_id)- continue-- plan = copy_plan.get(source_name)- if plan is None:+ if source not in slices:raise ValueError(f"Unexpected GPT-OSS expert source: {name}")- bank_name, source_slice, copy_fn = plan- raw = _read_safetensor_slice(f, name, source_slice)- copy_fn(banks[bank_name][layer_id], raw)- if tracker is not None:- tracker.note(layer_id)-- if layer_sink is not None:- _load(layer_sink)- elif _hb:- with PinPipeline() as pins:- _load(pins)- else:- _load(None)-- expected = {- (layer_id, source_name)- for layer_id in range(config.num_layers)- for source_name in all_expert_sources- }- missing = expected - seen- if missing:- raise ValueError(f"Missing GPT-OSS expert tensors: {sorted(missing)[:8]}")- assert all(- len(per_layer) == config.num_layers- and all(t.is_contiguous() and t.size(0) == config.num_experts for t in per_layer)- for per_layer in banks.values()- )- return banks---def load_mxfp4_triton_banks_streaming_parallel(- model_path: str,- model_config=None,- *,- dtype: torch.dtype,- tp_info=None,- workers: int = 8,- chunk: int = 8 << 20,- layer_sink=None,-) -> dict[str, list[torch.Tensor]]:- """parallel counterpart of load_mxfp4_triton_banks_streaming: same banks + copy plan,- but bulk expert tensors come from the common chunked multi-threaded O_DIRECT reader.- source_slice applied to the whole tensor == the serial result, so correct for any TP.- ``layer_sink``: see :func:`load_mxfp4_triton_banks_streaming`."""- from freetoken.models.weight import iter_expert_tensors_parallel-- config = model_config if model_config is not None else parse_config(cached_load_hf_config(model_path))- if config.moe_weight_format != "mxfp4":- raise ValueError("GPT-OSS offload requires MXFP4 expert weights")- if tp_info is None:- tp_info = get_tp_info()- tp_start, tp_end, _local = local_mxfp4_intermediate_range(- config.moe_intermediate_size, rank=tp_info.rank, world_size=tp_info.size- )- tp_end = max(tp_start, tp_end)- banks, _hb = _empty_mxfp4_triton_banks(config, dtype=dtype, tp_info=tp_info)- all_expert_sources = (- "gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias",- "down_proj_blocks", "down_proj_scales", "down_proj_bias",- )- copy_plan = {- "gate_up_proj_blocks": ("gate_up_blocks",- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None), slice(None)),- _copy_transposed_mxfp4_blocks),- "gate_up_proj_scales": ("gate_up_scales",- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None)),- _copy_transposed_mxfp4_scales),- "gate_up_proj_bias": ("gate_up_bias",- (slice(None), slice(2 * tp_start, 2 * tp_end)), _copy_prefix),- "down_proj_blocks": ("down_blocks",- (slice(None), slice(None), slice(tp_start // 32, tp_end // 32), slice(None)),- _copy_transposed_mxfp4_blocks),- "down_proj_scales": ("down_scales",- (slice(None), slice(None), slice(tp_start // 32, tp_end // 32)),- _copy_transposed_mxfp4_scales),- }-- def _is_expert(name: str) -> bool:- info = _expert_layer_and_name(name)- return info is not None and 0 <= info[0] < config.num_layers-- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline-- seen: set[tuple[int, str]] = set()-- def _load(sink) -> None:- # _hb is empty in the CPU-only fallback (no banks to pin/stream): no-op tracker.- tracker = LayerCompletionTracker(len(all_expert_sources), _hb, sink) if _hb else None- for name, whole in iter_expert_tensors_parallel(model_path, _is_expert, workers=workers, chunk=chunk):- layer_id, source_name = _expert_layer_and_name(name)- seen.add((layer_id, source_name))- if source_name == "down_proj_bias":- if tp_info.rank == 0:- banks["down_bias"][layer_id].copy_(whole)- if tracker is not None:- tracker.note(layer_id)- continue- bank_name, source_slice, copy_fn = copy_plan[source_name]- copy_fn(banks[bank_name][layer_id], whole[source_slice])- if tracker is not None:- tracker.note(layer_id)-- if layer_sink is not None:- _load(layer_sink)- elif _hb:- with PinPipeline() as pins:- _load(pins)- else:- _load(None)-- expected = {(layer_id, src) for layer_id in range(config.num_layers) for src in all_expert_sources}- missing = expected - seen- if missing:- raise ValueError(f"Missing GPT-OSS expert tensors: {sorted(missing)[:8]}")- assert all(- len(per_layer) == config.num_layers- and all(t.is_contiguous() and t.size(0) == config.num_experts for t in per_layer)- for per_layer in banks.values()- )- return banks---def setup_offload_expert_banks(- model_path: str,- model_config,- *,- device: torch.device,- dtype: torch.dtype,- dummy: bool = False,- tp_info=None,- parallel: bool = False,- workers: int = 8,- chunk: int = 8 << 20,- layer_sink=None,-):- from freetoken.moe.expert_banks import ExpertBanks-- if model_config.moe_weight_format != "mxfp4":- raise ValueError("GPT-OSS offload currently supports MXFP4 expert weights")- if dummy:- sources = _make_dummy_mxfp4_triton_banks(model_config, dtype=dtype, tp_info=tp_info)- streamed = False- elif parallel: # parallel: common chunked multi-threaded O_DIRECT reader- sources = load_mxfp4_triton_banks_streaming_parallel(- model_path, model_config, dtype=dtype, tp_info=tp_info, workers=workers, chunk=chunk,- layer_sink=layer_sink,- )- streamed = layer_sink is not None- else:- sources = load_mxfp4_triton_banks_streaming(- model_path, model_config, dtype=dtype, tp_info=tp_info, layer_sink=layer_sink- )- streamed = layer_sink is not None- return ExpertBanks("mxfp4_triton", sources, streamed=streamed)+ yield name, _read_safetensor_slice(f, name, slices[source][1])++ def _pieces():+ pending: dict[int, dict[str, torch.Tensor]] = {}+ for name, tensor in _tensors():+ layer_id, source = _expert_layer_and_name(name)+ role = slices[source][0]+ if role == "down_bias" and tp_info.rank != 0:+ tensor = torch.zeros_like(tensor) # the TP reduction adds the bias once, on rank 0+ piece = pending.setdefault(layer_id, {})+ piece[role] = tensor+ if len(piece) == len(_EXPERT_SOURCES):+ del pending[layer_id]+ yield layer_id, 0, num_experts, piece+ if pending:+ raise ValueError(f"Missing GPT-OSS expert tensors for layers {sorted(pending)[:8]}")++ return _pieces()__all__ = ["iter_weights",+ "iter_expert_pieces","local_mxfp4_intermediate_range",- "load_mxfp4_triton_banks_streaming",- "setup_offload_expert_banks","shard_gpt_oss_tensor",]diff --git a/python/freetoken/models/llama/attention.py b/python/freetoken/models/llama/attention.pyindex 49516910..52b7e969 100644--- a/python/freetoken/models/llama/attention.py+++ b/python/freetoken/models/llama/attention.py@@ -22,6 +22,7 @@ class LlamaAttention(BaseOP):*,has_attn_bias: bool = False,has_qk_norm: bool = False,+ prefix: str = "",):head_dim = config.head_dimself.layer_id = layer_id@@ -37,6 +38,8 @@ class LlamaAttention(BaseOP):num_qo_heads=config.num_qo_heads,num_kv_heads=config.num_kv_heads,has_bias=has_attn_bias,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",)if has_qk_norm:self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)@@ -59,6 +62,8 @@ class LlamaAttention(BaseOP):head_dim * config.num_qo_heads,config.hidden_size,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.o_proj",)@nvtx_annotate("MHA")diff --git a/python/freetoken/models/llama/model.py b/python/freetoken/models/llama/model.pyindex a3d80171..eb7ff28f 100644--- a/python/freetoken/models/llama/model.py+++ b/python/freetoken/models/llama/model.py@@ -16,9 +16,9 @@ if TYPE_CHECKING:class LlamaDecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = LlamaAttn(config, layer_id)- self.mlp = LlamaMLP(config)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = LlamaAttn(config, layer_id, prefix=f"{prefix}.self_attn")+ self.mlp = LlamaMLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")self.input_layernorm = RMSNormFused(size=config.hidden_size,eps=config.rms_norm_eps,@@ -44,13 +44,16 @@ class LlamaDecoderLayer(BaseOP):class LlamaModel(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [LlamaDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ LlamaDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size,@@ -73,6 +76,8 @@ class LlamaForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/loader.py b/python/freetoken/models/loader.pyindex 49419364..0a2b69ae 100644--- a/python/freetoken/models/loader.py+++ b/python/freetoken/models/loader.py@@ -289,146 +289,6 @@ def iter_stacked_experts()-def _packed_expert_source_info(key: str) -> tuple[int, str] | None:- parts = key.split(".")- if len(parts) < 5 or parts[0] != "model" or parts[1] != "layers":- return None- if parts[-2] != "experts" or parts[-1] not in {"gate_up_proj", "down_proj"}:- return None- try:- return int(parts[2]), parts[-1]- except ValueError:- return None---class _PlainBank:- """CPU-only fallback bank: a plain unpinned tensor with a no-op pin (no CUDA)."""-- __slots__ = ("tensor",)-- def __init__(self, tensor: torch.Tensor):- self.tensor = tensor-- def pin(self) -> None:- pass---def _alloc_expert_bank(shape: tuple[int, ...], *, dtype: torch.dtype):- """Allocate an UNPINNED bank (lazy host mmap), to be filled then pinned at the end --- pin-after-fill. Registering already-resident pages skips cudaHostAlloc's slow commit- (~2.8 GiB/s) / zero-fill. Returns a bank object exposing ``.tensor`` and ``.pin()``."""- if torch.cuda.is_available():- from freetoken.moe.host_banks import HostBank-- return HostBank(tuple(shape), dtype)- return _PlainBank(torch.empty(shape, dtype=dtype))---def _copy_expert_layer_into_bank(- banks: dict[str, list],- row_shape: dict[str, tuple[int, ...]],- seen_layers: dict[str, set[int]],- *,- bank_name: str,- tensor: torch.Tensor,- layer: int,- config,- dtype: torch.dtype,-) -> None:- if layer < 0 or layer >= config.num_layers:- raise ValueError(- f"Unexpected MoE expert layer {layer}; expected [0, {config.num_layers})"- )- if tensor.size(0) != config.num_experts:- raise ValueError(- f"Unexpected {bank_name} expert count {tensor.size(0)}; "- f"expected {config.num_experts}"- )- expected_shape = row_shape.setdefault(bank_name, tuple(tensor.shape[1:]))- if tuple(tensor.shape[1:]) != expected_shape:- raise ValueError(- f"Inconsistent {bank_name} expert shape {tuple(tensor.shape[1:])}; "- f"expected {expected_shape}"- )-- bank = banks[bank_name][layer]- if bank is None:- banks[bank_name][layer] = bank = _alloc_expert_bank(- (config.num_experts, *tensor.shape[1:]), dtype=dtype- )- bank.tensor.copy_(tensor) # whole-layer arrival; pinned later, after fully resident- seen_layers[bank_name].add(layer)---def stream_moe_expert_sources(- tensors: Iterable[tuple[str, torch.Tensor]],- config,- *,- dtype: torch.dtype,- layer_sink=None,-) -> tuple[list[torch.Tensor], list[torch.Tensor]]:- """Stream packed per-layer BF16 expert tensors into final offload banks.-- Model adapters normalize ordinary MoE expert weights to- ``...experts.gate_up_proj`` and ``...experts.down_proj`` with shape- ``[num_experts, ...]``. Each arrives whole-layer, so it's written directly into- its own ``[num_experts, ...]`` per-layer bank (independent allocation).-- ``layer_sink=None`` (serving): pin each layer's banks as its writes complete,- via an internally-owned :class:`PinPipeline`. ``layer_sink`` given (converter):- the tracker fires into it instead -- nothing is pinned, and the sink may release- banks it has written out, so the returned tensors are only valid until then (the- caller owns that tradeoff).- """- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline-- banks: dict[str, list] = { # name -> per-layer [bank obj (HostBank/_PlainBank) or None]- "gate_up": [None] * config.num_layers,- "down": [None] * config.num_layers,- }- row_shape: dict[str, tuple[int, ...]] = {}- seen_layers: dict[str, set[int]] = {"gate_up": set(), "down": set()}-- def _load(sink) -> None:- tracker = LayerCompletionTracker(2, banks, sink) # gate_up + down per layer- for name, tensor in tensors:- expert_info = _packed_expert_source_info(name)- if expert_info is None:- raise ValueError(f"Unexpected expert weight key: {name}")- layer, packed_name = expert_info- bank_name = "gate_up" if packed_name == "gate_up_proj" else "down"- _copy_expert_layer_into_bank(- banks,- row_shape,- seen_layers,- bank_name=bank_name,- tensor=tensor,- layer=layer,- config=config,- dtype=dtype,- )- tracker.note(layer)-- expected_layers = set(range(config.num_layers))- missing = {- name: sorted(expected_layers - seen)- for name, seen in seen_layers.items()- if seen != expected_layers- }- if missing:- raise ValueError(f"Missing MoE expert source layers: {missing}")-- if layer_sink is not None:- _load(layer_sink)- else:- with PinPipeline() as pins:- _load(pins)- return (- [bank.tensor for bank in banks["gate_up"]],- [bank.tensor for bank in banks["down"]],- )--__all__ = ["MergeRule","iter_root_safetensor_files_from_index",@@ -436,5 +296,4 @@ __all__ = ["iter_merged_tensors","iter_stacked_experts","shard_tensor",- "stream_moe_expert_sources",]diff --git a/python/freetoken/models/minimax_m2/__init__.py b/python/freetoken/models/minimax_m2/__init__.pyindex e4e1211c..ad276f84 100644--- a/python/freetoken/models/minimax_m2/__init__.py+++ b/python/freetoken/models/minimax_m2/__init__.py@@ -1,11 +1,10 @@from .config import parse_configfrom .model import MiniMaxM2ForCausalLM-from .weight import iter_weights, load_nvfp4_expert_sources, load_nvfp4_expert_sources_parallel+from .weight import iter_weights, nvfp4_expert_spec__all__ = [+ "nvfp4_expert_spec","MiniMaxM2ForCausalLM","parse_config","iter_weights",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel",]diff --git a/python/freetoken/models/minimax_m2/attention.py b/python/freetoken/models/minimax_m2/attention.pyindex fffe3603..320ec9ae 100644--- a/python/freetoken/models/minimax_m2/attention.py+++ b/python/freetoken/models/minimax_m2/attention.py@@ -26,7 +26,7 @@ class MiniMaxM2Attention(BaseOP):of each head are rotated."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):head_dim = config.head_dimself.layer_id = layer_idtp_size = get_tp_info().size@@ -41,6 +41,8 @@ class MiniMaxM2Attention(BaseOP):num_qo_heads=config.num_qo_heads,num_kv_heads=config.num_kv_heads,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",)if config.use_qk_norm:self.q_norm = RMSNorm(self.qo_attn_dim, eps=config.rms_norm_eps)@@ -63,6 +65,8 @@ class MiniMaxM2Attention(BaseOP):head_dim * config.num_qo_heads,config.hidden_size,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.o_proj",)@nvtx_annotate("MHA")diff --git a/python/freetoken/models/minimax_m2/model.py b/python/freetoken/models/minimax_m2/model.pyindex 671ea361..457ab565 100644--- a/python/freetoken/models/minimax_m2/model.py+++ b/python/freetoken/models/minimax_m2/model.py@@ -16,9 +16,9 @@ if TYPE_CHECKING:class MiniMaxM2DecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = MiniMaxM2Attention(config, layer_id)- self.block_sparse_moe = MiniMaxM2SparseMoeBlock(config, layer_id)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = MiniMaxM2Attention(config, layer_id, prefix=f"{prefix}.self_attn")+ self.block_sparse_moe = MiniMaxM2SparseMoeBlock(config, layer_id, prefix=f"{prefix}.block_sparse_moe")self.input_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)self.post_attention_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps@@ -37,13 +37,16 @@ class MiniMaxM2DecoderLayer(BaseOP):class MiniMaxM2Model(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [MiniMaxM2DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ MiniMaxM2DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)@@ -63,6 +66,8 @@ class MiniMaxM2ForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/minimax_m2/moe.py b/python/freetoken/models/minimax_m2/moe.pyindex 5d319802..1410f388 100644--- a/python/freetoken/models/minimax_m2/moe.py+++ b/python/freetoken/models/minimax_m2/moe.py@@ -16,7 +16,7 @@ class MiniMaxM2SparseMoeBlock(BaseOP):NVFP4 expert banks in the unified offload cache (the expert GEMM backend follows``cache.quant_format``; see :meth:`OffloadMoELayer.routed_forward`)."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self.top_k = config.num_experts_per_tokself.num_experts = config.num_expertsself.norm_topk_prob = config.norm_topk_prob@@ -27,6 +27,8 @@ class MiniMaxM2SparseMoeBlock(BaseOP):config,layer_id=layer_id,renormalize=config.norm_topk_prob,+ quant_config=config.quant,+ prefix=f"{prefix}.experts",)def _route(self, router_logits: torch.Tensor) -> TopK:diff --git a/python/freetoken/models/minimax_m2/weight.py b/python/freetoken/models/minimax_m2/weight.pyindex 103f8ec5..1b91ead8 100644--- a/python/freetoken/models/minimax_m2/weight.py+++ b/python/freetoken/models/minimax_m2/weight.py@@ -15,7 +15,6 @@ from freetoken.models.loader import ()from freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,- load_nvfp4_expert_source_banks,)from freetoken.utils import cached_load_hf_configfrom tqdm import tqdm@@ -49,7 +48,7 @@ def iter_weights(include_moe_experts: bool,include_non_moe: bool,) -> Iterator[tuple[str, torch.Tensor]]:- """yield dense BF16 weights only; NVFP4 experts go to the offload cache via load_nvfp4_expert_sources."""+ """yield dense BF16 weights only; NVFP4 experts go to the offload cache from their pieces."""assert not include_moe_experts, ("MiniMax-M2 stores experts as NVFP4 and only supports the offload MoE backend; ""experts are loaded into the offload cache, not the dense model."@@ -88,39 +87,8 @@ def iter_weights(yield from iter_merged_tensors(raw(), _MERGE_RULES, model_name="minimax_m2")-def load_nvfp4_expert_sources(- model_path: str,- config,- *,- layer_sink=None,-) -> dict[str, torch.Tensor]:- """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""- return load_nvfp4_expert_source_banks(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- layer_sink=layer_sink,- )---def load_nvfp4_expert_sources_parallel(- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None-):- """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel-- return load_nvfp4_expert_source_banks_parallel(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- workers=workers,- chunk=chunk,- layer_sink=layer_sink,- )+def nvfp4_expert_spec(model_path: str, config):+ return _NVFP4_SOURCE_SPEC-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]+__all__ = ["iter_weights", "nvfp4_expert_spec"]diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.pyindex 71c0bb51..cfcebb6f 100644--- a/python/freetoken/models/minimax_m3/__init__.py+++ b/python/freetoken/models/minimax_m3/__init__.py@@ -1,15 +1,13 @@from .config import parse_configfrom .model import MiniMaxM3ForCausalLMfrom .weight import (+ nvfp4_expert_spec,iter_weights,- load_nvfp4_expert_sources,- load_nvfp4_expert_sources_parallel,)__all__ = [+ "nvfp4_expert_spec","MiniMaxM3ForCausalLM","parse_config","iter_weights",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel",]diff --git a/python/freetoken/models/minimax_m3/attention.py b/python/freetoken/models/minimax_m3/attention.pyindex fdf090b7..0d55c072 100644--- a/python/freetoken/models/minimax_m3/attention.py+++ b/python/freetoken/models/minimax_m3/attention.py@@ -22,12 +22,10 @@ from typing import TYPE_CHECKINGfrom freetoken.core import get_global_ctxfrom freetoken.distributed import get_tp_info-from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm+from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm, LinearReplicatedfrom freetoken.layers.rotary import get_ropefrom freetoken.utils import nvtx_annotate-from .mlp import make_proj-if TYPE_CHECKING:import torch@@ -35,7 +33,7 @@ if TYPE_CHECKING:class MiniMaxM3Attention(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.m3_argsassert get_tp_info().size == 1, "MiniMax-M3 currently supports TP=1 only"self.layer_id = layer_id@@ -45,11 +43,14 @@ class MiniMaxM3Attention(BaseOP):self.qo_attn_dim = self.num_qo_heads * self.head_dimself.kv_attn_dim = self.num_kv_heads * self.head_dim- quant = config.attn_quant- self.qkv_proj = make_proj(- quant, args.hidden_size, self.qo_attn_dim + 2 * self.kv_attn_dim+ self.qkv_proj = LinearReplicated(+ args.hidden_size, self.qo_attn_dim + 2 * self.kv_attn_dim, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.qkv_proj",+ )+ self.o_proj = LinearReplicated(+ self.qo_attn_dim, args.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.o_proj",)- self.o_proj = make_proj(quant, self.qo_attn_dim, args.hidden_size)# Per-head Gemma (1+w) q/k norms (qk_norm_type == "per_head").self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=args.norm_eps)@@ -70,8 +71,9 @@ class MiniMaxM3Attention(BaseOP):self.index_dim = args.index_dimself.index_q_dim = self.num_index_heads * self.index_dim# Merged [index_q | index_k] projection (one shared index KEY head).- self.index_qk_proj = make_proj(- quant, args.hidden_size, self.index_q_dim + self.index_dim+ self.index_qk_proj = LinearReplicated(+ args.hidden_size, self.index_q_dim + self.index_dim, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.index_qk_proj",)self.index_q_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)self.index_k_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)diff --git a/python/freetoken/models/minimax_m3/config.py b/python/freetoken/models/minimax_m3/config.pyindex 906870e3..7df81d79 100644--- a/python/freetoken/models/minimax_m3/config.py+++ b/python/freetoken/models/minimax_m3/config.py@@ -16,14 +16,12 @@ ablation: plain MHAKVCache, every layer attends its whole history through a geneFULL backend). ``page_size`` is pinned to the 128-token sparse block by the backend's``page_sizes`` declaration, so one KV page == one sparse block.-Resident-weight quantization is resolved HERE (from the FREETOKEN_M3_*_MXFP8 env-switches, default on) into the standard ``ModelConfig`` fields -- ``attn_quant`` /-``dense_quant`` = ``"mxfp8"`` or ``"none"`` -- and every consumer (the module-constructors and the weight loader) reads those fields, so the resolved config is the-single record of what the served weights actually are. The routed experts are NVFP4+The resident projections are served in the precision the checkpoint stores (MXFP8 for the+official export); ``attn_quant`` / ``dense_quant`` record what was detected for the weight+loader. The routed experts are NVFP4(same ModelOpt layout as MiniMax-M2 / GLM) and always live in the offload cache;their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernels-(``select_nvfp4_backend``). lm_head / embeddings are BF16 in the checkpoint.+(the NVFP4 MoE kernel objects). lm_head / embeddings are BF16 in the checkpoint."""from __future__ import annotations@@ -85,22 +83,8 @@ def parse_config(hf_config: Any) -> ModelConfig:)args = load_args(text, num_layers, sparse_enabled=sparse_enabled)- # W8A16 MXFP8 (the checkpoint's native dense quantization) for the resident- # weights; decode is weight-bandwidth bound and the freed VRAM densifies the- # expert cache. =0 dequantizes to bf16 at load (bring-up / ablation). Read at- # PARSE time so the resolved config is the single record of the served weights- # -- an FTW checkpoint converted under one setting must be served under the- # same one (see weight.py) -- and logged HERE so the FTW serve path (which- # never runs iter_weights) still leaves a serve-time record of the modes.- attn_mxfp8 = os.getenv("FREETOKEN_M3_ATTN_MXFP8", "1") != "0"- mlp_mxfp8 = os.getenv("FREETOKEN_M3_MLP_MXFP8", "1") != "0"- _log_mode_once(- f"quant={attn_mxfp8}/{mlp_mxfp8}",- f"MiniMax-M3 resident quant: attn={'mxfp8' if attn_mxfp8 else 'none'} "- f"dense={'mxfp8' if mlp_mxfp8 else 'none'} lm_head=none "- "(FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; an FTW checkpoint "- "converted under one setting must be served under the same one).",- )+ # the quantized release stores the dense projections MXFP8; the reader keeps them native and the layers take their scheme from the QuantConfig+ dense_mxfp8 = getattr(hf_config, "quantization_config", None) is not None# Leading dense-FFN layers: M3's moe_layer_freq is a contiguous 0-prefix; the# offload cache and the generic num_moe_layers arithmetic rely on that shape.@@ -191,8 +175,8 @@ def parse_config(hf_config: Any) -> ModelConfig:# resolved config describes the served weights; the constructors and# iter_weights read these fields, never the env directly. lm_head stays bf16# (the checkpoint excludes it from quantization).- attn_quant="mxfp8" if attn_mxfp8 else "none",- dense_quant="mxfp8" if mlp_mxfp8 else "none",+ attn_quant="mxfp8" if dense_mxfp8 else "none",+ dense_quant="mxfp8" if dense_mxfp8 else "none",lm_head_quant="none",m3_args=args,)diff --git a/python/freetoken/models/minimax_m3/mlp.py b/python/freetoken/models/minimax_m3/mlp.pyindex b08f6935..d5332eed 100644--- a/python/freetoken/models/minimax_m3/mlp.py+++ b/python/freetoken/models/minimax_m3/mlp.py@@ -1,11 +1,9 @@"""SwiGLU-OAI MLP for MiniMax-M3's leading dense layers and per-layer shared experts.The checkpoint ships these MXFP8 (fp8-e4m3 weight + uint8 e8m0 block-32-``weight_scale_inv``); serving default keeps them native W8A16-(``ModelConfig.dense_quant == "mxfp8"``, from FREETOKEN_M3_MLP_MXFP8) -- decode reads-every shared expert each token, so this halves the resident-MLP weight traffic and-frees the same VRAM for expert-cache slots. ``FREETOKEN_M3_MLP_MXFP8=0`` dequantizes-to bf16 at load (bring-up / ablation).+``weight_scale_inv``) and they are served native W8A16 through the checkpoint's+QuantConfig -- decode reads every shared expert each token, so this halves the+resident-MLP weight traffic and frees the same VRAM for expert-cache slots.gate/up are stored merged (``gate_up_proj``, [gate; up] halves -- the loaderconcatenates the two checkpoint projections output-wise; MXFP8 scales are per-output-@@ -25,28 +23,25 @@ if TYPE_CHECKING:import torch-def make_proj(quant: str, in_features: int, out_features: int) -> BaseOP:- """A resident projection in the model's resolved quant mode: ``"mxfp8"`` (W8A16,- the checkpoint's native block-32 e8m0 format -- see weight.py) or bf16."""- if quant == "mxfp8":- from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear-- return Mxfp8Linear(in_features, out_features, has_bias=False)- return LinearReplicated(in_features, out_features, has_bias=False)--class MiniMaxM3MLP(BaseOP):def __init__(self,hidden_size: int,intermediate_size: int,*,- quant: str,alpha: float,limit: float,+ quant_config=None,+ prefix: str = "",):- self.gate_up_proj = make_proj(quant, hidden_size, 2 * intermediate_size)- self.down_proj = make_proj(quant, intermediate_size, hidden_size)+ self.gate_up_proj = LinearReplicated(+ hidden_size, 2 * intermediate_size, has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.gate_up_proj",+ )+ self.down_proj = LinearReplicated(+ intermediate_size, hidden_size, has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.down_proj",+ )self._alpha = alphaself._limit = limitdiff --git a/python/freetoken/models/minimax_m3/model.py b/python/freetoken/models/minimax_m3/model.pyindex 86c47328..e8b6e5d0 100644--- a/python/freetoken/models/minimax_m3/model.py+++ b/python/freetoken/models/minimax_m3/model.py@@ -27,19 +27,22 @@ class MiniMaxM3DecoderLayer(BaseOP):``block_sparse_moe`` -- matching the checkpoint's naming. All layernorms areGemma-style (1+w) RMSNorm (``use_gemma_norm``)."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.m3_args- self.self_attn = MiniMaxM3Attention(config, layer_id)+ self.self_attn = MiniMaxM3Attention(config, layer_id, prefix=f"{prefix}.self_attn")self.is_moe_layer = layer_id in args.moe_layer_idsif self.is_moe_layer:- self.block_sparse_moe: BaseOP = MiniMaxM3SparseMoeBlock(config, layer_id)+ self.block_sparse_moe: BaseOP = MiniMaxM3SparseMoeBlock(+ config, layer_id, prefix=f"{prefix}.block_sparse_moe"+ )else:self.mlp = MiniMaxM3MLP(config.hidden_size,args.dense_intermediate_size,- quant=config.dense_quant,alpha=args.swiglu_alpha,limit=args.swiglu_limit,+ quant_config=config.quant,+ prefix=f"{prefix}.mlp",)self.input_layernorm = GemmaPlusOneRMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps@@ -62,13 +65,16 @@ class MiniMaxM3DecoderLayer(BaseOP):class MiniMaxM3Model(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [MiniMaxM3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ MiniMaxM3DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = GemmaPlusOneRMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)@@ -89,6 +95,8 @@ class MiniMaxM3ForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/minimax_m3/moe.py b/python/freetoken/models/minimax_m3/moe.pyindex 9f54c156..fe1139b8 100644--- a/python/freetoken/models/minimax_m3/moe.py+++ b/python/freetoken/models/minimax_m3/moe.py@@ -4,8 +4,8 @@ Routing follows the DeepSeek/GLM family (``GlmMoeDsaSparseBlock``): fp32 sigmoidscores + a selection-only ``e_score_correction_bias``, gather the unbiased scores,renormalize, scale by ``routed_scaling_factor`` (2.0). No group-limited top-k(n_group == 1). The routed experts (NVFP4, swigluoai) go through ``make_moe_layer``-(the offload family; the swigluoai alpha/limit ride the layer via ``extra_attrs``,-gpt-oss precedent) and the always-on shared expert (MXFP8, swigluoai) is added.+(the offload family; the swigluoai alpha/limit are the layer's ``alpha`` /+``limit``) and the always-on shared expert (MXFP8, swigluoai) is added.The checkpoint keeps ``e_score_correction_bias`` fp32 and the reference applies itfp32; the bias sits on the top-k boundary, so it is stored fp32 here too (unlike@@ -29,7 +29,7 @@ TopK = Tuple[torch.Tensor, torch.Tensor]class MiniMaxM3SparseMoeBlock(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):args = config.m3_argsself.top_k = config.num_experts_per_tokself.num_experts = config.num_experts@@ -55,17 +55,18 @@ class MiniMaxM3SparseMoeBlock(BaseOP):layer_id=layer_id - config.first_k_dense_replace,renormalize=config.norm_topk_prob,activation="swigluoai",- extra_attrs={- "hidden_act_alpha": args.swiglu_alpha,- "swiglu_limit": args.swiglu_limit,- },+ alpha=args.swiglu_alpha,+ limit=args.swiglu_limit,+ quant_config=config.quant,+ prefix=f"{prefix}.experts",)self.shared_experts = MiniMaxM3MLP(config.hidden_size,config.shared_expert_intermediate_size * max(1, config.n_shared_experts),- quant=config.dense_quant,alpha=args.swiglu_alpha,limit=args.swiglu_limit,+ quant_config=config.quant,+ prefix=f"{prefix}.shared_experts",)def _route(self, hidden_states: torch.Tensor) -> TopK:diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.pyindex e10a7237..3aea2347 100644--- a/python/freetoken/models/minimax_m3/weight.py+++ b/python/freetoken/models/minimax_m3/weight.py@@ -6,22 +6,14 @@ The checkpoint is the multimodal wrapper layout: every text-tower tensor carries``multi_modal_projector.`` / ``patch_merge_mlp.``) is never read (text-only serving).Resident (non routed-expert) dense projections are MXFP8 in the checkpoint-(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``). What this loader-yields follows the quant modes RESOLVED IN ``parse_config`` (``ModelConfig.attn_quant``-/ ``dense_quant``, from the FREETOKEN_M3_*_MXFP8 switches, default on): in the default-mode the fp8 weight and its scale codes stream through verbatim (merged output-wise for-the fused qkv / index-qk / gate-up projections -- the scales are per-output-row, so-fusion is exact); with a switch off the projections are dequantized to bf16 at load.+(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``): the fp8 weight and+its scale codes stream through verbatim (merged output-wise for the fused qkv /+index-qk / gate-up projections -- the scales are per-output-row, so fusion is exact).Norms, the router gate, ``e_score_correction_bias`` (kept fp32), embeddings andlm_head are unquantized and stream through verbatim. Routed experts are NVFP4(``w1/w3/w2`` = gate/up/down, same ModelOpt layout as MiniMax-M2) and go to the-offload cache via ``load_nvfp4_expert_sources``; expert ``input_scale`` calibration+offload cache from their NVFP4 pieces; expert ``input_scale`` calibrationtensors are unused (W4A16) and never match the bank spec.--FTW caveat: an FTW checkpoint stores whatever iter_weights yielded at CONVERSION time,-and the model is built from the env at SERVE time -- the two must agree (a mismatch-fails loudly in load_state_dict on the ``*.weight_scale_inv`` keys). The active modes-are logged at load so conversion logs record the choice."""from __future__ import annotations@@ -37,7 +29,6 @@ from freetoken.distributed import get_tp_infofrom freetoken.models.loader import drop_page_cachefrom freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,- load_nvfp4_expert_source_banks,)from freetoken.utils import cached_load_hf_config, download_hf_weightfrom tqdm import tqdm@@ -153,7 +144,7 @@ def iter_weights() -> Iterator[tuple[str, torch.Tensor]]:assert not include_moe_experts, ("MiniMax-M3 stores routed experts as NVFP4 and only supports the offload MoE "- "backend; experts are loaded into the offload cache via load_nvfp4_expert_sources()."+ "backend; experts are loaded into the offload cache from their NVFP4 pieces.")assert include_non_moeconfig = parse_config(cached_load_hf_config(model_path))@@ -170,8 +161,7 @@ def iter_weights(init_logger(__name__).info(f"MiniMax-M3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "- f"lm_head={config.lm_head_quant} (FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; "- "an FTW conversion records these choices implicitly -- serve with the same flags)"+ f"lm_head={config.lm_head_quant}")try:for layer in tqdm(@@ -248,36 +238,8 @@ def iter_weights(reader.close()-def load_nvfp4_expert_sources(- model_path: str, config, *, layer_sink=None-) -> dict[str, list[torch.Tensor]]:- """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""- return load_nvfp4_expert_source_banks(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- layer_sink=layer_sink,- )---def load_nvfp4_expert_sources_parallel(- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None-):- """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel-- return load_nvfp4_expert_source_banks_parallel(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- workers=workers,- chunk=chunk,- layer_sink=layer_sink,- )+def nvfp4_expert_spec(model_path: str, config):+ return _NVFP4_SOURCE_SPEC-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]+__all__ = ["iter_weights", "nvfp4_expert_spec"]diff --git a/python/freetoken/models/mistral/attention.py b/python/freetoken/models/mistral/attention.pyindex 14ac92f7..01a48388 100644--- a/python/freetoken/models/mistral/attention.py+++ b/python/freetoken/models/mistral/attention.py@@ -22,6 +22,7 @@ class MistralAttention(BaseOP):*,has_attn_bias: bool = False,has_qk_norm: bool = False,+ prefix: str = "",):head_dim = config.head_dimself.layer_id = layer_id@@ -37,6 +38,8 @@ class MistralAttention(BaseOP):num_qo_heads=config.num_qo_heads,num_kv_heads=config.num_kv_heads,has_bias=has_attn_bias,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",)if has_qk_norm:self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)@@ -59,6 +62,8 @@ class MistralAttention(BaseOP):head_dim * config.num_qo_heads,config.hidden_size,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.o_proj",)@nvtx_annotate("MHA")diff --git a/python/freetoken/models/mistral/model.py b/python/freetoken/models/mistral/model.pyindex 5d90ec22..8d258bcf 100644--- a/python/freetoken/models/mistral/model.py+++ b/python/freetoken/models/mistral/model.py@@ -16,9 +16,9 @@ if TYPE_CHECKING:class MistralDecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = MistralAttn(config, layer_id)- self.mlp = MistralMLP(config)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = MistralAttn(config, layer_id, prefix=f"{prefix}.self_attn")+ self.mlp = MistralMLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")self.input_layernorm = RMSNormFused(size=config.hidden_size,eps=config.rms_norm_eps,@@ -44,13 +44,16 @@ class MistralDecoderLayer(BaseOP):class MistralModel(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [MistralDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ MistralDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size,@@ -73,6 +76,8 @@ class MistralForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/muse_glimmer/attention.py b/python/freetoken/models/muse_glimmer/attention.pyindex 29764dcf..7c814d4c 100644--- a/python/freetoken/models/muse_glimmer/attention.py+++ b/python/freetoken/models/muse_glimmer/attention.py@@ -5,12 +5,11 @@ from typing import TYPE_CHECKINGimport torchfrom freetoken.attention import AttentionSpecfrom freetoken.core import get_global_ctx-from freetoken.layers import BaseOP, GemmaRMSNorm+from freetoken.layers import BaseOP, GemmaRMSNorm, LinearColParallelMerged, LinearReplicatedfrom freetoken.layers.rotary import get_ropefrom freetoken.models.config import SWAAttentionGroupConfigfrom freetoken.utils import nvtx_annotate-from freetoken.models.quant_linear import make_col_merged, make_replicatedif TYPE_CHECKING:from freetoken.models.config import ModelConfig@@ -31,7 +30,7 @@ class MuseGlimmerAttention(BaseOP):the same normed layer input as q/k/v, so it rides the fused projection."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self.layer_id = layer_idgroup = config.attention_group_for_layer(layer_id)self.is_swa = isinstance(group, SWAAttentionGroupConfig)@@ -43,10 +42,14 @@ class MuseGlimmerAttention(BaseOP):self.kv_attn_dim = self.num_kv * head_dimself._qkvg_split = [self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim, self.qo_attn_dim]- self.qkvg_proj = make_col_merged(- config, config.hidden_size, self._qkvg_split, has_bias=False+ self.qkvg_proj = LinearColParallelMerged(+ config.hidden_size, self._qkvg_split, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.qkvg_proj",+ )+ self.o_proj = LinearReplicated(+ self.qo_attn_dim, config.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.o_proj",)- self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False)# Weightless: the checkpoint carries no q/k norm weights; the runtime ones vector# is intentionally not part of state_dict.self.qk_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps, with_scale=False)diff --git a/python/freetoken/models/muse_glimmer/model.py b/python/freetoken/models/muse_glimmer/model.pyindex ba632c0b..1e1072ec 100644--- a/python/freetoken/models/muse_glimmer/model.py+++ b/python/freetoken/models/muse_glimmer/model.py@@ -9,6 +9,8 @@ from freetoken.layers import (GemmaPlusOneRMSNorm,GemmaPlusOneRMSNormFused,GemmaRMSNorm,+ LinearColParallelMerged,+ LinearReplicated,OPList,ParallelLMHead,RMSNorm,@@ -19,24 +21,25 @@ from freetoken.models.blocks import BaseLLMModelfrom freetoken.utils import nvtx_annotatefrom .attention import MuseGlimmerAttention-from freetoken.models.quant_linear import make_col_merged, make_replicatedif TYPE_CHECKING:from freetoken.models.config import ModelConfigclass MuseGlimmerMLP(BaseOP):- """SwiGLU MLP (gate|up fused), NVFP4 (W4A16) on the quantized checkpoint else bf16."""+ """SwiGLU MLP (gate|up fused); the projections take the checkpoint's scheme for their prefix."""- def __init__(self, config: ModelConfig):- self.gate_up_proj = make_col_merged(- config,+ def __init__(self, config: ModelConfig, *, prefix: str = ""):+ self.gate_up_proj = LinearColParallelMerged(config.hidden_size,[config.intermediate_size, config.intermediate_size],has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.gate_up_proj",)- self.down_proj = make_replicated(- config, config.intermediate_size, config.hidden_size, has_bias=False+ self.down_proj = LinearReplicated(+ config.intermediate_size, config.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.down_proj",)@nvtx_annotate("MLP")@@ -56,10 +59,10 @@ class MuseGlimmerDecoderLayer(BaseOP):exists to keep."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self._layer_id = layer_id- self.self_attn = MuseGlimmerAttention(config, layer_id)- self.mlp = MuseGlimmerMLP(config)+ self.self_attn = MuseGlimmerAttention(config, layer_id, prefix=f"{prefix}.self_attn")+ self.mlp = MuseGlimmerMLP(config, prefix=f"{prefix}.mlp")H = config.hidden_sizepost_eps = config.post_norm_eps if config.post_norm_eps is not None else config.rms_norm_epsself.input_layernorm = GemmaPlusOneRMSNorm(H, eps=config.rms_norm_eps)@@ -81,7 +84,7 @@ class MuseGlimmerDecoderLayer(BaseOP):class MuseGlimmerModel(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,@@ -93,7 +96,10 @@ class MuseGlimmerModel(BaseOP):config.hidden_size, eps=config.rms_norm_eps, with_scale=False)self.layers = OPList(- [MuseGlimmerDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ MuseGlimmerDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])# Final norm scales by the raw checkpoint weight (plain RMSNorm, not the# centered (1+w) form the decoder norms use).@@ -114,6 +120,8 @@ class MuseGlimmerForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)self._output_multiplier = config.output_multiplierself._final_logit_softcapping = config.final_logit_softcappingdiff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.pyindex 6b933ff1..1d1b4285 100644--- a/python/freetoken/models/nvfp4_banks.py+++ b/python/freetoken/models/nvfp4_banks.py@@ -60,284 +60,80 @@ def _bank_layer(spec: Nvfp4ExpertSourceSpec, layer: int, config) -> int | None:return bank_layer-def _alloc_nvfp4_host_banks(num_layers: int, E: int, H: int, I: int):- """6 NVFP4 source banks, one ``[E, ...]`` tensor per layer (independent allocations),- unpinned (pin-after-fill): register only after fill to skip cudaHostAlloc's slow- commit. Caller fills each layer's ``.tensor`` then pins it (per-layer, via- ``PinPipeline``, as its writes complete)."""- from freetoken.moe.host_banks import alloc_layer_banks+def _kind_suffix(kind: str) -> str:+ return {"weight": "", "weight_scale": "_scale", "weight_scale_2": "_global"}[kind]- fp8 = torch.float8_e4m3fn- return alloc_layer_banks({- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),- "gate_up_scale": ((E, 2 * I, H // 16), fp8),- "gate_up_global": ((E, 2 * I), torch.float16),- "down_packed": ((E, H, I // 2), torch.uint8),- "down_scale": ((E, H, I // 16), fp8),- "down_global": ((E, H), torch.float16),- }, num_layers)--def load_nvfp4_expert_source_banks(+def iter_nvfp4_expert_pieces(model_path: str,config,spec: Nvfp4ExpertSourceSpec,*,- drop_page_cache: DropPageCache,- primary: bool,- layer_sink=None,-) -> dict[str, list[torch.Tensor]]:- """Build the 6 native NVFP4 source banks by streaming checkpoint shards (serial per-shard read).-- ModelOpt row layout: gate/up fused on the output-row axis, down separate; the per-tensor- global scale (weight_scale_2) is kept as a separate per-output-row FP16 bank (``*_global``),- so dequant is ``fp4 * block_scale * global``. Each bank is one ``[E, ...]`` tensor per- layer, indexed by ``[bank_layer][expert]``. (The marlin/b12x backends repack these and- fold the global into per-expert alphas; see moe/nvfp4_backends.py.)-- ``layer_sink=None`` (serving): pin each bank layer as its writes complete, via an- internally-owned :class:`PinPipeline`. ``layer_sink`` given (converter; for- marlin/b12x the provider wraps it in a per-layer repacking sink first): the- completion tracker fires into it instead -- nothing here is pinned, and the sink- may release banks it has written out, so the returned tensors are only valid- until then (the caller owns that tradeoff).+ parallel: bool = False,+ workers: int = 8,+ chunk: int = 8 << 20,+ drop_page_cache: DropPageCache | None = None,+ primary: bool = True,+):+ """One piece per routed expert: ``gate`` / ``up`` / ``down`` codes plus their ``_scale``+ (fp8 block scales) and ``_global`` (the per-tensor scale, reciprocal for quant-side dialects,+ fp16) companions, straight from the safetensors shards.++ Serial reads walk the shards in order; ``parallel`` uses the chunked O_DIRECT reader. Either+ way tensors of one expert may span shards, so they are grouped by (layer, expert) as they land."""+ from freetoken.models.loader import drop_page_cache as _drop+ from freetoken.moe.expert_pieces import per_expert_pieces++ drop = drop_page_cache or _dropfolder = download_hf_weight(model_path)- index_path = os.path.join(folder, "model.safetensors.index.json")- with open(index_path, encoding="utf-8") as f:+ with open(os.path.join(folder, "model.safetensors.index.json"), encoding="utf-8") as f:weight_map = json.load(f)["weight_map"]- E = config.num_experts- H = config.hidden_size- I = config.moe_intermediate_size- num_layers = _num_moe_layers(config)-- for shard in sorted(set(weight_map.values())):- drop_page_cache(os.path.join(folder, shard))-- weight_shards: dict[str, list[tuple[str, re.Match[str], int]]] = collections.defaultdict(list)- global_shards: dict[str, list[tuple[str, re.Match[str], int]]] = collections.defaultdict(list)- for name, shard in weight_map.items():+ wanted: dict[str, tuple[int, int, str]] = {}+ for name in weight_map:match = spec.key_pattern.match(name)if match is None:continue- layer = int(match.group("layer"))- bank_layer = _bank_layer(spec, layer, config)+ bank_layer = _bank_layer(spec, int(match.group("layer")), config)if bank_layer is None:continueproj = match.group("proj")if proj not in spec.proj_to_role:raise ValueError(f"{spec.desc}: unknown NVFP4 expert projection {proj!r}")kind = _canon_kind(spec, match.group("kind"))- if kind == "weight_scale_2":- global_shards[shard].append((name, match, bank_layer))- elif kind in {"weight", "weight_scale"}:- weight_shards[shard].append((name, match, bank_layer))- else:+ if kind not in ("weight", "weight_scale", "weight_scale_2"):raise ValueError(f"{spec.desc}: unknown NVFP4 expert tensor kind {kind!r}")-- globals_map: dict[tuple[int, int, str], torch.Tensor] = {}- for shard in sorted(global_shards):- path = os.path.join(folder, shard)- with safetensors.safe_open(path, framework="pt", device="cpu") as f:- for name, match, _bank_layer_id in global_shards[shard]:- key = (- int(match.group("layer")),- int(match.group("expert")),- match.group("proj"),- )- globals_map[key] = _ingest_global(spec, f.get_tensor(name))- drop_page_cache(path)-- _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill- gate_up_packed = [b.tensor for b in _hb["gate_up_packed"]]- gate_up_scale = [b.tensor for b in _hb["gate_up_scale"]]- gate_up_global = [b.tensor for b in _hb["gate_up_global"]]- down_packed = [b.tensor for b in _hb["down_packed"]]- down_scale = [b.tensor for b in _hb["down_scale"]]- down_global = [b.tensor for b in _hb["down_global"]]-- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline-- def _load(sink) -> int:- tracker = LayerCompletionTracker(E * 6, _hb, sink)- placed = 0- for shard in tqdm(sorted(weight_shards), desc=f"Loading {spec.desc}", disable=not primary):+ wanted[name] = (bank_layer, int(match.group("expert")), spec.proj_to_role[proj] + _kind_suffix(kind))+ expected = _num_moe_layers(config) * config.num_experts * 9+ if len(wanted) != expected:+ raise ValueError(f"{spec.desc}: found {len(wanted)} expert tensors, expected {expected}")++ def _serial():+ by_shard: dict[str, list[str]] = collections.defaultdict(list)+ for name, shard in weight_map.items():+ if name in wanted:+ by_shard[shard].append(name)+ for shard in tqdm(sorted(by_shard), desc=f"Loading {spec.desc}", disable=not primary):path = os.path.join(folder, shard)+ drop(path)with safetensors.safe_open(path, framework="pt", device="cpu") as f:- for name, match, bank_layer_id in weight_shards[shard]:- layer = int(match.group("layer"))- expert = int(match.group("expert"))- proj = match.group("proj")- role = spec.proj_to_role[proj]- kind = _canon_kind(spec, match.group("kind"))+ for name in by_shard[shard]:tensor = f.get_tensor(name)- if kind == "weight":- if role == "gate":- gate_up_packed[bank_layer_id][expert, :I] = tensor- elif role == "up":- gate_up_packed[bank_layer_id][expert, I:] = tensor- elif role == "down":- down_packed[bank_layer_id][expert] = tensor- else:- raise ValueError(f"{spec.desc}: unknown projection role {role!r}")- else:- global_scale = globals_map[(layer, expert, proj)]- if role == "gate":- gate_up_scale[bank_layer_id][expert, :I] = tensor- gate_up_global[bank_layer_id][expert, :I] = global_scale- elif role == "up":- gate_up_scale[bank_layer_id][expert, I:] = tensor- gate_up_global[bank_layer_id][expert, I:] = global_scale- elif role == "down":- down_scale[bank_layer_id][expert] = tensor- down_global[bank_layer_id][expert] = global_scale- else:- raise ValueError(f"{spec.desc}: unknown projection role {role!r}")- tracker.note(bank_layer_id)- placed += 1- drop_page_cache(path)- return placed-- if layer_sink is not None:- placed = _load(layer_sink)- else:- with PinPipeline() as pins:- placed = _load(pins)-- expected = num_layers * E * 6- assert placed == expected, f"{spec.desc}: loaded {placed} expert tensors, expected {expected}"- return {- "gate_up_packed": gate_up_packed,- "gate_up_scale": gate_up_scale,- "gate_up_global": gate_up_global,- "down_packed": down_packed,- "down_scale": down_scale,- "down_global": down_global,- }---def load_nvfp4_expert_source_banks_parallel(- model_path: str,- config,- spec: Nvfp4ExpertSourceSpec,- *,- drop_page_cache: DropPageCache,- primary: bool,- workers: int = 8,- chunk: int = 8 << 20,- layer_sink=None,-) -> dict[str, list[torch.Tensor]]:- """parallel counterpart of :func:`load_nvfp4_expert_source_banks`, byte-for-byte same- placement. bulk weight/weight_scale read via chunked multi-threaded O_DIRECT reader- (iter_expert_tensors_parallel); tiny globals (``weight_scale_2``) stay serial (negligible- bytes). ``layer_sink``: see :func:`load_nvfp4_expert_source_banks`."""- from freetoken.models.weight import iter_expert_tensors_parallel-- folder = download_hf_weight(model_path)- with open(os.path.join(folder, "model.safetensors.index.json"), encoding="utf-8") as f:- weight_map = json.load(f)["weight_map"]-- E = config.num_experts- H = config.hidden_size- I = config.moe_intermediate_size- num_layers = _num_moe_layers(config)-- weight_info: dict[str, tuple[re.Match[str], int]] = {} # name -> (match, bank_layer)- global_names_by_shard: dict[str, list[str]] = collections.defaultdict(list)- for name, shard in weight_map.items():- match = spec.key_pattern.match(name)- if match is None:- continue- bank_layer = _bank_layer(spec, int(match.group("layer")), config)- if bank_layer is None:- continue- kind = _canon_kind(spec, match.group("kind"))- if kind == "weight_scale_2":- global_names_by_shard[shard].append(name)- elif kind in {"weight", "weight_scale"}:- weight_info[name] = (match, bank_layer)- else:- raise ValueError(f"{spec.desc}: unknown NVFP4 expert tensor kind {kind!r}")-- # Pass 1: tiny per-tensor global scales (serial; data is scalar-per-expert).- globals_map: dict[tuple[int, int, str], torch.Tensor] = {}- for shard in sorted(global_names_by_shard):- path = os.path.join(folder, shard)- drop_page_cache(path)- with safetensors.safe_open(path, framework="pt", device="cpu") as f:- for name in global_names_by_shard[shard]:- m = spec.key_pattern.match(name)- globals_map[(int(m.group("layer")), int(m.group("expert")), m.group("proj"))] = (- _ingest_global(spec, f.get_tensor(name))- )- drop_page_cache(path)-- _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill- gate_up_packed = [b.tensor for b in _hb["gate_up_packed"]]- gate_up_scale = [b.tensor for b in _hb["gate_up_scale"]]- gate_up_global = [b.tensor for b in _hb["gate_up_global"]]- down_packed = [b.tensor for b in _hb["down_packed"]]- down_scale = [b.tensor for b in _hb["down_scale"]]- down_global = [b.tensor for b in _hb["down_global"]]-- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline+ if wanted[name][2].endswith("_global"):+ tensor = _ingest_global(spec, tensor)+ yield name, tensor+ drop(path)- # Pass 2: bulk weight/weight_scale via the common parallel reader; place by name.- def _load(sink) -> int:- tracker = LayerCompletionTracker(E * 6, _hb, sink)- placed = 0- for name, tensor in iter_expert_tensors_parallel(- folder, lambda n: n in weight_info, workers=workers, chunk=chunk- ):- match, bank_layer_id = weight_info[name]- layer = int(match.group("layer"))- expert = int(match.group("expert"))- proj = match.group("proj")- role = spec.proj_to_role[proj]- kind = _canon_kind(spec, match.group("kind"))- if kind == "weight":- if role == "gate":- gate_up_packed[bank_layer_id][expert, :I] = tensor- elif role == "up":- gate_up_packed[bank_layer_id][expert, I:] = tensor- else:- down_packed[bank_layer_id][expert] = tensor- else:- g = globals_map[(layer, expert, proj)]- if role == "gate":- gate_up_scale[bank_layer_id][expert, :I] = tensor- gate_up_global[bank_layer_id][expert, :I] = g- elif role == "up":- gate_up_scale[bank_layer_id][expert, I:] = tensor- gate_up_global[bank_layer_id][expert, I:] = g- else:- down_scale[bank_layer_id][expert] = tensor- down_global[bank_layer_id][expert] = g- tracker.note(bank_layer_id)- placed += 1- return placed+ def _parallel():+ from freetoken.models.weight import iter_expert_tensors_parallel- if layer_sink is not None:- placed = _load(layer_sink)- else:- with PinPipeline() as pins:- placed = _load(pins)+ for name, tensor in iter_expert_tensors_parallel(folder, lambda n: n in wanted, workers=workers, chunk=chunk):+ if wanted[name][2].endswith("_global"):+ tensor = _ingest_global(spec, tensor)+ yield name, tensor- expected = num_layers * E * 6- assert placed == expected, f"{spec.desc}: loaded {placed} expert tensors, expected {expected}"- return {- "gate_up_packed": gate_up_packed,- "gate_up_scale": gate_up_scale,- "gate_up_global": gate_up_global,- "down_packed": down_packed,- "down_scale": down_scale,- "down_global": down_global,- }+ return per_expert_pieces(_parallel() if parallel else _serial(), wanted.get, tensors_per_expert=9)-__all__ = [- "Nvfp4ExpertSourceSpec",- "load_nvfp4_expert_source_banks",- "load_nvfp4_expert_source_banks_parallel",-]+__all__ = ["Nvfp4ExpertSourceSpec", "iter_nvfp4_expert_pieces"]diff --git a/python/freetoken/models/quant_linear.py b/python/freetoken/models/quant_linear.pydeleted file mode 100644index 6ba67a9e..00000000--- a/python/freetoken/models/quant_linear.py+++ /dev/null@@ -1,78 +0,0 @@-"""Quant-aware dense-linear factories, shared by the models that serve quantized-dense projections (qwen3_5_moe, muse_glimmer).--Maps the model's quant config (``expert_quant`` for the dense MLP / shared-expert path,-``attn_quant`` for attention + GatedDeltaNet projections) to the right ``BaseOP`` linear:-block-FP8, per-tensor-FP8 and NVFP4 implementations live under ``freetoken.kernel.triton``;-the bf16 fallback is the framework's TP-aware ``freetoken.layers``. Only the *dispatch*-(config -> layer class) lives here.-"""--from __future__ import annotations---def make_col_merged_quant(expert_quant: str, attn_quant: str, in_f: int,- output_sizes: list[int], has_bias: bool = False):- """Column-merged linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""- if expert_quant == "fp8_block":- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged-- return Fp8BlockColMerged(in_f, output_sizes, has_bias)- if attn_quant == "fp8_pertensor":- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged-- return Fp8PerTensorColMerged(in_f, output_sizes, has_bias)- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention (q/k/v fused)- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged-- return Nvfp4DenseColMerged(in_f, output_sizes, has_bias)- from freetoken.layers import LinearColParallelMerged-- return LinearColParallelMerged(in_f, output_sizes, has_bias=has_bias)---def make_replicated_quant(expert_quant: str, attn_quant: str, in_f: int, out_f: int,- has_bias: bool = False):- """Replicated linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""- if expert_quant == "fp8_block":- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockLinear-- return Fp8BlockLinear(in_f, out_f, has_bias)- if attn_quant == "fp8_pertensor":- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear-- return Fp8PerTensorLinear(in_f, out_f, has_bias)- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention o_proj / GDN out_proj- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseLinear-- return Nvfp4DenseLinear(in_f, out_f, has_bias)- from freetoken.layers import LinearReplicated-- return LinearReplicated(in_f, out_f, has_bias=has_bias)---def make_replicated(config, in_f: int, out_f: int, has_bias: bool = False):- """Config-driven replicated linear: ``Fp8BlockLinear`` under block-fp8, ``Fp8PerTensorLinear``- under per-tensor-fp8 attention, ``Nvfp4DenseLinear`` under nvfp4, else ``LinearReplicated``."""- return make_replicated_quant(- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),- in_f, out_f, has_bias,- )---def make_col_merged(config, in_f: int, output_sizes: list[int], has_bias: bool = False):- """Config-driven column-merged linear: ``Fp8BlockColMerged`` under block-fp8,- ``Fp8PerTensorColMerged`` under per-tensor-fp8 attention, ``Nvfp4DenseColMerged`` under- nvfp4, else ``LinearColParallelMerged``."""- return make_col_merged_quant(- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),- in_f, output_sizes, has_bias,- )---__all__ = [- "make_col_merged_quant",- "make_replicated_quant",- "make_replicated",- "make_col_merged",-]diff --git a/python/freetoken/models/qwen2/attention.py b/python/freetoken/models/qwen2/attention.pyindex 5ac6c2c0..60bb2624 100644--- a/python/freetoken/models/qwen2/attention.py+++ b/python/freetoken/models/qwen2/attention.py@@ -22,6 +22,7 @@ class Qwen2Attention(BaseOP):*,has_attn_bias: bool = False,has_qk_norm: bool = False,+ prefix: str = "",):head_dim = config.head_dimself.layer_id = layer_id@@ -37,6 +38,8 @@ class Qwen2Attention(BaseOP):num_qo_heads=config.num_qo_heads,num_kv_heads=config.num_kv_heads,has_bias=has_attn_bias,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",)if has_qk_norm:self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)@@ -59,6 +62,8 @@ class Qwen2Attention(BaseOP):head_dim * config.num_qo_heads,config.hidden_size,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.o_proj",)@nvtx_annotate("MHA")diff --git a/python/freetoken/models/qwen2/model.py b/python/freetoken/models/qwen2/model.pyindex 762529da..ed56fde2 100644--- a/python/freetoken/models/qwen2/model.py+++ b/python/freetoken/models/qwen2/model.py@@ -16,9 +16,9 @@ if TYPE_CHECKING:class Qwen2DecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = Qwen2Attn(config, layer_id, has_qk_norm=False, has_attn_bias=True)- self.mlp = Qwen2MLP(config)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = Qwen2Attn(config, layer_id, has_qk_norm=False, has_attn_bias=True, prefix=f"{prefix}.self_attn")+ self.mlp = Qwen2MLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")self.input_layernorm = RMSNormFused(size=config.hidden_size,eps=config.rms_norm_eps,@@ -42,13 +42,16 @@ class Qwen2DecoderLayer(BaseOP):class Qwen2Model(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [Qwen2DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ Qwen2DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size,@@ -71,6 +74,8 @@ class Qwen2ForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/qwen3/attention.py b/python/freetoken/models/qwen3/attention.pyindex 74b6891e..039dc50d 100644--- a/python/freetoken/models/qwen3/attention.py+++ b/python/freetoken/models/qwen3/attention.py@@ -22,6 +22,7 @@ class Qwen3Attention(BaseOP):*,has_attn_bias: bool = False,has_qk_norm: bool = False,+ prefix: str = "",):head_dim = config.head_dimself.layer_id = layer_id@@ -37,6 +38,8 @@ class Qwen3Attention(BaseOP):num_qo_heads=config.num_qo_heads,num_kv_heads=config.num_kv_heads,has_bias=has_attn_bias,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",)if has_qk_norm:self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)@@ -59,6 +62,8 @@ class Qwen3Attention(BaseOP):head_dim * config.num_qo_heads,config.hidden_size,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.o_proj",)@nvtx_annotate("MHA")diff --git a/python/freetoken/models/qwen3/model.py b/python/freetoken/models/qwen3/model.pyindex d8becaa0..dc35930d 100644--- a/python/freetoken/models/qwen3/model.py+++ b/python/freetoken/models/qwen3/model.py@@ -16,9 +16,9 @@ if TYPE_CHECKING:class Qwen3DecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True)- self.mlp = Qwen3MLP(config)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True, prefix=f"{prefix}.self_attn")+ self.mlp = Qwen3MLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")self.input_layernorm = RMSNormFused(size=config.hidden_size,eps=config.rms_norm_eps,@@ -42,13 +42,16 @@ class Qwen3DecoderLayer(BaseOP):class Qwen3Model(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [Qwen3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ Qwen3DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size,@@ -71,6 +74,8 @@ class Qwen3ForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.pyindex 98936e9f..8620ce9c 100644--- a/python/freetoken/models/qwen3_5_moe/__init__.py+++ b/python/freetoken/models/qwen3_5_moe/__init__.py@@ -1,19 +1,12 @@from .config import parse_configfrom .model import Qwen3_5MoEForCausalLM-from .weight import (- iter_weights,- iter_weights_parallel,- load_nvfp4_expert_sources,- load_nvfp4_expert_sources_parallel,- setup_offload_expert_banks,-)+from .weight import iter_expert_pieces, iter_weights, iter_weights_parallel, nvfp4_expert_spec__all__ = ["Qwen3_5MoEForCausalLM","parse_config","iter_weights","iter_weights_parallel",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel",- "setup_offload_expert_banks",+ "iter_expert_pieces",+ "nvfp4_expert_spec",]diff --git a/python/freetoken/models/qwen3_5_moe/attention.py b/python/freetoken/models/qwen3_5_moe/attention.pyindex 2421264e..12a91bb3 100644--- a/python/freetoken/models/qwen3_5_moe/attention.py+++ b/python/freetoken/models/qwen3_5_moe/attention.py@@ -4,12 +4,10 @@ from typing import TYPE_CHECKINGimport torchfrom freetoken.core import get_global_ctx-from freetoken.layers import BaseOP, GemmaRMSNorm+from freetoken.layers import BaseOP, GemmaRMSNorm, LinearColParallelMerged, LinearReplicatedfrom freetoken.layers.rotary import get_ropefrom freetoken.utils import nvtx_annotate-from .quant_linear import make_col_merged, make_replicated-if TYPE_CHECKING:from freetoken.models.config import ModelConfig@@ -27,7 +25,7 @@ class Qwen3_5Attention(BaseOP):column/row-parallel for tensor parallelism later."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):head_dim = config.head_dimself.layer_id = layer_idself.num_q = config.num_qo_heads@@ -39,10 +37,10 @@ class Qwen3_5Attention(BaseOP):# Fused q/k/v projection (one GEMM instead of three); q half is 2x for the# output gate. Split sizes: [num_q*head_dim*2, num_kv*head_dim, num_kv*head_dim].self._qkv_split = [self.num_q * head_dim * 2, self.kv_attn_dim, self.kv_attn_dim]- # Block-fp8 (Fp8BlockColMerged) when the checkpoint is quantized, else bf16- # LinearColParallelMerged. q/k/v out dims are all /128, so the merged fp8 weight +- # weight_scale_inv concatenate cleanly along the output dim.- self.qkv_proj = make_col_merged(config, config.hidden_size, self._qkv_split, has_bias=False)+ self.qkv_proj = LinearColParallelMerged(+ config.hidden_size, self._qkv_split, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.qkv_proj",+ )# Qwen3.5 uses Gemma-style (1+weight) RMSNorm; the weight loader bakes the +1# into the stored weight (GemmaRMSNorm scales by the raw weight).self.q_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps)@@ -58,7 +56,10 @@ class Qwen3_5Attention(BaseOP):else None),)- self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False)+ self.o_proj = LinearReplicated(+ self.qo_attn_dim, config.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.o_proj",+ )def _project(self, x: torch.Tensor):"""Returns (q, k, v, gate): q [N, num_q, head_dim] post qk-norm+rope,diff --git a/python/freetoken/models/qwen3_5_moe/config.py b/python/freetoken/models/qwen3_5_moe/config.pyindex 2ac4b607..dbcabc74 100644--- a/python/freetoken/models/qwen3_5_moe/config.py+++ b/python/freetoken/models/qwen3_5_moe/config.py@@ -44,7 +44,7 @@ def _expert_quant(hf_config: Any) -> str:offload cache). The nvidia/modelopt checkpoints are either plain NVFP4 (``quant_algo````NVFP4``) or ``MIXED_PRECISION`` (per-layer ``quantized_layers`` map); in the mixedcase the routed experts carry their own ``W4A16_NVFP4``/``FP8`` algo. Dense quantized- weights (attention/shared-expert/lm_head) are handled separately by dequant-at-load."""+ weights (attention / shared expert / lm_head) are served through their own schemes."""get = _quant_accessor(hf_config)if get is None:return "none"@@ -109,7 +109,7 @@ def _attn_quant(hf_config: Any) -> str:``MIXED_PRECISION`` checkpoints tag ``self_attn.{q,k,v,o}_proj`` and``linear_attn.{in_proj_qkv,in_proj_z,out_proj}`` with ``quant_algo`` ``FP8`` (fp8-e4m3weight + a scalar ``weight_scale``; W8A16). Returns ``"fp8_pertensor"`` when present,- else ``"none"`` (NVFP4 dense weights -- shared_expert/lm_head -- stay dequant-at-load)."""+ else ``"none"`` (NVFP4 dense weights are covered by ``dense_quant`` / ``lm_head_quant``)."""get = _quant_accessor(hf_config)if get is None:return "none"diff --git a/python/freetoken/models/qwen3_5_moe/gdn.py b/python/freetoken/models/qwen3_5_moe/gdn.pyindex 2e732005..d5fe2aa8 100644--- a/python/freetoken/models/qwen3_5_moe/gdn.py+++ b/python/freetoken/models/qwen3_5_moe/gdn.py@@ -4,13 +4,10 @@ import torchimport torch.nn.functional as Ffrom freetoken.core import get_global_ctxfrom freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen-from freetoken.layers import BaseOP, LinearColParallelMerged--from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged-from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged+from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicated+from freetoken.layers.quantization import QuantConfigfrom .gdn_kernels import gdn_decode_fla, gdn_prefill_chunk_fla-from .quant_linear import make_replicated_quantclass _DepthwiseConv1d(BaseOP):@@ -20,26 +17,6 @@ class _DepthwiseConv1d(BaseOP):self.weight = torch.empty(conv_dim, 1, kernel)-class _GatedRMSNorm(BaseOP):- """RMSNorm of x followed by a silu(z) gate (HF Qwen3_5MoeRMSNormGated).-- Uses the fused fla ``rms_norm_gated`` triton kernel (norm(x) * silu(z) in one- kernel) instead of the unfused pow/mean/rsqrt/mul/silu chain, matching sglang's- ``RMSNormGated`` -- collapses ~8 elementwise kernels per GDN layer into one."""-- def __init__(self, dim: int, eps: float):- self.weight = torch.empty(dim)- self.eps = eps-- def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:- from freetoken.kernel.fla import rms_norm_gated-- return rms_norm_gated(- x=x, weight=self.weight, bias=None, z=z, eps=self.eps,- is_rms_norm=True, norm_before_gate=True, activation="silu",- )--class Qwen3_5GatedDeltaNet(BaseOP):"""GatedDeltaNet op using the vendored flash-linear-attention triton kernels(``freetoken.kernel.fla``) for the recurrence and a per-request@@ -52,8 +29,8 @@ class Qwen3_5GatedDeltaNet(BaseOP):def __init__(self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,- conv_kernel_size, rms_norm_eps, layer_id, expert_quant: str = "none",- attn_quant: str = "none",+ conv_kernel_size, rms_norm_eps, layer_id, *, quant_config: QuantConfig | None = None,+ prefix: str = "",):self.layer_id = layer_id# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as@@ -71,25 +48,27 @@ class Qwen3_5GatedDeltaNet(BaseOP):self.value_dim = num_v_heads * head_v_dimself.conv_dim = 2 * self.key_dim + self.value_dimself.conv_kernel_size = conv_kernel_size- # qkv|z carry a weight scale (block-fp8 weight_scale_inv, or per-tensor FP8- # weight_scale); b|a stay bf16. Both quant modes therefore split the four-way- # fusion into an fp8 qkvz GEMM + a bf16 ba GEMM (matches sglang/vLLM).- self._block_fp8 = expert_quant == "fp8_block"- self._pertensor_fp8 = attn_quant == "fp8_pertensor"- self._fp8 = self._block_fp8 or self._pertensor_fp8+ # quantized checkpoints quantize qkv|z but not b|a, so the fusion splits into a qkvz GEMM and a ba GEMM with their own schemes (matches sglang / vLLM)+ self._split_in_proj = (+ quant_config is not None and quant_config.scheme_for(f"{prefix}.in_proj_qkvz") is not None+ )self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads]- if self._fp8:- ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged- self.in_proj_qkvz = ColMerged(- hidden_size, [self.conv_dim, self.value_dim], has_bias=False+ if self._split_in_proj:+ self.in_proj_qkvz = LinearColParallelMerged(+ hidden_size, [self.conv_dim, self.value_dim], has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.in_proj_qkvz",)self.in_proj_ba = LinearColParallelMerged(- hidden_size, [num_v_heads, num_v_heads], has_bias=False+ hidden_size, [num_v_heads, num_v_heads], has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.in_proj_ba",)else:# Fused input projection (one GEMM instead of four): qkv | z | b | a.- self.in_proj = LinearColParallelMerged(hidden_size, self._in_proj_split, has_bias=False)+ self.in_proj = LinearColParallelMerged(+ hidden_size, self._in_proj_split, has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.in_proj",+ )self.conv1d = _DepthwiseConv1d(self.conv_dim, conv_kernel_size)# Recurrence-gating params kept in fp32 (exp/softplus is precision-sensitive,# and the fla kernel reads them as fp32) -- matches HF/sglang, and avoids a@@ -97,12 +76,10 @@ class Qwen3_5GatedDeltaNet(BaseOP):# *.A_log / *.dt_bias from the model-dtype downcast.self.dt_bias = torch.empty(num_v_heads, dtype=torch.float32)self.A_log = torch.empty(num_v_heads, dtype=torch.float32)- self.norm = _GatedRMSNorm(head_v_dim, eps=rms_norm_eps)- # out_proj follows the checkpoint quant: block-fp8 / per-tensor-fp8 / compressed-tensors- # NVFP4 (W4A16) / bf16. in_proj_* stay bf16 in every mode (above), so a compressed-tensors- # NVFP4 checkpoint (attn_quant=="nvfp4") only makes out_proj native FP4.- self.out_proj = make_replicated_quant(- expert_quant, attn_quant, self.value_dim, hidden_size, has_bias=False+ self.norm = GatedRMSNorm(head_v_dim, eps=rms_norm_eps)+ self.out_proj = LinearReplicated(+ self.value_dim, hidden_size, has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.out_proj",)def _gate_params(self, a: torch.Tensor, b: torch.Tensor):@@ -161,7 +138,7 @@ class Qwen3_5GatedDeltaNet(BaseOP):fla = build_fla_metadata(batch, hidden_states.device)batch.fla_metadata = fla- if self._fp8:+ if self._split_in_proj:qkvz = self.in_proj_qkvz.forward(hidden_states)conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1)ba = self.in_proj_ba.forward(hidden_states)diff --git a/python/freetoken/models/qwen3_5_moe/model.py b/python/freetoken/models/qwen3_5_moe/model.pyindex eba7fd24..d34793ad 100644--- a/python/freetoken/models/qwen3_5_moe/model.py+++ b/python/freetoken/models/qwen3_5_moe/model.py@@ -27,7 +27,7 @@ class Qwen3_5DecoderLayer(BaseOP):where the mixer is a GatedDeltaNet (linear layers) or gated attention (full layers).All norms are Gemma-style (1+weight)."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):self._layer_id = layer_idself._is_linear = config.is_linear_layer(layer_id)if self._is_linear:@@ -42,14 +42,18 @@ class Qwen3_5DecoderLayer(BaseOP):conv_kernel_size=g.conv_kernel_dim,rms_norm_eps=config.rms_norm_eps,layer_id=layer_id,- expert_quant=config.expert_quant,- attn_quant=config.attn_quant,+ quant_config=config.quant,+ prefix=f"{prefix}.linear_attn",)else:- self.self_attn = Qwen3_5Attention(config, layer_id)+ self.self_attn = Qwen3_5Attention(config, layer_id, prefix=f"{prefix}.self_attn")# Dense variants (num_experts==0, e.g. Qwen3.6-27B) use a plain SwiGLU MLP instead of# the routed MoE block; both expose ``forward(hidden)->hidden`` and the same key prefix.- self.mlp = Qwen3_5MoE(config, layer_id) if config.moe_enabled else Qwen3_5DenseMLP(config)+ self.mlp = (+ Qwen3_5MoE(config, layer_id, prefix=f"{prefix}.mlp")+ if config.moe_enabled+ else Qwen3_5DenseMLP(config, prefix=f"{prefix}.mlp")+ )self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)@@ -69,13 +73,16 @@ class Qwen3_5DecoderLayer(BaseOP):class Qwen3_5Model(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [Qwen3_5DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ Qwen3_5DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)@@ -91,22 +98,14 @@ class Qwen3_5Model(BaseOP):class Qwen3_5MoEForCausalLM(BaseLLMModel):def __init__(self, config: ModelConfig):self.model = Qwen3_5Model(config)- if getattr(config, "lm_head_quant", "none") == "nvfp4":- # checkpoint stores the (untied) lm_head as NVFP4: keep it native (W4A16) -- the- # bf16 dequant of this ~1 GB matrix was the single largest decode kernel.- from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead-- assert not config.tie_word_embeddings, "NVFP4 lm_head assumes untied embeddings"- self.lm_head = Nvfp4LMHead(- num_embeddings=config.vocab_size, embedding_dim=config.hidden_size- )- else:- self.lm_head = ParallelLMHead(- num_embeddings=config.vocab_size,- embedding_dim=config.hidden_size,- tie_word_embeddings=config.tie_word_embeddings,- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,- )+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",+ )super().__init__()def forward(self) -> torch.Tensor:diff --git a/python/freetoken/models/qwen3_5_moe/moe.py b/python/freetoken/models/qwen3_5_moe/moe.pyindex b5ab1cf3..766059e5 100644--- a/python/freetoken/models/qwen3_5_moe/moe.py+++ b/python/freetoken/models/qwen3_5_moe/moe.py@@ -12,8 +12,6 @@ from freetoken.layers import (silu_and_mul,)-from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged, Fp8BlockLinear-if TYPE_CHECKING:from freetoken.models.config import ModelConfig@@ -21,25 +19,17 @@ if TYPE_CHECKING:class _SharedExpert(BaseOP):"""Always-present shared SwiGLU expert of width ``shared_expert_intermediate_size``."""- def __init__(self, config: ModelConfig, hidden_size: int, intermediate_size: int):- if getattr(config, "expert_quant", "none") == "fp8_block":- self.gate_up_proj = Fp8BlockColMerged(- hidden_size, [intermediate_size, intermediate_size], has_bias=False- )- self.down_proj = Fp8BlockLinear(intermediate_size, hidden_size, has_bias=False)- elif getattr(config, "dense_quant", "none") == "nvfp4":- # NVFP4 checkpoint: keep the shared expert's NVFP4 weights native (W4A16).- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear-- self.gate_up_proj = Nvfp4DenseColMerged(- hidden_size, [intermediate_size, intermediate_size], has_bias=False- )- self.down_proj = Nvfp4DenseLinear(intermediate_size, hidden_size, has_bias=False)- else:- self.gate_up_proj = LinearColParallelMerged(- hidden_size, [intermediate_size, intermediate_size], has_bias=False- )- self.down_proj = LinearRowParallel(intermediate_size, hidden_size, has_bias=False)+ def __init__(+ self, config: ModelConfig, hidden_size: int, intermediate_size: int, *, prefix: str = ""+ ):+ self.gate_up_proj = LinearColParallelMerged(+ hidden_size, [intermediate_size, intermediate_size], has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.gate_up_proj",+ )+ self.down_proj = LinearRowParallel(+ intermediate_size, hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.down_proj",+ )def forward(self, x: torch.Tensor) -> torch.Tensor:return self.down_proj.forward(silu_and_mul(self.gate_up_proj.forward(x)))@@ -47,13 +37,12 @@ class _SharedExpert(BaseOP):class Qwen3_5DenseMLP(_SharedExpert):"""Dense (non-MoE) SwiGLU MLP for dense Qwen3.x checkpoints (e.g. 27B): ``gate_up_proj``- (fused gate|up) + ``down_proj`` at full ``intermediate_size``. Same structure (and quant- dispatch) as the shared expert -- NVFP4 (W4A16) when ``dense_quant=="nvfp4"``, else bf16 --- so it reuses ``_SharedExpert`` directly and keeps the state-dict keys flat+ (fused gate|up) + ``down_proj`` at full ``intermediate_size``. Same structure as the shared+ expert, so it reuses ``_SharedExpert`` directly and keeps the state-dict keys flat(``...layers.N.mlp.{gate_up_proj,down_proj}``)."""- def __init__(self, config: ModelConfig):- super().__init__(config, config.hidden_size, config.intermediate_size)+ def __init__(self, config: ModelConfig, *, prefix: str = ""):+ super().__init__(config, config.hidden_size, config.intermediate_size, prefix=prefix)class Qwen3_5MoE(BaseOP):@@ -64,19 +53,19 @@ class Qwen3_5MoE(BaseOP):Router softmaxes over all experts, takes top-k, and renormalizes (HF semantics)."""- def __init__(self, config: ModelConfig, layer_id: int | None = None):- weight_format = (- "fp8_block" if getattr(config, "expert_quant", "none") == "fp8_block" else "bf16"- )+ def __init__(self, config: ModelConfig, layer_id: int | None = None, *, prefix: str = ""):self.experts = make_moe_layer(config,layer_id=layer_id,renormalize=config.norm_topk_prob,- weight_format=weight_format,+ quant_config=config.quant,+ prefix=f"{prefix}.experts",)+ # routers stay bf16 whatever the checkpoint quantizesself.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)self.shared_expert = _SharedExpert(- config, config.hidden_size, config.shared_expert_intermediate_size+ config, config.hidden_size, config.shared_expert_intermediate_size,+ prefix=f"{prefix}.shared_expert",)self.shared_expert_gate = LinearReplicated(config.hidden_size, 1, has_bias=False)diff --git a/python/freetoken/models/qwen3_5_moe/quant_linear.py b/python/freetoken/models/qwen3_5_moe/quant_linear.pydeleted file mode 100644index cd0443b6..00000000--- a/python/freetoken/models/qwen3_5_moe/quant_linear.py+++ /dev/null@@ -1,19 +0,0 @@-"""Re-export of the shared quant-aware dense-linear factories (models/quant_linear.py).--The dispatch used to live here; muse_glimmer needing the same NVFP4/bf16 pair promoted-it to a model-agnostic home. Kept as a module so existing imports stay valid.-"""--from freetoken.models.quant_linear import (- make_col_merged,- make_col_merged_quant,- make_replicated,- make_replicated_quant,-)--__all__ = [- "make_col_merged_quant",- "make_replicated_quant",- "make_replicated",- "make_col_merged",-]diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex d07f18cd..45adb48b 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -7,6 +7,7 @@ from typing import Iteratorimport safetensorsimport torch+from freetoken.layers.quantization import QuantKindfrom freetoken.distributed import get_tp_infofrom freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4from freetoken.models.loader import (@@ -18,10 +19,7 @@ from freetoken.models.loader import (iter_weight_files,nvfp4_parts_ct,)-from freetoken.models.nvfp4_banks import (- Nvfp4ExpertSourceSpec,- load_nvfp4_expert_source_banks,-)+from freetoken.models.nvfp4_banks import Nvfp4ExpertSourceSpecfrom freetoken.utils import cached_load_hf_config, download_hf_weightfrom tqdm import tqdm@@ -191,7 +189,7 @@ def iter_weights(if config.expert_quant == "fp8_block":# Dense (attn/GDN/shared-expert) weights are always block-fp8; routed experts are# yielded here only for the resident path (include_moe_experts=True). Under offload- # they are excluded and loaded by setup_offload_expert_banks instead.+ # they are excluded and loaded from expert pieces instead.yield from _iter_weights_fp8(model_path, device,include_non_moe=include_non_moe, include_moe_experts=include_moe_experts,@@ -230,7 +228,7 @@ def iter_weights(with safetensors.safe_open(file, framework="pt", device=str(device)) as f:keyset = set(f.keys())for raw_name in f.keys():- # Per-expert NVFP4 tensors go to the offload cache (load_nvfp4_expert_sources),+ # Per-expert NVFP4 tensors go to the offload cache (expert pieces),# not the dense pass. bf16-base stacked experts (experts.gate_up_proj) have no# ``.mlp.experts.<int>.`` so they are unaffected and still hit _PACKED_EXPERT.if _NVFP4_EXPERT_RE.search(raw_name):@@ -670,7 +668,7 @@ def iter_weights_parallel(Qwen3.5 stores experts pre-fused/pre-stacked per layer (already ``[E, ...]``), so nomerge/stack -- just rename and yield; bank builder places by name as the serial path."""assert include_moe_experts and not include_non_moe, (- "qwen3_5_moe parallel reader is experts-only (used by load_moe_expert_sources)"+ "qwen3_5_moe parallel reader is experts-only (used by the expert piece reader)")from freetoken.models.weight import iter_expert_tensors_parallel@@ -760,7 +758,7 @@ def _iter_weights_fp8(q/k/v -> qkv_proj, GDN in_proj_qkv|z -> in_proj_qkvz (fp8) and in_proj_b|a -> in_proj_ba(bf16), shared_expert gate|up -> gate_up_proj; Gemma (1+w) norms get +1.- Routed experts: skipped under offload (loaded by setup_offload_expert_banks). Under the+ Routed experts: skipped under offload (loaded from expert pieces). Under theresident (non-offload) path ``include_moe_experts`` is True -> per-layer stacked fp8experts for the Fp8ResidentMoE buffers are yielded too."""if get_tp_info().size > 1:@@ -787,20 +785,40 @@ def _iter_weights_fp8(assert not fuse_buf, f"Incomplete fp8 fusions: {sorted(k for k, _ in fuse_buf)}"if include_moe_experts:- # Resident (non-offload) experts: build the stacked fp8 banks once via the shared- # parallel reader (pageable host -- the engine copies the per-layer slices to GPU),- # then yield per-layer views into the Fp8ResidentMoE buffers.+ # resident experts: stack the per-expert pieces into the per-layer tensors the resident MoE method declares (pageable host; the engine copies to GPU)config = parse_config(cached_load_hf_config(model_path))if not config.is_moe:return # dense checkpoint: no routed experts to build as resident banks- L, E, H, I, dense = _moe_dims(config)- banks = _build_fp8_expert_banks(model_path, config, dummy=False, pin=False)- for li in range(L):+ yield from _resident_fp8_experts(model_path, config)+++def _resident_fp8_experts(model_path, config):+ from freetoken.kernel.triton.fp8_block_linear import FP8++ B = 128+ L, E, H, I, dense = _moe_dims(config)+ shapes = {+ "gate_up_proj": ((E, 2 * I, H), FP8),+ "gate_up_scale_inv": ((E, 2 * I // B, H // B), torch.bfloat16),+ "down_proj": ((E, H, I), FP8),+ "down_scale_inv": ((E, H // B, I // B), torch.bfloat16),+ }+ layers: dict[int, dict[str, torch.Tensor]] = {}+ placed = [0] * L+ for li, e0, e1, piece in iter_expert_pieces(model_path, config, "fp8_block", parallel=None):+ stack = layers.setdefault(li, {n: torch.empty(shape, dtype=dt) for n, (shape, dt) in shapes.items()})+ stack["gate_up_proj"][e0:e1, :I] = piece["gate"]+ stack["gate_up_proj"][e0:e1, I:] = piece["up"]+ stack["gate_up_scale_inv"][e0:e1, : I // B] = piece["gate_scale"]+ stack["gate_up_scale_inv"][e0:e1, I // B :] = piece["up_scale"]+ stack["down_proj"][e0:e1] = piece["down"]+ stack["down_scale_inv"][e0:e1] = piece["down_scale"]+ placed[li] += e1 - e0+ if placed[li] == E:pre = f"model.layers.{dense + li}.mlp.experts"- yield f"{pre}.gate_up_proj", banks["gate_up"][li]- yield f"{pre}.gate_up_scale_inv", banks["gate_up_scale"][li]- yield f"{pre}.down_proj", banks["down"][li]- yield f"{pre}.down_scale_inv", banks["down_scale"][li]+ for name, tensor in layers.pop(li).items():+ yield f"{pre}.{name}", tensor+ assert not layers, f"incomplete resident fp8 experts for layers {sorted(layers)}"class _ShardReader:@@ -831,49 +849,6 @@ class _ShardReader:self._handles.clear()-def setup_offload_expert_banks(- model_path: str, model_config, *, device: torch.device, dtype: torch.dtype,- dummy: bool = False, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20,- decode_target: str = "gpu", layer_sink=None,-):- """Build the routed-expert offload banks. The qwen3_5_moe module always exports this hook,- so it intercepts *every* qwen3_5_moe offload load -- defer non-block-fp8 checkpoints (plain- BF16, or modelopt NVFP4) to the generic provider for that ``expert_quant``.-- block-fp8 default ("fp8"): keep experts block-fp8 -- ``gate_up``/``down`` fp8 banks + their- bf16 ``weight_scale_inv`` banks (half the host/cache bytes; routed rows are dequantized on- demand in ``_expert_gemm``). ``FREETOKEN_FP8_EXPERTS=bf16`` instead dequantizes every expert- to bf16 at load (reuses the bf16 offload path; ~2x the memory). Both modes build per-layer- :class:`HostBank` banks (pin-after-fill), so the converter's ``layer_sink`` streams each- completed layer straight through; ``layer_sink`` also reaches the deferred (nvfp4/none)- providers for non-block-fp8 checkpoints.-- ``decode_target`` is forwarded so the cpu backend gets CPU-readable (native, non-- GPU-tiled) bank layouts -- e.g. native ``nvfp4`` rows rather than marlin/b12x."""- eq = getattr(model_config, "expert_quant", "none")- if eq != "fp8_block":- from freetoken.moe.expert_banks import _PROVIDERS # nvfp4 -> _nvfp4_banks, none -> _bf16_banks-- return _PROVIDERS[eq](model_path, model_config, device, dtype, dummy,- parallel=parallel, workers=workers, chunk=chunk,- decode_target=decode_target, layer_sink=layer_sink)- if get_tp_info().size > 1:- raise NotImplementedError("qwen3_5_moe fp8 expert banks support TP=1 only")- from freetoken.moe.expert_banks import ExpertBanks-- mode = os.environ.get("FREETOKEN_FP8_EXPERTS", "fp8").strip().lower()- if mode not in ("fp8", "bf16"):- raise ValueError(f"FREETOKEN_FP8_EXPERTS must be 'fp8' or 'bf16', got {mode!r}")- sink = None if dummy else layer_sink- if mode == "bf16":- return _setup_bf16_dequant_banks(model_path, model_config, device, dummy, layer_sink=sink)- banks = _build_fp8_expert_banks(- model_path, model_config, dummy=dummy, parallel=parallel, workers=workers, chunk=chunk,- pin=True, layer_sink=sink,- )- return ExpertBanks("fp8_block", banks, streamed=sink is not None)--def _moe_dims(model_config):L = model_config.num_moe_layersreturn (@@ -889,214 +864,59 @@ def _expert_reader(model_path, device):return _ShardReader(folder, weight_map, device)-def _build_fp8_expert_banks(- model_path, config, *, dummy: bool, parallel: bool | None = None,- workers: int = 8, chunk: int = 8 << 20, pin: bool = True, layer_sink=None,-) -> dict[str, list[torch.Tensor]]:- """The single block-fp8 routed-expert reader, shared by offload (``pin=True`` -> per-layer- :class:`HostBank` banks, pin-after-fill / streamable) and resident (``pin=False`` -> plain- pageable tensors, never pinned or streamed). Reads via the common chunked multi-threaded- O_DIRECT reader (drops page cache, no per-tensor serial overhead) when the experts are- scattered per-tensor; else a serial shard fallback. Stacks gate|up -> gate_up into one- ``[E, ...]`` tensor per layer per bank.-- Each expert contributes 6 ``place()`` writes per layer ({gate,up,down} x {weight,- weight_scale_inv}), so a layer completes after ``E * 6`` writes. ``layer_sink=None``- (serving) pins each layer's 4 banks as it completes via an internally-owned- :class:`PinPipeline`; ``layer_sink`` given (converter) fires the completion tracker into it- instead (nothing pinned; released banks -- caller owns that tradeoff). ``pin=False`` and the- CUDA-less host stay on the plain materialize path (no pin, no stream)."""- from freetoken.kernel.triton.fp8_block_linear import FP8+def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):+ """Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their+ ``_scale`` (bf16 block ``weight_scale_inv``) companions. Other expert kinds use the generic readers."""+ if kind is not QuantKind.FP8_BLOCK:+ return None+ if get_tp_info().size > 1:+ raise NotImplementedError("qwen3_5_moe fp8 expert banks support TP=1 only")from freetoken.models.weight import experts_scattered, iter_expert_tensors_parallel+ from freetoken.moe.expert_pieces import per_expert_pieces- B = 128L, E, H, I, dense = _moe_dims(config)+ suffix = {"weight": "", "weight_scale_inv": "_scale"}- # 16B-align the per-expert scale rows (Qwen3.8: down_scale is 20x5 bf16 = 200 B) so the- # fused multi-bank copy engages; the GEMMs read scales through explicit strides, so the- # padding is inert. Unconditional: one layout per format, shared with the byte formulas.- from freetoken.moe.offload_cache import fp8_block_scale_pad as _pad_cols-- specs = {- "gate_up": ((E, 2 * I, H), FP8),- "gate_up_scale": ((E, 2 * I // B, _pad_cols(2 * I // B, H // B)), torch.bfloat16),- "down": ((E, H, I), FP8),- "down_scale": ((E, H // B, _pad_cols(H // B, I // B)), torch.bfloat16),- }- hb = None- if pin:- from freetoken.moe.host_banks import alloc_layer_banks-- hb = alloc_layer_banks(specs, L) # lazy anon mmaps (unpinned)- banks = {name: [b.tensor for b in hb[name]] for name in specs}- else: # resident dequant source: plain pageable tensors (never pinned / streamed)- banks = {name: [torch.empty(shape, dtype=dt) for _ in range(L)] for name, (shape, dt) in specs.items()}- gate_up, gate_up_scale, down, down_scale = (- banks["gate_up"], banks["gate_up_scale"], banks["down"], banks["down_scale"]- )- if dummy:- for li in range(L):- gate_up[li].view(torch.uint8).random_(0, 16) # small fp8 codes (avoid NaN/inf)- down[li].view(torch.uint8).random_(0, 16)- gate_up_scale[li].fill_(1.0)- down_scale[li].fill_(1.0)- if hb is not None and torch.cuda.is_available():- from freetoken.moe.host_banks import pin_banks-- pin_banks(hb) # pin-after-fill (match the other dummies)- return banks-- def place(raw_name: str, t: torch.Tensor) -> int | None:+ def locate(raw_name: str):m = _FP8_EXPERT_RE.match(raw_name)if m is None:return None- li, e = int(m["layer"]) - dense, int(m["expert"])- proj, kind = m["proj"], m["kind"]- if kind == "weight":- (gate_up[li][e, :I] if proj == "gate" else- gate_up[li][e, I:] if proj == "up" else down[li][e]).copy_(t)- else: # weight_scale_inv- (gate_up_scale[li][e, : I // B, : H // B] if proj == "gate" else- gate_up_scale[li][e, I // B :, : H // B] if proj == "up" else- down_scale[li][e, :, : I // B]).copy_(t)- return li+ li = int(m["layer"]) - dense+ if not 0 <= li < L:+ raise ValueError(f"unexpected routed-expert layer in {raw_name}")+ return li, int(m["expert"]), m["proj"] + suffix[m["kind"]]if parallel is None:parallel = experts_scattered(model_path)+ if parallel:+ tensors = iter_expert_tensors_parallel(+ model_path, lambda n: _FP8_EXPERT_RE.match(n) is not None, workers=workers, chunk=chunk+ )+ return per_expert_pieces(tensors, locate, tensors_per_expert=6)- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline-- def _load(sink) -> None:- # {gate,up,down} x {weight, weight_scale_inv} per expert -> E*6 writes/layer.- tracker = LayerCompletionTracker(E * 6, hb, sink) if sink is not None else None- if parallel:- for raw_name, t in iter_expert_tensors_parallel(- model_path, lambda n: _FP8_EXPERT_RE.match(n) is not None, workers=workers, chunk=chunk- ):- li = place(raw_name, t)- if tracker is not None and li is not None:- tracker.note(li)- else:- reader = _expert_reader(model_path, torch.device("cpu"))- primary = get_tp_info().is_primary()- try:- for li in tqdm(range(L), desc="Loading fp8 experts (serial)", disable=not primary):- layer = dense + li- for e in range(E):- p = f"model.language_model.layers.{layer}.mlp.experts.{e}"- for proj in ("gate", "up", "down"):- for kind in ("weight", "weight_scale_inv"):- key = f"{p}.{proj}_proj.{kind}"- place(key, reader.get(key))- if tracker is not None:- tracker.note(li)- finally:- reader.close()-- if not pin:- assert layer_sink is None, "pin=False (resident source) cannot stream to a layer_sink"- _load(None)- elif layer_sink is not None:- _load(layer_sink)- elif torch.cuda.is_available():- with PinPipeline() as pins:- _load(pins)- else:- _load(None) # CUDA-less: mmap banks stay pageable, never pinned- return banks---def _setup_bf16_dequant_banks(model_path, model_config, device, dummy: bool, *, layer_sink=None):- from freetoken.models.weight import dummy_moe_expert_sources- from freetoken.moe.expert_banks import ExpertBanks-- if dummy:- gate_up, down = dummy_moe_expert_sources(model_config, dtype=torch.bfloat16)- return ExpertBanks("bf16", {"gate_up": gate_up, "down": down})-- from freetoken.kernel.triton.fp8_block_linear import dequant_block_fp8- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks-- L, E, H, I, dense = _moe_dims(model_config)- specs = {"gate_up": ((E, 2 * I, H), torch.bfloat16), "down": ((E, H, I), torch.bfloat16)}- hb = alloc_layer_banks(specs, L) # lazy anon mmaps (unpinned)- banks = {name: [b.tensor for b in hb[name]] for name in specs}- gate_up, down = banks["gate_up"], banks["down"]- reader = _expert_reader(model_path, device)- primary = get_tp_info().is_primary()-- def _deq(prefix: str) -> torch.Tensor:- return dequant_block_fp8(reader.get(f"{prefix}.weight"), reader.get(f"{prefix}.weight_scale_inv"))-- def _load(sink) -> None:- # Whole-layer gate_up + down copies -> 2 writes/layer.- tracker = LayerCompletionTracker(2, hb, sink) if sink is not None else None+ def _serial():+ reader = _expert_reader(model_path, torch.device("cpu"))try:- for li in tqdm(range(L), desc="Loading fp8 experts (dequant->bf16)", disable=not primary):- layer = dense + li- gu_rows = torch.empty(E, 2 * I, H, dtype=torch.bfloat16, device=device)- dn_rows = torch.empty(E, H, I, dtype=torch.bfloat16, device=device)+ for li in tqdm(range(L), desc="Loading fp8 experts (serial)", disable=not get_tp_info().is_primary()):for e in range(E):- p = f"model.language_model.layers.{layer}.mlp.experts.{e}"- gu_rows[e, :I] = _deq(f"{p}.gate_proj")- gu_rows[e, I:] = _deq(f"{p}.up_proj")- dn_rows[e] = _deq(f"{p}.down_proj")- gate_up[li].copy_(gu_rows)- if tracker is not None:- tracker.note(li)- down[li].copy_(dn_rows)- if tracker is not None:- tracker.note(li)+ base = f"model.language_model.layers.{dense + li}.mlp.experts.{e}"+ for proj in ("gate", "up", "down"):+ for kind, suf in suffix.items():+ name = f"{base}.{proj}_proj.{kind}"+ yield name, reader.get(name)finally:reader.close()- if layer_sink is not None:- _load(layer_sink)- elif torch.cuda.is_available():- with PinPipeline() as pins:- _load(pins)- else:- _load(None) # CUDA-less: mmap banks stay pageable, never pinned- return ExpertBanks("bf16", banks, streamed=layer_sink is not None)---def load_nvfp4_expert_sources(- model_path: str, config, *, layer_sink=None-) -> dict[str, torch.Tensor]:- """Build the CPU NVFP4 expert source banks for the offload cache (gate/up fused on the- output-row axis, down separate; weight_scale_2 carried as the per-row global scale)."""- return load_nvfp4_expert_source_banks(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- layer_sink=layer_sink,- )+ return per_expert_pieces(_serial(), locate, tensors_per_expert=6)-def load_nvfp4_expert_sources_parallel(- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None-):- """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel-- return load_nvfp4_expert_source_banks_parallel(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- workers=workers,- chunk=chunk,- layer_sink=layer_sink,- )+def nvfp4_expert_spec(model_path: str, config):+ return _NVFP4_SOURCE_SPEC__all__ = ["iter_weights","iter_weights_parallel",- "setup_offload_expert_banks",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel",+ "iter_expert_pieces",+ "nvfp4_expert_spec",]diff --git a/python/freetoken/models/qwen3_moe/attention.py b/python/freetoken/models/qwen3_moe/attention.pyindex d1fef63c..8a737b27 100644--- a/python/freetoken/models/qwen3_moe/attention.py+++ b/python/freetoken/models/qwen3_moe/attention.py@@ -22,6 +22,7 @@ class Qwen3MoeAttention(BaseOP):*,has_attn_bias: bool = False,has_qk_norm: bool = False,+ prefix: str = "",):head_dim = config.head_dimself.layer_id = layer_id@@ -37,6 +38,8 @@ class Qwen3MoeAttention(BaseOP):num_qo_heads=config.num_qo_heads,num_kv_heads=config.num_kv_heads,has_bias=has_attn_bias,+ quant_config=config.quant,+ prefix=f"{prefix}.qkv_proj",)if has_qk_norm:self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)@@ -59,6 +62,8 @@ class Qwen3MoeAttention(BaseOP):head_dim * config.num_qo_heads,config.hidden_size,has_bias=False,+ quant_config=config.quant,+ prefix=f"{prefix}.o_proj",)@nvtx_annotate("MHA")diff --git a/python/freetoken/models/qwen3_moe/model.py b/python/freetoken/models/qwen3_moe/model.pyindex e37437d2..4918092e 100644--- a/python/freetoken/models/qwen3_moe/model.py+++ b/python/freetoken/models/qwen3_moe/model.py@@ -17,9 +17,9 @@ if TYPE_CHECKING:class Qwen3DecoderLayer(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int):- self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True)- self.mlp = Qwen3MLP(config, layer_id)+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):+ self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True, prefix=f"{prefix}.self_attn")+ self.mlp = Qwen3MLP(config, layer_id, prefix=f"{prefix}.mlp")self.input_layernorm = RMSNormFused(size=config.hidden_size,eps=config.rms_norm_eps,@@ -43,13 +43,16 @@ class Qwen3DecoderLayer(BaseOP):class Qwen3Model(BaseOP):- def __init__(self, config: ModelConfig):+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [Qwen3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ Qwen3DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])self.norm = RMSNormFused(size=config.hidden_size,@@ -72,6 +75,8 @@ class Qwen3MoeForCausalLM(BaseLLMModel):embedding_dim=config.hidden_size,tie_word_embeddings=config.tie_word_embeddings,tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",)super().__init__()diff --git a/python/freetoken/models/qwen3_moe/moe.py b/python/freetoken/models/qwen3_moe/moe.pyindex 973cfd95..c5a73305 100644--- a/python/freetoken/models/qwen3_moe/moe.py+++ b/python/freetoken/models/qwen3_moe/moe.py@@ -11,8 +11,10 @@ if TYPE_CHECKING:class Qwen3MoeMLP(BaseOP):- def __init__(self, config: ModelConfig, layer_id: int | None = None):- self.experts = make_moe_layer(config, layer_id=layer_id)+ def __init__(self, config: ModelConfig, layer_id: int | None = None, *, prefix: str = ""):+ self.experts = make_moe_layer(+ config, layer_id=layer_id, quant_config=config.quant, prefix=f"{prefix}.experts"+ )self.gate = LinearReplicated(config.hidden_size,config.num_experts,diff --git a/python/freetoken/models/qwen3_moe/weight.py b/python/freetoken/models/qwen3_moe/weight.pyindex a147640c..a8b480d0 100644--- a/python/freetoken/models/qwen3_moe/weight.py+++ b/python/freetoken/models/qwen3_moe/weight.py@@ -89,7 +89,7 @@ def iter_weights_parallel("""experts-only iter_weights: raw experts read via the common chunked multi-threadedO_DIRECT reader, then same merge+stack pipeline."""assert include_moe_experts and not include_non_moe, (- "qwen3_moe parallel reader is experts-only (used by load_moe_expert_sources)"+ "qwen3_moe parallel reader is experts-only (used by the expert piece reader)")from freetoken.models.weight import iter_expert_tensors_paralleldiff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.pyindex 04c29d77..1a74d4ae 100644--- a/python/freetoken/models/qwen4_exp/__init__.py+++ b/python/freetoken/models/qwen4_exp/__init__.py@@ -12,24 +12,20 @@ Contracts shared across modules (do not rename):from .config import parse_configfrom .model import Qwen4ExpForCausalLMfrom .weight import (+ nvfp4_expert_spec,iter_weights,- load_nvfp4_expert_sources,- load_nvfp4_expert_sources_parallel,load_ple_table,)# Official FP8 checkpoints share qwen3_5_moe's block-fp8 expert layout (same-# model.language_model.layers.* keys), so reuse its bank hook; for every other-# expert_quant it defers to the per-quant providers, which resolve this module's-# load_nvfp4_expert_sources via the model spec.-from freetoken.models.qwen3_5_moe.weight import setup_offload_expert_banks+# model.language_model.layers.* keys), so reuse its expert reader.+from freetoken.models.qwen3_5_moe.weight import iter_expert_pieces__all__ = [+ "nvfp4_expert_spec","Qwen4ExpForCausalLM","iter_weights",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel","load_ple_table","parse_config",- "setup_offload_expert_banks",+ "iter_expert_pieces",]diff --git a/python/freetoken/models/qwen4_exp/attention.py b/python/freetoken/models/qwen4_exp/attention.pyindex d6ab2867..66211c7c 100644--- a/python/freetoken/models/qwen4_exp/attention.py+++ b/python/freetoken/models/qwen4_exp/attention.py@@ -76,7 +76,7 @@ class QSAAttentionBackend(Protocol):class Qwen4ExpIndexer(BaseOP):"""QSA indexer weights (checkpoint prefix ``self_attn.indexer``); the scoring lives in the backend."""- def __init__(self, config: ModelConfig, layer_id: int) -> None:+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = "") -> None:args = config.qwen4_argsself.layer_id = layer_idself.num_heads = args.index_n_heads@@ -84,7 +84,10 @@ class Qwen4ExpIndexer(BaseOP):self.head_dim = args.index_head_dimself.eps = config.rms_norm_epsself._split = [self.num_heads * self.head_dim, self.num_kv_heads * self.head_dim]- self.index_qk_proj = LinearReplicated(args.hidden_size, sum(self._split), has_bias=False)+ self.index_qk_proj = LinearReplicated(+ args.hidden_size, sum(self._split), has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.index_qk_proj",+ )self.q_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)self.k_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)@@ -113,7 +116,7 @@ class Qwen4ExpAttention(BaseOP):(both zero-centered, loaded RAW), ``indexer.*``."""- def __init__(self, config: ModelConfig, layer_id: int) -> None:+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = "") -> None:self.layer_id = layer_idself.num_q = config.num_qo_headsself.num_kv = config.num_kv_heads@@ -122,9 +125,13 @@ class Qwen4ExpAttention(BaseOP):self.kv_attn_dim = self.num_kv * self.head_dimself._qkv_split = [self.qo_attn_dim * 2, self.kv_attn_dim, self.kv_attn_dim]self.qkv_proj = LinearColParallelMerged(- config.hidden_size, self._qkv_split, has_bias=False+ config.hidden_size, self._qkv_split, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.qkv_proj",+ )+ self.o_proj = LinearReplicated(+ self.qo_attn_dim, config.hidden_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.o_proj",)- self.o_proj = LinearReplicated(self.qo_attn_dim, config.hidden_size, has_bias=False)self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)self.k_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)rotary = config.rotary_config@@ -135,7 +142,7 @@ class Qwen4ExpAttention(BaseOP):base=rotary.base,rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,)- self.indexer = Qwen4ExpIndexer(config, layer_id)+ self.indexer = Qwen4ExpIndexer(config, layer_id, prefix=f"{prefix}.indexer")@nvtx_annotate("QSA")def forward(self, x: torch.Tensor, batch: Batch) -> torch.Tensor:diff --git a/python/freetoken/models/qwen4_exp/gdn.py b/python/freetoken/models/qwen4_exp/gdn.pyindex 69838153..ed49dd42 100644--- a/python/freetoken/models/qwen4_exp/gdn.py+++ b/python/freetoken/models/qwen4_exp/gdn.py@@ -4,15 +4,9 @@ import torchimport torch.nn.functional as Ffrom freetoken.core import get_global_ctxfrom freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen-from freetoken.layers import BaseOP, LinearColParallelMerged--from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged-from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged+from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicated+from freetoken.layers.quantization import QuantConfigfrom freetoken.models.qwen3_5_moe.gdn_kernels import gdn_decode_fla, gdn_prefill_chunk_fla-from freetoken.models.quant_linear import make_replicated_quant---_GATE_ACTIVATIONS = ("silu", "swish", "sigmoid")class _DepthwiseConv1d(BaseOP):@@ -22,30 +16,6 @@ class _DepthwiseConv1d(BaseOP):self.weight = torch.empty(conv_dim, 1, kernel)-class _GatedRMSNorm(BaseOP):- """RMSNorm of x followed by an ``activation(z)`` gate (HF Qwen4ExpTextRMSNormGated).-- Uses the fused fla ``rms_norm_gated`` triton kernel (norm(x) * act(z) in one- kernel) instead of the unfused pow/mean/rsqrt/mul/act chain, matching sglang's- ``RMSNormGated`` -- collapses ~8 elementwise kernels per GDN layer into one.- Qwen3.8-Flash-Next gates with sigmoid where Qwen3.5 gates with silu."""-- def __init__(self, dim: int, eps: float, activation: str):- # rms_norm_gated drops the gate entirely (no error) for a name it does not know.- assert activation in _GATE_ACTIVATIONS, f"unsupported GDN output gate {activation!r}"- self.weight = torch.empty(dim)- self.eps = eps- self.activation = activation-- def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:- from freetoken.kernel.fla import rms_norm_gated-- return rms_norm_gated(- x=x, weight=self.weight, bias=None, z=z, eps=self.eps,- is_rms_norm=True, norm_before_gate=True, activation=self.activation,- )--class Qwen4ExpGatedDeltaNet(BaseOP):"""GatedDeltaNet op using the vendored flash-linear-attention triton kernels(``freetoken.kernel.fla``) for the recurrence and a per-request@@ -62,7 +32,7 @@ class Qwen4ExpGatedDeltaNet(BaseOP):def __init__(self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,conv_kernel_size, rms_norm_eps, layer_id, output_gate: str = "sigmoid",- expert_quant: str = "none", attn_quant: str = "none",+ *, quant_config: QuantConfig | None = None, prefix: str = "",):self.layer_id = layer_id# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as@@ -80,25 +50,27 @@ class Qwen4ExpGatedDeltaNet(BaseOP):self.value_dim = num_v_heads * head_v_dimself.conv_dim = 2 * self.key_dim + self.value_dimself.conv_kernel_size = conv_kernel_size- # qkv|z carry a weight scale (block-fp8 weight_scale_inv, or per-tensor FP8- # weight_scale); b|a stay bf16. Both quant modes therefore split the four-way- # fusion into an fp8 qkvz GEMM + a bf16 ba GEMM (matches sglang/vLLM).- self._block_fp8 = expert_quant == "fp8_block"- self._pertensor_fp8 = attn_quant == "fp8_pertensor"- self._fp8 = self._block_fp8 or self._pertensor_fp8+ # quantized checkpoints quantize qkv|z but not b|a, so the fusion splits into a qkvz GEMM and a ba GEMM with their own schemes (matches sglang / vLLM)+ self._split_in_proj = (+ quant_config is not None and quant_config.scheme_for(f"{prefix}.in_proj_qkvz") is not None+ )self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads]- if self._fp8:- ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged- self.in_proj_qkvz = ColMerged(- hidden_size, [self.conv_dim, self.value_dim], has_bias=False+ if self._split_in_proj:+ self.in_proj_qkvz = LinearColParallelMerged(+ hidden_size, [self.conv_dim, self.value_dim], has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.in_proj_qkvz",)self.in_proj_ba = LinearColParallelMerged(- hidden_size, [num_v_heads, num_v_heads], has_bias=False+ hidden_size, [num_v_heads, num_v_heads], has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.in_proj_ba",)else:# Fused input projection (one GEMM instead of four): qkv | z | b | a.- self.in_proj = LinearColParallelMerged(hidden_size, self._in_proj_split, has_bias=False)+ self.in_proj = LinearColParallelMerged(+ hidden_size, self._in_proj_split, has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.in_proj",+ )self.conv1d = _DepthwiseConv1d(self.conv_dim, conv_kernel_size)# Recurrence-gating params kept in fp32 (exp/softplus is precision-sensitive,# and the fla kernel reads them as fp32) -- matches HF/sglang, and avoids a@@ -106,12 +78,10 @@ class Qwen4ExpGatedDeltaNet(BaseOP):# *.A_log / *.dt_bias from the model-dtype downcast.self.dt_bias = torch.empty(num_v_heads, dtype=torch.float32)self.A_log = torch.empty(num_v_heads, dtype=torch.float32)- self.norm = _GatedRMSNorm(head_v_dim, eps=rms_norm_eps, activation=output_gate)- # out_proj follows the checkpoint quant: block-fp8 / per-tensor-fp8 / compressed-tensors- # NVFP4 (W4A16) / bf16. in_proj_* stay bf16 in every mode (above), so a compressed-tensors- # NVFP4 checkpoint (attn_quant=="nvfp4") only makes out_proj native FP4.- self.out_proj = make_replicated_quant(- expert_quant, attn_quant, self.value_dim, hidden_size, has_bias=False+ self.norm = GatedRMSNorm(head_v_dim, eps=rms_norm_eps, activation=output_gate)+ self.out_proj = LinearReplicated(+ self.value_dim, hidden_size, has_bias=False,+ quant_config=quant_config, prefix=f"{prefix}.out_proj",)def _gate_params(self, a: torch.Tensor, b: torch.Tensor):@@ -170,7 +140,7 @@ class Qwen4ExpGatedDeltaNet(BaseOP):fla = build_fla_metadata(batch, hidden_states.device)batch.fla_metadata = fla- if self._fp8:+ if self._split_in_proj:qkvz = self.in_proj_qkvz.forward(hidden_states)conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1)ba = self.in_proj_ba.forward(hidden_states)diff --git a/python/freetoken/models/qwen4_exp/hc.py b/python/freetoken/models/qwen4_exp/hc.pyindex 6fed76c0..f13808dc 100644--- a/python/freetoken/models/qwen4_exp/hc.py+++ b/python/freetoken/models/qwen4_exp/hc.py@@ -85,7 +85,7 @@ class GatedResidual(BaseOP):Launch budget on CUDA: ``mix`` is 3 kernels around 2 GEMMs, ``combine`` is 1."""- def __init__(self, config: ModelConfig, use_combine: bool = True) -> None:+ def __init__(self, config: ModelConfig, use_combine: bool = True, *, prefix: str = "") -> None:args = config.qwen4_argsself.hc_count = args.hc_countself.hidden_size = args.hidden_size@@ -97,12 +97,19 @@ class GatedResidual(BaseOP):# 16-row alignment for the merged skinny GEMM (vLLM hyperconnection.py:98)self.pad_size = (-(self.lowrank + self.hc_count)) % 16self.input_mix_weight_down_block_inject = LinearReplicated(- width, self.lowrank + self.hc_count + self.pad_size, has_bias=False+ width, self.lowrank + self.hc_count + self.pad_size, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_down_block_inject",)else:self.pad_size = 0- self.input_mix_weight_down = LinearReplicated(width, self.lowrank, has_bias=False)- self.input_mix_weight_up = LinearReplicated(self.lowrank, width, has_bias=False)+ self.input_mix_weight_down = LinearReplicated(+ width, self.lowrank, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_down",+ )+ self.input_mix_weight_up = LinearReplicated(+ self.lowrank, width, has_bias=False,+ quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_up",+ )def _down(self, rn: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:"""Run the down GEMM and split off the raw inject logits; the pad columns are dropped."""diff --git a/python/freetoken/models/qwen4_exp/model.py b/python/freetoken/models/qwen4_exp/model.pyindex 19c31e2f..a4b63362 100644--- a/python/freetoken/models/qwen4_exp/model.py+++ b/python/freetoken/models/qwen4_exp/model.py@@ -33,7 +33,7 @@ if TYPE_CHECKING:from freetoken.models.config import ModelConfig-def build_linear_mixer(config: ModelConfig, layer_id: int) -> BaseOP:+def build_linear_mixer(config: ModelConfig, layer_id: int, prefix: str) -> BaseOP:"""GDN mixer of a linear_attention layer (Qwen3.5's GDN with a configurable output gate)."""from .gdn import Qwen4ExpGatedDeltaNet@@ -48,28 +48,26 @@ def build_linear_mixer(config: ModelConfig, layer_id: int) -> BaseOP:rms_norm_eps=config.rms_norm_eps,layer_id=layer_id,output_gate=g.output_gate,- # Qwen3.8's block-fp8 checkpoint keeps the GDN projections bf16 (only the routed- # experts are quantized), so do not let expert_quant flip them to Fp8Block.- expert_quant="none" if config.expert_quant == "fp8_block" else config.expert_quant,- attn_quant=config.attn_quant,+ quant_config=config.quant,+ prefix=prefix,)class Qwen4ExpDecoderLayer(BaseOP):"""One decoder layer over the hyper-connection streams (see the module docstring for the flow)."""- def __init__(self, config: ModelConfig, layer_id: int) -> None:+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = "") -> None:self._layer_id = layer_idself._is_linear = config.is_linear_layer(layer_id)if self._is_linear:- self.linear_attn = build_linear_mixer(config, layer_id)+ self.linear_attn = build_linear_mixer(config, layer_id, f"{prefix}.linear_attn")else:- self.self_attn = Qwen4ExpAttention(config, layer_id)- self.mlp = Qwen4ExpMoE(config, layer_id)- self.attn_hyper_connection = GatedResidual(config)- self.mlp_hyper_connection = GatedResidual(config)+ self.self_attn = Qwen4ExpAttention(config, layer_id, prefix=f"{prefix}.self_attn")+ self.mlp = Qwen4ExpMoE(config, layer_id, prefix=f"{prefix}.mlp")+ self.attn_hyper_connection = GatedResidual(config, prefix=f"{prefix}.attn_hyper_connection")+ self.mlp_hyper_connection = GatedResidual(config, prefix=f"{prefix}.mlp_hyper_connection")self.ple = (- PLELayer(config, layer_id) if layer_id in config.qwen4_args.ple_layer_ids else None+ PLELayer(config, layer_id, prefix=f"{prefix}.ple") if layer_id in config.qwen4_args.ple_layer_ids else None)@nvtx_annotate("Layer_{}", layer_id_field="_layer_id")@@ -87,16 +85,19 @@ class Qwen4ExpDecoderLayer(BaseOP):class Qwen4ExpModel(BaseOP):- def __init__(self, config: ModelConfig) -> None:+ def __init__(self, config: ModelConfig, *, prefix: str = "model") -> None:self.hc_count = config.qwen4_args.hc_countself.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,)self.layers = OPList(- [Qwen4ExpDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ [+ Qwen4ExpDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")+ for layer_id in range(config.num_layers)+ ])- self.hyper_connection_mixer = GatedResidual(config, use_combine=False)+ self.hyper_connection_mixer = GatedResidual(config, use_combine=False, prefix=f"{prefix}.hyper_connection_mixer")# plain tuple (not an OP child), so it never shows up in the state dictself._ple = tuple(layer.ple for layer in self.layers.op_list if layer.ple is not None)@@ -127,20 +128,14 @@ class Qwen4ExpForCausalLM(BaseLLMModel):def __init__(self, config: ModelConfig) -> None:self._config = configself.model = Qwen4ExpModel(config)- if getattr(config, "lm_head_quant", "none") == "nvfp4":- from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead-- assert not config.tie_word_embeddings, "NVFP4 lm_head assumes untied embeddings"- self.lm_head = Nvfp4LMHead(- num_embeddings=config.vocab_size, embedding_dim=config.hidden_size- )- else:- self.lm_head = ParallelLMHead(- num_embeddings=config.vocab_size,- embedding_dim=config.hidden_size,- tie_word_embeddings=config.tie_word_embeddings,- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,- )+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ quant_config=config.quant,+ prefix="lm_head",+ )super().__init__()def load_host_tables(self, engine_config) -> int:diff --git a/python/freetoken/models/qwen4_exp/moe.py b/python/freetoken/models/qwen4_exp/moe.pyindex 9ef65c0a..3687f280 100644--- a/python/freetoken/models/qwen4_exp/moe.py+++ b/python/freetoken/models/qwen4_exp/moe.py@@ -1,11 +1,9 @@from __future__ import annotations-from dataclasses import replacefrom typing import TYPE_CHECKINGimport torchfrom freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid-from freetoken.layers.moe import make_moe_layerfrom freetoken.models.qwen3_5_moe.moe import Qwen3_5MoEif TYPE_CHECKING:@@ -18,21 +16,6 @@ class Qwen4ExpMoE(Qwen3_5MoE):Same weights, same state dict. The gate reduction stays ahead of the routed experts, which may write into ``hidden_states`` in place."""- def __init__(self, config: ModelConfig, layer_id: int | None = None) -> None:- if getattr(config, "expert_quant", "none") != "fp8_block":- super().__init__(config, layer_id=layer_id)- return- # Qwen3.8's block-fp8 checkpoint quantizes only the routed experts; the shared- # expert stays bf16, so hide expert_quant from _SharedExpert's fp8 branch and- # rebuild the routed experts with the fp8_block bank layout.- super().__init__(replace(config, expert_quant="none"), layer_id=layer_id)- self.experts = make_moe_layer(- config,- layer_id=layer_id,- renormalize=config.norm_topk_prob,- weight_format="fp8_block",- )-def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:num_tokens, hidden_dim = hidden_states.shapehidden_states = hidden_states.view(-1, hidden_dim)diff --git a/python/freetoken/models/qwen4_exp/ple.py b/python/freetoken/models/qwen4_exp/ple.pyindex 5100229a..13157cb8 100644--- a/python/freetoken/models/qwen4_exp/ple.py+++ b/python/freetoken/models/qwen4_exp/ple.py@@ -542,7 +542,7 @@ class PLELayer(BaseOP):"""def __init__(- self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None+ self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None, *, prefix: str = "") -> None:args = config.qwen4_argsself.args = args@@ -554,8 +554,12 @@ class PLELayer(BaseOP):self.state_len = args.ple_conv_state_lenwidth = args.ple_state_widthself.ple_embedding = NGramEmbedding(args, table)- self.key_proj = LinearReplicated(args.ple_embed_dim, width, has_bias=False)- self.value_proj = LinearReplicated(args.ple_embed_dim, args.hidden_size, has_bias=False)+ self.key_proj = LinearReplicated(+ args.ple_embed_dim, width, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.key_proj"+ )+ self.value_proj = LinearReplicated(+ args.ple_embed_dim, args.hidden_size, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.value_proj"+ )self.norm_key = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)self.norm_query = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)self.norm_conv = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.pyindex f8d2a749..da4bea48 100644--- a/python/freetoken/models/qwen4_exp/weight.py+++ b/python/freetoken/models/qwen4_exp/weight.py@@ -1,10 +1,10 @@-"""Qwen3.8-Flash-Next (RadixArk NVFP4) checkpoint reader.+"""Qwen3.8-Flash-Next checkpoint reader (the NVFP4 and the official block-fp8 releases).Three separate paths, because the checkpoint's three weight classes live in different places:* :func:`iter_weights` -- every dense (non-expert) tensor, with the ``model.language_model.`` prefix stripped and fused where the model expects one buffer. See ``_FUSIONS``.* :func:`load_ple_table` -- the 47.7 GiB FP8 n-gram table, 128 checkpoint shards concatenated into one pinned :class:`HostBank`.-* :func:`load_nvfp4_expert_sources` -- the routed NVFP4 experts, into the offload cache's source banks.+* :func:`nvfp4_expert_spec` -- how the routed NVFP4 experts are named, for the offload cache's expert reader.Dropped: ``mtp.*`` (speculative head, including its stacked ``mtp.layers.0.mlp.experts.*``) and ``model.visual.*`` (served text-only)."""@@ -24,7 +24,6 @@ from freetoken.distributed import get_tp_infofrom freetoken.models.loader import drop_page_cache, iter_weight_filesfrom freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,- load_nvfp4_expert_source_banks,)from freetoken.moe.host_banks import HostBank, read_range_intofrom freetoken.utils import download_hf_weight@@ -147,15 +146,16 @@ def iter_weights("""Yield the dense (non-expert) weights, prefix-stripped and fused to the model's buffers.Keys keep the checkpoint's module names below the stripped prefix, so the emitted set is the- model's state dict minus the routed experts. Nothing here is quantized: the modelopt- ``ignore`` list covers everything except those experts, so attention, GDN, HC, PLE, the shared- expert and lm_head are all plain bf16 (the n-gram hash constants stay int64). Fusions:+ model's state dict minus the routed experts. Nothing here is quantized: every release's skip+ list (modelopt ``ignore``, fp8 ``modules_to_not_convert``) covers everything except those experts,+ so attention, GDN, HC, PLE, the shared expert and lm_head are all plain bf16 (the n-gram hash+ constants stay int64). Fusions:attention q|k|v -> ``qkv_proj``, GDN ``in_proj_{qkv,z,b,a}`` -> ``in_proj``, shared-expertgate|up -> ``gate_up_proj``, and each per-layer HC's ``input_mix_weight_down`` |``block_inject_weight`` -> a zero-padded ``input_mix_weight_down_block_inject``.``include_moe_experts`` is accepted for the loader contract but never yields anything: the- routed experts are NVFP4 and always come from :func:`load_nvfp4_expert_sources`.+ routed experts are NVFP4 and always come from the offload cache's expert reader."""if get_tp_info().size > 1:raise NotImplementedError("qwen4_exp weight loading supports TP=1 only")@@ -289,40 +289,13 @@ def load_ple_table(model_path: str, qwen4_args, *, pin: bool = True,# ======================================================================================-def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None) -> dict:- """Build the CPU NVFP4 expert source banks for the offload cache (gate/up fused on the output-row axis, down separate; weight_scale_2 carried as the per-row global scale)."""- return load_nvfp4_expert_source_banks(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- layer_sink=layer_sink,- )---def load_nvfp4_expert_sources_parallel(- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None-) -> dict:- """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel-- return load_nvfp4_expert_source_banks_parallel(- model_path,- config,- _NVFP4_SOURCE_SPEC,- drop_page_cache=drop_page_cache,- primary=get_tp_info().is_primary(),- workers=workers,- chunk=chunk,- layer_sink=layer_sink,- )+def nvfp4_expert_spec(model_path: str, config):+ return _NVFP4_SOURCE_SPEC__all__ = [+ "nvfp4_expert_spec","PleTable","iter_weights",- "load_nvfp4_expert_sources",- "load_nvfp4_expert_sources_parallel","load_ple_table",]diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex b8c17322..3778d3c3 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -14,28 +14,75 @@ class ModelSpec:model_cls: strparse_config: str = "parse_config"iter_weights: str = "iter_weights"+ # attribute-path root -> checkpoint root, for quantization_config lookups+ checkpoint_roots: tuple[tuple[str, str], ...] = ()+ # inner attribute path -> checkpoint path (feed_forward.shared_mlp -> mlp)+ checkpoint_segments: tuple[tuple[str, str], ...] = ()+ # fused leaf -> source leaves (qkv_proj -> q_proj, k_proj, v_proj)+ packed_modules_mapping: tuple[tuple[str, tuple[str, ...]], ...] = ()+ # checkpoint-name globs the family serves in bf16 although the quantization_config covers them+ unquantized_modules: tuple[str, ...] = ()+# Multimodal wrappers store the text tower under model.language_model.+_LANGUAGE_MODEL_ROOT = (("model", "model.language_model"),)+# Fused projections split back into their HF leaves so the quantization_config lookup sees the stored names.+_DENSE_PACKED = (+ ("qkv_proj", ("q_proj", "k_proj", "v_proj")),+ ("gate_up_proj", ("gate_proj", "up_proj")),+)+# per-expert checkpoints: probe the first expert's projections for the experts container+_EXPERTS_PACKED = (("experts", ("experts.0.gate_proj", "experts.0.up_proj", "experts.0.down_proj")),)+_EXPERTS_W123_PACKED = (("experts", ("experts.0.w1", "experts.0.w2", "experts.0.w3")),)+_QWEN3_5_PACKED = _DENSE_PACKED + (+ ("in_proj_qkvz", ("in_proj_qkv", "in_proj_z")),+ ("in_proj_ba", ("in_proj_b", "in_proj_a")),+ ("in_proj", ("in_proj_qkv", "in_proj_z", "in_proj_b", "in_proj_a")),+) + _EXPERTS_PACKED+# Qwen3.8's per-layer hyper-connections fuse the down projection with the block-inject rows.+_QWEN4_EXP_PACKED = _QWEN3_5_PACKED + (+ ("input_mix_weight_down_block_inject", ("input_mix_weight_down", "block_inject_weight")),+)+# GLM-5.3-Flash's KDA fuses its six input projections.+_GLM5_NEXT_PACKED = _EXPERTS_PACKED + (+ ("in_proj", ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")),+)+# Gemma 4 keeps HF's flat layer children (mlp / experts / router) under one feed_forward block.+_GEMMA4_SEGMENTS = (+ ("feed_forward.shared_mlp", "mlp"),+ ("feed_forward.experts", "experts"),+ ("feed_forward.router", "router"),+)+_GEMMA4_PACKED = _DENSE_PACKED + _EXPERTS_PACKED+_MINIMAX_M3_PACKED = _DENSE_PACKED + (+ ("index_qk_proj", ("index_q_proj", "index_k_proj")),+) + _EXPERTS_W123_PACKED+_MODEL_REGISTRY: dict[str, ModelSpec] = {"LlamaForCausalLM": ModelSpec("freetoken.models.llama","LlamaForCausalLM",+ packed_modules_mapping=_DENSE_PACKED,),"Qwen2ForCausalLM": ModelSpec("freetoken.models.qwen2","Qwen2ForCausalLM",+ packed_modules_mapping=_DENSE_PACKED,),"Qwen3ForCausalLM": ModelSpec("freetoken.models.qwen3","Qwen3ForCausalLM",+ packed_modules_mapping=_DENSE_PACKED,),"Qwen3MoeForCausalLM": ModelSpec("freetoken.models.qwen3_moe","Qwen3MoeForCausalLM",+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,),"MiniMaxM2ForCausalLM": ModelSpec("freetoken.models.minimax_m2","MiniMaxM2ForCausalLM",+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_W123_PACKED,),# MiniMax-M3 (model_type minimax_m3_vl): multimodal wrapper config (text tower in# text_config, weights under language_model.); served text-only. GQA + block-sparse@@ -44,19 +91,29 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"MiniMaxM3SparseForConditionalGeneration": ModelSpec("freetoken.models.minimax_m3","MiniMaxM3ForCausalLM",+ checkpoint_roots=(("model", "language_model.model"), ("lm_head", "language_model.lm_head")),+ packed_modules_mapping=_MINIMAX_M3_PACKED,),# Text-only sibling (the text_config's own architectures entry)."MiniMaxM3SparseForCausalLM": ModelSpec("freetoken.models.minimax_m3","MiniMaxM3ForCausalLM",+ packed_modules_mapping=_MINIMAX_M3_PACKED,),"DeepseekV4ForCausalLM": ModelSpec("freetoken.models.deepseek_v4","DeepseekV4ForCausalLM",+ # the checkpoint has no ``model.`` root+ checkpoint_roots=(("model.layers", "layers"), ("model.head", "head")),+ packed_modules_mapping=_EXPERTS_W123_PACKED,+ # the head, the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert+ unquantized_modules=("head", "*.compressor.wkv", "*.compressor.wgate", "*.indexer.weights_proj"),),"Qwen3_5MoeForConditionalGeneration": ModelSpec("freetoken.models.qwen3_5_moe","Qwen3_5MoEForCausalLM",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=_QWEN3_5_PACKED,),# Qwen3.8-Flash-Next (model_type qwen4_exp): multimodal wrapper config (text tower in# text_config, weights under model.language_model.); served text-only. 36 GDN + 12 QSA@@ -65,6 +122,8 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"Qwen4ExpForConditionalGeneration": ModelSpec("freetoken.models.qwen4_exp","Qwen4ExpForCausalLM",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=_QWEN4_EXP_PACKED,),# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the@@ -72,6 +131,8 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"Qwen3_5ForConditionalGeneration": ModelSpec("freetoken.models.qwen3_5_moe","Qwen3_5MoEForCausalLM",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=_QWEN3_5_PACKED,),# Muse-Glimmer-30B (model_type muse_glimmer): multimodal wrapper config (text tower in# text_config, weights under model.language_model.); served text-only. Dense gated GQA@@ -81,10 +142,16 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"MuseGlimmerForConditionalGeneration": ModelSpec("freetoken.models.muse_glimmer","MuseGlimmerForCausalLM",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=(+ ("qkvg_proj", ("q_proj", "k_proj", "v_proj", "gate_proj")),+ ("gate_up_proj", ("gate_proj", "up_proj")),+ ),),"MistralForCausalLM": ModelSpec("freetoken.models.mistral","MistralForCausalLM",+ packed_modules_mapping=_DENSE_PACKED,),"Mistral3ForConditionalGeneration": ModelSpec("freetoken.models.mistral",@@ -93,20 +160,30 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"Gemma4ForConditionalGeneration": ModelSpec("freetoken.models.gemma4","Gemma4ForCausalLM",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ checkpoint_segments=_GEMMA4_SEGMENTS,+ packed_modules_mapping=_GEMMA4_PACKED,),"Gemma4ForCausalLM": ModelSpec("freetoken.models.gemma4","Gemma4ForCausalLM",+ checkpoint_segments=_GEMMA4_SEGMENTS,+ packed_modules_mapping=_GEMMA4_PACKED,),# Dense text tower of the gemma-4-12B "Unified"/omni model (model_type gemma4_unified_text).# Same decoder as gemma4; the dense feed-forward is selected via config.is_moe."Gemma4UnifiedForConditionalGeneration": ModelSpec("freetoken.models.gemma4","Gemma4ForCausalLM",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ checkpoint_segments=_GEMMA4_SEGMENTS,+ packed_modules_mapping=_GEMMA4_PACKED,),"Gemma4UnifiedForCausalLM": ModelSpec("freetoken.models.gemma4","Gemma4ForCausalLM",+ checkpoint_segments=_GEMMA4_SEGMENTS,+ packed_modules_mapping=_GEMMA4_PACKED,),# GGUF (native Q4_0/Q6_K) gemma4: same model classes, GGUF config + weight loaders."Gemma4GGUFForCausalLM": ModelSpec(@@ -118,10 +195,12 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"GptOssForCausalLM": ModelSpec("freetoken.models.gpt_oss","GptOssForCausalLM",+ packed_modules_mapping=_DENSE_PACKED,),"Glm4MoeForCausalLM": ModelSpec("freetoken.models.glm4_moe","Glm4MoeForCausalLM",+ packed_modules_mapping=_EXPERTS_PACKED,),# GLM-5.2 (model_type glm_moe_dsa): DeepSeek-V3.2-class MLA + DSA sparse attention# with GLM-4-style sigmoid/noaux_tc MoE routing; NVFP4 routed experts served from@@ -129,6 +208,7 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"GlmMoeDsaForCausalLM": ModelSpec("freetoken.models.glm_moe_dsa","GlmMoeDsaForCausalLM",+ packed_modules_mapping=_EXPERTS_PACKED,),# GLM-5.3-Flash (model_type glm5_next): hybrid KDA linear attention (34/45 layers)# + NoPE-MLA/DSA with a kpool-compressed indexer (11/45), mHC x4 residual streams,@@ -137,11 +217,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"Glm5NextForConditionalGeneration": ModelSpec("freetoken.models.glm5_next","Glm5NextForCausalLM",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=_GLM5_NEXT_PACKED,),# Text-only sibling (the text_config's own architectures entry)."Glm5NextForCausalLM": ModelSpec("freetoken.models.glm5_next","Glm5NextForCausalLM",+ packed_modules_mapping=_GLM5_NEXT_PACKED,),}diff --git a/python/freetoken/models/weight.py b/python/freetoken/models/weight.pyindex 6a34f3b9..03b11a1d 100644--- a/python/freetoken/models/weight.py+++ b/python/freetoken/models/weight.py@@ -22,10 +22,7 @@ from concurrent.futures import ThreadPoolExecutorfrom typing import Callable, Iterator, Tupleimport torch-from freetoken.distributed import get_tp_info-from freetoken.kernel.pinned import alloc_pinned_tensor, copy_to_pinned_tensor-from freetoken.models.loader import stream_moe_expert_sources-from freetoken.utils import cached_load_hf_config, div_even+from freetoken.utils import cached_load_hf_configfrom .register import _load_attr, get_model_spec@@ -256,74 +253,6 @@ def load_weight()-def load_moe_expert_sources(- model_path: str,- *,- dtype: torch.dtype,- dummy: bool = False,- parallel: bool = False,- workers: int = 8,- chunk: int = 8 << 20,- layer_sink=None,-) -> Tuple[list[torch.Tensor], list[torch.Tensor]]:- config, spec = _spec_for_model_path(model_path)- if not config.is_moe:- raise ValueError(- f"{config.architectures[0]} does not provide MoE expert source loading"- )- if dummy:- builder = _model_override(spec, "dummy_moe_expert_sources") or dummy_moe_expert_sources- return builder(config, dtype=dtype)- if parallel: # parallel: experts read via the common chunked multi-threaded O_DIRECT reader- iter_weights = _model_override(spec, "iter_weights_parallel")- if iter_weights is None: # model has no parallel reader -> let the caller fall back to serial- raise NotImplementedError(- f"{spec.module} provides no iter_weights_parallel")- src = iter_weights(model_path, torch.device("cpu"), include_moe_experts=True,- include_non_moe=False, workers=workers, chunk=chunk)- else:- iter_weights = _load_attr(spec.module, spec.iter_weights)- src = iter_weights(model_path, torch.device("cpu"), include_moe_experts=True,- include_non_moe=False)- return stream_moe_expert_sources(- src,- config,- dtype=dtype,- layer_sink=layer_sink,- )---def load_nvfp4_moe_expert_sources(- model_path: str,- model_config,- *,- dummy: bool = False,- parallel: bool = False,- workers: int = 8,- chunk: int = 8 << 20,- layer_sink=None,-) -> dict:- """Load (or fabricate, with ``dummy=True``) packed NVFP4 expert source banks.- ``parallel=True`` uses the model's ``load_nvfp4_expert_sources_parallel`` (common- chunked multi-threaded O_DIRECT reader). ``layer_sink``: see- ``models.nvfp4_banks.load_nvfp4_expert_source_banks``; forwarded to the per-model- loader, which forwards it on."""- _config, spec = _spec_for_model_path(model_path)- if dummy:- builder = (- _model_override(spec, "dummy_nvfp4_expert_sources") or dummy_nvfp4_expert_sources- )- return builder(model_config)- if parallel:- loader = _model_override(spec, "load_nvfp4_expert_sources_parallel")- if loader is None: # no parallel reader -> let the caller fall back to serial- raise NotImplementedError(- f"{spec.module} provides no load_nvfp4_expert_sources_parallel")- return loader(model_path, model_config, workers=workers, chunk=chunk, layer_sink=layer_sink)- loader = _load_attr(spec.module, "load_nvfp4_expert_sources")- return loader(model_path, model_config, layer_sink=layer_sink)--def load_q4_0_moe_expert_sources(model_path: str,model_config,@@ -342,72 +271,8 @@ def load_q4_0_moe_expert_sources(return loader(model_path, model_config, layer_sink=layer_sink)-def _num_moe_layers(config) -> int:- value = getattr(config, "num_moe_layers", None)- if value is not None:- return int(value)- return int(config.num_layers) - int(getattr(config, "first_k_dense_replace", 0))---def dummy_moe_expert_sources(- config, *, dtype: torch.dtype-) -> Tuple[list[torch.Tensor], list[torch.Tensor]]:- """Random BF16 expert banks shaped like ``stream_moe_expert_sources`` output:- one independently allocated ``[num_experts, ...]`` tensor per layer."""- num_layers = _num_moe_layers(config)- intermediate_size = div_even(config.moe_intermediate_size, get_tp_info().size)- gate_up = [- torch.randn(config.num_experts, 2 * intermediate_size, config.hidden_size, dtype=dtype)- for _ in range(num_layers)- ]- down = [- torch.randn(config.num_experts, config.hidden_size, intermediate_size, dtype=dtype)- for _ in range(num_layers)- ]- if torch.cuda.is_available():- gate_up = [copy_to_pinned_tensor(t) for t in gate_up]- down = [copy_to_pinned_tensor(t) for t in down]- return gate_up, down---def dummy_nvfp4_expert_sources(config) -> dict[str, list[torch.Tensor]]:- """Random NVFP4 (ModelOpt-layout) expert banks shaped like the real loader's.-- Same pinned allocation and per-layer bank shapes as ``load_nvfp4_expert_sources``,- so the repack/offload path downstream is exercised unchanged. Packed codes are- random nibbles; block scales are 1.0 and globals small because random e4m3 bytes- reach 448 (and include NaN encodings), which would blow up the dummy activations.- """- num_layers = _num_moe_layers(config)- E = config.num_experts- H, I = config.hidden_size, config.moe_intermediate_size- fp8 = torch.float8_e4m3fn-- def bank(*shape: int, dtype: torch.dtype) -> list[torch.Tensor]:- return [alloc_pinned_tensor(*shape, dtype=dtype) for _ in range(num_layers)]-- sources = {- "gate_up_packed": bank(E, 2 * I, H // 2, dtype=torch.uint8),- "gate_up_scale": bank(E, 2 * I, H // 16, dtype=fp8),- "gate_up_global": bank(E, 2 * I, dtype=torch.float16),- "down_packed": bank(E, H, I // 2, dtype=torch.uint8),- "down_scale": bank(E, H, I // 16, dtype=fp8),- "down_global": bank(E, H, dtype=torch.float16),- }- for t in sources["gate_up_packed"] + sources["down_packed"]:- t.random_(0, 256)- for t in sources["gate_up_scale"] + sources["down_scale"]:- t.fill_(1.0)- for t in sources["gate_up_global"] + sources["down_global"]:- t.fill_(0.01)- return sources--__all__ = ["load_weight",- "load_moe_expert_sources",- "load_nvfp4_moe_expert_sources",- "dummy_moe_expert_sources",- "dummy_nvfp4_expert_sources",+ "load_q4_0_moe_expert_sources","iter_expert_tensors_parallel",]diff --git a/python/freetoken/moe/__init__.py b/python/freetoken/moe/__init__.pyindex 5fc41911..8794abce 100644--- a/python/freetoken/moe/__init__.py+++ b/python/freetoken/moe/__init__.py@@ -1,74 +1,17 @@from __future__ import annotations-from typing import Protocol+# Where routed experts live. The offload family (offload / cpu / hybrid) serves experts from+# pinned host banks through an ``OffloadMoeCache`` -- the GPU only holds the two-layer prefill+# double buffer -- and differs only in how decode gets the experts: ``offload`` streams the+# missing experts over PCIe into a GPU slot cache, ``cpu`` computes them on the CPU from the+# host banks, ``hybrid`` fetches at most K missing experts per layer and computes the rest on+# the CPU, overlapped. ``fused`` keeps every expert resident on the GPU.+MOE_STRATEGIES = ("fused", "offload", "cpu", "hybrid")+OFFLOAD_MOE_STRATEGIES = frozenset({"offload", "cpu", "hybrid"})-from freetoken.utils import Registry, init_logger-from .base import BaseMoeBackend+def is_offload_moe_strategy(strategy: str) -> bool:+ return strategy in OFFLOAD_MOE_STRATEGIES-logger = init_logger(__name__)--class MoeBackendCreator(Protocol):- def __call__(self) -> BaseMoeBackend: ...---SUPPORTED_MOE_BACKENDS = Registry[MoeBackendCreator]("MoE Backend")--# Backends that serve experts from CPU (pinned) host banks through an-# ``OffloadMoeCache`` -- the GPU only holds the two-layer prefill double buffer.-# They differ only in how *decode* gets the experts: ``offload`` streams the-# missing experts over PCIe into a GPU slot cache and runs the GEMM on the GPU;-# ``cpu`` ships the activations to the CPU, computes the experts there (high RAM-# bandwidth), and ships the results back; ``hybrid`` keeps a full GPU slot cache-# AND a CPU executor -- it fetches at most K missing experts/layer over PCIe-# (computed on the GPU with the cache hits) and computes the remaining misses on-# the CPU, overlapped, then merges (capped PCIe + CPU overflow). All build their-# cache the same way, so model layer construction and the engine wiring key off-# this set rather than a bare ``== "offload"`` check.-OFFLOAD_MOE_BACKENDS = frozenset({"offload", "cpu", "hybrid"})---def is_offload_moe_backend(backend: str) -> bool:- return backend in OFFLOAD_MOE_BACKENDS---@SUPPORTED_MOE_BACKENDS.register("fused")-def create_fused_moe_backend():- from .fused import FusedMoe-- return FusedMoe()---@SUPPORTED_MOE_BACKENDS.register("offload")-def create_offload_moe_backend():- from .offload import OffloadMoeBackend-- return OffloadMoeBackend()---@SUPPORTED_MOE_BACKENDS.register("cpu")-def create_cpu_moe_backend():- from .cpu_offload import CpuOffloadMoeBackend-- return CpuOffloadMoeBackend()---@SUPPORTED_MOE_BACKENDS.register("hybrid")-def create_hybrid_moe_backend():- from .cpu_offload import HybridMoeBackend-- return HybridMoeBackend()---def create_moe_backend(backend: str) -> BaseMoeBackend:- return SUPPORTED_MOE_BACKENDS[backend]()---__all__ = [- "BaseMoeBackend",- "create_moe_backend",- "SUPPORTED_MOE_BACKENDS",- "OFFLOAD_MOE_BACKENDS",- "is_offload_moe_backend",-]+__all__ = ["MOE_STRATEGIES", "OFFLOAD_MOE_STRATEGIES", "is_offload_moe_strategy"]diff --git a/python/freetoken/moe/base.py b/python/freetoken/moe/base.pydeleted file mode 100644index f5a2ce7a..00000000--- a/python/freetoken/moe/base.py+++ /dev/null@@ -1,18 +0,0 @@-from abc import ABC, abstractmethod--import torch---class BaseMoeBackend(ABC):- @abstractmethod- def forward(- self,- hidden_states: torch.Tensor,- w1: torch.Tensor,- w2: torch.Tensor,- gating_output: torch.Tensor,- topk: int,- renormalize: bool,- activation: str,- apply_router_weight_on_input: bool,- ) -> torch.Tensor: ...diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex d9cb43c1..e2e48d14 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -1,4 +1,4 @@-"""Python wrapper around the ``_cpu_moe`` C++ executor (--moe-backend cpu).+"""Python wrapper around the ``_cpu_moe`` C++ executor (--moe-strategy cpu).Owns the persistent CPU worker pool, the per-batch-size pinned IO buffers, andthe per-(layer, batch-size) host-func task descriptors. ``decode`` issues the@@ -157,15 +157,17 @@ class CpuMoeExecutor:device: torch.device,swiglu_alpha: float = 1.702,swiglu_limit: float | None = None,+ fmt: str | None = None,) -> None:from freetoken.kernel import _cpu_moe+ from freetoken.moe.legacy_format import canonical_role- fmt = cache.quant_format+ fmt = fmt or cache.quant_formatif fmt not in _WFMT_IDS:raise NotImplementedError(- f"--moe-backend cpu/hybrid computes experts on the CPU and supports "+ f"--moe-strategy cpu/hybrid computes experts on the CPU and supports "f"{sorted(_WFMT_IDS)} formats, but this checkpoint's experts are "- f"{fmt!r}; use --moe-backend offload (GPU-side dequant) instead."+ f"{fmt!r}; use --moe-strategy offload (GPU-side dequant) instead.")if activation not in _ACT_IDS:raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")@@ -193,7 +195,9 @@ class CpuMoeExecutor:# The per-layer tensors and their pointer tables must outlive the executor# (C++ holds raw addresses into both).self._banks: list[torch.Tensor] = []- ptrs, (self.H, self.I) = self._resolve_banks(cache.bank_sources, fmt)+ ptrs, (self.H, self.I) = self._resolve_banks(+ {canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()}, fmt+ )# Decide the flag handshake up front (env + device + a functional stream-memop# probe): its coordinator needs a core of its own, which the auto thread sizing@@ -375,8 +379,8 @@ class CpuMoeExecutor:return self._resolve_dsfp4_banks(banks)# nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.- gup, gus, gug = banks["gate_up_packed"], banks["gate_up_scale"], banks["gate_up_global"]- dnp, dns, dng = banks["down_packed"], banks["down_scale"], banks["down_global"]+ gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]+ dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"assert gug[0].dtype == torch.float16 and dng[0].dtype == torch.float16, (gug[0].dtype, dng[0].dtype)@@ -432,8 +436,8 @@ class CpuMoeExecutor:(N innermost) + per-output-row biases. The C++ kernel streams K andaccumulates a contiguous N-block, so the GPU-tiled layout is read in place(no repack, no extra host memory). Block scales are e8m0 (1 byte / 32 K)."""- gub, gus, gob = banks["gate_up_blocks"], banks["gate_up_scales"], banks["gate_up_bias"]- dnb, dns, dob = banks["down_blocks"], banks["down_scales"], banks["down_bias"]+ gub, gus, gob = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_bias"]+ dnb, dns, dob = banks["down"], banks["down_scale"], banks["down_bias"]assert gub[0].dtype == torch.uint8 and dnb[0].dtype == torch.uint8, (gub[0].dtype, dnb[0].dtype)assert gus[0].dtype == torch.uint8 and dns[0].dtype == torch.uint8, (gus[0].dtype, dns[0].dtype)assert gob[0].dtype == torch.bfloat16 and dob[0].dtype == torch.bfloat16, (gob[0].dtype, dob[0].dtype)@@ -463,8 +467,8 @@ class CpuMoeExecutor:scales, no global, no bias. Layout matches nvfp4 (K contiguous per output row),so the C++ GEMV reads it in place. The kernel additionally FP8-round-trips theactivations (block 128) to match DSV4's W4A8 reference, hence the %128 dims."""- gup, gus = banks["gate_up_packed"], banks["gate_up_scale"]- dnp, dns = banks["down_packed"], banks["down_scale"]+ gup, gus = banks["gate_up"], banks["gate_up_scale"]+ dnp, dns = banks["down"], banks["down_scale"]assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"I = int(gup[0].shape[1] // 2)diff --git a/python/freetoken/moe/cpu_offload.py b/python/freetoken/moe/cpu_offload.pydeleted file mode 100644index 8d2738ad..00000000--- a/python/freetoken/moe/cpu_offload.py+++ /dev/null@@ -1,58 +0,0 @@-from __future__ import annotations--import torch--from .base import BaseMoeBackend---class CpuOffloadMoeBackend(BaseMoeBackend):- """Marker backend for CPU-compute expert offload.-- Like :class:`OffloadMoeBackend`, the real work lives in ``OffloadMoELayer``:- prefill streams whole expert layers into the GPU double buffer and runs the- GEMM on the GPU (identical to ``offload``), while decode ships the- activations to the CPU, computes the experts there (reading the pinned host- banks at full RAM bandwidth), and ships the results back. The whole decode- path is CUDA-graph capturable via ``cudaLaunchHostFunc`` submit/sync nodes.- """-- def forward(- self,- hidden_states: torch.Tensor,- w1: torch.Tensor,- w2: torch.Tensor,- gating_output: torch.Tensor,- topk: int,- renormalize: bool,- activation: str,- apply_router_weight_on_input: bool,- ) -> torch.Tensor:- raise RuntimeError("CPU-offload MoE is handled by OffloadMoELayer")---class HybridMoeBackend(BaseMoeBackend):- """Marker backend for hybrid GPU-cache + CPU-overflow expert decode.-- Prefill is identical to ``offload`` (stream whole layers into the GPU double- buffer, GEMM on the GPU). Decode keeps a full GPU slot cache but caps PCIe- fetches per (layer, step) -- at ``hybrid_max_fetch`` experts, or at the- bandwidth-matched ``hybrid_fetch_fraction`` of that step's misses (default:- the benched pcie_bw/cpu_bw ratio, so the fetch and the CPU compute overlap- perfectly): the cache hits plus the freshly-fetched experts run on the GPU,- the remaining misses run on the CPU (reading the pinned host banks), and the- two partial results are summed. The real work lives in the model's MoE layer- (``OffloadMoELayer`` / DSV4 ``MoE``).- """-- def forward(- self,- hidden_states: torch.Tensor,- w1: torch.Tensor,- w2: torch.Tensor,- gating_output: torch.Tensor,- topk: int,- renormalize: bool,- activation: str,- apply_router_weight_on_input: bool,- ) -> torch.Tensor:- raise RuntimeError("Hybrid MoE is handled by the model's MoE layer")diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.pyindex 8b6116ba..e3a768b6 100644--- a/python/freetoken/moe/expert_banks.py+++ b/python/freetoken/moe/expert_banks.py@@ -1,29 +1,24 @@-"""Per-quant-format expert bank providers for the offload MoE cache.--A provider owns everything between "checkpoint on disk" and "banks ready for-``OffloadMoeCache.set_bank_sources``" for one ``expert_quant`` value: loading the-pinned host banks, picking a kernel backend, repacking into its layout, and-extracting the GPU-resident alphas if the format folds its global scales. The-engine stays quant-agnostic: it calls :func:`load_expert_banks` and wires the-returned bundle into the cache.--A format is fully described by three per-format tables: ``_BANK_SCHEMAS``-(offload_cache, bank layout), ``_PROVIDERS`` (here, loading/repack), and-``OffloadMoELayer._expert_gemm`` (kernel dispatch). Adding a format touches those-three places and nothing else.+"""Expert banks for the offload MoE cache: load, pack and pin the routed experts.++The expert kernel (``QuantMethod.kernel``) owns the bank layout and the pack step; the+checkpoint side delivers pieces (``moe.expert_pieces``) and this module fills the pinned host+banks from them (``build_expert_banks``). The GGUF q4_0 experts still+use their own providers until they get a method."""from __future__ import annotationsimport glob+import mathimport os-import threadingfrom dataclasses import dataclass, fieldimport torch+from freetoken.layers.quantization import QuantKindfrom freetoken.utils import init_logger+from .host_banks import alloc_layer_banksfrom .offload_cache import _BANK_BYTES_PER_EXPERT, _BANK_SCHEMASlogger = init_logger(__name__)@@ -49,187 +44,112 @@ class ExpertBanks:# streamed straight to its sink instead of staying materialized here) -- set by# convert.py's per-format streaming gate; ``sources`` may hold released tensors.streamed: bool = False+ # the expert (kind, kernel) the banks were packed for; None for the legacy providers+ kind: QuantKind | None = None+ kernel: str | None = None+ layout: dict | None = None+++def _dummy_fill(role: str, tensor: torch.Tensor) -> None:+ """Random but finite bank contents for --use-dummy-weight."""+ if role.endswith("_scale"):+ if tensor.dtype is torch.uint8:+ tensor.fill_(127) # e8m0 exponent code for 1.0+ else:+ tensor.fill_(1.0)+ elif role.endswith("_global"):+ tensor.fill_(0.01)+ elif tensor.dtype in (torch.uint8, torch.int32):+ tensor.view(torch.uint8).random_(0, 256)+ elif tensor.dtype is torch.float8_e4m3fn:+ tensor.view(torch.uint8).random_(0, 16) # small codes, no NaN / inf+ else:+ tensor.normal_()-_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader-+def build_expert_banks(+ method,+ num_layers: int,+ pieces,+ *,+ device: torch.device,+ layer_sink=None,+ dummy: bool = False,+) -> ExpertBanks:+ """Fill host banks in the kernel's layout from a stream of expert pieces.-def _v4_unsupported(quant):- raise NotImplementedError(- f"parallel expert reader not implemented for quant {quant!r} yet; "- "add a load_*_expert_sources_parallel using freetoken.models.weight."- "iter_expert_tensors_parallel (only ds_fp4 implemented so far)"- )+ ``pieces`` yields ``(layer_id, e0, e1, {role: tensor[e1 - e0, ...]})`` in any order;+ each batch is packed in place into rows ``e0:e1`` of that layer's banks. A layer is+ complete once its ``num_experts`` rows have arrived: with ``layer_sink=None`` its banks+ are pinned in the background, otherwise the sink receives them (converter). ``dummy``+ skips the pieces and fills the banks with finite random contents.+ """+ from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, pin_banks+ from freetoken.moe.legacy_format import legacy_format_for++ kernel = method.kernel+ layout = method.layout()+ E = method.cfg.num_experts+ specs = {role: ((E, *spec.shape), spec.dtype) for role, spec in layout.items() if not spec.resident}+ hb = alloc_layer_banks(specs, num_layers)+ banks = {role: [b.tensor for b in hb[role]] for role in specs}+ alphas = {+ role: torch.empty(num_layers * E, dtype=spec.dtype, device=device)+ for role, spec in layout.items() if spec.resident+ }+ if dummy:+ for role, per_layer in banks.items():+ for tensor in per_layer:+ _dummy_fill(role, tensor)+ for alpha in alphas.values():+ alpha.fill_(1.0)+ if torch.cuda.is_available():+ pin_banks(hb)+ return ExpertBanks(+ legacy_format_for(method.kind, kernel.name), banks,+ gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),+ kind=method.kind, kernel=kernel.name, layout=layout,+ )-def _bf16_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:- from freetoken.models.weight import load_moe_expert_sources+ def _fill(sink) -> None:+ tracker = LayerCompletionTracker(E, hb, sink) if sink is not None else None+ # a reader that skips a layer or mislabels a piece must fail here, not serve uninitialized rows+ written = torch.zeros(num_layers, E, dtype=torch.int32)+ for layer_id, e0, e1, piece in pieces:+ if not (0 <= layer_id < num_layers and 0 <= e0 < e1 <= E):+ raise ValueError(f"expert piece out of range: layer {layer_id}, experts {e0}:{e1} of {num_layers} x {E}")+ # refuse before writing: a duplicate row would also complete the layer early and hand the sink a half-filled bank+ if written[layer_id, e0:e1].any():+ raise ValueError(f"expert rows written more than once: layer {layer_id}, experts {e0}:{e1}")+ written[layer_id, e0:e1] = 1+ out = {role: banks[role][layer_id][e0:e1] for role in specs}+ got = method.pack(piece, out)+ for role, values in got.items():+ alphas[role][layer_id * E + e0 : layer_id * E + e1] = values.to(alphas[role].dtype)+ if tracker is not None:+ for _ in range(e1 - e0):+ tracker.note(layer_id)+ missing = (written == 0).nonzero().tolist()+ if missing:+ raise ValueError(f"expert banks were not filled: {len(missing)} (layer, expert) rows missing (first {missing[:4]})")++ if layer_sink is not None:+ _fill(layer_sink)+ elif torch.cuda.is_available():+ with PinPipeline() as pins:+ _fill(pins)+ else:+ _fill(None)- # bf16 banks are written as-loaded -> streamable (dummy fabricates in one shot, so a- # sink given alongside dummy=True would never fire; keep it materialize-only there).- sink = None if dummy else layer_sink- gate_up_source, down_source = load_moe_expert_sources(- model_path, dtype=dtype, dummy=dummy, parallel=parallel, workers=workers, chunk=chunk,- layer_sink=sink,- )return ExpertBanks(- "bf16", {"gate_up": gate_up_source, "down": down_source}, streamed=sink is not None- )---class _RepackedBank:- """A repacked per-layer bank forwarded to the converter's outer sink: the post-repack- tensor (a reinterpreted view of the native source's storage) plus a ``release`` that- frees that native source. Exposes just the ``.tensor``/``.nbytes``/``.release()`` the- :class:`~freetoken.checkpoint.convert._ConvertSink` reads."""-- __slots__ = ("tensor", "nbytes", "_src")-- def __init__(self, tensor: torch.Tensor, src) -> None:- self.tensor = tensor- self.nbytes = tensor.numel() * tensor.element_size()- self._src = src # the native HostBank whose storage `tensor` reinterprets-- def release(self) -> None:- self._src.release()---class _Nvfp4RepackSink:- """Streaming layer sink (converter only) for the nvfp4 marlin/b12x backends: repack- each completed layer's 6 native banks into the backend's 4-bank layout in place, then- forward the renamed banks to the outer FTW sink and stash the per-layer alphas.-- Runs from the loader's reader threads and calls CUDA ops (the repack), so the whole- per-layer body -- ``set_device`` + repack + forward + stash -- is serialized under one- lock (conversion is disk/compute-bound, not concurrency-bound). The two native- ``*_global`` banks fold into the alphas, so they are released here (the outer sink only- sees, and releases, the 4 weight banks)."""-- def __init__(self, repack_layer, config, device: torch.device, outer) -> None:- from freetoken.moe.nvfp4_backends import _POST_NVFP4_BANKS-- self._repack_layer = repack_layer- self._config = config- self._device = device- self._outer = outer- self._post_names = _POST_NVFP4_BANKS- self._lock = threading.Lock()- self._gate_up_alpha: dict[int, torch.Tensor] = {}- self._down_alpha: dict[int, torch.Tensor] = {}- # post-repack per-layer views, kept only to reassemble ExpertBanks.sources (they- # alias storage the outer sink released after writing -- released-tensor caveat).- self._post: dict[str, dict[int, torch.Tensor]] = {n: {} for n in self._post_names}-- def __call__(self, layer_id: int, banks: dict) -> None:- from freetoken.moe.nvfp4_backends import _NATIVE_NVFP4_BANKS-- with self._lock:- if self._device.type == "cuda":- torch.cuda.set_device(self._device)- layer_tensors = {name: banks[name].tensor for name in _NATIVE_NVFP4_BANKS}- post, gate_up_alpha, down_alpha = self._repack_layer(- layer_tensors, self._config, self._device- )- self._gate_up_alpha[layer_id] = gate_up_alpha- self._down_alpha[layer_id] = down_alpha- forwarded = {}- for name in self._post_names:- self._post[name][layer_id] = post[name]- # post[name] reinterprets banks[name]'s storage in place; release it via that bank.- forwarded[name] = _RepackedBank(post[name], banks[name])- # the two *_global banks were consumed into the alphas; they are not forwarded,- # so release them here instead of leaving them resident.- banks["gate_up_global"].release()- banks["down_global"].release()- self._outer(layer_id, forwarded)-- def assemble(self, num_layers: int):- """After the load: flat layer-major ``[L*E]`` alphas + the 4-name post-repack- per-layer source lists. Asserts every layer streamed through."""- for by_layer, what in (- (self._gate_up_alpha, "gate_up_alpha"),- (self._down_alpha, "down_alpha"),- ):- missing = [l for l in range(num_layers) if l not in by_layer]- assert not missing, f"nvfp4 repack sink never saw layers {missing} ({what})"- gate_up_alpha = torch.cat([self._gate_up_alpha[l] for l in range(num_layers)])- down_alpha = torch.cat([self._down_alpha[l] for l in range(num_layers)])- sources = {- name: [self._post[name][l] for l in range(num_layers)] for name in self._post_names- }- return sources, gate_up_alpha, down_alpha---def _nvfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:- from freetoken.models.weight import load_nvfp4_moe_expert_sources- from freetoken.moe.nvfp4_backends import (- b12x_repack_layer,- b12x_repack_sources_inplace,- marlin_repack_layer,- marlin_repack_sources_inplace,- select_nvfp4_backend,+ legacy_format_for(method.kind, kernel.name), banks,+ gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),+ streamed=layer_sink is not None, kind=method.kind, kernel=kernel.name, layout=layout,)- # Backend pick is a pure hardware/config decision, independent of the loaded data, so- # it's resolved up front. The native "nvfp4" layout (decode_target=="cpu", or the- # "triton" backend) is written as-loaded straight through the sink. marlin/b12x repack- # per expert: when converting (layer_sink given), a wrapper sink repacks each layer as- # it completes and forwards the renamed banks to the outer sink; serving (no sink)- # repacks the whole materialized bank set in place after load.- native = decode_target == "cpu"- backend = None- if not native:- backend = select_nvfp4_backend(device, getattr(model_config, "moe_intermediate_size", None),- getattr(model_config, "nvfp4_backend", "auto"),- activation=getattr(model_config, "hidden_act", "silu"))- native = backend == "triton"-- repack_sink = None- if not native and not dummy and layer_sink is not None:- repack_layer = {"marlin": marlin_repack_layer, "b12x": b12x_repack_layer}[backend]- repack_sink = _Nvfp4RepackSink(repack_layer, model_config, device, layer_sink)-- if native:- sink = None if dummy else layer_sink- else:- sink = repack_sink # None for serving/dummy marlin/b12x; the repack wrapper when converting- # parallel only parallelizes the source read; the backend repack below is reused unchanged.- sources = load_nvfp4_moe_expert_sources(- model_path, model_config, dummy=dummy, parallel=parallel, workers=workers, chunk=chunk,- layer_sink=sink,- )- # CPU-compute decode (cpu/hybrid) reads the native ModelOpt rows directly (its- # dequant-in-GEMV kernel), so keep the native "nvfp4" layout and skip the GPU-tiled- # marlin/b12x repacks (which only the GPU W4A16 kernels can read).- if decode_target == "cpu":- return ExpertBanks("nvfp4", {name: sources[name] for name in _BANK_SCHEMAS["nvfp4"]},- streamed=sink is not None)- # Pick the expert-GEMM backend by compute capability (and MoE width: auto keeps- # small-I MoE on the Triton M=1 GEMV, which beats b12x's tensor cores at single-stream- # decode) and repack the banks (in place; the tiled blocks are byte-identical per- # expert) into that backend's layout. One layout per process: prefill and decode read it.- logger.info(f"NVFP4 expert backend: {backend}")- if backend == "triton":- return ExpertBanks("nvfp4", {name: sources[name] for name in _BANK_SCHEMAS["nvfp4"]},- streamed=sink is not None)- quant_format = f"nvfp4_{backend}"- if repack_sink is not None:- # Streamed conversion: each layer was already repacked + written by the wrapper.- num_layers = len(sources["gate_up_packed"])- post_sources, gate_up_alpha, down_alpha = repack_sink.assemble(num_layers)- return ExpertBanks(- quant_format, post_sources,- gate_up_alpha=gate_up_alpha, down_alpha=down_alpha, streamed=True,- )- repack = {"marlin": marlin_repack_sources_inplace, "b12x": b12x_repack_sources_inplace}- sources = repack[backend](sources, model_config, device)- return ExpertBanks(- quant_format,- {name: sources[name] for name in _BANK_SCHEMAS[quant_format]},- gate_up_alpha=sources["gate_up_alpha"],- down_alpha=sources["down_alpha"],- )+_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel readerdef _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:@@ -252,91 +172,18 @@ def _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False,)-def _dsfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:- args = model_config.dsv4_args- assert args is not None, "ds_fp4 expert banks require dsv4_args on the model config"- # DeepSeek-FP4: packed e2m1 + e8m0 per-32 block scales, no global scale -> 4 banks,- # no alphas. DeepSeek-V4's own grouped GEMV kernels read them via bank_views().- # Written as-loaded -> streamable (dummy fabricates in one shot; never streamed).- sink = None if dummy else layer_sink- if dummy:- from freetoken.models.deepseek_v4.weight import dummy_dsfp4_expert_sources-- banks = dummy_dsfp4_expert_sources(args)- elif parallel: # parallel: common chunked multi-threaded O_DIRECT reader- from freetoken.models.deepseek_v4.weight import load_dsfp4_expert_sources_parallel-- banks = load_dsfp4_expert_sources_parallel(- model_path, args, workers=workers, chunk=chunk, layer_sink=sink- )- else:- from freetoken.models.deepseek_v4.weight import load_dsfp4_expert_sources-- banks = load_dsfp4_expert_sources(model_path, args, layer_sink=sink)- return ExpertBanks(- "ds_fp4", {name: banks[name] for name in _BANK_SCHEMAS["ds_fp4"]}, streamed=sink is not None- )---def _model_setup_override(model_config):- architectures = getattr(model_config, "architectures", None)- if not architectures:- return None-- from freetoken.models.register import _load_attr, get_model_spec-- try:- spec = get_model_spec(architectures[0])- except ValueError:- return None- try:- return _load_attr(spec.module, "setup_offload_expert_banks")- except AttributeError:- return None---# ModelConfig.expert_quant -> provider+# expert formats that still load through their own provider (GGUF)_PROVIDERS = {- "none": _bf16_banks,- "nvfp4": _nvfp4_banks,- "ds_fp4": _dsfp4_banks,"q4_0": _q4_0_banks,}-def _build_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk, decode_target="gpu", layer_sink=None) -> ExpertBanks:- """Dispatch to the model's setup-override or the per-quant provider. ``parallel=True``- is the parallel read; a provider that hasn't implemented it raises NotImplementedError (the- caller falls back to serial). ``decode_target`` lets the cpu backend force CPU-readable- (native, non-GPU-tiled) bank layouts. ``layer_sink`` (converter only) is forwarded to- setups/providers that declare the parameter; the rest ignore it and stay on the- materialize-and-write path (``ExpertBanks.streamed`` reports which happened)."""- setup = _model_setup_override(model_config)- if setup is not None:- import inspect-- params = inspect.signature(setup).parameters- supports_parallel = "parallel" in params- if parallel and not supports_parallel:- arch = getattr(model_config, "architectures", ["?"])[0]- raise NotImplementedError(- f"parallel reader not implemented for {arch} (model owns expert setup "- "via setup_offload_expert_banks; add a parallel path there)"- )- kw = dict(device=device, dtype=dtype, dummy=dummy)- if supports_parallel:- kw.update(parallel=parallel, workers=workers, chunk=chunk)- if "decode_target" in params:- kw["decode_target"] = decode_target- if "layer_sink" in params and layer_sink is not None:- kw["layer_sink"] = layer_sink- return setup(model_path, model_config, **kw)-+def _legacy_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk, decode_target="gpu", layer_sink=None) -> ExpertBanks:expert_quant = model_config.expert_quantif expert_quant not in _PROVIDERS:raise ValueError(- f"no expert-bank provider for expert_quant={expert_quant!r} "- f"(known: {sorted(_PROVIDERS)})"+ f"{expert_quant!r} experts load through their MoE quant method; "+ f"only {sorted(_PROVIDERS)} still have a format provider")return _PROVIDERS[expert_quant](model_path, model_config, device, dtype, dummy,@@ -345,6 +192,18 @@ def _build_expert_banks(model_path, model_config, device, dtype, dummy, parallel)+def _method_expert_banks(model_path, model_config, method, device, dummy, parallel, workers, chunk, layer_sink=None) -> ExpertBanks:+ from freetoken.moe.expert_pieces import iter_expert_pieces++ num_layers = model_config.num_moe_layers+ if dummy:+ return build_expert_banks(method, num_layers, None, device=device, dummy=True)+ pieces = iter_expert_pieces(+ model_path, model_config, method.kind, parallel=parallel, workers=workers, chunk=chunk+ )+ return build_expert_banks(method, num_layers, pieces, device=device, layer_sink=layer_sink)++def _host_ram_fits_parallel(model_path: str) -> bool:"""Best-effort: can free host RAM hold the expert banks plus the parallel reader's oneextra (non-reclaimable) whole-shard buffer? Unknown (non-local path / no /proc) -> True,@@ -386,11 +245,19 @@ def ftw_bank_bytes(model_path: str) -> int | None:return sum(t["nbytes"] for t in tensors if t.get("kind") == "experts_bank")-def bank_bytes_estimate(model_config) -> int | None:- """Estimated total expert-bank bytes of a raw checkpoint, from the model config alone.+def bank_bytes_estimate(model_config, method=None) -> int | None:+ """Estimated total expert-bank bytes of a raw checkpoint before loading it.- Sizes the pin-budget decisions where FTW metadata is not available; ``None`` for unknown formats or missing dims (callers then skip the pre-load sizing).- nvfp4 uses the native-row formula, a slight over-estimate for the repacked backends."""+ With a bound expert ``method`` the kernel's layout gives the exact host bytes; otherwise the+ format-tag table sizes the GGUF format. ``None`` for unknown formats or missing dims+ (callers then skip the pre-load sizing)."""+ layers = getattr(model_config, "num_moe_layers", None)+ if method is not None and layers:+ per_expert = sum(+ math.prod(spec.shape) * torch.empty((), dtype=spec.dtype).element_size()+ for spec in method.layout().values() if not spec.resident+ )+ return layers * method.cfg.num_experts * per_expertexpert_quant = getattr(model_config, "expert_quant", "none")fmt = expert_quant if expert_quant != "none" else (getattr(model_config, "moe_weight_format", None) or "bf16"@@ -409,6 +276,7 @@ def load_expert_banks(model_path: str,model_config,*,+ method=None,device: torch.device,dtype: torch.dtype,dummy: bool = False,@@ -436,6 +304,10 @@ def load_expert_banks(provider only engages it (and reports ``ExpertBanks.streamed=True``) for its ownstreamable formats, so callers must check ``streamed`` rather than assume it fired.+ ``method`` (the bound expert quant method of the model's offload layers) selects+ the generic path: the family's pieces packed by the method's kernel. Without it only the+ GGUF q4_0 format loads, through its own provider.+``layer_residency``: per-layer ``HostResidency`` labels applied at settle time -- explicitly on the FTW fast path, ambiently (``requested_residency``) in the slow-path providers.Applied labels are echoed on ``ExpertBanks.layer_residency``; a loader that settles some other way leaves it ``None`` (CPU-layer decode still works on pinned banks, it just saves no pin quota)."""@@ -480,16 +352,19 @@ def load_expert_banks(# allocation) falls back to serial.from freetoken.moe.host_banks import requested_residency+ def _build(par: bool) -> ExpertBanks:+ if method is not None:+ return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink)+ return _legacy_expert_banks(model_path, model_config, device, dtype, dummy, par, workers, chunk, decode_target, layer_sink)+with requested_residency(layer_residency) as residency_plan:try:- banks = _build_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk,- decode_target, layer_sink)+ banks = _build(parallel)except NotImplementedError as exc:if not parallel:raiselogger.warning_rank0(f"parallel reader unavailable ({exc}); falling back to serial build")- banks = _build_expert_banks(model_path, model_config, device, dtype, dummy, False, workers, chunk,- decode_target, layer_sink)+ banks = _build(False)return _echo_residency(banks, layer_residency, residency_plan)diff --git a/python/freetoken/moe/expert_pieces.py b/python/freetoken/moe/expert_pieces.pynew file mode 100644index 00000000..79f655a9--- /dev/null+++ b/python/freetoken/moe/expert_pieces.py@@ -0,0 +1,157 @@+"""Expert pieces: the checkpoint-side stream ``build_expert_banks`` packs into host banks.++A piece is ``(bank_layer, e0, e1, {role: tensor[e1 - e0, ...]})``: rows ``e0:e1`` of one MoE+layer's experts, in the checkpoint's own form (per-projection ``gate`` / ``up`` / ``down`` with+their ``_scale`` / ``_global`` companions, or an already fused ``gate_up``). The expert kernel's+``pack`` turns pieces into its bank layout, so readers never know the layout.+"""++from __future__ import annotations++from typing import Callable, Iterable, Iterator++import torch+from freetoken.layers.quantization import QuantKind+from freetoken.models.register import _load_attr, get_model_spec++Piece = tuple[int, int, int, dict[str, torch.Tensor]]+++def num_moe_layers(config) -> int:+ value = getattr(config, "num_moe_layers", None)+ if value is not None:+ return int(value)+ return int(config.num_layers) - int(getattr(config, "first_k_dense_replace", 0) or 0)+++def bank_layer_of(config, layer: int) -> int | None:+ """MoE-layer index of checkpoint layer ``layer``; None for the leading dense layers."""+ bank_layer = layer - int(getattr(config, "first_k_dense_replace", 0) or 0)+ return bank_layer if 0 <= bank_layer < num_moe_layers(config) else None+++def _model_hook(spec, name: str):+ try:+ return _load_attr(spec.module, name)+ except AttributeError:+ return None+++def iter_expert_pieces(+ model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20+) -> Iterator[Piece]:+ """The pieces of ``model_path``'s routed experts, stored as ``kind``.++ A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, parallel, workers,+ chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16+ experts come from the family's stacked ``iter_weights`` and NVFP4 experts from its+ ``nvfp4_expert_spec``. The reader is resolved here, before any bank is allocated, so a+ missing parallel reader raises ``NotImplementedError`` while a serial fallback is still cheap.+ """+ spec = get_model_spec(config.architectures[0])+ hook = _model_hook(spec, "iter_expert_pieces")+ if hook is not None:+ pieces = hook(model_path, config, kind, parallel=parallel, workers=workers, chunk=chunk)+ if pieces is not None:+ return pieces+ if kind is QuantKind.NONE:+ return _bf16_pieces(model_path, config, spec, parallel=parallel, workers=workers, chunk=chunk)+ if kind is QuantKind.NVFP4:+ spec_hook = _model_hook(spec, "nvfp4_expert_spec")+ if spec_hook is None:+ raise NotImplementedError(f"{spec.module} provides no nvfp4_expert_spec")+ from freetoken.models.nvfp4_banks import iter_nvfp4_expert_pieces++ return iter_nvfp4_expert_pieces(+ model_path, config, spec_hook(model_path, config), parallel=parallel, workers=workers, chunk=chunk+ )+ raise NotImplementedError(f"{spec.module} provides no expert reader for {kind!r} experts")+++def packed_expert_source_info(key: str) -> tuple[int, str] | None:+ """``model.layers.N....experts.{gate_up_proj,down_proj}`` -> (N, leaf); None otherwise."""+ parts = key.split(".")+ if len(parts) < 5 or parts[0] != "model" or parts[1] != "layers":+ return None+ if parts[-2] != "experts" or parts[-1] not in {"gate_up_proj", "down_proj"}:+ return None+ try:+ return int(parts[2]), parts[-1]+ except ValueError:+ return None+++def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config) -> Iterator[Piece]:+ """Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer."""+ pending: dict[int, dict[str, torch.Tensor]] = {}+ num_experts = config.num_experts+ for name, tensor in tensors:+ info = packed_expert_source_info(name)+ if info is None:+ raise ValueError(f"Unexpected expert weight key: {name}")+ layer, packed_name = info+ bank_layer = bank_layer_of(config, layer)+ if bank_layer is None:+ raise ValueError(f"Unexpected MoE expert layer {layer}; expected a routed-expert layer")+ if tensor.size(0) != num_experts:+ raise ValueError(f"Unexpected {packed_name} expert count {tensor.size(0)}; expected {num_experts}")+ piece = pending.setdefault(bank_layer, {})+ piece["gate_up" if packed_name == "gate_up_proj" else "down"] = tensor+ if len(piece) == 2:+ del pending[bank_layer]+ yield bank_layer, 0, num_experts, piece+ if pending:+ raise ValueError(f"Missing MoE expert source layers: {sorted(pending)}")+++def _bf16_pieces(model_path: str, config, spec, *, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:+ device = torch.device("cpu")+ if parallel:+ iter_weights = _model_hook(spec, "iter_weights_parallel")+ if iter_weights is None:+ raise NotImplementedError(f"{spec.module} provides no iter_weights_parallel")+ tensors = iter_weights(+ model_path, device, include_moe_experts=True, include_non_moe=False, workers=workers, chunk=chunk+ )+ else:+ iter_weights = _load_attr(spec.module, spec.iter_weights)+ tensors = iter_weights(model_path, device, include_moe_experts=True, include_non_moe=False)+ return stacked_expert_pieces(tensors, config)+++def per_expert_pieces(+ tensors: Iterable[tuple[str, torch.Tensor]],+ locate: Callable[[str], tuple[int, int, str] | None],+ *,+ tensors_per_expert: int,+) -> Iterator[Piece]:+ """Group per-expert tensors into one piece per expert, in whatever order they arrive.++ ``locate(name)`` -> ``(bank_layer, expert, role)`` or None to skip; a role tensor is one+ expert's ``[...]`` row and becomes the piece's ``[1, ...]`` entry. Globals may be scalars.+ """+ pending: dict[tuple[int, int], dict[str, torch.Tensor]] = {}+ for name, tensor in tensors:+ where = locate(name)+ if where is None:+ continue+ bank_layer, expert, role = where+ piece = pending.setdefault((bank_layer, expert), {})+ piece[role] = tensor.reshape(1, -1) if role.endswith("_global") else tensor.unsqueeze(0)+ if len(piece) == tensors_per_expert:+ del pending[(bank_layer, expert)]+ yield bank_layer, expert, expert + 1, piece+ if pending:+ missing = sorted(pending)[:8]+ raise ValueError(f"Incomplete expert tensors in checkpoint (layer, expert): {missing}")+++__all__ = [+ "Piece",+ "bank_layer_of",+ "iter_expert_pieces",+ "num_moe_layers",+ "packed_expert_source_info",+ "per_expert_pieces",+ "stacked_expert_pieces",+]diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.pyindex ecc45eae..a90aa4c7 100644--- a/python/freetoken/moe/fused.py+++ b/python/freetoken/moe/fused.py@@ -5,7 +5,6 @@ from pathlib import Pathfrom typing import Dict, Tupleimport torch-from freetoken.moe import BaseMoeBackendfrom freetoken.utils import div_ceil@@ -198,13 +197,15 @@ def fused_experts_impl(topk_ids: torch.Tensor,activation: str = "silu",apply_router_weight_on_input: bool = False,+ act_alpha: float = 1.0,+ act_limit: float = float("inf"),) -> torch.Tensor:"""Returns ``hidden_states`` itself, overwritten with the routed output. A caller thatstill needs the input afterwards (a shared expert, a residual) must read it BEFORE thiscall or pass a copy. ``fused_experts_decode_impl`` allocates instead, so the contract isnot shared; the resident bf16 path routes decode through here too."""from freetoken.kernel import fused_moe_kernel_triton, moe_sum_reduce_triton- from freetoken.layers import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul+ from freetoken.layers import gated_act_and_mulpadded_size = 0assert hidden_states.shape[1] == w1.shape[2] - padded_size, "Hidden size mismatch"@@ -273,8 +274,7 @@ def fused_experts_impl(config,compute_type=compute_type,)- FN_MAP = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}- FN_MAP[activation](intermediate_cache1.view(-1, N), intermediate_cache2)+ gated_act_and_mul(activation, intermediate_cache1.view(-1, N), intermediate_cache2, alpha=act_alpha, limit=act_limit)fused_moe_kernel_triton(intermediate_cache2,w2,@@ -305,9 +305,11 @@ def fused_experts_decode_impl(topk_ids: torch.Tensor,activation: str = "silu",apply_router_weight_on_input: bool = False,+ act_alpha: float = 1.0,+ act_limit: float = float("inf"),) -> torch.Tensor:from freetoken.kernel import fused_moe_decode_kernel_triton, moe_sum_reduce_triton- from freetoken.layers import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul+ from freetoken.layers import gated_act_and_mulassert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"assert w1.shape[0] == w2.shape[0], "Expert cache size mismatch"@@ -323,9 +325,6 @@ def fused_experts_decode_impl(assert w1.dtype == hidden_states.dtype and w2.dtype == hidden_states.dtypeassert topk_weights.dtype == torch.float32assert topk_ids.dtype == torch.int32- if activation not in {"silu", "gelu", "gelu_tanh"}:- raise ValueError(f"Unsupported activation: {activation}")-M, _ = hidden_states.shapetop_k = topk_ids.shape[1]gate_up_dim = w1.shape[1]@@ -358,8 +357,7 @@ def fused_experts_decode_impl(device=hidden_states.device,dtype=hidden_states.dtype,)- FN_MAP = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}- FN_MAP[activation](intermediate_cache1.view(-1, gate_up_dim), intermediate_cache2)+ gated_act_and_mul(activation, intermediate_cache1.view(-1, gate_up_dim), intermediate_cache2, alpha=act_alpha, limit=act_limit)intermediate_cache3 = torch.empty((M, top_k, w2.shape[1]),@@ -381,32 +379,3 @@ def fused_experts_decode_impl(out_hidden_states = torch.empty_like(hidden_states)moe_sum_reduce_triton(intermediate_cache3, out_hidden_states)return out_hidden_states---class FusedMoe(BaseMoeBackend):- def forward(- self,- hidden_states: torch.Tensor,- w1: torch.Tensor,- w2: torch.Tensor,- gating_output: torch.Tensor,- topk: int,- renormalize: bool,- activation: str = "silu",- apply_router_weight_on_input: bool = False,- ) -> torch.Tensor:- topk_weights, topk_ids = fused_topk(- hidden_states=hidden_states,- gating_output=gating_output,- topk=topk,- renormalize=renormalize,- )- return fused_experts_impl(- hidden_states,- w1,- w2,- topk_weights,- topk_ids,- activation,- apply_router_weight_on_input=apply_router_weight_on_input,- )diff --git a/python/freetoken/moe/fused_fp8_block.py b/python/freetoken/moe/fused_fp8_block.pyindex bd3dbdc1..4c7a58d4 100644--- a/python/freetoken/moe/fused_fp8_block.py+++ b/python/freetoken/moe/fused_fp8_block.py@@ -16,7 +16,7 @@ from __future__ import annotationsdef fused_experts_fp8_block(hidden_states, gate_up, gate_up_scale, down, down_scale,topk_weights, topk_ids, num_experts, activation="silu",- apply_router_weight_on_input=False,+ apply_router_weight_on_input=False, act_alpha=1.0, act_limit=float("inf"),):"""Prefill: W8A8 fused grouped GEMM over the materialized-layer banks(``[num_experts, ...]``, position == expert id)."""@@ -25,13 +25,14 @@ def fused_experts_fp8_block(return fused_experts_fp8_blockscale(hidden_states, gate_up, gate_up_scale, down, down_scale,- topk_weights, topk_ids, num_experts, activation,+ topk_weights, topk_ids, num_experts, activation, act_alpha, act_limit,)def fused_experts_decode_fp8_block(hidden_states, gate_up, gate_up_scale, down, down_scale,topk_weights, topk_ids, activation="silu", apply_router_weight_on_input=False,+ act_alpha=1.0, act_limit=float("inf"),):"""Decode: W8A16 fused inline-dequant grouped GEMV -- reads the routed experts' fp8 rowsdirectly (``topk_ids`` index the banks) and dequantizes in the K-loop. CUDA-graph safe."""@@ -40,7 +41,7 @@ def fused_experts_decode_fp8_block(return fused_experts_decode_fp8_blockscale(hidden_states, gate_up, gate_up_scale, down, down_scale,- topk_weights, topk_ids, activation,+ topk_weights, topk_ids, activation, act_alpha, act_limit,)diff --git a/python/freetoken/moe/fused_mxfp4.py b/python/freetoken/moe/fused_mxfp4.pyindex 82088aa8..ac92190b 100644--- a/python/freetoken/moe/fused_mxfp4.py+++ b/python/freetoken/moe/fused_mxfp4.py@@ -9,7 +9,7 @@ import torchfrom freetoken.moe.fused import moe_align_block_size, try_get_optimal_moe_config-# Token-count threshold used by GptOssMxfp4TritonMoELayer.forward to dispatch: batches at+# Token-count threshold used by the resident gpt-oss mxfp4 kernel to dispatch: batches at# or below this bound take the gather decode path (no sort), larger batches take the# grouped/sorted prefill kernel. GPT-OSS decode targets small batches (max running# requests), matching the gather path.diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.pyindex f736b80c..7988fa9e 100644--- a/python/freetoken/moe/fused_nvfp4.py+++ b/python/freetoken/moe/fused_nvfp4.py@@ -21,38 +21,9 @@ from freetoken.kernel.triton.nvfp4_fused_moe import (_e2m1_lut,_prefill_nvfp4_moe_kernel,)-from freetoken.layers import (- gelu_and_mul,- gelu_tanh_and_mul,- silu_and_mul,- swiglu_clamp_and_mul,- swigluoai_and_mul,-)+from freetoken.layers import gated_act_and_mulfrom freetoken.moe.fused import moe_align_block_size-_ACT = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}---def _run_act(- activation: str,- gate_up: torch.Tensor,- out: torch.Tensor,- act_alpha: float,- act_limit: float,-) -> None:- """gemm1 -> gemm2 activation dispatch. ``swigluoai`` (MiniMax-M3, clamped- gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) and- ``swiglu_clamp`` (GLM-5.3, same clamp without the +1 up bias) carry the- per-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kinds- ignore them."""- if activation == "swigluoai":- swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)- return- if activation == "swiglu_clamp":- swiglu_clamp_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)- return- _ACT[activation](gate_up, out)-# Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune,# which benchmarks at run time). Tuned offline against the NVFP4 decode kernels.# These drive the original LUT-gather decode (_decode_gemm), kept only for A/B.@@ -196,7 +167,7 @@ def _fused_experts_decode_nvfp4(ic1, topk_weights, topk_ids, apply_router_weight_on_input, False,)ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)- _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)gemm_fn(ic2, down_packed, down_scale, down_global,@@ -346,7 +317,7 @@ def fused_experts_nvfp4(apply_router_weight_on_input, cfg,)ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)- _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)_prefill_gemm(ic2, down_packed, down_scale, down_global, ic3,diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.pyindex 436f52d2..91e26487 100644--- a/python/freetoken/moe/host_banks.py+++ b/python/freetoken/moe/host_banks.py@@ -36,6 +36,10 @@ logger = init_logger(__name__)_BLK = 4096 # O_DIRECT alignment (page size)+class PinFailed(RuntimeError):+ """cudaHostRegister refused a bank: the host is out of pinnable RAM or over its pin quota."""++class HostResidency(str, Enum):"""Residency class of a host bank layer.@@ -136,9 +140,7 @@ class HostBank:try:host_register(self.addr, len(self._buf))except RuntimeError as exc:- raise RuntimeError(- f"cudaHostRegister failed for {len(self._buf) / 2**30:.1f} GiB"- ) from exc+ raise PinFailed(f"cudaHostRegister failed for {len(self._buf) / 2**30:.1f} GiB") from excself._pinned = Truedef release(self) -> None:diff --git a/python/freetoken/moe/legacy_format.py b/python/freetoken/moe/legacy_format.pynew file mode 100644index 00000000..973a3dba--- /dev/null+++ b/python/freetoken/moe/legacy_format.py@@ -0,0 +1,46 @@+"""Bank and format names on disk (FTW), in the CPU executor and in the GGUF provider; the kernels use the canonical roles."""++from __future__ import annotations++from freetoken.layers.quantization import QuantKind++CANONICAL_ROLE = {+ "gate_up_packed": "gate_up",+ "gate_up_blocks": "gate_up",+ "gate_up_scales": "gate_up_scale",+ "down_packed": "down",+ "down_blocks": "down",+ "down_scales": "down_scale",+}+++def canonical_role(bank_name: str) -> str:+ return CANONICAL_ROLE.get(bank_name, bank_name)+++def legacy_bank_names(quant_format: str) -> dict[str, str]:+ """Canonical role -> the bank name FTW files store for ``quant_format``."""+ from freetoken.moe.offload_cache import _BANK_SCHEMAS++ return {canonical_role(name): name for name in _BANK_SCHEMAS[quant_format]}+++# (kind, kernel) <-> the quant_format tag+LEGACY_FORMAT = {+ (QuantKind.NONE, "fused"): "bf16",+ (QuantKind.FP8_BLOCK, "triton"): "fp8_block",+ (QuantKind.NVFP4, "triton"): "nvfp4",+ (QuantKind.NVFP4, "marlin"): "nvfp4_marlin",+ (QuantKind.NVFP4, "b12x"): "nvfp4_b12x",+ (QuantKind.MXFP4, "triton_gptoss"): "mxfp4_triton",+ (QuantKind.MXFP4, "triton"): "ds_fp4",+}+_KIND_KERNEL = {fmt: kk for kk, fmt in LEGACY_FORMAT.items()}+++def legacy_format_for(kind: QuantKind, kernel: str) -> str:+ return LEGACY_FORMAT[(kind, kernel)]+++def kind_kernel_for(legacy_format: str) -> tuple[QuantKind, str]:+ return _KIND_KERNEL[legacy_format]diff --git a/python/freetoken/moe/nvfp4_backends.py b/python/freetoken/moe/nvfp4_backends.pydeleted file mode 100644index e8a3c6f6..00000000--- a/python/freetoken/moe/nvfp4_backends.py+++ /dev/null@@ -1,790 +0,0 @@-"""NVFP4 expert-GEMM backends borrowed from vLLM (Marlin) and flashinfer (b12x).--FreeToken stores MoE experts as ModelOpt NVFP4 (packed e2m1 codes + fp8-e4m3 per-16-block scales + per-tensor global scale) in pinned host banks and gathers the routed-experts into a GPU slot cache. This module picks the fastest fused-MoE kernel for-that format by compute capability and owns the matching bank layout:--========== ========================== =============================================-backend compute capability kernel / weight layout-========== ========================== =============================================-marlin sm_80 .. sm_99 vLLM ``fused_marlin_moe`` (W4A16- dequant-in-kernel); Marlin-tiled weights-b12x sm_120+ and CUDA>=13 ``flashinfer`` SM12x CuTe-DSL MoE (W4A16);- b12x-packed weights-triton anything (fallback) FreeToken's own Triton kernels; the native- row-major ModelOpt layout (used on sm_120 +- CUDA 12.x, where b12x cannot run)-========== ========================== =============================================--The Marlin kernels come from vLLM (not sgl-kernel) deliberately: vLLM ships the-NVFP4 instantiations in its AOT wheel together with the matching host-side layout-transforms, so the pair stays consistent by construction. sgl-kernel's AOT wheel-excludes the NVFP4 MoE Marlin kernels to cut wheel size (its python transforms pair-with sglang's *JIT* kernel tree, whose scale encoding has since diverged).--Hard rules:--* A backend owns ``{load-time pack, forward}`` as a unit. Prefill and decode consume- the *same* bank layout -- there is never a second copy of the experts.-* Every pack keeps the expert dimension outermost with per-expert blocks contiguous- and byte-identical in size to the native layout, so the banks are repacked *in- place* and the offload cache's slot gather (``copy_missing``) works unchanged.-* The per-tensor global scales never enter the offload banks: they are tiny- ``[L*E]`` vectors kept resident on the GPU and gathered per forward call- (device-side, CUDA-graph safe).--The small layout-transform helpers are ported from sglang-(``srt/layers/quantization/marlin_utils{,_fp4}.py``, Apache-2.0); the kernels-themselves are imported, not vendored.-"""--from __future__ import annotations--import os--import torch--from freetoken.utils import init_logger--logger = init_logger(__name__)--# The 6 native ModelOpt NVFP4 source bank names (== _BANK_SCHEMAS["nvfp4"]) the repack-# reads, and the 4 post-repack bank names both marlin and b12x emit (== the "nvfp4_marlin"-# / "nvfp4_b12x" schemas); the two *_global banks fold into per-expert alphas, not banks.-_NATIVE_NVFP4_BANKS = (- "gate_up_packed", "gate_up_scale", "gate_up_global",- "down_packed", "down_scale", "down_global",-)-_POST_NVFP4_BANKS = ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale")--# ----------------------------------------------------------------------------# Backend selection-# ------------------------------------------------------------------------------def _donor_symbols_ok(backend: str) -> bool:- """Probe the exact donor symbols the pack/forward paths below use.-- ``find_spec`` only proves the top-level package exists; an incompatible donor- version would otherwise crash mid-init, after minutes of bank loading. Probing- at selection time degrades to triton before any bank is touched.- """- try:- if backend == "marlin":- from vllm import _custom_ops # noqa: F401- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( # noqa: F401- fused_marlin_moe,- )- from vllm.model_executor.layers.quantization.utils.marlin_utils import ( # noqa: F401- marlin_permute_scales,- )- from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401- nvfp4_marlin_process_global_scale,- nvfp4_marlin_process_scales,- )- from vllm.scalar_type import scalar_types # noqa: F401- else:- assert backend == "b12x"- # Probe exactly what the pack/forward use: the row-major->swizzled scale- # transform, the W4A16 prepare, and the prepared-weights launch (which also- # transitively imports the CuTe-DSL kernel + cutlass).- from flashinfer import nvfp4_block_scale_interleave # noqa: F401- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import ( # noqa: F401- _launch_sm120_w4a16_moe,- )- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import ( # noqa: F401- W4A16PackedWeights,- _make_workspace,- prepare_w4a16_packed_weights,- )- except Exception as exc:- logger.warning(- f"NVFP4 {backend} backend is installed but unusable ({exc!r}); "- "falling back to the Triton inline-dequant backend"- )- return False- return True---def _flashinfer_cuda_major() -> int | None:- """flashinfer's own toolkit-CUDA major; None if it can't be determined. Only a- fallback proxy when the driver version is unavailable -- the b12x kernel is JIT- PTX-compiled through the driver, so the toolkit major is *not* what gates it."""- try:- from flashinfer.jit.cpp_ext import get_cuda_version-- return int(get_cuda_version().major)- except Exception:- return None---def _b12x_unusable_reason(cc: tuple[int, int]) -> str | None:- """None if the flashinfer b12x decode/prefill kernel can actually run on this device,- else a human-readable reason. The b12x *pack* works anywhere, but the SM12x CuTe-DSL- kernel itself requires sm_120+ and a CUDA>=13 *driver* (it JIT-compiles PTX through the- driver), so selection must check the runtime here -- not just that flashinfer imports --- or the model loads in the b12x layout and then crashes on the first decode."""- import importlib.util-- if cc < (12, 0):- return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"- if importlib.util.find_spec("flashinfer") is None:- return "flashinfer is not installed"- from freetoken.kernel.backend import driver_cuda_version-- drv = driver_cuda_version()- if drv is not None:- if drv < 13000:- return (- "b12x fused MoE requires a CUDA>=13 driver "- f"(driver supports CUDA {drv // 1000}.{(drv % 1000) // 10})"- )- else:- # Driver version undetermined: fall back to flashinfer's toolkit major as a- # conservative proxy rather than risk loading the b12x layout and crashing.- major = _flashinfer_cuda_major()- if major is not None and major < 13:- return (- "b12x fused MoE requires CUDA>=13 (driver undetermined; "- f"flashinfer toolkit is CUDA {major}.x)"- )- if not _donor_symbols_ok("b12x"):- return "flashinfer b12x donor symbols are unusable"- return None---def _b12x_min_intermediate() -> int:- """Smallest ``moe_intermediate_size`` for which ``auto`` prefers b12x over Triton.-- One bank layout is chosen per process (b12x's prepared/tiled weights and Triton's- native row-major layout are not byte-compatible, so they can't coexist without a 2x- expert-memory copy), so this is a *load-time* pick by shape. The default (1024) is the- M=1 single-stream decode crossover measured on RTX 5090 (CUDA-graph %HBM, top_k=8):-- I=512 (Qwen3.x-35B-A3B): triton 43% vs b12x 27% -> triton- I=768 (Qwen3-30B-A3B): triton 41% vs b12x 41% -> triton (tie; b12x wins batched)- I=1536 (MiniMax-M2): triton 65% vs b12x 82% -> b12x-- b12x always wins *batched* decode (M>=~4) thanks to tensor cores, so a throughput- deployment of a small-I model should force it with ``--nvfp4-backend flashinfer``.- Override the threshold with ``FREETOKEN_NVFP4_B12X_MIN_I``."""- raw = os.environ.get("FREETOKEN_NVFP4_B12X_MIN_I")- if raw is None:- return 1024- try:- return int(raw)- except ValueError as exc:- raise ValueError(- f"bad FREETOKEN_NVFP4_B12X_MIN_I={raw!r}; expected an integer"- ) from exc---def select_nvfp4_backend(- device: torch.device,- intermediate_size: int | None = None,- requested: str = "auto",- activation: str = "silu",-) -> str:- """Pick the NVFP4 expert-GEMM backend for ``device`` (and, in ``auto``, ``intermediate_size``).-- ``requested`` is the ``--nvfp4-backend`` choice; a forced backend fails loudly (never- silently degrades) when it cannot run on this hardware/toolkit:-- * ``auto`` (default) -- capability pick: ``marlin`` on sm_80-99 (+ vLLM), flashinfer- ``b12x`` on sm_120+ (CUDA>=13 driver) when the MoE is wide enough that its tensor cores- beat Triton at single-stream M=1 decode (``intermediate_size >= _b12x_min_intermediate()``),- else the portable Triton inline-dequant GEMV.- * ``marlin`` / ``flashinfer`` / ``triton`` -- force that backend (and its bank layout),- raising if it cannot run rather than degrading.-- ``activation`` is the model's routed-expert activation: the borrowed marlin/b12x- kernels hard-code silu (their fused epilogue), so any other activation (MiniMax-M3's- ``swigluoai``) resolves ``auto`` to the Triton kernels -- which dispatch the- activation as a separate elementwise op -- and rejects a forced marlin/flashinfer.-- Returns the internal backend name (``marlin`` / ``b12x`` / ``triton``); ``flashinfer``- maps to ``b12x``.- """- import importlib.util-- requested = (requested or "auto").strip().lower()- if requested not in ("auto", "marlin", "flashinfer", "triton"):- raise ValueError(- f"bad --nvfp4-backend={requested!r}; expected auto, marlin, flashinfer or triton"- )- if requested == "triton":- return "triton"- if activation != "silu":- if requested != "auto":- raise RuntimeError(- f"--nvfp4-backend={requested} only supports silu routed experts (its fused "- f"epilogue); this model's experts use {activation!r} -- use "- "--nvfp4-backend triton (or auto)."- )- logger.info(- f"NVFP4 auto backend: routed experts use {activation!r}, which only the "- "Triton kernels support; using the Triton inline-dequant kernels"- )- return "triton"- if requested == "marlin":- if device.type != "cuda":- raise RuntimeError("--nvfp4-backend=marlin requires a CUDA device")- if not _donor_symbols_ok("marlin"):- raise RuntimeError(- "--nvfp4-backend=marlin is unusable: vLLM Marlin donor symbols are unavailable"- )- return "marlin"- if requested == "flashinfer":- cc = torch.cuda.get_device_capability(device) if device.type == "cuda" else (0, 0)- reason = _b12x_unusable_reason(cc)- if reason is not None:- raise RuntimeError(f"--nvfp4-backend=flashinfer is unavailable: {reason}")- return "b12x"-- # auto: capability pick, but only land on marlin/b12x when they can really run.- if device.type != "cuda":- return "triton"- cc = torch.cuda.get_device_capability(device)- if (- (8, 0) <= cc < (10, 0)- and importlib.util.find_spec("vllm") is not None- and _donor_symbols_ok("marlin")- ):- return "marlin"- if cc >= (12, 0):- reason = _b12x_unusable_reason(cc)- if reason is None:- thr = _b12x_min_intermediate()- if intermediate_size is None or intermediate_size >= thr:- return "b12x"- logger.info(- f"NVFP4 auto backend: b12x is runnable but moe_intermediate_size="- f"{intermediate_size} < {thr}; using the Triton decode kernels "- "(faster at single-stream M=1 decode for small-I MoE; set "- "--nvfp4-backend flashinfer to force b12x for batched throughput)"- )- return "triton"- logger.info(- f"NVFP4 auto backend: flashinfer b12x unavailable ({reason}); "- "using the Triton inline-dequant decode kernels"- )- return "triton" # sm_100/103, b12x unusable, or the donor package is missing---# ----------------------------------------------------------------------------# Marlin pack (kernels and layout transforms borrowed from vLLM; mirrors its-# prepare_moe_fp4_layer_for_marlin so the host prep always matches the AOT op)-# ------------------------------------------------------------------------------def _marlin_pack_proj(- packed: torch.Tensor,- scale: torch.Tensor,- row_global: torch.Tensor,- *,- size_k: int,- size_n: int,-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:- """Pack one expert projection (on GPU) into the Marlin layout.-- ``packed`` ``[N, K//2]`` uint8 e2m1 pairs, ``scale`` ``[N, K//16]`` e4m3,- ``row_global`` ``[N]`` fp16 per-row global (rows of a merged gate_up may carry- w1's and w3's different globals). Returns ``(qweight [K//16, 2N] int32,- scales [K//16, N] e4m3-coded, global scalar bf16)``.-- Marlin takes one global scalar per projection, so when the row globals differ- the ratio is folded into the fp16 block scales before they are re-encoded- (exact when all rows share one global, <=1 scale ulp otherwise).- """- from vllm import _custom_ops as ops- from vllm.model_executor.layers.quantization.utils.marlin_utils import (- marlin_permute_scales,- )- from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (- nvfp4_marlin_process_global_scale,- nvfp4_marlin_process_scales,- )-- assert size_n % 64 == 0, f"Marlin requires N % 64 == 0, got {size_n}"- qweight = ops.gptq_marlin_repack(- b_q_weight=packed.view(torch.int32).T.contiguous(),- perm=torch.empty(0, dtype=torch.int, device=packed.device),- size_k=size_k,- size_n=size_n,- num_bits=4,- )-- g = row_global.to(torch.float32)- g_max = g.max()- s = scale.to(torch.bfloat16)- if not torch.all(g == g_max):- s = (s.float() * (g / g_max).unsqueeze(1)).to(torch.bfloat16)- s = marlin_permute_scales(s.T, size_k=size_k, size_n=size_n, group_size=16)- s = nvfp4_marlin_process_scales(s)- if isinstance(s, tuple): # newer vLLM returns (scales, scale_factor)- s, factor = s- g_max = g_max / factor- g_out = nvfp4_marlin_process_global_scale(g_max.to(torch.bfloat16).reshape(1))- return qweight, s, g_out--[email protected]_grad()-def marlin_repack_layer(- layer_banks: dict[str, torch.Tensor],- config,- device: torch.device,- *,- chunk: int = 32,-) -> tuple[dict[str, torch.Tensor], torch.Tensor, torch.Tensor]:- """Repack ONE layer's 6 native NVFP4 banks into the Marlin layout, in place.-- ``layer_banks`` are that layer's native ``[E, ...]`` tensors keyed by the 6 native- NVFP4 bank names (``gate_up_packed/scale/global``, ``down_packed/scale/global``).- The Marlin blocks are byte-identical in size, so the weight/scale storage is- reinterpreted rather than reallocated; the two ``*_global`` banks are consumed into- per-expert alphas and are not part of the output layout.-- Returns ``({post-repack bank name -> reinterpreted tensor}, gate_up_alpha, down_alpha)``- where the dict is keyed by the 4 ``nvfp4_marlin`` bank names and the two alphas are- ``[E]`` bf16 on ``device``. Staging ``.to(device, non_blocking=True)`` from the native- source works whether or not it is pinned (pageable -> synchronous copy)."""- H = config.hidden_size- I = config.moe_intermediate_size- gu_packed_l = layer_banks["gate_up_packed"]- gu_scale_l = layer_banks["gate_up_scale"]- gu_global_l = layer_banks["gate_up_global"]- dn_packed_l = layer_banks["down_packed"]- dn_scale_l = layer_banks["down_scale"]- dn_global_l = layer_banks["down_global"]- E = gu_packed_l.size(0)-- gate_up_q = gu_packed_l.view(torch.int32).view(E, H // 16, 4 * I)- gate_up_s = gu_scale_l.view(E, H // 16, 2 * I)- down_q = dn_packed_l.view(torch.int32).view(E, I // 16, 2 * H)- down_s = dn_scale_l.view(E, I // 16, H)- gate_up_alpha = torch.empty(E, dtype=torch.bfloat16, device=device)- down_alpha = torch.empty(E, dtype=torch.bfloat16, device=device)-- for start in range(0, E, chunk):- end = min(start + chunk, E)- # Stage the native rows on GPU before their (in-place) storage is overwritten.- gu_p = gu_packed_l[start:end].to(device, non_blocking=True)- gu_s = gu_scale_l[start:end].to(device, non_blocking=True)- gu_g = gu_global_l[start:end].to(device, non_blocking=True)- dn_p = dn_packed_l[start:end].to(device, non_blocking=True)- dn_s = dn_scale_l[start:end].to(device, non_blocking=True)- dn_g = dn_global_l[start:end].to(device, non_blocking=True)- for i in range(end - start):- qw, sc, al = _marlin_pack_proj(gu_p[i], gu_s[i], gu_g[i], size_k=H, size_n=2 * I)- gate_up_q[start + i].copy_(qw, non_blocking=True)- gate_up_s[start + i].copy_(sc.view(torch.float8_e4m3fn), non_blocking=True)- gate_up_alpha[start + i] = al[0]- qw, sc, al = _marlin_pack_proj(dn_p[i], dn_s[i], dn_g[i], size_k=I, size_n=H)- down_q[start + i].copy_(qw, non_blocking=True)- down_s[start + i].copy_(sc.view(torch.float8_e4m3fn), non_blocking=True)- down_alpha[start + i] = al[0]- torch.cuda.synchronize(device)-- return (- {- "gate_up_packed": gate_up_q,- "gate_up_scale": gate_up_s,- "down_packed": down_q,- "down_scale": down_s,- },- gate_up_alpha,- down_alpha,- )--[email protected]_grad()-def marlin_repack_sources_inplace(- sources: dict[str, list[torch.Tensor]],- config,- device: torch.device,- *,- chunk: int = 32,-) -> dict[str, list[torch.Tensor] | torch.Tensor]:- """Convert the native NVFP4 banks from ``load_nvfp4_expert_sources`` to the- Marlin layout, in place per layer (thin per-layer loop over- :func:`marlin_repack_layer`; the Marlin blocks are byte-identical in size, so each- layer's bank tensor is reinterpreted rather than reallocated).-- Returns the per-layer bank lists for ``OffloadMoeCache.set_bank_sources`` plus- the processed per-expert global-scale vectors (``gate_up_alpha`` / ``down_alpha``,- flat ``[L*E]`` bf16 on ``device``, layer-major) that the forward path gathers- per slot.- """- num_layers = len(sources["gate_up_packed"])-- gate_up_q_layers: list[torch.Tensor] = []- gate_up_s_layers: list[torch.Tensor] = []- down_q_layers: list[torch.Tensor] = []- down_s_layers: list[torch.Tensor] = []- gate_up_alpha_layers: list[torch.Tensor] = []- down_alpha_layers: list[torch.Tensor] = []-- for layer_id in range(num_layers):- layer_banks = {name: sources[name][layer_id] for name in _NATIVE_NVFP4_BANKS}- post, gate_up_alpha, down_alpha = marlin_repack_layer(- layer_banks, config, device, chunk=chunk- )- gate_up_q_layers.append(post["gate_up_packed"])- gate_up_s_layers.append(post["gate_up_scale"])- down_q_layers.append(post["down_packed"])- down_s_layers.append(post["down_scale"])- gate_up_alpha_layers.append(gate_up_alpha)- down_alpha_layers.append(down_alpha)-- return {- "gate_up_packed": gate_up_q_layers,- "gate_up_scale": gate_up_s_layers,- "down_packed": down_q_layers,- "down_scale": down_s_layers,- "gate_up_alpha": torch.cat(gate_up_alpha_layers),- "down_alpha": torch.cat(down_alpha_layers),- }---# ----------------------------------------------------------------------------# Marlin forward (vLLM's fused_marlin_moe, borrowed wholesale)-# ------------------------------------------------------------------------------def marlin_fused_experts(- hidden_states: torch.Tensor,- gate_up_q: torch.Tensor,- gate_up_s: torch.Tensor,- gate_up_alpha: torch.Tensor,- down_q: torch.Tensor,- down_s: torch.Tensor,- down_alpha: torch.Tensor,- topk_weights: torch.Tensor,- topk_ids: torch.Tensor,- activation: str,- apply_router_weight_on_input: bool,-) -> torch.Tensor:- """Marlin W4A16 fused MoE (two grouped GEMMs + activation + reduce).-- Two calling regimes share this entry: decode passes the full ``[S]`` slot cache- with ``topk_ids`` rewritten to slot ids; full-layer prefill passes banks whose- position == expert id (the materialized ``[:E]`` slot view or the overlap double- buffer views), so the raw routing ids arrive unmapped. The ``*_alpha`` vectors- are the matching per-row global scales in both regimes.- vLLM's implementation is device-side only (no host syncs), so the decode call is- CUDA-graph capturable.- """- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe- from vllm.scalar_type import scalar_types-- assert activation == "silu", "Marlin NVFP4 backend supports gated silu only"- return fused_marlin_moe(- hidden_states,- gate_up_q,- down_q,- None, # bias1- None, # bias2- gate_up_s,- down_s,- gating_output=None,- topk_weights=topk_weights,- topk_ids=topk_ids,- quant_type_id=scalar_types.float4_e2m1f.id,- apply_router_weight_on_input=apply_router_weight_on_input,- global_num_experts=gate_up_q.size(0),- activation=activation,- global_scale1=gate_up_alpha,- global_scale2=down_alpha,- )---# ----------------------------------------------------------------------------# b12x (flashinfer SM12x CuTe-DSL, W4A16) -- the pack (prepare_w4a16_packed_weights)-# runs on any CUDA build, but the fused-MoE kernel needs sm_120/121 AND a CUDA>=13-# *driver* (it JIT-compiles PTX at runtime). select_nvfp4_backend gates on that, so-# this path is only reached on hardware that can actually run it.-#-# Layout contract (verified against flashinfer 0.6.12 prepare_w4a16_packed_weights):-# * prepare() takes the block scales already *swizzled* (it calls unswizzle_expert_-# scales internally), so the native row-major fp8 scales are run through-# nvfp4_block_scale_interleave first.-# * prepare() reorders the w13 rows with reorder_w13_to_gate_up == cat([second, first]),-# i.e. it expects the merged proj as [up, gate] and emits [gate, up]. FreeToken stores-# [gate, up], so the halves are swapped before prepare() to come out [gate, up] again-# (the kernel computes silu(first) * second == silu(gate) * up).-# * modelopt globals are passed straight through (no 1/g inversion).-# The fused forward reconstructs the W4A16PackedWeights from the banks and uses the-# prepared-weights launch (_launch_sm120_w4a16_moe) directly, so weights are prepared-# exactly once at load time (b12x_fused_moe would re-prepare + ptr-cache every call).-# ------------------------------------------------------------------------------def _b12x_swizzle_block_scales(scale: torch.Tensor) -> torch.Tensor:- """Row-major per-16 fp8 block scales ``[n, N, K//16]`` -> flashinfer's expert-leading- *swizzled* storage, which is what :func:`prepare_w4a16_packed_weights` expects on input- (it unswizzles internally). The 128x4 interleave is per-expert; ``N`` is a multiple of- 128 and ``K//16`` a multiple of 4 for the supported models, so the shape is preserved."""- from flashinfer import nvfp4_block_scale_interleave-- pieces = [- nvfp4_block_scale_interleave(scale[e].view(torch.uint8).contiguous())- for e in range(scale.shape[0])- ]- return torch.stack(pieces, dim=0).view(torch.float8_e4m3fn)--[email protected]_grad()-def b12x_repack_layer(- layer_banks: dict[str, torch.Tensor],- config,- device: torch.device,- *,- chunk: int = 32,-) -> tuple[dict[str, torch.Tensor], torch.Tensor, torch.Tensor]:- """Repack ONE layer's 6 native NVFP4 banks into flashinfer's SM12x W4A16 layout, in place.-- Mirrors :func:`marlin_repack_layer`; the b12x packed blocks are also byte-identical- per expert (4-bit weights + e4m3-coded scales), so the weight/scale storage is- reinterpreted and the two ``*_global`` banks fold into ``[E]`` fp32 alphas. Uses- flashinfer's own prepare helpers so the layout always matches the installed kernel.-- Returns ``({post-repack bank name -> reinterpreted tensor}, gate_up_alpha, down_alpha)``- keyed by the 4 ``nvfp4_b12x`` bank names. Staging from the native source works whether- or not it is pinned (pageable ``.to(device)`` is a synchronous copy)."""- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (- prepare_w4a16_packed_weights,- )-- I = config.moe_intermediate_size- gu_packed_l = layer_banks["gate_up_packed"]- gu_scale_l = layer_banks["gate_up_scale"]- gu_global_l = layer_banks["gate_up_global"]- dn_packed_l = layer_banks["down_packed"]- dn_scale_l = layer_banks["down_scale"]- dn_global_l = layer_banks["down_global"]- E = gu_packed_l.size(0)-- gate_up_alpha = torch.empty(E, dtype=torch.float32, device=device)- down_alpha = torch.empty(E, dtype=torch.float32, device=device)- gate_up_q = gate_up_s = down_q = down_s = None-- for start in range(0, E, chunk):- end = min(start + chunk, E)- n = end - start- gu_g = gu_global_l[start:end].to(device)- # The merged gate_up rows carry w1/w3 globals; b12x also takes one alpha per- # expert, so fold any ratio into the block scales like the Marlin pack does.- g = gu_g.float()- g_max = g.max(dim=1, keepdim=True).values- gu_s_native = gu_scale_l[start:end].to(device).to(torch.float16)- ratio = g / g_max- if not torch.all(ratio == 1.0):- gu_s_native = (gu_s_native.float() * ratio.unsqueeze(-1)).to(torch.float16)- # The down bank stores one w2 global broadcast per output row; b12x takes a- # single alpha per expert, so a row-varying global cannot be represented (the- # marlin pack folds the ratio into the block scales instead -- do not let such- # a checkpoint be silently truncated to row 0 here).- dn_g = dn_global_l[start:end].to(device)- if not torch.all(dn_g == dn_g[:, :1]):- raise ValueError("b12x pack requires a row-constant per-expert down_proj global scale")- # prepare() emits [gate, up] from a [up, gate] input (reorder_w13_to_gate_up),- # so swap FreeToken's native [gate, up] halves here; swap the (ratio-folded)- # block scales identically to keep weight/scale rows aligned.- gu_p = gu_packed_l[start:end].to(device)- gu_p = torch.cat([gu_p[:, I:], gu_p[:, :I]], dim=1).contiguous()- gu_s_native = torch.cat(- [gu_s_native[:, I:], gu_s_native[:, :I]], dim=1- ).contiguous()- prepared = prepare_w4a16_packed_weights(- gu_p,- _b12x_swizzle_block_scales(gu_s_native.to(torch.float8_e4m3fn)),- g_max.squeeze(1),- dn_packed_l[start:end].to(device),- _b12x_swizzle_block_scales(dn_scale_l[start:end].to(device)),- dn_g[:, 0].float(),- activation="silu",- params_dtype=torch.bfloat16,- source_format="modelopt",- )- if gate_up_q is None:- def _as_bank(bank: torch.Tensor, t: torch.Tensor) -> torch.Tensor:- assert t[0].nbytes * E == bank.nbytes, (- f"b12x block size mismatch: {t[0].nbytes} * {E} != {bank.nbytes}"- )- return bank.view(t.dtype).view(E, *t.shape[1:])-- gate_up_q = _as_bank(gu_packed_l, prepared.w13)- gate_up_s = _as_bank(gu_scale_l, prepared.w13_scale)- down_q = _as_bank(dn_packed_l, prepared.w2)- down_s = _as_bank(dn_scale_l, prepared.w2_scale)- gate_up_q[start:end].copy_(prepared.w13[:n])- gate_up_s[start:end].copy_(prepared.w13_scale[:n])- down_q[start:end].copy_(prepared.w2[:n])- down_s[start:end].copy_(prepared.w2_scale[:n])- gate_up_alpha[start:end] = prepared.w13_global_scale[:n].float()- down_alpha[start:end] = prepared.w2_global_scale[:n].float()-- return (- {- "gate_up_packed": gate_up_q,- "gate_up_scale": gate_up_s,- "down_packed": down_q,- "down_scale": down_s,- },- gate_up_alpha,- down_alpha,- )--[email protected]_grad()-def b12x_repack_sources_inplace(- sources: dict[str, list[torch.Tensor]],- config,- device: torch.device,- *,- chunk: int = 32,-) -> dict[str, list[torch.Tensor] | torch.Tensor]:- """Convert the native NVFP4 banks to flashinfer's SM12x W4A16 layout in place, per- layer (thin per-layer loop over :func:`b12x_repack_layer`).-- Mirrors :func:`marlin_repack_sources_inplace`; the b12x packed blocks are also- byte-identical per expert (4-bit weights + e4m3-coded scales), so each layer's- bank tensor is reinterpreted.- """- num_layers = len(sources["gate_up_packed"])-- gate_up_q_layers: list[torch.Tensor] = []- gate_up_s_layers: list[torch.Tensor] = []- down_q_layers: list[torch.Tensor] = []- down_s_layers: list[torch.Tensor] = []- gate_up_alpha_layers: list[torch.Tensor] = []- down_alpha_layers: list[torch.Tensor] = []-- for layer_id in range(num_layers):- layer_banks = {name: sources[name][layer_id] for name in _NATIVE_NVFP4_BANKS}- post, gate_up_alpha, down_alpha = b12x_repack_layer(- layer_banks, config, device, chunk=chunk- )- gate_up_q_layers.append(post["gate_up_packed"])- gate_up_s_layers.append(post["gate_up_scale"])- down_q_layers.append(post["down_packed"])- down_s_layers.append(post["down_scale"])- gate_up_alpha_layers.append(gate_up_alpha)- down_alpha_layers.append(down_alpha)-- return {- "gate_up_packed": gate_up_q_layers,- "gate_up_scale": gate_up_s_layers,- "down_packed": down_q_layers,- "down_scale": down_s_layers,- "gate_up_alpha": torch.cat(gate_up_alpha_layers),- "down_alpha": torch.cat(down_alpha_layers),- }---# Per-device scratch (sized sms*4+2) that run_w4a16_moe reads off the prepared object;-# tiny and shape-independent, so one per device is reused across all layers/decodes.-_B12X_WORKSPACE: dict[torch.device, torch.Tensor] = {}---def _b12x_small_workspace(device: torch.device) -> torch.Tensor:- ws = _B12X_WORKSPACE.get(device)- if ws is None:- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (- _make_workspace,- )-- ws = _make_workspace(device, max_blocks_per_sm=4)- _B12X_WORKSPACE[device] = ws- return ws---def b12x_fused_experts(- hidden_states: torch.Tensor,- gate_up_q: torch.Tensor,- gate_up_s: torch.Tensor,- gate_up_alpha: torch.Tensor,- down_q: torch.Tensor,- down_s: torch.Tensor,- down_alpha: torch.Tensor,- topk_weights: torch.Tensor,- topk_ids: torch.Tensor,- activation: str,- apply_router_weight_on_input: bool,-) -> torch.Tensor:- """SM12x W4A16 fused MoE over cache slots (same calling convention as- :func:`marlin_fused_experts`). Requires sm_120/121 + a CUDA>=13 driver; reached only- when :func:`select_nvfp4_backend` has confirmed the runtime supports it.-- The banks already hold flashinfer's prepared (tiled) layout from- :func:`b12x_repack_sources_inplace`, so this wraps them back into a- ``W4A16PackedWeights`` and calls the prepared-weights launch directly -- the public- ``b12x_fused_moe`` would re-prepare the raw modelopt weights (and ptr-cache them) on- every call, which both costs a prepare per step and breaks for the offload cache,- whose slot contents move between calls.-- CUDA-graph note: the launch resolves a (shape-keyed, module-cached) scratch workspace- on first use and flashinfer raises if that happens *during* capture, so the decode- path must be warmed once eagerly before graph capture (FreeToken already does)."""- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import (- _launch_sm120_w4a16_moe,- )- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (- W4A16PackedWeights,- )-- assert activation == "silu", "b12x backend supports gated silu only"- assert not apply_router_weight_on_input- num_experts = gate_up_q.size(0)- hidden_size = hidden_states.size(-1)- # down bank is the prepared w2 == [E, K_tiles, ...] with K_tiles == intermediate//16.- intermediate_size = down_q.size(1) * 16- prepared = W4A16PackedWeights(- w13=gate_up_q,- w13_scale=gate_up_s,- w13_global_scale=gate_up_alpha,- w2=down_q,- w2_scale=down_s,- w2_global_scale=down_alpha,- workspace=_b12x_small_workspace(hidden_states.device),- hidden_size=hidden_size,- intermediate_size=intermediate_size,- num_experts=num_experts,- is_gated=True,- params_dtype=hidden_states.dtype,- source_format="modelopt",- )- out = torch.empty(- hidden_states.size(0), hidden_size, dtype=hidden_states.dtype, device=hidden_states.device- )- _launch_sm120_w4a16_moe(- a=hidden_states,- topk_ids=topk_ids,- topk_weights=topk_weights,- w1_weight=gate_up_q,- w1_weight_sf=gate_up_s,- w1_alpha=gate_up_alpha,- w2_weight=down_q,- w2_weight_sf=down_s,- w2_alpha=down_alpha,- num_experts=num_experts,- top_k=topk_ids.size(1),- num_local_experts=num_experts,- scatter_output=out,- activation="silu",- source_format="modelopt",- _prepared_weights=prepared,- )- return out---__all__ = [- "select_nvfp4_backend",- "marlin_repack_layer",- "marlin_repack_sources_inplace",- "marlin_fused_experts",- "b12x_repack_layer",- "b12x_repack_sources_inplace",- "b12x_fused_experts",-]diff --git a/python/freetoken/moe/offload.py b/python/freetoken/moe/offload.pydeleted file mode 100644index 0e688a7d..00000000--- a/python/freetoken/moe/offload.py+++ /dev/null@@ -1,20 +0,0 @@-from __future__ import annotations--import torch--from .base import BaseMoeBackend---class OffloadMoeBackend(BaseMoeBackend):- def forward(- self,- hidden_states: torch.Tensor,- w1: torch.Tensor,- w2: torch.Tensor,- gating_output: torch.Tensor,- topk: int,- renormalize: bool,- activation: str,- apply_router_weight_on_input: bool,- ) -> torch.Tensor:- raise RuntimeError("Offload MoE is handled by OffloadMoELayer")diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex e1f20dd2..7abb9c80 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -141,18 +141,22 @@ class OffloadMoeCache:# pcie_bw / cpu_bw ratio so the PCIe fetch and the CPU overflow GEMV take equal# time (perfect overlap): fetched : cpu = pcie : cpu - pcie.hybrid_fetch_fraction: float = 0.0+ # bank layout from the expert kernel (a BankSpec per role); when given it replaces the _BANK_SCHEMAS lookup and the slot cap comes from max_slots+ layout: dict | None = None+ max_slots: int | None = Nonedef __post_init__(self) -> None:policy_ids = {"lru": 0}assert self.cache_policy in policy_idsassert self.decode_target in ("gpu", "cpu", "hybrid"), self.decode_target- assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"+ if self.layout is None:+ assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"# Attached by the engine for decode_target == "cpu" (CpuMoeExecutor); None# for the GPU decode path.self.cpu_executor = None# MoE layer ids whose decode runs on the CPU executor; the rest use the GPU# offload/PCIe path. Set by the engine after construction (empty = all-GPU,- # all layers = the plain --moe-backend cpu case).+ # all layers = the plain --moe-strategy cpu case).self.cpu_layer_ids: frozenset = frozenset()# num_experts floor + nvfp4_marlin slot cap, shared with the runtime-rebuild path.self.validate_rebuild(self.cache_size)@@ -199,7 +203,10 @@ class OffloadMoeCache:# carry independent host attributes -- see layer_residency) and their GPU# slot caches, keyed by the format's bank schema (attached by# set_bank_sources). The GPU slot cache stays one unified pool per bank.- self.bank_schema = _BANK_SCHEMAS[self.quant_format]+ if self.layout is not None:+ self.bank_schema = tuple(role for role, spec in self.layout.items() if not spec.resident)+ else:+ self.bank_schema = _BANK_SCHEMAS[self.quant_format]self.bank_sources: dict[str, list[torch.Tensor]] = {}self.bank_caches: dict[str, torch.Tensor] = {}# per-layer host residency: the GPU movement paths require "pinned"; LOCKED/PAGEABLE layers decode on the CPU executor and prefill via copy_missing's pageable branch@@ -294,18 +301,22 @@ class OffloadMoeCache:per layer (independent allocations, so each layer can carry its own hostattributes); each slot cache mirrors the bank's row shape and dtype as oneunified GPU pool. The row layouts are produced by the weight loaders /- repackers (see ``_BANK_SCHEMAS`` and :mod:`freetoken.moe.nvfp4_backends`)+ repackers (see ``_BANK_SCHEMAS`` and :mod:`freetoken.layers.quantization.moe.nvfp4`)-- the cache machinery is layout-agnostic and just moves rows.``layer_residency`` labels each layer with a ``HostResidency`` value (default: all pinned).Non-pinned (LOCKED/PAGEABLE) layers have no device address: they must already be routed to the CPU executor (``cpu_layer_ids``, set BEFORE this call), the copy plan skips their rows, and their only movement is ``copy_missing``'s whole-layer pageable prefill branch -- which is why prefill overlap is incompatible with them."""+ from freetoken.moe.legacy_format import canonical_rolefrom freetoken.moe.host_banks import HostResidency- assert set(sources) == set(self.bank_schema), (- f"banks {sorted(sources)} do not match the {self.quant_format!r} "- f"schema {self.bank_schema}"- )+ # loaders and FTW files may still name the banks the old way (gate_up_packed, ...)+ by_role = {canonical_role(name): per_layer for name, per_layer in sources.items()}+ if set(by_role) != {canonical_role(n) for n in self.bank_schema}:+ raise AssertionError(+ f"banks {sorted(sources)} do not match the {self.quant_format!r} schema {self.bank_schema}"+ )+ sources = {name: by_role[canonical_role(name)] for name in self.bank_schema}residency = layer_residency or [HostResidency.PINNED.value] * self.num_layersassert len(residency) == self.num_layers, (len(residency), self.num_layers)unpinned = frozenset(@@ -329,6 +340,13 @@ class OffloadMoeCache:per_layer = sources[name]assert len(per_layer) == self.num_layers, (name, len(per_layer))head = per_layer[0]+ if self.layout is not None:+ spec = self.layout[name]+ if tuple(head.shape[1:]) != tuple(spec.shape) or head.dtype != spec.dtype:+ raise ValueError(+ f"bank {name!r} rows are {tuple(head.shape[1:])} {head.dtype} but the expert kernel's layout "+ f"wants {tuple(spec.shape)} {spec.dtype}; the banks were packed for another kernel"+ )for layer_id, source in enumerate(per_layer):assert source.is_contiguous(), f"bank {name!r} layer {layer_id} must be contiguous"assert source.size(0) == self.num_experts, (name, layer_id, source.shape)@@ -432,11 +450,16 @@ class OffloadMoeCache:"""if cache_size < self.num_experts:raise ValueError(f"cache_size {cache_size} < num_experts {self.num_experts}")- if self.quant_format == "nvfp4_marlin" and cache_size > MARLIN_MAX_CACHE_SIZE:+ if self.max_slots is not None and cache_size > self.max_slots:+ raise ValueError(+ f"moe_cache_size={cache_size} exceeds the expert kernel's slot limit of {self.max_slots}; "+ f"pass --moe-cache-size {self.max_slots} or less, or let the default kernel serve the experts"+ )+ if self.layout is None and self.quant_format == "nvfp4_marlin" and cache_size > MARLIN_MAX_CACHE_SIZE:raise ValueError(f"moe_cache_size={cache_size} exceeds the marlin backend's slot limit of "f"{MARLIN_MAX_CACHE_SIZE} (vLLM moe_align_block_size caps padded experts at "- "1024); reduce moe_cache_size or force --nvfp4-backend triton"+ "1024); reduce moe_cache_size or force --quant-backend moe.nvfp4=triton")def rebuild(self, cache_size: int) -> None:@@ -1033,14 +1056,6 @@ class OffloadMoeCache:def iter_offload_moe_layers(model) -> Iterator:from freetoken.layers import BaseOP, OffloadMoELayer- # A model whose MoE blocks are bespoke nn.Modules (not OffloadMoELayer) declares its- # offload layers explicitly via this hook (e.g. DeepSeek-V4-Flash); attach_offload_moe_cache- # then sets .offload_cache on each yielded layer just like the OffloadMoELayer walk.- hook = getattr(model, "_iter_offload_moe_layers", None)- if hook is not None:- yield from hook()- return-if isinstance(model, OffloadMoELayer):yield modeldiff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex 5b4db587..32e8d126 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -10,6 +10,27 @@ from freetoken.distributed import DistributedInfofrom freetoken.scheduler import SchedulerConfigfrom freetoken.utils import init_logger+logger = init_logger(__name__)+++class _DeprecatedAlias(argparse.Action):+ """An old flag: warns at parse time, converts the value if asked, stores it."""++ def __init__(self, *args, new_flag: str, convert=None, **kwargs):+ self.new_flag, self.convert = new_flag, convert+ super().__init__(*args, **kwargs)++ def __call__(self, parser, namespace, values, option_string=None):+ logger.warning("%s is deprecated; use %s", option_string, self.new_flag)+ setattr(namespace, self.dest, self.convert(values) if self.convert else values)+++def _nvfp4_entry(value: str) -> str:+ """The --quant-backend entry an old --nvfp4-backend value stands for; auto stands for none."""+ if value == "auto":+ return ""+ return "moe.nvfp4=" + {"flashinfer": "b12x"}.get(value, value)+@dataclass(frozen=True)class ServerArgs(SchedulerConfig):@@ -93,7 +114,16 @@ def parse_args("""from freetoken.attention import validate_attn_backendfrom freetoken.kvcache import SUPPORTED_CACHE_MANAGER- from freetoken.moe import SUPPORTED_MOE_BACKENDS+ from freetoken.moe import MOE_STRATEGIES++ def _parse_quant_backend(value: str) -> str:+ from freetoken.layers.quantization import QuantBackend++ try:+ QuantBackend.parse(value)+ except ValueError as exc:+ raise argparse.ArgumentTypeError(str(exc)) from None+ return valuedef _parse_moe_cache_rate(value: str) -> float:try:@@ -467,16 +497,37 @@ def parse_args()parser.add_argument(- "--moe-backend",- default=ServerArgs.moe_backend,- choices=["auto"] + SUPPORTED_MOE_BACKENDS.supported_names(),+ "--moe-strategy",+ default=ServerArgs.moe_strategy,+ choices=["auto", *MOE_STRATEGIES],help=(- "The MoE backend to use. 'auto' resolves a MoE model to the offload family "+ "How the routed experts are served. 'auto' resolves a MoE model to the offload family ""(offload, or hybrid when a `ft bench bw` profile recommends it); resident ""'fused' experts must be requested explicitly."),)+ parser.add_argument(+ "--moe-backend",+ dest="moe_strategy",+ action=_DeprecatedAlias,+ new_flag="--moe-strategy",+ default=argparse.SUPPRESS,+ choices=["auto", *MOE_STRATEGIES],+ help="[Deprecated] Use --moe-strategy.",+ )++ parser.add_argument(+ "--quant-backend",+ default=None,+ type=_parse_quant_backend,+ help=(+ "Kernel per quantized layer type: comma-separated layer[.kind]=name entries, e.g. "+ "'linear=marlin,moe=b12x' or 'moe.nvfp4=triton'. A layer-level entry applies to every "+ "kind whose kernel table lists the name; unlisted tables stay automatic."+ ),+ )+parser.add_argument("--ple-backend",default=ServerArgs.ple_backend,@@ -489,13 +540,12 @@ def parse_args(parser.add_argument("--nvfp4-backend",- default=ServerArgs.nvfp4_backend,+ action=_DeprecatedAlias,+ new_flag="--quant-backend moe.nvfp4=<marlin|b12x|triton>",+ convert=_nvfp4_entry,+ default=argparse.SUPPRESS,choices=["auto", "marlin", "flashinfer", "triton"],- help=(- "NVFP4 routed-expert GEMM backend (default: triton, the portable inline-dequant "- "kernel). auto picks by GPU (marlin on sm80-99 + vLLM; flashinfer b12x on sm120+ "- "& CUDA>=13; else triton). Force one to override; it fails loudly if it cannot run."- ),+ help="[Deprecated] Use --quant-backend moe.nvfp4=<marlin|b12x|triton> ('flashinfer' is b12x).",)parser.add_argument(@@ -552,7 +602,7 @@ def parse_args(type=int,default=ServerArgs.moe_cpu_threads,help=(- "Number of CPU worker threads for --moe-backend cpu decode experts. "+ "Number of CPU worker threads for --moe-strategy cpu decode experts. ""0 = auto (physical cores)."),)@@ -562,13 +612,15 @@ def parse_args(type=str,default=ServerArgs.moe_cpu_layers,help=(- "With --moe-backend offload/hybrid: which MoE layers compute on the "+ "With --moe-strategy offload/hybrid: which MoE layers compute on the ""CPU executor instead of the GPU offload/PCIe path (where CUDA pinning ""is quota-capped, e.g. WSL, their banks are OS-locked instead of pinned). Explicit id list ('3,7,11'), a count ('8' = 8 "- "layers evenly strided), or a fraction ('0.5'). Unset = automatic where "- "CUDA pinning is quota-capped, e.g. WSL (locks just enough head+tail "- "layers when the banks exceed the pin budget, none otherwise); '0' "- "forces all layers on GPU."+ "layers evenly strided), a fraction ('0.5'), or 'auto'. 'auto' is for Windows/WSL "+ "only, where CUDA pinned memory is capped: it locks just enough head+tail layers "+ "for the banks over the pin budget. Any value, 'auto' included, commits to CPU "+ "decode before the model is built, so the expert format must have a CPU executor "+ "path (bf16, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the "+ "GPU; a boot whose banks exceed a known pin budget stops and asks for this flag."),)@@ -577,7 +629,7 @@ def parse_args(type=int,default=ServerArgs.moe_hybrid_max_fetch,help=(- "For --moe-backend hybrid: max experts fetched over PCIe per (layer, decode "+ "For --moe-strategy hybrid: max experts fetched over PCIe per (layer, decode ""step); the rest of that step's misses are computed on the CPU, overlapped. ""-1 (default) = auto: fetch the benched pcie/cpu bandwidth fraction of each ""step's misses (perfect overlap; needs an `ft bench bw` profile, else 1). "@@ -661,6 +713,14 @@ def parse_args(kwargs["max_running_req"] = 1kwargs["silent_output"] = True+ # the old flag stands in for one --quant-backend entry; next to the real flag it is a usage error+ entry = kwargs.pop("nvfp4_backend", None)+ if entry is not None:+ if kwargs["quant_backend"] is not None:+ parser.error("--nvfp4-backend cannot be combined with --quant-backend; write --quant-backend moe.nvfp4=... instead")+ if entry:+ kwargs["quant_backend"] = entry+if kwargs["model_path"].startswith("~"):kwargs["model_path"] = os.path.expanduser(kwargs["model_path"])@@ -681,14 +741,14 @@ def parse_args(# sizing flag at all, default to --moe-cache-auto so a bare `ft serve <FTW MoE>` works# out of the box (the scheduler resolves the size from free VRAM). Explicit# size/rate/auto is preserved.- from freetoken.moe import is_offload_moe_backend+ from freetoken.moe import is_offload_moe_strategy_no_cache_flag = (kwargs["moe_cache_size"] == 0and not kwargs["moe_cache_auto"]and (kwargs["moe_cache_rate"] is None or kwargs["moe_cache_rate"] == 0))- if is_offload_moe_backend(kwargs["moe_backend"]) and _no_cache_flag:+ if is_offload_moe_strategy(kwargs["moe_strategy"]) and _no_cache_flag:kwargs["moe_cache_auto"] = Trueif kwargs["model_source"] == "modelscope":@@ -726,6 +786,5 @@ def parse_args(del kwargs["tensor_parallel_size"]result = ServerArgs(**kwargs)- logger = init_logger(__name__)logger.info(f"Parsed arguments:\n{result}")return result, run_shelldiff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.pyindex dbc38679..a28146e6 100644--- a/python/freetoken/utils/hf.py+++ b/python/freetoken/utils/hf.py@@ -6,6 +6,7 @@ from typing import Anyfrom typing import FrozenSetfrom huggingface_hub import hf_hub_download, snapshot_download+from huggingface_hub.utils import EntryNotFoundErrorfrom tqdm.asyncio import tqdmfrom transformers import (AutoConfig,@@ -170,6 +171,17 @@ class RawConfigShim:return json.loads(json.dumps(self._data)) # deep copy, callers may mutate+def optional_hf_file(model_path: str, filename: str) -> str | None:+ """Local path of ``filename`` in a checkpoint dir or Hub repo; None when the checkpoint has no such file."""+ if os.path.isdir(model_path):+ path = os.path.join(model_path, filename)+ return path if os.path.isfile(path) else None+ try:+ return hf_hub_download(repo_id=model_path, filename=filename)+ except EntryNotFoundError:+ return None++def _raw_config_json(model_path: str) -> dict:if os.path.isdir(model_path):path = os.path.join(model_path, "config.json")diff --git a/tests/conftest.py b/tests/conftest.pynew file mode 100644index 00000000..aa33bef8--- /dev/null+++ b/tests/conftest.py@@ -0,0 +1,10 @@+import pytest++[email protected](autouse=True)+def _default_quant_backend():+ """Tests that install kernel requests must not leak them into the next test."""+ yield+ from freetoken.layers.quantization import QuantBackend, set_quant_backend++ set_quant_backend(QuantBackend())diff --git a/tests/e2e/test_aime.py b/tests/e2e/test_aime.pyindex f2803539..3dde4cad 100644--- a/tests/e2e/test_aime.py+++ b/tests/e2e/test_aime.py@@ -154,6 +154,16 @@ def recommended_sampling(model_dir: Path) -> tuple[SamplingParams, int]:return sp, n_samples+def format_chat(tokenizer, messages: list[dict]) -> str:+ """The thinking-mode prompt: the checkpoint's chat template, or DeepSeek-V4's encoder module."""+ from freetoken.tokenizer.tokenize import _apply_dsv4_chat_encoder, _load_dsv4_encoder_if_needed++ encoder = _load_dsv4_encoder_if_needed(tokenizer)+ if encoder is not None:+ return _apply_dsv4_chat_encoder(encoder, messages, None, {"enable_thinking": True})+ return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=True)++def build_llm(model_path: Path) -> LLM:"""Resident by default; set FREETOKEN_TEST_MOE_CACHE_SIZE>0 to run an offload model(fp8 Qwen3.5-MoE, GLM-4.7, MiniMax, ... -- routed experts that do not fit in HBM)."""@@ -167,12 +177,15 @@ def build_llm(model_path: Path) -> LLM:max_running_req=1,max_extend_tokens=8192,)+ if quant_backend := os.environ.get("FREETOKEN_TEST_QUANT_BACKEND"):+ kwargs["quant_backend"] = quant_backend # --quant-backend syntax, e.g. moe.nvfp4=b12xcache_size = int(os.environ.get("FREETOKEN_TEST_MOE_CACHE_SIZE", "0"))cache_auto = os.environ.get("FREETOKEN_TEST_MOE_CACHE_AUTO") == "1"if cache_size > 0 or cache_auto:kwargs.update(- moe_backend=os.environ.get("FREETOKEN_TEST_MOE_BACKEND", "offload"),+ moe_strategy=os.environ.get("FREETOKEN_TEST_MOE_STRATEGY", "offload"),moe_cpu_threads=int(os.environ.get("FREETOKEN_TEST_MOE_CPU_THREADS", "0")),+ moe_cpu_layers=os.environ.get("FREETOKEN_TEST_MOE_CPU_LAYERS"),moe_cache_auto=cache_auto,moe_cache_size=cache_size,kv_reserve_tokens=int(os.environ.get("FREETOKEN_TEST_KV_RESERVE", "8192")),@@ -251,13 +264,7 @@ def test_aime():for idx in selected:row = rows[idx]prompt, expected = row["prompt"], str(row["answer"])- messages = [{"role": "user", "content": prompt}]- formatted_prompt = tokenizer.apply_chat_template(- messages,- tokenize=False,- add_generation_prompt=True,- enable_thinking=True,- )+ formatted_prompt = format_chat(tokenizer, [{"role": "user", "content": prompt}])# Decode at the model's recommended params and check pass@N: the expected answer must# appear in at least one of N samples (single greedy/sampled runs are brittle -- see# recommended_sampling docstring).diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.pyindex 9ac2a4f4..a4e47aa0 100644--- a/tests/engine/test_cache_budget.py+++ b/tests/engine/test_cache_budget.py@@ -100,24 +100,24 @@ def test_expert_bytes_per_slot_sums_row_bytes_over_banks():assert expert_bytes_per_slot(sources) == 512 + 256-def test_resolve_auto_applies_ratio_once_and_marlin_cap():+def test_resolve_auto_applies_ratio_once():# baseline 1000, weights 100, ratio 0.9 -> budget = 900 - 100 - 0(fixed) = 800size, pages, overlap = resolve_moe_cache_auto(baseline_free=1000, weights_bytes=100, memory_ratio=0.9,cache_per_page=10, fixed_cache_size=0, per_expert_bytes=50,num_experts=4, total_experts=8, prefill_overlap=True,- kv_reserve_tokens=0, page_size=1, quant_format="bf16",+ kv_reserve_tokens=0, page_size=1,)# budget 800: experts cap at 8 -> 400 bytes; KV = 400//10 = 40 pagesassert size == 8 and pages == 40 and overlap is True-def test_resolve_auto_marlin_caps_slots():+def test_resolve_auto_caps_slots_at_the_kernel_limit():size, _, _ = resolve_moe_cache_auto(baseline_free=10_000_000, weights_bytes=0, memory_ratio=1.0,cache_per_page=10, fixed_cache_size=0, per_expert_bytes=100,num_experts=128, total_experts=4000, prefill_overlap=False,- kv_reserve_tokens=0, page_size=1, quant_format="nvfp4_marlin",+ kv_reserve_tokens=0, page_size=1, max_slots=992,)assert size == 992@@ -136,7 +136,7 @@ def _dsv4_adjust_cfg(**over):moe_cache_auto = Truemoe_cache_size = 0moe_cache_rate = None- moe_backend = "offload"+ moe_strategy = "offload"max_running_req = 1cuda_graph_max_bs = 1cuda_graph_bs = [1]@@ -145,7 +145,6 @@ def _dsv4_adjust_cfg(**over):page_size = 1attention_backend = "dsv4_sparse"moe_cpu_layers = None- nvfp4_backend = "auto"num_page_override = Nonenum_token_override = None@@ -166,7 +165,7 @@ def test_adjust_config_allows_auto_for_dsv4():cfg = _dsv4_adjust_cfg()_adjust_config(cfg) # must not raise- assert cfg.moe_backend == "offload"+ assert cfg.moe_strategy == "offload"assert cfg.moe_cache_auto is True # resolved later at engine init, not hereassert cfg.page_size == 128 # DSV4's KV page is the P-token window page@@ -212,14 +211,13 @@ def test_adjust_config_resolves_num_tokens_generic():moe_cache_auto = Falsemoe_cache_size = 0moe_cache_rate = None- moe_backend = "auto"+ moe_strategy = "auto"max_running_req = 4cuda_graph_max_bs = 2cuda_graph_bs = [1, 2]max_seq_len = 1024page_size = 1attention_backend = "fi"- nvfp4_backend = "auto"num_page_override = Nonenum_token_override = 5000@@ -303,7 +301,6 @@ def test_engine_resolve_auto_moe_cache_size_maps_kwargs():size = 1class StubBanks:- quant_format = "bf16"# 2 layers (num_moe_layers above) x 4 experts each -- per-layer host bank contract.sources = {"gate_up": [torch.zeros(4, 32, 8, dtype=torch.float16)] * 2, # row = 32*8*2 = 512@@ -329,10 +326,17 @@ def test_engine_resolve_auto_moe_cache_size_maps_kwargs():cache_per_page=cache_per_page, fixed_cache_size=fixed,per_expert_bytes=expert_bytes_per_slot(StubBanks.sources),num_experts=4, total_experts=8, prefill_overlap=True,- kv_reserve_tokens=0, page_size=16, quant_format="bf16",+ kv_reserve_tokens=0, page_size=16,)assert (size, pages, overlap) == expected+ class StubMethod:+ def slot_limit(self):+ return 5++ size, _, _ = engine._resolve_auto_moe_cache_size(StubConfig(), StubBanks(), StubMethod())+ assert size == 5+# ---------------------------------------------------------------------------# offload-cache sizing guard + auto-resolution (_require_offload_cache_size / _adjust_config),@@ -342,7 +346,7 @@ def test_engine_resolve_auto_moe_cache_size_maps_kwargs():def _offload_engine_config(**overrides):"""A frozen EngineConfig for a quantized-experts MoE checkpoint in the bare-invocation state- (moe_backend="auto") unless overridden — the shared fixture for the _adjust_config tests."""+ (moe_strategy="auto") unless overridden — the shared fixture for the _adjust_config tests."""from freetoken.distributed import DistributedInfofrom freetoken.engine.config import EngineConfig@@ -364,7 +368,7 @@ def _offload_engine_config(**overrides):num_moe_layers=10,num_experts=8,expert_quant="nvfp4", # quantized experts -> must resolve to an offload backend- moe_backend="auto",+ moe_strategy="auto",),)return config@@ -389,7 +393,7 @@ def test_adjust_config_defaults_moe_cache_auto_for_auto_resolved_offload_backend"""Bare `ft serve <FTW MoE checkpoint>`: no --moe-backend, no --moe-cache-* flags at all.args.py's parse-time default only fires when the backend is *already*- offload-family at parse time -- but a bare invocation leaves moe_backend="auto" at parse+ offload-family at parse time -- but a bare invocation leaves moe_strategy="auto" at parsetime, and the "auto" -> offload/cpu/hybrid resolution only happens later, in _adjust_config,once the model_config (and its expert_quant) is known. This proves the engine-levelresolution: a quantized-experts model auto-resolving to an offload-family backend also gets@@ -397,7 +401,7 @@ def test_adjust_config_defaults_moe_cache_auto_for_auto_resolved_offload_backendreached with moe_cache_size still 0."""from freetoken.engine.engine import _adjust_config- from freetoken.moe import is_offload_moe_backend+ from freetoken.moe import is_offload_moe_strategyconfig = _offload_engine_config()_adjust_config(config)@@ -405,7 +409,7 @@ def test_adjust_config_defaults_moe_cache_auto_for_auto_resolved_offload_backend# Which member of the family gets picked is not this test's claim, and is not ours to# decide: a bare "auto" consults ~/.cache/freetoken/benchbw.json, so a box that has run# `ft bench bw` resolves nvfp4 experts to hybrid instead. Assert the family, not the member.- assert is_offload_moe_backend(config.moe_backend)+ assert is_offload_moe_strategy(config.moe_strategy)assert config.moe_cache_auto is Trueassert config.moe_cache_size == 0 # still unresolved -- the scheduler sizes it from VRAM@@ -439,14 +443,13 @@ def _generic_rotary_cfg(max_position, override):moe_cache_auto = Falsemoe_cache_size = 0moe_cache_rate = None- moe_backend = "auto"+ moe_strategy = "auto"max_running_req = 4cuda_graph_max_bs = 2cuda_graph_bs = [1, 2]max_seq_len = 1024page_size = 1attention_backend = "triton"- nvfp4_backend = "auto"num_page_override = Nonenum_token_override = Nonemax_seq_len_override = Nonediff --git a/tests/engine/test_moe_cpu_layers.py b/tests/engine/test_moe_cpu_layers.pyindex dee36a11..feb19300 100644--- a/tests/engine/test_moe_cpu_layers.py+++ b/tests/engine/test_moe_cpu_layers.py@@ -46,7 +46,7 @@ def test_out_of_range_raises(spec):def _cfg(backend, spec=None):- return SimpleNamespace(moe_backend=backend, moe_cpu_layers=spec)+ return SimpleNamespace(moe_strategy=backend, moe_cpu_layers=spec)def test_resolve_backend_dispatch():diff --git a/tests/kernels/test_fp8_blockscale_moe.py b/tests/kernels/test_fp8_blockscale_moe.pynew file mode 100644index 00000000..d776104f--- /dev/null+++ b/tests/kernels/test_fp8_blockscale_moe.py@@ -0,0 +1,74 @@+"""The block-fp8 MoE kernels against a torch reference, for each gate-up epilogue they accept."""++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++E, H, I, TOPK, M = 4, 256, 256, 2, 8+BLOCK = 128+++def _quant_block(w):+ n, k = w.shape+ blocks = w.float().view(n // BLOCK, BLOCK, k // BLOCK, BLOCK)+ scale = blocks.abs().amax(dim=(1, 3), keepdim=True).clamp(min=1e-12) / 448.0+ q = (blocks / scale).to(torch.float8_e4m3fn).view(n, k)+ return q, scale.squeeze(1).squeeze(-1).to(torch.bfloat16)+++def _dequant(q, s):+ n, k = q.shape+ return (q.float().view(n // BLOCK, BLOCK, k // BLOCK, BLOCK) * s.float()[:, None, :, None]).view(n, k)+++def _experts():+ torch.manual_seed(0)+ gu = [_quant_block(torch.randn(2 * I, H, device="cuda") / H**0.5) for _ in range(E)]+ dn = [_quant_block(torch.randn(H, I, device="cuda") / I**0.5) for _ in range(E)]+ stack = lambda xs, i: torch.stack([x[i] for x in xs]).contiguous()+ return stack(gu, 0), stack(gu, 1), stack(dn, 0), stack(dn, 1)+++def _routing():+ ids = torch.stack([torch.randperm(E, device="cuda")[:TOPK] for _ in range(M)]).to(torch.int32)+ w = torch.softmax(torch.randn(M, TOPK, device="cuda"), dim=-1)+ return w, ids+++def _reference(x, gate_up, gate_up_scale, down, down_scale, w, ids, activation, alpha, limit):+ from freetoken.layers import gated_act_and_mul++ out = torch.zeros(M, H, device="cuda", dtype=torch.float32)+ for e in range(E):+ gu = _dequant(gate_up[e], gate_up_scale[e])+ dn = _dequant(down[e], down_scale[e])+ for t, k in zip(*torch.nonzero(ids == e, as_tuple=True)):+ h = (x[t].float() @ gu.T).to(x.dtype).view(1, -1)+ a = torch.empty(1, I, device="cuda", dtype=x.dtype)+ gated_act_and_mul(activation, h, a, alpha=alpha, limit=limit)+ out[t] += w[t, k] * (a[0].float() @ dn.T)+ return out.to(x.dtype)++[email protected]("activation, alpha, limit", [("silu", 1.0, float("inf")), ("swiglu_clamp", 1.0, 0.5), ("gelu_tanh", 1.0, float("inf"))])+def test_fp8_block_moe_epilogues_match_the_reference(activation, alpha, limit):+ from freetoken.kernel.triton.fp8_blockscale_moe import fused_experts_decode_fp8_blockscale, fused_experts_fp8_blockscale++ gate_up, gate_up_scale, down, down_scale = _experts()+ w, ids = _routing()+ x = torch.randn(M, H, device="cuda", dtype=torch.bfloat16)+ ref = _reference(x, gate_up, gate_up_scale, down, down_scale, w, ids, activation, alpha, limit).float()+ plain = _reference(x, gate_up, gate_up_scale, down, down_scale, w, ids, "silu", 1.0, float("inf")).float()+ if activation != "silu":+ assert not torch.allclose(ref, plain, rtol=1e-2, atol=1e-3), "the epilogue under test must change the result"++ decode = fused_experts_decode_fp8_blockscale(x, gate_up, gate_up_scale, down, down_scale, w, ids, activation, alpha, limit).float()+ # W8A16 decode: only bf16 accumulation-order differences remain+ assert torch.nn.functional.cosine_similarity(decode.flatten(), ref.flatten(), dim=0) > 0.999+ assert (decode - ref).abs().max() <= 2e-2 * ref.abs().max() + 1e-3++ prefill = fused_experts_fp8_blockscale(x, gate_up, gate_up_scale, down, down_scale, w, ids, E, activation, alpha, limit).float()+ # W8A8 prefill quantizes the activations per 128-group, so the tolerance is the fp8 activation error+ assert torch.nn.functional.cosine_similarity(prefill.flatten(), ref.flatten(), dim=0) > 0.99+ assert (prefill - ref).abs().max() <= 8e-2 * ref.abs().max() + 1e-3diff --git a/tests/kernels/test_fp8_pertensor_linear.py b/tests/kernels/test_fp8_pertensor_linear.pyindex 441da408..b03ed374 100644--- a/tests/kernels/test_fp8_pertensor_linear.py+++ b/tests/kernels/test_fp8_pertensor_linear.py@@ -105,32 +105,6 @@ def test_batch_size_does_not_change_the_numeric_scheme():assert torch.equal(alone, batched[:1])-def test_layer_load_marks_uniform_scale_and_optional_input_scale():- from freetoken.kernel.triton.fp8_pertensor_linear import (- Fp8PerTensorColMerged,- Fp8PerTensorLinear,- )-- K = 512- w8, scale = _quant_parts([256, 64, 64], K)- merged = Fp8PerTensorColMerged(K, [256, 64, 64])- merged.load_state_dict({- "weight": w8, "weight_scale": scale, "input_scale": torch.tensor(0.01, device=DEV),- })- assert merged._uniform_scale is False- assert merged.input_scale is not None-- single = Fp8PerTensorLinear(K, 384)- flat = scale[:1].expand(384).contiguous()- single.load_state_dict({"weight": w8, "weight_scale": flat}) # no input_scale- assert single._uniform_scale is True- assert single.input_scale is None-- # a reload must not trip over the input_scale it kept from the first load- single.load_state_dict({"weight": w8, "weight_scale": flat})- assert single.input_scale is None--@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+")@pytest.mark.parametrize("M", [1, 4, 64, 300])def test_per_part_path_matches_rowwise(M: int, monkeypatch):@@ -160,16 +134,19 @@ def test_fused_layer_forward_on_a_side_stream_completes():stall fails the test instead of hanging the session."""script = textwrap.dedent("""import os, time, torch- from freetoken.kernel.triton.fp8_pertensor_linear import FP8, Fp8PerTensorColMerged+ from freetoken.kernel.triton.fp8_pertensor_linear import (+ FP8, fp8_pertensor_linear, rowwise_scaled_mm_ok, weight_scale_segments,+ )torch.manual_seed(0)K, parts = 2048, [8192, 512, 512] # a prefill-sized fused qkvw8 = (torch.randn(sum(parts), K, device="cuda") * 8).clamp(-448, 448).to(FP8)scale = torch.cat([torch.full((p,), 0.01 * (i + 1), device="cuda")for i, p in enumerate(parts)])- layer = Fp8PerTensorColMerged(K, parts)- layer.load_state_dict({"weight": w8, "weight_scale": scale,- "input_scale": torch.tensor(0.02, device="cuda")})+ input_scale = torch.tensor(0.02, device="cuda")+ # same load-time decisions the torch fp8_tensor kernel makes in finalize+ segments = weight_scale_segments(scale)+ rowwise_scaled_mm_ok()x = torch.randn(2010, K, device="cuda", dtype=torch.bfloat16) # #182 shapetorch.cuda.synchronize()@@ -177,7 +154,7 @@ def test_fused_layer_forward_on_a_side_stream_completes():events = []with torch.cuda.stream(stream):for _ in range(128):- layer.forward(x)+ fp8_pertensor_linear(x, w8, scale, None, input_scale, False, scale_segments=segments)ev = torch.cuda.Event()ev.record(stream)events.append(ev)diff --git a/tests/kernels/test_mxfp8_linear.py b/tests/kernels/test_mxfp8_linear.pyindex 1f1557d4..f762cb38 100644--- a/tests/kernels/test_mxfp8_linear.py+++ b/tests/kernels/test_mxfp8_linear.py@@ -42,18 +42,6 @@ def test_mxfp8_linear_matches_dequant_reference(M: int, N: int, K: int):assert rel.item() < 2e-2, rel.item()-def test_mxfp8_module_shapes_and_forward():- from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear-- lin = Mxfp8Linear(6144, 512)- assert lin.weight.shape == (512, 6144) and lin.weight.dtype == torch.float8_e4m3fn- assert lin.weight_scale_inv.shape == (512, 192) and lin.weight_scale_inv.dtype == torch.uint8- w8, codes = _make_mxfp8(512, 6144, seed=42)- lin.weight, lin.weight_scale_inv = w8, codes- x = torch.randn(3, 6144, device=DEV, dtype=torch.bfloat16)- assert lin.forward(x).shape == (3, 512)--def test_gemma_plus_one_norm_matches_flashinfer_semantics():"""Triton fallback vs the (1+w) definition; per-head 3D strided in-place."""from freetoken.kernel.triton.norm import gemma_fused_add_rmsnorm, gemma_rmsnormdiff --git a/tests/models/qwen4_exp/test_skeleton.py b/tests/models/qwen4_exp/test_skeleton.pyindex 00f7c43f..fa75d6ee 100644--- a/tests/models/qwen4_exp/test_skeleton.py+++ b/tests/models/qwen4_exp/test_skeleton.py@@ -367,7 +367,7 @@ def test_qsa_layer_matches_hf_dense():class _StubLinearMixer(BaseOP):"""Stands in for the GDN layer; same [T, hidden] -> [T, hidden] shape."""- def __init__(self, config, layer_id):+ def __init__(self, config, layer_id, prefix=""):self.out_proj = LinearReplicated(config.hidden_size, config.hidden_size, has_bias=False)def forward(self, x):@@ -379,7 +379,6 @@ def test_shared_expert_gate_fusion_matches_eager():"""Qwen4ExpMoE only swaps qwen3_5's gemv+sigmoid+mul+add gate chain for two triton kernels."""from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoEfrom freetoken.models.qwen4_exp.moe import Qwen4ExpMoE- from freetoken.moe.fused import FusedMoefrom freetoken.utils.torch_utils import torch_dtypeconfig = _config()@@ -387,7 +386,7 @@ def test_shared_expert_gate_fusion_matches_eager():with torch.device(device), torch_dtype(dtype):moe = Qwen4ExpMoE(config, 0)_fill(moe, torch.Generator(device=device).manual_seed(21), scale=0.2)- _fresh_ctx(moe_backend=FusedMoe())+ _fresh_ctx(_batch=SimpleNamespace(is_prefill=True))x = torch.randn(6, config.hidden_size, device=device, dtype=dtype) * 0.5fused = moe.forward(x.clone())@@ -433,7 +432,6 @@ def test_decoder_stack_prefill_and_decode(monkeypatch):from freetoken.models.qwen4_exp import model as model_modulefrom freetoken.models.qwen4_exp.attention import TorchDenseQSAReferencefrom freetoken.models.qwen4_exp.ple import GpuResidentTable- from freetoken.moe.fused import FusedMoefrom freetoken.utils.torch_utils import torch_dtypetorch.manual_seed(8)@@ -462,7 +460,6 @@ def test_decoder_stack_prefill_and_decode(monkeypatch):prompts = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12], [9, 10, 11, 12, 13]]ctx = _fresh_ctx(attn_backend=TorchDenseQSAReference(config, num_slots, max_len, device, dtype),- moe_backend=FusedMoe(),linear_state_pool=pool,)reqs = [diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.pyindex 747d22f2..82f9a01d 100644--- a/tests/models/qwen4_exp/test_weight_ckpt.py+++ b/tests/models/qwen4_exp/test_weight_ckpt.py@@ -19,7 +19,7 @@ import torchfrom freetoken.distributed import set_tp_info, try_get_tp_infofrom freetoken.kernel.aot_models import expert_bank_row_bytes-from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks+from freetoken.models.nvfp4_banks import iter_nvfp4_expert_piecesfrom freetoken.models.qwen4_exp.config import parse_configfrom freetoken.models.qwen4_exp.weight import (_NVFP4_SOURCE_SPEC,@@ -27,6 +27,7 @@ from freetoken.models.qwen4_exp.weight import (iter_weights,load_ple_table,)+from freetoken.moe.expert_banks import build_expert_banksfrom freetoken.moe.host_banks import HostResidencyfrom freetoken.utils import cached_load_hf_config@@ -287,9 +288,23 @@ def layer0_expert_banks():)config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,num_moe_layers=1)- return load_nvfp4_expert_source_banks(+ layer = _triton_layer()+ pieces = iter_nvfp4_expert_pieces(MODEL_PATH, config, spec, drop_page_cache=lambda path: None, primary=False)+ return build_expert_banks(layer.quant_method, 1, pieces, device=torch.device("cuda")).sources+++def _triton_layer():+ """An offload NVFP4 layer bound to the Triton kernel, whose banks are the native rows."""+ from freetoken.layers.moe import OffloadMoELayer+ from freetoken.layers.quantization import QuantBackend, QuantConfig, set_quant_backend++ if try_get_tp_info() is None:+ set_tp_info(0, 1)+ set_quant_backend(QuantBackend.parse("moe.nvfp4=triton"))+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})+ return OffloadMoELayer(0, E, 10, H, I, quant_config=quant, prefix="model.layers.0.mlp.experts")@pytest.mark.slow@@ -297,11 +312,11 @@ def test_sampled_experts_match_the_checkpoint(layer0_expert_banks, reader):banks = layer0_expert_banksfor expert in random.Random(1).sample(range(E), 8):base = f"{LM}.layers.0.mlp.experts.{expert}"- assert torch.equal(banks["gate_up_packed"][0][expert, :I],+ assert torch.equal(banks["gate_up"][0][expert, :I],reader.get(f"{base}.gate_proj.weight"))- assert torch.equal(banks["gate_up_packed"][0][expert, I:],+ assert torch.equal(banks["gate_up"][0][expert, I:],reader.get(f"{base}.up_proj.weight"))- assert torch.equal(banks["down_packed"][0][expert],+ assert torch.equal(banks["down"][0][expert],reader.get(f"{base}.down_proj.weight"))for proj, bank, rows in (("gate_proj", "gate_up_scale", slice(0, I)),("up_proj", "gate_up_scale", slice(I, 2 * I)),@@ -320,18 +335,9 @@ def test_expert_bank_bytes_match_the_aot_row_table(layer0_expert_banks):@pytest.mark.skipif(not torch.cuda.is_available(), reason="dummy banks are pinned")-def test_dummy_expert_sources_have_the_real_bank_shapes(layer0_expert_banks):- from freetoken.models.weight import _model_override, dummy_nvfp4_expert_sources- from freetoken.models.register import get_model_spec-- spec = get_model_spec("Qwen4ExpForConditionalGeneration")- # No dummy_* override, so --use-dummy-weight goes through the generic builders.- for hook in ("dummy_nvfp4_expert_sources", "dummy_moe_expert_sources", "dummy_q4_0_expert_sources"):- assert _model_override(spec, hook) is None-- config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,- num_moe_layers=1)- dummy = dummy_nvfp4_expert_sources(config)+def test_dummy_expert_banks_have_the_real_bank_shapes(layer0_expert_banks):+ layer = _triton_layer()+ dummy = build_expert_banks(layer.quant_method, 1, None, device=torch.device("cuda"), dummy=True).sourcesassert set(dummy) == set(layer0_expert_banks)for name, banks in dummy.items():assert banks[0].shape == layer0_expert_banks[name][0].shapediff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.pyindex 8f0bc518..54060e16 100644--- a/tests/models/test_glm5_next_config.py+++ b/tests/models/test_glm5_next_config.py@@ -202,8 +202,7 @@ def test_moe_and_scalars():assert cfg.attn_sm_scale == pytest.approx(256**-0.5)assert cfg.num_moe_layers == _NUM_LAYERS - 3assert cfg.is_moe- # Checkpoint-faithful default; the FREETOKEN_GLM5_*_FP8 env flags opt into- # the W8A16 fp8 load.+ # checkpoint-faithful: the resident projections follow the quantization_config, never a switchassert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3# Text-only serving: the vision tower is never built.assert cfg.vision_config is Nonediff --git a/tests/models/test_glm5_next_kda_op.py b/tests/models/test_glm5_next_kda_op.pyindex 11adbf11..588e5e00 100644--- a/tests/models/test_glm5_next_kda_op.py+++ b/tests/models/test_glm5_next_kda_op.py@@ -53,7 +53,7 @@ def _make_args():def _make_op(seed=0):from freetoken.models.glm5_next.kda import Glm5NextKDA- cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none")+ cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none", quant=None)op = Glm5NextKDA(cfg, layer_id=0)torch.manual_seed(seed)dev, dt = "cuda", torch.bfloat16diff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.pyindex 9cd8c7fb..1062261e 100644--- a/tests/models/test_minimax_m3.py+++ b/tests/models/test_minimax_m3.py@@ -11,6 +11,7 @@ from __future__ import annotationsimport pytestimport torch+from freetoken.layers.quantization import MoEConfigfrom freetoken.attention.base import AttnTypefrom freetoken.models.minimax_m3.config import parse_config@@ -148,7 +149,7 @@ def test_parse_config_full_model(monkeypatch):assert cfg.hidden_act_alpha == pytest.approx(1.702)assert cfg.swiglu_limit == pytest.approx(7.0)assert cfg.expert_quant == "nvfp4"- # Default env: MXFP8 dense native, bf16 lm_head.+ # quantized release: MXFP8 dense native, bf16 lm_head.assert cfg.attn_quant == "mxfp8" and cfg.dense_quant == "mxfp8"assert cfg.lm_head_quant == "none"@@ -212,13 +213,6 @@ def test_sparse_ablation_env(monkeypatch):assert resolve_pool_class(cfg) is MHAKVCache-def test_mxfp8_ablation_env(monkeypatch):- monkeypatch.setenv("FREETOKEN_M3_ATTN_MXFP8", "0")- monkeypatch.setenv("FREETOKEN_M3_MLP_MXFP8", "0")- cfg = parse_config(_hf_config())- assert cfg.attn_quant == "none" and cfg.dense_quant == "none"--def test_registry_resolves_both_architectures():from freetoken.models.register import get_model_spec@@ -238,13 +232,19 @@ def test_auto_backend_resolution():assert attention_backend_info("m3_sparse").page_sizes == (128,)-def test_nvfp4_backend_restricted_to_triton_for_swigluoai():- from freetoken.moe.nvfp4_backends import select_nvfp4_backend+def test_nvfp4_experts_restricted_to_triton_for_swigluoai(monkeypatch):+ """The borrowed marlin / b12x kernels hard-code silu; swigluoai experts land on the Triton kernels."""+ from freetoken.kernel import backend+ from freetoken.layers.quantization import KernelSelectionError, ModelOptConfig, select_kernel+ from freetoken.layers.quantization.moe import Nvfp4MoEMethod- dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")- assert select_nvfp4_backend(dev, 3072, "auto", activation="swigluoai") == "triton"- with pytest.raises(RuntimeError):- select_nvfp4_backend(dev, 3072, "marlin", activation="swigluoai")+ monkeypatch.setattr(backend, "device_capability", lambda: (12, 0))+ monkeypatch.setattr(backend, "is_vllm_installed", lambda: True)+ monkeypatch.setattr(backend, "is_flashinfer_installed", lambda: True)+ cfg = MoEConfig(num_experts=256, hidden=6144, intermediate=3072, top_k=4, scheme=ModelOptConfig.SCHEMES["NVFP4"], strategy="offload", activation="swigluoai", alpha=1.702, limit=7.0)+ assert select_kernel(Nvfp4MoEMethod.candidates, "auto", cfg).name == "triton"+ with pytest.raises(KernelSelectionError):+ select_kernel(Nvfp4MoEMethod.candidates, "marlin", cfg)def test_expert_source_spec_layer_to_bank():diff --git a/tests/models/test_models_loader.py b/tests/models/test_models_loader.pyindex 7254739c..b0ac2b13 100644--- a/tests/models/test_models_loader.py+++ b/tests/models/test_models_loader.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport refrom types import SimpleNamespace+import pytestimport torch@@ -181,8 +182,8 @@ def test_stack_expert_tensors_after_all_experts_arrive():assert packed[0][1][1].tolist() == [[11.0, 11.0]]-def test_stream_moe_expert_sources_writes_layers_into_final_banks():- from freetoken.models.loader import stream_moe_expert_sources+def test_stacked_expert_pieces_pair_each_layer_in_arrival_order():+ from freetoken.moe.expert_pieces import stacked_expert_piecesconfig = SimpleNamespace(num_layers=2, num_experts=2)tensors = [@@ -192,16 +193,11 @@ def test_stream_moe_expert_sources_writes_layers_into_final_banks():("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),]- gate_up_source, down_source = stream_moe_expert_sources(- tensors,- config,- dtype=torch.bfloat16,- )+ pieces = list(stacked_expert_pieces(tensors, config))- assert len(gate_up_source) == 2 and all(t.shape == (2, 3, 4) for t in gate_up_source)- assert len(down_source) == 2 and all(t.shape == (2, 4, 3) for t in down_source)- assert all(t.dtype == torch.bfloat16 for t in gate_up_source + down_source)- torch.testing.assert_close(gate_up_source[0], torch.full_like(gate_up_source[0], 2.0))- torch.testing.assert_close(gate_up_source[1], torch.full_like(gate_up_source[1], 3.0))- torch.testing.assert_close(down_source[0], torch.full_like(down_source[0], 10.0))- torch.testing.assert_close(down_source[1], torch.full_like(down_source[1], 11.0))+ assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(1, 0, 2), (0, 0, 2)]+ assert torch.equal(pieces[0][3]["gate_up"], torch.full((2, 3, 4), 3.0))+ assert torch.equal(pieces[0][3]["down"], torch.full((2, 4, 3), 11.0))+ assert torch.equal(pieces[1][3]["gate_up"], torch.full((2, 3, 4), 2.0))+ with pytest.raises(ValueError, match="Missing MoE expert source layers"):+ list(stacked_expert_pieces(tensors[:3], config))diff --git a/tests/models/test_muse_glimmer.py b/tests/models/test_muse_glimmer.pyindex cbd579e4..e9b51ce6 100644--- a/tests/models/test_muse_glimmer.py+++ b/tests/models/test_muse_glimmer.py@@ -59,6 +59,7 @@ def _hf_config(num_layers: int = 52, quantized: bool = False) -> _Cfg:data["quantization_config"] = {"quant_method": "compressed-tensors","format": "nvfp4-pack-quantized",+ "ignore": ["lm_head"],"config_groups": {"group_0": {"format": "nvfp4-pack-quantized",@@ -447,16 +448,22 @@ def test_model_state_dict_matches_loader_keys():assert layers[3].self_attn.attn_spec.sliding_window is Noneassert layers[0].self_attn.attn_spec.sm_scale == pytest.approx(3.87 * 128**-0.5)- # NVFP4 build swaps every text Linear for the W4A16 kernels.- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear+ # NVFP4 build: every text Linear the config targets gets the W4A16 method.+ from freetoken.layers.quantization import NameMap, QuantConfig+ from freetoken.layers.quantization.linear import Nvfp4LinearMethod+ from freetoken.layers.quantization.linear import UnquantizedLinearMethod+ from freetoken.models.register import get_model_spec- qcfg = parse_config(_hf_config(num_layers=4, quantized=True))+ hf_q = _hf_config(num_layers=4, quantized=True)+ qcfg = parse_config(hf_q)+ spec = get_model_spec("MuseGlimmerForConditionalGeneration")+ object.__setattr__(qcfg, "quant", QuantConfig.from_hf(+ hf_q, name_map=NameMap(roots=spec.checkpoint_roots, packed=spec.packed_modules_mapping)+ ))qmodel = MuseGlimmerForCausalLM(qcfg)attn = qmodel.model.layers.op_list[0].self_attnmlp = qmodel.model.layers.op_list[0].mlp- assert isinstance(attn.qkvg_proj, Nvfp4DenseColMerged)- assert isinstance(attn.o_proj, Nvfp4DenseLinear)- assert isinstance(mlp.gate_up_proj, Nvfp4DenseColMerged)- assert isinstance(mlp.down_proj, Nvfp4DenseLinear)- assert type(qmodel.lm_head).__name__ == "ParallelLMHead" # lm_head stays bf16+ for proj in (attn.qkvg_proj, attn.o_proj, mlp.gate_up_proj, mlp.down_proj):+ assert type(proj.quant_method) is Nvfp4LinearMethod+ assert type(qmodel.lm_head.quant_method) is UnquantizedLinearMethod # lm_head stays bf16del model, qmodel, torchdiff --git a/tests/models/test_quant_config.py b/tests/models/test_quant_config.pynew file mode 100644index 00000000..b0043cde--- /dev/null+++ b/tests/models/test_quant_config.py@@ -0,0 +1,523 @@+"""What each checkpoint's quant config resolves to: the dialect, the method behind every layer, and agreement with the stored tensors.++Rows need the checkpoint's config.json locally (/mnt/nvme/models or the HF cache); absent ones skip.+"""++from __future__ import annotations++import glob+import json+import os+import re+import struct+from collections import defaultdict+from dataclasses import dataclass, field+from pathlib import Path+from types import SimpleNamespace+from typing import Any, Callable++import pytest+import torch++from freetoken.distributed.info import set_tp_info, try_get_tp_info+from freetoken.engine.config import EngineConfig, checkpoint_quant_config+from freetoken.layers import set_rope_device+from freetoken.layers.quantization import (+ CompressedTensorsConfig,+ Fp8BlockConfig,+ KernelSelectionError,+ ModelOptConfig,+ Mxfp4Config,+ NoQuantConfig,+ QuantConfig,+ QuantKind,+)+from freetoken.layers.quantization.linear import (+ Fp8BlockLinearMethod,+ Fp8TensorLinearMethod,+ Mxfp8LinearMethod,+ Nvfp4LinearMethod,+ UnquantizedLinearMethod,+)+from freetoken.layers.quantization.moe import Fp8BlockMoEMethod, Mxfp4MoEMethod, Nvfp4MoEMethod, UnquantizedMoEMethod+from freetoken.models import create_model+from freetoken.models.gpt_oss.moe import GptOssMoELayer, GptOssOffloadMoELayer+from freetoken.models.register import get_model_spec+from freetoken.utils.torch_utils import torch_dtype++MODELS = "/mnt/nvme/models"+HF_CACHE = os.path.expanduser("~/.cache/huggingface/hub")++BF16, FP8B, FP8T, MXFP8, NVFP4 = UnquantizedLinearMethod, Fp8BlockLinearMethod, Fp8TensorLinearMethod, Mxfp8LinearMethod, Nvfp4LinearMethod+++def model_dir(name: str) -> str | None:+ path = os.path.join(MODELS, name.split("/")[-1])+ if os.path.isfile(os.path.join(path, "config.json")): # a download in progress has the dir but no config yet+ return path+ snaps = glob.glob(os.path.join(HF_CACHE, "models--" + name.replace("/", "--"), "snapshots", "*", "config.json"))+ return os.path.dirname(snaps[0]) if snaps else None+++def op_at(model, path: str):+ op = model+ for part in path.split("."):+ op = op.op_list[int(part)] if part.isdigit() else getattr(op, part)+ return op+++def build_meta_model(path: str, strategy: str):+ if try_get_tp_info() is None:+ set_tp_info(0, 1)+ set_rope_device(torch.device("cpu"))+ config = EngineConfig(model_path=path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy=strategy)+ from freetoken.engine.engine import _decode_target++ object.__setattr__(config.model_config, "moe_strategy", strategy)+ object.__setattr__(config.model_config, "decode_target", _decode_target(config))+ with torch.device("meta"), torch_dtype(torch.bfloat16):+ return create_model(config.model_config), config.model_config+++def _layer_indices(model) -> dict[str, int | None]:+ """First layer of each kind, for the {moe} {dense} {lin} {full} {dsa} {kda} {ple} placeholders in probe paths."""+ layers = model.model.layers.op_list++ def first(pred):+ return next((i for i, layer in enumerate(layers) if pred(layer)), None)++ def has_experts(layer):+ if getattr(layer, "is_moe_layer", None) is not None:+ return layer.is_moe_layer+ block = next((getattr(layer, n) for n in ("mlp", "feed_forward", "block_sparse_moe", "ffn") if hasattr(layer, n)), None)+ return block is not None and hasattr(block, "experts")++ def is_linear_attn(layer):+ return getattr(layer, "_is_linear", False) or hasattr(layer, "linear_attn")++ attn = lambda layer: getattr(layer, "self_attn", None)+ ple = getattr(getattr(model, "_config", None), "qwen4_args", None)+ return {+ "moe": first(has_experts),+ "dense": first(lambda l: not has_experts(l)),+ "lin": first(is_linear_attn),+ "full": first(lambda l: not is_linear_attn(l)),+ "dsa": first(lambda l: getattr(attn(l), "indexer", None) is not None),+ "kda": first(lambda l: hasattr(attn(l), "in_proj")),+ "ple": ple.ple_layer_ids[0] if ple is not None else None,+ }+++@dataclass(frozen=True)+class Case:+ name: str+ dialect: type+ methods: dict[str, Any] # probe path -> method class, (class, kernel name), or None for a layer without a method+ strategy: str = "offload"+ raises: type | None = None+ skip_if: Callable[[], bool] | None = None+ env: dict[str, str] = field(default_factory=dict)+ check: Callable[[Any, dict], None] | None = None # family extras beyond the method table+++def _installed(module: str) -> bool:+ try:+ __import__(module)+ return True+ except ImportError:+ return False+++def _qwen35_fp8(model, idx):+ assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj+++def _qwen35_nvfp4(model, idx):+ assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj+ assert op_at(model, f"model.layers.{idx['lin']}.linear_attn.in_proj_qkvz").quant_method.scheme.has("input_scale")+ assert model.lm_head.weight.dtype is torch.uint8+++def _no_split(model, idx):+ assert not model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj+++def _dsv4(model, idx):+ block = op_at(model, "model.layers.2")+ for proj in (block.attn.wq_a, block.ffn.shared_experts.w1):+ assert proj.weight.dtype == torch.float8_e4m3fn and proj.weight_scale_inv.dtype == torch.float8_e8m0fnu+ assert proj.weight_scale_inv.shape == (proj.out_features // 128, proj.in_features // 128) and proj.bias is None+ assert set(block.attn.compressor.wkv.state_dict()) == {"weight"}+ experts = block.ffn.experts+ assert experts.limit == 10.0 and experts.quant_method.cpu_format == "ds_fp4" and not experts.has_bias+ H, I = 4096, 2048+ assert {r: (s.shape, s.dtype) for r, s in experts.quant_method.layout().items()} == {+ "gate_up": ((2 * I, H // 2), torch.uint8), "gate_up_scale": ((2 * I, H // 32), torch.float8_e8m0fnu),+ "down": ((H, I // 2), torch.uint8), "down_scale": ((H, I // 32), torch.float8_e8m0fnu),+ }+ keys = set(model.state_dict())+ assert "model.layers.2.attn.wq_a.weight_scale_inv" in keys and "model.layers.2.attn.wq_a.scale" not in keys+ assert "model.layers.2.attn.compressor.wkv.weight_scale_inv" not in keys+ assert "model.head.weight" in keys and not any(".ffn.experts." in k for k in keys)+++def _gpt_oss(cls):+ def check(model, idx):+ experts = op_at(model, "model.layers.0.mlp.experts")+ assert type(experts) is cls+ assert experts.interleaved and experts.alpha == pytest.approx(1.702) and experts.limit == 7.0+ if cls is GptOssMoELayer:+ assert experts.gate_up_proj_blocks.dtype is torch.uint8 and experts.gate_up_proj_blocks.dim() == 4++ return check+++def _m3_sparse(model, idx):+ layers = model.model.layers.op_list+ sparse = next((i for i, l in enumerate(layers) if getattr(l.self_attn, "is_sparse", False)), None)+ if sparse is not None:+ assert type(op_at(model, f"model.layers.{sparse}.self_attn.index_qk_proj").quant_method) is MXFP8+++def _gemma_tied_head(model, idx):+ assert model.lm_head.quant_method is None+++CASES = [+ Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,+ "model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,+ "model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8B,+ "model.layers.{lin}.mlp.shared_expert.down_proj": FP8B, "model.layers.{lin}.mlp.experts": Fp8BlockMoEMethod,+ "model.layers.{lin}.mlp.gate": BF16, "model.layers.{lin}.mlp.shared_expert_gate": BF16, "lm_head": BF16,+ }, check=_qwen35_fp8),+ Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,+ "model.layers.{full}.self_attn.qkv_proj": FP8T, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4,+ "model.layers.{lin}.mlp.shared_expert.down_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,+ "lm_head": NVFP4,+ }, check=_qwen35_nvfp4),+ # resident NVFP4 experts have no kernel without vLLM's Marlin+ Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {}, strategy="fused", raises=KernelSelectionError,+ skip_if=lambda: not _installed("flashinfer") or _installed("vllm")),+ Case("Qwen3.8-27B-NVFP4", ModelOptConfig, {+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{full}.self_attn.o_proj": FP8T,+ "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,+ }, strategy="auto"),+ Case("Qwen3.6-27B", NoQuantConfig, {+ "model.layers.{lin}.linear_attn.in_proj": BF16, "model.layers.{lin}.mlp.gate_up_proj": BF16, "lm_head": BF16,+ }, strategy="auto", check=_no_split),+ Case("Qwen3.6-35B-A3B", NoQuantConfig, {+ "model.layers.{lin}.mlp.experts": UnquantizedMoEMethod, "model.layers.{lin}.mlp.shared_expert.down_proj": BF16,+ }),+ Case("nvidia/Qwen3.6-27B-NVFP4", ModelOptConfig, {+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,+ "model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.qkv_proj": FP8T,+ "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,+ }, strategy="auto"),+ *[Case(name, Fp8BlockConfig, {+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,+ "model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,+ "model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.gate_up_proj": FP8B,+ "model.layers.{lin}.mlp.down_proj": FP8B, "lm_head": BF16,+ }, strategy="auto") for name in ("Qwen/Qwen3.8-27B-FP8", "Qwen/Qwen3.6-27B-FP8")],+ Case("RedHatAI/Muse-Glimmer-30B-NVFP4", CompressedTensorsConfig, {+ "model.layers.0.self_attn.qkvg_proj": NVFP4, "model.layers.0.self_attn.o_proj": NVFP4,+ "model.layers.0.mlp.gate_up_proj": NVFP4, "model.layers.0.mlp.down_proj": NVFP4, "lm_head": BF16,+ }, strategy="auto"),+ Case("nvidia/Gemma-4-31B-IT-NVFP4", ModelOptConfig, {+ "model.layers.0.feed_forward.shared_mlp.gate_up_proj": NVFP4, "model.layers.0.feed_forward.shared_mlp.down_proj": NVFP4,+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,+ }, strategy="auto", check=_gemma_tied_head),+ Case("google/gemma-4-12B-it", NoQuantConfig, {+ "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16, "model.layers.0.self_attn.qkv_proj": BF16,+ }, strategy="auto", check=_gemma_tied_head),+ Case("Qwen/Qwen3-4B", NoQuantConfig, {+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,+ "model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,+ }, strategy="auto"),+ *[Case(name, NoQuantConfig, {+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,+ "model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,+ }, strategy="auto") for name in ("Qwen/Qwen2.5-1.5B-Instruct", "unsloth/Llama-3.2-1B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3")],+ *[Case("Qwen/Qwen3-30B-A3B", NoQuantConfig, {+ "model.layers.0.mlp.experts": UnquantizedMoEMethod, "model.layers.0.mlp.gate": BF16,+ "model.layers.0.self_attn.qkv_proj": BF16, "lm_head": BF16,+ }, strategy=strategy) for strategy in ("fused", "offload")],+ Case("RadixArk/Qwen3.8-Flash-Next-NVFP4", ModelOptConfig, {+ "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.linear_attn.in_proj": BF16,+ "model.layers.{lin}.linear_attn.out_proj": BF16, "model.layers.{full}.self_attn.qkv_proj": BF16,+ "model.layers.{full}.self_attn.o_proj": BF16, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": BF16,+ "model.layers.{lin}.mlp.shared_expert.down_proj": BF16, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,+ # the indexer, hyper-connection and PLE projections ask the config too+ "model.layers.{full}.self_attn.indexer.index_qk_proj": BF16,+ "model.layers.{lin}.attn_hyper_connection.input_mix_weight_down_block_inject": BF16,+ "model.layers.{lin}.mlp_hyper_connection.input_mix_weight_up": BF16,+ "model.hyper_connection_mixer.input_mix_weight_down": BF16,+ "model.layers.{ple}.ple.key_proj": BF16, "model.layers.{ple}.ple.value_proj": BF16,+ }, check=_no_split),+ Case("deepseek-ai/DeepSeek-V4-Flash-0731", Fp8BlockConfig, {+ **{f"model.layers.2.{p}": (FP8B, "dsv4") for p in (+ "attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "attn.indexer.wq_b",+ "ffn.shared_experts.w1", "ffn.shared_experts.w2", "ffn.shared_experts.w3")},+ # the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert+ **{f"model.layers.2.{p}": (BF16, "torch") for p in (+ "attn.compressor.wkv", "attn.compressor.wgate", "attn.indexer.weights_proj", "attn.indexer.compressor.wkv")},+ "model.head": (BF16, "torch"), "model.layers.2.ffn.experts": (Mxfp4MoEMethod, "triton"),+ }, check=_dsv4),+ Case("nvidia/MiniMax-M2.5-NVFP4", ModelOptConfig, {+ "model.layers.3.block_sparse_moe.experts": Nvfp4MoEMethod, "model.layers.3.self_attn.qkv_proj": BF16,+ "model.layers.3.self_attn.o_proj": BF16, "model.layers.3.block_sparse_moe.gate": BF16, "lm_head": BF16,+ }),+ Case("nvidia/MiniMax-M3-NVFP4", ModelOptConfig, {+ "model.layers.{moe}.block_sparse_moe.experts": Nvfp4MoEMethod,+ "model.layers.{moe}.block_sparse_moe.shared_experts.gate_up_proj": MXFP8,+ "model.layers.{moe}.block_sparse_moe.shared_experts.down_proj": MXFP8, "model.layers.{dense}.mlp.gate_up_proj": MXFP8,+ "model.layers.{moe}.self_attn.qkv_proj": MXFP8, "model.layers.{moe}.self_attn.o_proj": MXFP8, "lm_head": BF16,+ }, env={"FREETOKEN_M3_MAX_LAYERS": "8"}, check=_m3_sparse),+ Case("nvidia/Gemma-4-26B-A4B-NVFP4", ModelOptConfig, {+ "model.layers.0.feed_forward.experts": Nvfp4MoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,+ "model.layers.0.feed_forward.shared_mlp.down_proj": BF16, "model.layers.0.feed_forward.router.proj": BF16,+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,+ }, check=_gemma_tied_head),+ Case("google/gemma-4-26B-A4B-it", NoQuantConfig, {+ "model.layers.0.feed_forward.experts": UnquantizedMoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,+ }),+ Case("RedHatAI/GLM-5.3-Flash-NVFP4", CompressedTensorsConfig, {+ "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": BF16,+ "model.layers.{moe}.mlp.shared_experts.down_proj": BF16, "model.layers.{dense}.mlp.gate_proj": BF16,+ "model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,+ # the indexer, the MLA kv_b_proj and the KDA projections ask the config too+ "model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,+ "model.layers.{kda}.self_attn.in_proj": BF16, "model.layers.{kda}.self_attn.f_b_proj": BF16,+ "model.layers.{kda}.self_attn.g_b_proj": BF16,+ }),+ Case("zai-org/GLM-5.3-Flash", Fp8BlockConfig, {+ "model.layers.{moe}.mlp.experts": (Fp8BlockMoEMethod, "triton"), "model.layers.{moe}.mlp.shared_experts.gate_proj": FP8B,+ "model.layers.{moe}.mlp.shared_experts.down_proj": FP8B, "model.layers.{dense}.mlp.gate_proj": FP8B,+ "model.layers.{dsa}.self_attn.q_b_proj": FP8B, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,+ "model.layers.{dsa}.self_attn.o_proj": FP8B, "model.layers.{kda}.self_attn.in_proj": BF16,+ "model.layers.{kda}.self_attn.o_proj": BF16, "lm_head": BF16,+ }),+ Case("nvidia/GLM-4.7-NVFP4", ModelOptConfig, {+ "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": NVFP4,+ "model.layers.{moe}.mlp.shared_experts.down_proj": NVFP4, "model.layers.{moe}.mlp.gate": BF16, "lm_head": BF16,+ }),+ Case("nvidia/GLM-5.3-NVFP4", ModelOptConfig, {+ "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.gate": BF16,+ "model.layers.{moe}.mlp.shared_experts.gate_proj": BF16, "model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,+ "model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{moe}.self_attn.kv_b_proj": BF16,+ }),+ *[Case("openai/gpt-oss-20b", Mxfp4Config, {+ "model.layers.0.mlp.experts": Mxfp4MoEMethod, "model.layers.0.self_attn.qkv_proj": BF16,+ "model.layers.0.self_attn.o_proj": BF16, "model.layers.0.mlp.router": BF16, "lm_head": BF16,+ }, strategy=strategy, check=_gpt_oss(cls)) for strategy, cls in (+ ("fused", GptOssMoELayer), ("offload", GptOssOffloadMoELayer), ("cpu", GptOssOffloadMoELayer), ("hybrid", GptOssOffloadMoELayer))],+ # hybrid decodes on the CPU executor too, so the expert kernel must have a CPU format+ Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="hybrid"),+ Case("Qwen3.6-35B-A3B", NoQuantConfig, {"model.layers.{lin}.mlp.experts": (UnquantizedMoEMethod, "fused")}, strategy="hybrid"),+ Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {}, strategy="hybrid", raises=KernelSelectionError),+]++[email protected]("case", CASES, ids=[f"{c.name.split('/')[-1]}:{c.strategy}" for c in CASES])+def test_probed_layers_get_the_method_their_config_says(case: Case, monkeypatch):+ path = model_dir(case.name)+ if path is None:+ pytest.skip(f"{case.name} not present locally")+ if case.skip_if is not None and case.skip_if():+ pytest.skip("kernel availability differs on this host")+ for key, value in case.env.items():+ monkeypatch.setenv(key, value)+ if case.raises is not None:+ with pytest.raises(case.raises):+ build_meta_model(path, case.strategy)+ return+ model, model_config = build_meta_model(path, case.strategy)+ assert type(model_config.quant) is case.dialect+ idx = _layer_indices(model)+ for template, expected in case.methods.items():+ probe = template.format(**{k: v for k, v in idx.items() if v is not None})+ op = op_at(model, probe)+ if getattr(op, "prefix", ""): # routers are built without one+ assert op.prefix == probe+ if expected is None:+ assert op.quant_method is None, probe+ continue+ cls, kernel = expected if isinstance(expected, tuple) else (expected, None)+ assert type(op.quant_method) is cls, f"{probe}: {type(op.quant_method).__name__}"+ if kernel is not None:+ assert op.quant_method.kernel.name == kernel, probe+ if case.check is not None:+ case.check(model, idx)+++# --------------------------------------------------------------------------- config without local weights+++def test_hub_ids_fetch_the_modelopt_sidecar(tmp_path, monkeypatch):+ """An old ModelOpt export keeps its quantization config only in hf_quant_config.json."""+ from huggingface_hub.utils import EntryNotFoundError++ sidecar = tmp_path / "hf_quant_config.json"+ sidecar.write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))++ def fake_download(repo_id, filename, **_):+ assert repo_id == "org/old-modelopt-fp8"+ if filename == "hf_quant_config.json":+ return str(sidecar)+ raise EntryNotFoundError(f"no {filename}")++ monkeypatch.setattr("freetoken.utils.hf.hf_hub_download", fake_download)+ quant = checkpoint_quant_config("org/old-modelopt-fp8", SimpleNamespace(architectures=["Qwen3MoeForCausalLM"]), get_model_spec("Qwen3MoeForCausalLM"))+ assert type(quant) is ModelOptConfig+ assert quant.scheme_for("model.layers.0.self_attn.q_proj").kind is QuantKind.FP8_TENSOR+ assert quant.scheme_for("lm_head") is None+++def test_unsupported_dialects_fail_closed(tmp_path):+ hf_config = SimpleNamespace(architectures=["Qwen3MoeForCausalLM"], quantization_config={"quant_method": "gptq", "bits": 4})+ with pytest.raises(NotImplementedError, match="gptq"):+ checkpoint_quant_config(str(tmp_path), hf_config, get_model_spec("Qwen3MoeForCausalLM"))+++# --------------------------------------------------------------------------- config against the stored tensors++# leaf names FreeToken builds as Linear / MoE layers; routers, norms, convs never ask for a scheme+_LINEAR_LEAVES = {+ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3",+ "in_proj_qkv", "in_proj_z", "in_proj_a", "in_proj_b", "in_proj_ba", "out_proj", "qkv_proj", "gate_up_proj",+ "lm_head", "q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "wq_a", "wq_b", "wkv_a", "wkv_b", "wo",+ "wkv", "wo_a", "wo_b", "wgate", "weights_proj", "experts",+ "b_proj", "f_a_proj", "f_b_proj", "g_a_proj", "g_b_proj", "index_qk_proj", "key_proj", "value_proj",+ "input_mix_weight_down", "input_mix_weight_up", "block_inject_weight",+}+_META_SUFFIXES = {"bias", "k_scale", "v_scale"}+_QUANT_SUFFIXES = {+ "weight_scale", "weight_scale_2", "weight_scale_inv", "weight_packed", "weight_global_scale",+ "input_scale", "input_scale_2", "input_global_scale", "scale",+ "gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias", "down_proj_blocks", "down_proj_scales", "down_proj_bias",+}+# unsupported dialects: from_hf must refuse them, nothing else is checked+_UNSUPPORTED = {"quark", "mxfp8", "fp_quant"}+# tables the model code reads itself; RadixArk lists the fp8 PLE table in ``ignore`` yet ships it quantized+_MODEL_READS = ("ple.ple_embedding",)+# non-Linear leaves whose parameters happen to be called scale+_NOT_LINEAR = {"router", "gate", "shared_expert_gate"}+++def _weight_map(p: Path) -> dict[str, str] | None:+ """Tensor name -> shard file, from the index or from a single model.safetensors."""+ if (p / "model.safetensors.index.json").exists():+ return json.load(open(p / "model.safetensors.index.json"))["weight_map"]+ if (p / "model.safetensors").exists():+ return {name: "model.safetensors" for name in _safetensors_header(p / "model.safetensors") if name != "__metadata__"}+ return None+++def _candidate_dirs() -> list[Path]:+ roots = glob.glob(os.path.join(HF_CACHE, "models--*/snapshots/*/")) + glob.glob(os.path.join(MODELS, "*/"))+ return [Path(d) for d in roots if (Path(d) / "config.json").exists() and _weight_map(Path(d)) is not None]+++def _label(p: Path) -> str:+ m = re.search(r"models--([^/]+)/snapshots", str(p))+ name = m.group(1).replace("--", "/") if m else f"nvme:{p.name}"+ # an HF cache entry may hold the config and index without the shards; the tensor check then only has the names+ shards = set(_weight_map(p).values())+ return name if all((p / s).exists() for s in shards) else f"{name}[index-only]"+++def _safetensors_header(path: Path) -> dict:+ with open(path, "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ return json.loads(f.read(n))+++def _tensor_info(p: Path, weight_map: dict[str, str]) -> dict[str, dict]:+ info: dict[str, dict] = {}+ for shard in sorted(set(weight_map.values())):+ if not (p / shard).exists():+ return {}+ for name, meta in _safetensors_header(p / shard).items():+ if name != "__metadata__":+ info[name] = meta+ return info+++def _expected_kinds(suffixes: set[str], tensors: dict[str, dict], module: str) -> set[QuantKind]:+ s = suffixes - _META_SUFFIXES+ if not s & {"weight", "weight_packed", "gate_up_proj_blocks"} or not s & _QUANT_SUFFIXES:+ return {QuantKind.NONE}+ if "weight_packed" in s or "weight_scale_2" in s:+ return {QuantKind.NVFP4}+ if "gate_up_proj_blocks" in s:+ return {QuantKind.MXFP4}++ def dtype(suffix):+ return (tensors.get(f"{module}.{suffix}") or {}).get("dtype")++ def shape(suffix):+ return (tensors.get(f"{module}.{suffix}") or {}).get("shape")++ if "weight_scale_inv" in s:+ if tensors:+ return {QuantKind.MXFP8} if dtype("weight_scale_inv") == "U8" else {QuantKind.FP8_BLOCK}+ return {QuantKind.FP8_BLOCK, QuantKind.MXFP8}+ if "scale" in s:+ if tensors:+ return {QuantKind.FP8_BLOCK} if dtype("weight") == "F8_E4M3" else {QuantKind.MXFP4}+ return {QuantKind.FP8_BLOCK, QuantKind.MXFP4}+ if "weight_scale" in s:+ if tensors:+ if dtype("weight") == "U8":+ return {QuantKind.MXFP4}+ sh = shape("weight_scale") or []+ return {QuantKind.FP8_BLOCK} if len(sh) == 2 and sh[0] > 1 and sh[1] > 1 else {QuantKind.FP8_TENSOR}+ return {QuantKind.FP8_TENSOR, QuantKind.FP8_BLOCK, QuantKind.MXFP4}+ return {QuantKind.NONE}+++def _probes(weight_map: dict[str, str]):+ mods: dict[str, set[str]] = defaultdict(set)+ for name in weight_map:+ module, _, suffix = name.rpartition(".")+ mods[module].add(suffix)+ for module, suffixes in mods.items():+ leaf = module.rpartition(".")[2]+ if leaf in _NOT_LINEAR or any(m in module for m in _MODEL_READS) or suffixes <= _META_SUFFIXES:+ continue+ if leaf in _LINEAR_LEAVES or suffixes & _QUANT_SUFFIXES:+ yield module, suffixes+++DIRS = _candidate_dirs()++[email protected](not DIRS, reason="no local checkpoint with config.json + safetensors")[email protected]("ckpt", DIRS, ids=_label)+def test_scheme_for_agrees_with_the_stored_tensors(ckpt: Path):+ cfg = json.load(open(ckpt / "config.json"))+ quant = cfg.get("quantization_config") or (cfg.get("text_config") or {}).get("quantization_config") or {}+ if str(quant.get("quant_method") or "").lower() in _UNSUPPORTED:+ with pytest.raises(NotImplementedError):+ QuantConfig.from_hf(cfg)+ return+ hfq = json.load(open(ckpt / "hf_quant_config.json")) if (ckpt / "hf_quant_config.json").exists() else None+ try:+ spec = get_model_spec((cfg.get("architectures") or [""])[0])+ except Exception:+ spec = None+ qc = QuantConfig.from_hf(cfg, hf_quant_config=hfq, unquantized=spec.unquantized_modules if spec else ())+ weight_map = _weight_map(ckpt)+ tensors = _tensor_info(ckpt, weight_map)+ mismatches, checked = [], 0+ for module, suffixes in _probes(weight_map):+ expected = _expected_kinds(suffixes, tensors, module)+ scheme = qc.scheme_for(module)+ kind = scheme.kind if scheme else QuantKind.NONE+ checked += 1+ if kind not in expected:+ mismatches.append(f"{module}: config says {kind}, tensors {sorted(suffixes)} say {sorted(k.value for k in expected)}")+ assert checked > 0+ assert not mismatches, f"{len(mismatches)}/{checked} modules disagree:\n " + "\n ".join(mismatches[:40])diff --git a/tests/moe/test_gpt_oss.py b/tests/moe/test_gpt_oss.pyindex eadbd4c1..1dbf1f16 100644--- a/tests/moe/test_gpt_oss.py+++ b/tests/moe/test_gpt_oss.py@@ -61,17 +61,28 @@ def tp1(monkeypatch):return tp_info+def _offload_layer(config, layer_id, cache):+ """gpt-oss offload experts as GptOssMLP builds them, kernel bound the way the engine does."""+ from freetoken.layers.quantization import QuantConfig+ from freetoken.models.gpt_oss.moe import GptOssOffloadMoELayer++ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "mxfp4", "modules_to_not_convert": []}})+ layer = GptOssOffloadMoELayer(+ layer_id, config.num_experts, config.num_experts_per_tok, config.hidden_size,+ config.moe_intermediate_size, renormalize=True, activation="gpt_oss_swiglu",+ alpha=config.hidden_act_alpha, limit=config.swiglu_limit, interleaved=True, has_bias=True,+ quant_config=quant, prefix=f"model.layers.{layer_id}.mlp.experts",+ )+ layer.offload_cache = cache+ return layer++def _make_offload_cache(config, device, *, cache_size=None, prefill_overlap=False):- from freetoken.moe.expert_banks import load_expert_banks+ from freetoken.moe.expert_banks import build_expert_banksfrom freetoken.moe.offload_cache import OffloadMoeCache- banks = load_expert_banks(- None,- config,- device=device,- dtype=torch.bfloat16,- dummy=True,- )+ probe = _offload_layer(config, 0, None)+ banks = build_expert_banks(probe.quant_method, config.num_layers, None, device=device, dummy=True)cache = OffloadMoeCache(num_layers=config.num_layers,num_experts=config.num_experts,@@ -80,6 +91,7 @@ def _make_offload_cache(config, device, *, cache_size=None, prefill_overlap=Falscache_policy="lru",prefill_overlap=prefill_overlap,quant_format=banks.quant_format,+ layout=banks.layout,)cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)@@ -188,7 +200,6 @@ def test_run_mxfp4_prefill_experts_t_matches_reference(M):@CUDAdef test_offload_decode_bit_identical_under_eviction(tp1):"""Offload decode matches direct split-K across a real evict+reload cycle."""- from freetoken.models.gpt_oss.moe import GptOssMxfp4OffloadMoELayerfrom freetoken.moe.fused_mxfp4 import (run_mxfp4_splitk_decode_experts as _run_mxfp4_splitk_decode_experts,)@@ -199,10 +210,8 @@ def test_offload_decode_bit_identical_under_eviction(tp1):cache = _make_offload_cache(c, dev, cache_size=E) # 4 slots, 8 total experts -> eviction- layer1 = GptOssMxfp4OffloadMoELayer(c, layer_id=1)- layer1.offload_cache = cache- layer0 = GptOssMxfp4OffloadMoELayer(c, layer_id=0)- layer0.offload_cache = cache+ layer1 = _offload_layer(c, 1, cache)+ layer0 = _offload_layer(c, 0, cache)torch.manual_seed(42)hidden = 0.1 * torch.randn(1, H, device=dev, dtype=torch.bfloat16)@@ -213,8 +222,8 @@ def test_offload_decode_bit_identical_under_eviction(tp1):tid_ref = torch.tensor([expert_ids], dtype=torch.int32, device=dev)return _run_mxfp4_splitk_decode_experts(hidden, tw, tid_ref,- g("gate_up_blocks"), g("gate_up_scales"), g("gate_up_bias"),- g("down_blocks"), g("down_scales"), g("down_bias"),+ g("gate_up"), g("gate_up_scale"), g("gate_up_bias"),+ g("down"), g("down_scale"), g("down_bias"),top_k=tk, hidden_act_alpha=c.hidden_act_alpha, swiglu_limit=c.swiglu_limit,)@@ -241,7 +250,7 @@ def test_offload_decode_bit_identical_under_eviction(tp1):@pytest.mark.parametrize("M", [16])def test_offload_prefill_overlap_matches_reference(M, tp1):"""Prefill overlap must not crash and must match the direct _t kernel."""- from freetoken.models.gpt_oss.moe import GptOssMxfp4OffloadMoELayer+ from freetoken.kernel import gpt_oss_fused_routingfrom freetoken.moe.fused_mxfp4 import (run_mxfp4_prefill_experts_t as _run_mxfp4_prefill_experts_t,)@@ -253,21 +262,20 @@ def test_offload_prefill_overlap_matches_reference(M, tp1):cache = _make_offload_cache(c, dev, prefill_overlap=True)# layer_id=0 is mandatory: _wait_prefill_overlap gates begin_prefill() on layer_id==0,# so only layer 0 exercises the previously-crashing code path.- layer = GptOssMxfp4OffloadMoELayer(c, layer_id=0)- layer.offload_cache = cache+ layer = _offload_layer(c, 0, cache)torch.manual_seed(2)hidden = 0.1 * torch.randn(M, H, device=dev, dtype=torch.bfloat16)logits = torch.randn(M, E, device=dev, dtype=torch.bfloat16)- tw, tid = layer._topk(logits.contiguous())+ tw, tid = gpt_oss_fused_routing(logits.contiguous(), tk)out_overlap = layer._prefill_routed(hidden, tw, tid.clone()) # must not raiseg = lambda k: cache.bank_sources[k][0].to(dev) # layer-0 source rowsout_ref = _run_mxfp4_prefill_experts_t(hidden, tw, tid,- g("gate_up_blocks"), g("gate_up_scales"), g("gate_up_bias"),- g("down_blocks"), g("down_scales"), g("down_bias"),+ g("gate_up"), g("gate_up_scale"), g("gate_up_bias"),+ g("down"), g("down_scale"), g("down_bias"),top_k=tk, hidden_act_alpha=c.hidden_act_alpha, swiglu_limit=c.swiglu_limit,)max_diff = (out_overlap - out_ref).abs().max().item()diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.pyindex 7005a1b4..0546fbcd 100644--- a/tests/moe/test_nvfp4_backends.py+++ b/tests/moe/test_nvfp4_backends.py@@ -9,15 +9,13 @@ Coverage by hardware:- Triton (any CUDA GPU): prefill + the production fast decode GEMV, plus a fast-vs-baseline-kernel equality guard. This is the path used on sm_120 + CUDA 12.x.- Marlin (sm_80..sm_99, e.g. H100): prefill + decode + overlap.- - b12x (sm_120 + CUDA>=13): pure-torch pack everywhere; the fused decode forward is- gated and skipped where the kernel cannot run.-The ``--nvfp4-backend`` selection + CUDA-13 gate is checked without a GPU.+ - b12x (sm_120 + CUDA>=13): pack + fused decode, skipped where the kernel cannot run.+Banks are built the way the loader builds them: the bound kernel object packs native pieces."""from __future__ import annotationsimport importlib.util-import typesimport pytestimport torch@@ -117,33 +115,63 @@ def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids, activation="silureturn out-def _marlin_cache(device, *, cache_size=S, prefill_overlap=False):- from freetoken.moe.nvfp4_backends import marlin_repack_sources_inplace- from freetoken.moe.offload_cache import OffloadMoeCache+def _bound_layer(kernel: str):+ """An NVFP4 offload layer with ``kernel`` bound, as the engine binds it."""+ from freetoken.distributed import set_tp_info, try_get_tp_info+ from freetoken.layers.moe import OffloadMoELayer+ from freetoken.layers.quantization import QuantBackend, QuantConfig, set_quant_backend- sources = _make_native_sources(device)- ref_sources = {k: [t.clone() for t in v] for k, v in sources.items()} # repack is in place- cfg = types.SimpleNamespace(hidden_size=H, moe_intermediate_size=I)- packed = marlin_repack_sources_inplace(sources, cfg, device, chunk=5)+ if try_get_tp_info() is None:+ set_tp_info(0, 1)+ set_quant_backend(QuantBackend.parse(f"moe.nvfp4={kernel}"))+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})+ return OffloadMoELayer(0, E, TOPK, H, I, quant_config=quant, prefix="model.layers.0.mlp.experts")++def _pieces(sources):+ for layer_id in range(L):+ yield layer_id, 0, E, {+ "gate_up": sources["gate_up_packed"][layer_id],+ "gate_up_scale": sources["gate_up_scale"][layer_id],+ "gate_up_global": sources["gate_up_global"][layer_id],+ "down": sources["down_packed"][layer_id],+ "down_scale": sources["down_scale"][layer_id],+ "down_global": sources["down_global"][layer_id],+ }+++def _packed_cache(device, kernel: str, *, seed=0, cache_size=S, prefill_overlap=False):+ """Native sources packed by ``kernel`` into an offload cache; the sources stay the dequant reference."""+ from freetoken.moe.expert_banks import build_expert_banks+ from freetoken.moe.offload_cache import OffloadMoeCache++ sources = _make_native_sources(device, seed=seed)+ layer = _bound_layer(kernel)+ banks = build_expert_banks(layer.quant_method, L, _pieces(sources), device=device)cache = OffloadMoeCache(num_layers=L,num_experts=E,cache_size=cache_size,device=device,- quant_format="nvfp4_marlin",+ quant_format=banks.quant_format,prefill_overlap=prefill_overlap,+ layout=banks.layout,+ max_slots=layer.quant_method.slot_limit(),)- cache.set_bank_sources({name: packed[name] for name in cache.bank_schema})- cache.set_alphas(packed["gate_up_alpha"], packed["down_alpha"])+ cache.set_bank_sources(banks.sources)+ cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)cache.reset()- return cache, ref_sources+ return cache, sources+++def _marlin_cache(device, *, cache_size=S, prefill_overlap=False):+ return _packed_cache(device, "marlin", cache_size=cache_size, prefill_overlap=prefill_overlap)@cuda@marlindef test_marlin_prefill_matches_dequant_reference():- from freetoken.moe.nvfp4_backends import marlin_fused_experts+ from freetoken.layers.quantization.moe.nvfp4 import marlin_fused_expertsdevice = torch.device("cuda")cache, ref_sources = _marlin_cache(device)@@ -172,7 +200,7 @@ def test_marlin_prefill_matches_dequant_reference():def test_marlin_decode_matches_dequant_reference_after_prefill_stomp():"""Decode through the slot cache, including the request-B-after-request-A patternthat B1 guarded against: a layer-1 full-layer prefill between two layer-0 decodes."""- from freetoken.moe.nvfp4_backends import marlin_fused_experts+ from freetoken.layers.quantization.moe.nvfp4 import marlin_fused_expertsdevice = torch.device("cuda")cache, ref_sources = _marlin_cache(device)@@ -211,7 +239,7 @@ def test_marlin_overlap_prefill_matches_dequant_reference():(cache_size == 2E, so every slot is buffer-backed): if the prefetch failed toinvalidate them, the post-prefill decode would "hit" stale mappings and read otherexperts' bytes; we assert it misses both experts instead."""- from freetoken.moe.nvfp4_backends import marlin_fused_experts+ from freetoken.layers.quantization.moe.nvfp4 import marlin_fused_expertsdevice = torch.device("cuda")cache, ref_sources = _marlin_cache(device, cache_size=2 * E, prefill_overlap=True)@@ -411,49 +439,18 @@ def test_triton_decode_marlin_matches_baseline_kernel():torch.testing.assert_close(marlin.float(), base.float(), rtol=2e-3, atol=2e-3)-def test_nvfp4_backend_selection():- """--nvfp4-backend selection + the flashinfer/marlin device gates -- runs without a GPU- via the CPU branch (forced backends need a usable device, so they error loudly there)."""- from freetoken.moe.nvfp4_backends import select_nvfp4_backend-- cpu = torch.device("cpu")- assert select_nvfp4_backend(cpu, None, "triton") == "triton"- assert select_nvfp4_backend(cpu, None, "auto") == "triton" # auto on CPU- with pytest.raises(RuntimeError):- select_nvfp4_backend(cpu, None, "flashinfer") # b12x needs a CUDA device- with pytest.raises(RuntimeError):- select_nvfp4_backend(cpu, None, "marlin") # marlin needs a CUDA device- with pytest.raises(ValueError):- select_nvfp4_backend(cpu, None, "bogus")--@cudadef test_b12x_decode_matches_dequant_reference():"""sm_120 + CUDA>=13 only: the flashinfer b12x W4A16 fused MoE over the slot cachevs the dequant reference (skipped on hardware/toolkits where b12x cannot run)."""- from freetoken.moe.nvfp4_backends import (- _b12x_unusable_reason,- b12x_fused_experts,- b12x_repack_sources_inplace,- )- from freetoken.moe.offload_cache import OffloadMoeCache+ from freetoken.layers.quantization.moe.nvfp4 import _b12x_unusable_reason, b12x_fused_expertsdevice = torch.device("cuda")reason = _b12x_unusable_reason(torch.cuda.get_device_capability(device))if reason is not None:pytest.skip(f"b12x not runnable here: {reason}")- sources = _make_native_sources(device, seed=8)- ref_sources = {k: [t.clone() for t in v] for k, v in sources.items()} # repack is in place- cfg = types.SimpleNamespace(hidden_size=H, moe_intermediate_size=I)- packed = b12x_repack_sources_inplace(sources, cfg, device, chunk=6)-- cache = OffloadMoeCache(- num_layers=L, num_experts=E, cache_size=S, device=device, quant_format="nvfp4_b12x"- )- cache.set_bank_sources({name: packed[name] for name in cache.bank_schema})- cache.set_alphas(packed["gate_up_alpha"], packed["down_alpha"])- cache.reset()+ cache, ref_sources = _packed_cache(device, "b12x", seed=8)torch.manual_seed(2)hidden = torch.randn(1, H, dtype=torch.bfloat16, device=device) / 4@@ -472,27 +469,23 @@ def test_b12x_decode_matches_dequant_reference():@cuda-def test_dummy_nvfp4_sources_match_loader_contract():- """--use-dummy-weight banks must match the real loader's shapes/dtypes/pinning so the- engine repack/offload path is exercised unchanged. The marlin repack + offload gather- tail (which needs vllm) lives in test_dummy_nvfp4_sources_marlin_repack."""- from freetoken.models.weight import dummy_nvfp4_expert_sources-- cfg = types.SimpleNamespace(- num_layers=L, num_experts=E, hidden_size=H, moe_intermediate_size=I- )- sources = dummy_nvfp4_expert_sources(cfg)+def test_dummy_nvfp4_banks_match_the_triton_layout():+ """--use-dummy-weight banks follow the bound kernel's layout (shapes, dtypes, pinning)."""+ from freetoken.moe.expert_banks import build_expert_banks++ layer = _bound_layer("triton")+ banks = build_expert_banks(layer.quant_method, L, None, device=torch.device("cuda"), dummy=True)expected = {- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),+ "gate_up": ((E, 2 * I, H // 2), torch.uint8),"gate_up_scale": ((E, 2 * I, H // 16), torch.float8_e4m3fn),"gate_up_global": ((E, 2 * I), torch.float16),- "down_packed": ((E, H, I // 2), torch.uint8),+ "down": ((E, H, I // 2), torch.uint8),"down_scale": ((E, H, I // 16), torch.float8_e4m3fn),"down_global": ((E, H), torch.float16),}- assert sources.keys() == expected.keys()+ assert banks.sources.keys() == expected.keys()for name, (shape, dtype) in expected.items():- layers = sources[name]+ layers = banks.sources[name]assert len(layers) == L, (name, len(layers))for t in layers:assert t.shape == shape and t.dtype == dtype and t.is_pinned(), name@@ -500,54 +493,42 @@ def test_dummy_nvfp4_sources_match_loader_contract():@cuda@marlin-def test_dummy_nvfp4_sources_marlin_repack():- """The --use-dummy-weight banks drop into the same marlin repack + offload path as the- real loader's (in-place repack). The gather kernel reads the banks zero-copy from the- GPU, which requires the allocator's memory to be device-mapped, not merely page-locked."""- from freetoken.models.weight import dummy_nvfp4_expert_sources- from freetoken.moe.nvfp4_backends import marlin_repack_sources_inplace+def test_dummy_nvfp4_banks_marlin_pack_gathers_zero_copy():+ """Dummy banks packed by marlin drop into the offload gather path like the real loader's."""+ from freetoken.moe.expert_banks import build_expert_banksfrom freetoken.moe.offload_cache import OffloadMoeCache- cfg = types.SimpleNamespace(- num_layers=L, num_experts=E, hidden_size=H, moe_intermediate_size=I- )- sources = dummy_nvfp4_expert_sources(cfg)-device = torch.device("cuda")- packed = marlin_repack_sources_inplace(sources, cfg, device, chunk=5)- assert torch.isfinite(packed["gate_up_alpha"].float()).all()- assert torch.isfinite(packed["down_alpha"].float()).all()+ layer = _bound_layer("marlin")+ banks = build_expert_banks(layer.quant_method, L, None, device=device, dummy=True)+ assert torch.isfinite(banks.gate_up_alpha.float()).all()+ assert torch.isfinite(banks.down_alpha.float()).all()cache = OffloadMoeCache(- num_layers=L, num_experts=E, cache_size=S, device=device, quant_format="nvfp4_marlin"+ num_layers=L, num_experts=E, cache_size=S, device=device, quant_format=banks.quant_format,+ layout=banks.layout, max_slots=layer.quant_method.kernel.max_slots,)- cache.set_bank_sources({name: packed[name] for name in cache.bank_schema})- cache.set_alphas(packed["gate_up_alpha"], packed["down_alpha"])+ cache.set_bank_sources(banks.sources)+ cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)cache.reset()cache.materialize_layer(0)cache.copy_missing()torch.cuda.synchronize()- assert torch.equal(cache.bank_caches["gate_up_packed"][:E].cpu(), packed["gate_up_packed"][0])+ assert torch.equal(cache.bank_caches["gate_up"][:E].cpu(), banks.sources["gate_up"][0])@cuda@pytest.mark.slow-def test_b12x_pack_is_byte_compatible_with_native_banks():- """The b12x kernel needs sm_120, but its pack is pure torch: verify the prepared- blocks drop into the native banks byte-for-byte (the in-place repack contract)."""- from freetoken.moe.nvfp4_backends import b12x_repack_sources_inplace+def test_b12x_pack_keeps_per_layer_banks_and_flat_alphas():+ """The b12x pack is pure torch: its banks stay per-layer lists with flat [L*E] alphas."""+ from freetoken.layers.quantization.moe.nvfp4 import _b12x_unusable_reasondevice = torch.device("cuda")- sources = _make_native_sources(device, seed=3)- cfg = types.SimpleNamespace(hidden_size=H, moe_intermediate_size=I)- try:- packed = b12x_repack_sources_inplace(sources, cfg, device, chunk=6)- except Exception as exc: # pragma: no cover - depends on flashinfer internals- pytest.skip(f"flashinfer w4a16 prepare unavailable off-target: {exc}")+ reason = _b12x_unusable_reason(torch.cuda.get_device_capability(device))+ if reason is not None:+ pytest.skip(f"b12x not runnable here: {reason}")+ cache, _ = _packed_cache(device, "b12x", seed=3)total = L * E- # packed banks stay per-layer lists; alphas are the one flat [L*E] exception (see- # cache_budget.expert_bytes_per_slot).- assert len(packed["gate_up_packed"]) == L- assert sum(t.shape[0] for t in packed["gate_up_packed"]) == total- assert packed["gate_up_alpha"].shape == (total,)- assert packed["down_packed"][0].dtype == torch.int32+ assert len(cache.bank_sources["gate_up"]) == L+ assert sum(t.shape[0] for t in cache.bank_sources["gate_up"]) == total+ assert cache.gate_up_alpha.shape == (total,)diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex 422ca867..b29ce531 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -4,6 +4,7 @@ import pytestimport torchfrom freetoken.distributed import set_tp_info, try_get_tp_info+from freetoken.layers.quantization import QuantKinddef _init_tp():@@ -11,18 +12,22 @@ def _init_tp():set_tp_info(rank=0, size=1)-def _make_layer_and_cache():+def _bf16_offload_layer(layer_id: int, num_experts: int, top_k: int, hidden_size: int, intermediate_size: int):+ """A bf16 offload layer on the fused kernel."""from freetoken.layers.moe import OffloadMoELayer+ from freetoken.layers.quantization import NoQuantConfig++ return OffloadMoELayer(+ layer_id, num_experts, top_k, hidden_size, intermediate_size,+ quant_config=NoQuantConfig(), prefix=f"model.layers.{layer_id}.mlp.experts",+ )+++def _make_layer_and_cache():from freetoken.moe.offload_cache import OffloadMoeCache_init_tp()- layer = OffloadMoELayer(- layer_id=0,- num_experts=4,- top_k=2,- hidden_size=8,- intermediate_size=16,- )+ layer = _bf16_offload_layer(0, 4, 2, 8, 16)cache = OffloadMoeCache(num_layers=1,num_experts=4,@@ -34,34 +39,41 @@ def _make_layer_and_cache():return layer, cache-def test_dummy_expert_sources_use_moe_layer_count(monkeypatch):- from types import SimpleNamespace+def test_dummy_expert_banks_follow_the_kernel_layout(monkeypatch):- import freetoken.models.weight as weight+ from freetoken.kernel import backend+ from freetoken.layers.quantization import QuantBackend, QuantConfig, set_quant_backend+ from freetoken.moe.expert_banks import build_expert_banks_init_tp()- config = SimpleNamespace(- num_layers=5,- num_moe_layers=3,- num_experts=4,- hidden_size=6,- moe_intermediate_size=8,- )-- gate_up, down = weight.dummy_moe_expert_sources(config, dtype=torch.float16)-- assert len(gate_up) == 3 and all(t.shape == (4, 16, 6) for t in gate_up)- assert len(down) == 3 and all(t.shape == (4, 6, 8) for t in down)-- monkeypatch.setattr(- weight,- "alloc_pinned_tensor",- lambda *shape, dtype: torch.empty(*shape, dtype=dtype),- )- banks = weight.dummy_nvfp4_expert_sources(config)-- assert {len(layers) for layers in banks.values()} == {3}- assert {t.shape[0] for layers in banks.values() for t in layers} == {4}+ L, E, H, I = 3, 4, 64, 32+ monkeypatch.setattr(backend, "device_capability", lambda: (0, 0))+ monkeypatch.setattr(backend, "is_vllm_installed", lambda: False)+ monkeypatch.setattr(backend, "is_flashinfer_installed", lambda: False)++ def _bound(quant):+ layer = _bf16_offload_layer(0, E, 2, H, I) if quant is None else None+ if layer is None:+ from freetoken.layers.moe import OffloadMoELayer++ layer = OffloadMoELayer(0, E, 2, H, I, quant_config=quant, prefix="model.layers.0.mlp.experts")+ return layer++ bf16 = _bound(None)+ banks = build_expert_banks(bf16.quant_method, L, None, device=torch.device("cpu"), dummy=True)+ assert banks.kind is QuantKind.NONE and set(banks.sources) == {"gate_up", "down"}+ assert len(banks.sources["gate_up"]) == L and all(t.shape == (E, 2 * I, H) for t in banks.sources["gate_up"])+ assert all(t.shape == (E, H, I) for t in banks.sources["down"])++ set_quant_backend(QuantBackend.parse("moe.nvfp4=triton"))+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})+ nvfp4 = _bound(quant)+ banks = build_expert_banks(nvfp4.quant_method, L, None, device=torch.device("cpu"), dummy=True)+ assert banks.kind is QuantKind.NVFP4 and banks.kernel == "triton"+ assert {len(layers) for layers in banks.sources.values()} == {L}+ assert {t.shape[0] for layers in banks.sources.values() for t in layers} == {E}+ assert torch.all(banks.sources["gate_up_scale"][0].float() == 1.0)+ assert torch.all(banks.sources["gate_up_global"][0].float() > 0)def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypatch):@@ -87,6 +99,8 @@ def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypagot_topk_ids,activation,apply_router_weight_on_input,+ act_alpha=1.0,+ act_limit=float("inf"),):calls["w1"] = w1calls["w2"] = w2@@ -94,7 +108,7 @@ def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypacalls["topk_ids"] = got_topk_ids.clone()return hidden_states- monkeypatch.setattr("freetoken.layers.moe.fused_experts_impl", fake_fused)+ monkeypatch.setattr("freetoken.moe.fused.fused_experts_impl", fake_fused)out = layer.prefill_forward(hidden_states, router_logits)@@ -112,22 +126,12 @@ def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypadef test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(monkeypatch):- from freetoken.layers.moe import OffloadMoELayerfrom freetoken.moe.offload_cache import OffloadMoeCache_init_tp()num_layers = 3num_experts = 4- layers = [- OffloadMoELayer(- layer_id=layer_id,- num_experts=num_experts,- top_k=2,- hidden_size=8,- intermediate_size=16,- )- for layer_id in range(num_layers)- ]+ layers = [_bf16_offload_layer(layer_id, num_experts, 2, 8, 16) for layer_id in range(num_layers)]cache = OffloadMoeCache(num_layers=num_layers,num_experts=num_experts,@@ -172,6 +176,8 @@ def test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(mogot_topk_ids,activation,apply_router_weight_on_input,+ act_alpha=1.0,+ act_limit=float("inf"),):layer_id = len(fused_calls)fused_calls.append(@@ -186,7 +192,7 @@ def test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(mo)return hidden_states + layer_id- monkeypatch.setattr("freetoken.layers.moe.fused_experts_impl", fake_fused)+ monkeypatch.setattr("freetoken.moe.fused.fused_experts_impl", fake_fused)out = hidden_statesfor layer in layers:@@ -353,6 +359,8 @@ def test_offload_moe_layer_decode_forward_uses_remapped_slot_ids(monkeypatch):got_topk_ids,activation,apply_router_weight_on_input,+ act_alpha=1.0,+ act_limit=float("inf"),):calls["w1"] = w1calls["w2"] = w2@@ -360,7 +368,7 @@ def test_offload_moe_layer_decode_forward_uses_remapped_slot_ids(monkeypatch):calls["topk_ids"] = got_topk_ids.clone()return hidden_states- monkeypatch.setattr("freetoken.layers.moe.fused_experts_decode_impl", fake_fused_decode)+ monkeypatch.setattr("freetoken.moe.fused.fused_experts_decode_impl", fake_fused_decode)out = layer.decode_forward(hidden_states, router_logits)@@ -429,17 +437,17 @@ def test_adjust_config_converts_moe_cache_rate_to_cache_size():num_moe_layers=10,num_experts=8,expert_quant="none",- moe_backend="auto",+ moe_strategy="auto",),)_adjust_config(config)- from freetoken.moe import is_offload_moe_backend+ from freetoken.moe import is_offload_moe_strategyassert config.moe_cache_size == 24# Family, not member: a box with a benchbw profile resolves bf16 experts to hybrid.- assert is_offload_moe_backend(config.moe_backend)+ assert is_offload_moe_strategy(config.moe_strategy)def test_graph_capture_reuses_warm_offload_cache_before_capture(monkeypatch):