Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

refactor(quant): config, scheme and method layers for quantization (#418)

Xiaoze Fan477c8601b248
diff --git a/benchmarks/bench_load_weight_generic.py b/benchmarks/bench_load_weight_generic.py
index 0e1905a0..4e4c5fe0 100644
--- a/benchmarks/bench_load_weight_generic.py
+++ b/benchmarks/bench_load_weight_generic.py
@@ -145,7 +145,7 @@ def _model_config(model_path: str):
dev = bind_assigned_gpu()
torch.zeros(1, device=dev) # init CUDA context (pinning / nvfp4 backend pick)
cfg = EngineConfig(model_path=model_path, tp_info=DistributedInfo(0, 1),
- dtype=torch.bfloat16, moe_backend="offload")
+ dtype=torch.bfloat16, moe_strategy="offload")
return cfg.model_config
diff --git a/docs/cli.md b/docs/cli.md
index ff4af382..37e44335 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -75,15 +75,17 @@ ft serve --model ... --gpu GPU-9e8d7c6b # the same card by UUID (a unique prefi
### MoE offload
-See [models.md](models.md#moe-backends) for what each backend does.
+See [models.md](models.md#moe-strategies) for what each strategy does.
| Flag | Default | Meaning |
|---|---|---|
-| `--moe-backend` | auto | `fused`/`offload`/`cpu`/`hybrid`; auto → offload, or hybrid with a `ft bench bw` profile |
-| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family backends) |
+| `--moe-strategy` | auto | `fused`/`offload`/`cpu`/`hybrid`; auto → offload, or hybrid with a `ft bench bw` profile. `--moe-backend` is the deprecated old spelling |
+| `--quant-backend` | auto | Kernel per quantized layer type, `layer[.kind]=name` entries: `linear=marlin,moe=b12x` or `moe.nvfp4=triton`. A layer-level entry applies to every kind whose table lists the name |
+| `--nvfp4-backend` | — | Deprecated: stands in for `--quant-backend moe.nvfp4=<marlin\|b12x\|triton>` (`flashinfer` means b12x); cannot be combined with `--quant-backend` |
+| `--moe-cache-size` / `--moe-cache-rate` / `--moe-cache-auto` | auto | GPU expert-cache size as slots / fraction of all experts / sized from free VRAM (mutually exclusive; auto is enabled by default for offload-family strategies) |
| `--kv-reserve-tokens` | 8192 | KV token floor reserved before `--moe-cache-auto` fills experts |
| `--moe-cpu-threads` | physical cores | CPU worker threads for the cpu/hybrid executor |
-| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, or a fraction) |
+| `--moe-cpu-layers` | all on GPU | With `offload`: which MoE layers decode on CPU (`3,7,11`, a count, a fraction, or `auto`). `auto` is for Windows/WSL only, where CUDA pinned memory is capped; every value needs an expert format the CPU executor serves (bf16, nvfp4, mxfp4), so fp8 experts cannot use it |
| `--moe-hybrid-max-fetch` | auto | With `hybrid`: max experts fetched over PCIe per layer per step; rest computed on CPU |
| `--moe-prefill-hit-d2d` | off | Prefill: copy cache-hit experts device-side, stream only misses (CUDA >= 13) |
| `--disable-moe-prefill-overlap` | overlap on | Disable the two-buffer prefill copy overlap |
@@ -146,7 +148,7 @@ environment so the agent cannot silently fall back to a paid endpoint.
## ft checkpoint
```bash
-ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8] [--gpu <uuid-or-index>]
+ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backend offload] [--quant-backend moe.nvfp4=b12x] [--shard-gib 8] [--gpu <uuid-or-index>]
```
Converts an HF safetensors checkpoint to FTW, FreeToken's self-contained
@@ -164,7 +166,7 @@ ft bench bw --gpu 1 # a specific GPU (UUID or nvidia-smi index, as
```
Measures host-RAM vs PCIe bandwidth with the real cpu/offload MoE kernels and writes a
-profile that `ft serve --moe-backend auto` and `--moe-hybrid-max-fetch -1` then read.
+profile that `ft serve --moe-strategy auto` and `--moe-hybrid-max-fetch -1` then read.
- One profile per GPU, at `~/.cache/freetoken/benchbw/<gpu-uuid>.json`.
- Keyed on expert format + GPU, so a profile from other hardware is ignored rather than
diff --git a/docs/models.md b/docs/models.md
index c9499163..ca3c1853 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -1,7 +1,6 @@
# Supported models
-FreeToken loads HF safetensors checkpoints directly (plus native GGUF for
-Gemma-4). The checkpoints below are known-good — the prebuilt kernels are tuned
+FreeToken loads HF safetensors checkpoints directly. The checkpoints below are known-good — the prebuilt kernels are tuned
for them; other checkpoints of the same architectures work too.
| Model | HF checkpoints |
@@ -19,9 +18,9 @@ for them; other checkpoints of the same architectures work too.
| MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) |
| Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) |
-## MoE backends
+## MoE strategies
-`ft serve --moe-backend {auto,fused,offload,cpu,hybrid}`:
+`ft serve --moe-strategy {auto,fused,offload,cpu,hybrid}` (`--moe-backend` is the deprecated old spelling):
- **fused** — experts resident on GPU (needs the VRAM); never auto-selected.
- **offload** — experts live in host RAM, an LRU cache of expert slots on GPU;
diff --git a/python/freetoken/checkpoint/__main__.py b/python/freetoken/checkpoint/__main__.py
index 2786173f..2b7e35f1 100644
--- a/python/freetoken/checkpoint/__main__.py
+++ b/python/freetoken/checkpoint/__main__.py
@@ -1,7 +1,7 @@
"""CLI: convert an HF safetensors checkpoint to a FreeToken Weight (FTW) checkpoint.
ft checkpoint --model <hf_dir> --out <ftw_dir> \
- [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8] [--gpu <uuid-or-index>]
+ [--dtype bfloat16] [--moe-backend offload] [--quant-backend moe.nvfp4=b12x] [--shard-gib 8] [--gpu <uuid-or-index>]
The output dir is self-contained: point the server's ``--model`` at it to load via the FTW
fast path (auto-detected).
@@ -21,6 +21,16 @@ from .convert import convert_checkpoint
_DTYPES = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}
+def _parse_quant_backend(value: str) -> str:
+ from freetoken.layers.quantization import QuantBackend
+
+ try:
+ QuantBackend.parse(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError(str(exc)) from None
+ return value
+
+
def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> int:
p = argparse.ArgumentParser(prog=prog, description=__doc__)
p.add_argument("--model", required=True, help="source HF safetensors checkpoint dir")
@@ -28,6 +38,9 @@ def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> i
p.add_argument("--dtype", choices=sorted(_DTYPES), default="bfloat16")
p.add_argument("--moe-backend", default="offload",
help="offload (experts -> banks) or e.g. triton (experts stay dense)")
+ p.add_argument("--quant-backend", type=_parse_quant_backend, default=None,
+ help="kernel per quantized layer type, as for ft serve; the expert banks are packed for the "
+ "chosen MoE kernel and the server picks that kernel back up from the FTW")
p.add_argument("--shard-gib", type=float, default=8.0, help="max shard size in GiB")
p.add_argument("--gpu", type=single_gpu_arg, default=None,
help="GPU for the repack: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or "
@@ -46,7 +59,7 @@ def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> i
t = time.perf_counter()
index = convert_checkpoint(
ns.model, ns.out, dtype=_DTYPES[ns.dtype],
- moe_backend=ns.moe_backend, shard_limit=shard_limit, device=device,
+ moe_backend=ns.moe_backend, quant_backend=ns.quant_backend, shard_limit=shard_limit, device=device,
)
dt = time.perf_counter() - t
c = index["counts"]
diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.py
index 420faf2c..9ead434f 100644
--- a/python/freetoken/checkpoint/convert.py
+++ b/python/freetoken/checkpoint/convert.py
@@ -119,9 +119,11 @@ class _ConvertSink:
under one lock (disk-bound anyway).
"""
- def __init__(self, writer: FTWWriter, desc: str = "Converting expert banks") -> None:
+ def __init__(self, writer: FTWWriter, desc: str = "Converting expert banks", *, names: dict[str, str] | None = None) -> None:
self._writer = writer
self._desc = desc
+ # canonical role -> the bank name the file stores
+ self._names = names or {}
self._bar = None
self._lock = threading.Lock()
self._seen: set[int] = set()
@@ -139,7 +141,7 @@ class _ConvertSink:
nbytes = 0
for bank_name, bank in banks.items():
self._writer.add_tensor(
- layer_bank_entry_name(bank_name, layer_id), bank.tensor, kind="experts_bank"
+ layer_bank_entry_name(self._names.get(bank_name, bank_name), layer_id), bank.tensor, kind="experts_bank"
)
nbytes += bank.nbytes
bank.release()
@@ -166,6 +168,7 @@ def convert_checkpoint(
*,
dtype: torch.dtype = torch.bfloat16,
moe_backend: str = "offload",
+ quant_backend: str | None = None,
shard_limit: int = DEFAULT_SHARD_LIMIT,
device: str | None = None,
) -> dict:
@@ -195,10 +198,17 @@ def convert_checkpoint(
torch.zeros(1, device=dev) # init CUDA context (needed by nvfp4 backend pick / pinning)
cfg = EngineConfig(model_path=model_path, tp_info=DistributedInfo(tp.rank, tp.size),
- dtype=dtype, moe_backend=moe_backend)
+ dtype=dtype, moe_strategy=moe_backend, quant_backend=quant_backend)
mc = cfg.model_config
offload = moe_backend == "offload" and getattr(mc, "is_moe", False)
include_moe_experts = not offload
+ method = None
+ if offload:
+ from freetoken.engine.engine import offload_expert_method
+ from freetoken.layers import set_rope_device
+
+ set_rope_device(dev)
+ method = offload_expert_method(cfg)
from freetoken.utils.progress import byte_bar, count_bar
@@ -220,14 +230,14 @@ def convert_checkpoint(
quant_format = None
num_layers = None
if offload:
- # Streamable formats (bf16, ds_fp4, nvfp4 on the triton backend, gpt-oss mxfp4, q4_0,
- # qwen3_5 fp8/bf16-dequant) write each layer to its own FTW entry as it completes (via
- # the sink) instead of materializing the whole bank set first; the non-streamable ones
- # (nvfp4 marlin/b12x -- repack mutates the whole bank set in place after load) ignore
- # the sink. Which happened is per-provider (e.g. nvfp4's backend pick), so it's read
- # back from ExpertBanks.streamed, not guessed here.
- sink = _ConvertSink(writer)
- banks = load_expert_banks(model_path, mc, device=dev, dtype=dtype, layer_sink=sink)
+ # every method-packed format streams each layer to its own FTW entry as it completes (via the sink); the GGUF provider reports through ExpertBanks.streamed whether it engaged the sink or materialized the whole bank set first
+ from freetoken.moe.legacy_format import legacy_bank_names, legacy_format_for
+
+ names = legacy_bank_names(legacy_format_for(method.kind, method.kernel.name)) if method is not None else {}
+ sink = _ConvertSink(writer, names=names)
+ banks = load_expert_banks(
+ model_path, mc, method=method, device=dev, dtype=dtype, layer_sink=sink
+ )
quant_format = banks.quant_format
if banks.streamed:
sink.close()
@@ -251,12 +261,13 @@ def convert_checkpoint(
# has whole-tensor add_tensor, so the per-layer sources reassemble into one
# flat tensor (a per-bank host RAM spike during conversion).
items = []
+ names = legacy_bank_names(quant_format)
for name, per_layer in banks.sources.items():
if num_layers is None:
num_layers = len(per_layer)
else:
assert len(per_layer) == num_layers, (name, len(per_layer), num_layers)
- items.append((name, torch.cat(per_layer, dim=0) if len(per_layer) > 1 else per_layer[0]))
+ items.append((names.get(name, name), torch.cat(per_layer, dim=0) if len(per_layer) > 1 else per_layer[0]))
for an in ("gate_up_alpha", "down_alpha"):
if getattr(banks, an, None) is not None:
items.append((an, getattr(banks, an)))
diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.py
index a365e3d5..207765b7 100644
--- a/python/freetoken/checkpoint/ftw.py
+++ b/python/freetoken/checkpoint/ftw.py
@@ -106,6 +106,14 @@ def is_ftw_checkpoint(path: str) -> bool:
return os.path.isfile(os.path.join(path, INDEX_NAME))
+def ftw_quant_format(path: str) -> str | None:
+ """The ``quant_format`` an FTW checkpoint's expert banks were packed for; None when ``path`` is not an FTW checkpoint or holds no banks."""
+ if not is_ftw_checkpoint(path):
+ return None
+ with open(os.path.join(path, INDEX_NAME)) as f:
+ return json.load(f).get("quant_format")
+
+
# ============================== writer ==============================
class FTWWriter:
"""Stream tensors into the FTW, rolling shard files at ``shard_limit``.
@@ -384,7 +392,8 @@ def iter_ftw_weights(path: str, *, kinds=("weight",), workers: int = 8,
reader.read_into(memoryview(buf), e, workers=workers, chunk=chunk)
dt = _dtype_of(e["dtype"])
t = torch.frombuffer(buf, dtype=dt, count=e["nbytes"] // _elsize(dt))
- if not _put((e["name"], t.view(*e["shape"]) if e["shape"] else t, buf, e["nbytes"])):
+ # a 0-d entry (a per-tensor scale) comes back 0-d, not [1]
+ if not _put((e["name"], t.view(*e["shape"]) if e["shape"] else t.view(()), buf, e["nbytes"])):
return
except BaseException as ex: # surface to consumer
err.append(ex)
@@ -590,8 +599,14 @@ def load_ftw_banks(
views.append(raw.view(num_experts, *row_shape) if row_shape else raw.view(num_experts))
sources[name] = views
+ from freetoken.moe.legacy_format import canonical_role, kind_kernel_for
from freetoken.moe.expert_banks import ExpertBanks
+ # the file names the banks the legacy way; the quant_format tag names the (kind, kernel) they were packed for
+ sources = {canonical_role(name): views for name, views in sources.items()}
+ quant_format = reader.meta("quant_format")
+ kind, kernel = kind_kernel_for(quant_format) if quant_format is not None else (None, None)
+
# a failed mlock leaves a LOCKED layer pageable; the log and labels report what the banks actually settled at
applied = list(residency)
for banks in row_hb.values():
@@ -628,8 +643,8 @@ def load_ftw_banks(
# (not a separate kind); everything else under experts_bank is a weight source.
alpha_kw = {n: alpha_hb[n].tensor for n in alpha_hb}
return ExpertBanks(
- reader.meta("quant_format"), sources, **alpha_kw,
- layer_residency=applied,
+ quant_format, sources, **alpha_kw,
+ layer_residency=applied, kind=kind, kernel=kernel,
)
diff --git a/python/freetoken/core.py b/python/freetoken/core.py
index ef0a539c..281ff41a 100644
--- a/python/freetoken/core.py
+++ b/python/freetoken/core.py
@@ -11,7 +11,6 @@ if TYPE_CHECKING:
from freetoken.attention.linear import FLAMetadata
from freetoken.kvcache import BaseCacheHandle, BaseKVCachePool
from freetoken.kvcache.linear_state_pool import LinearStatePool
- from freetoken.moe import BaseMoeBackend
from freetoken.moe.offload_cache import OffloadMoeCache
@@ -170,7 +169,6 @@ class Context:
# NOTE: this table always treat page_size = 1
page_table: torch.Tensor = field(init=False)
attn_backend: BaseAttnBackend = field(init=False)
- moe_backend: BaseMoeBackend = field(init=False)
moe_offload_cache: OffloadMoeCache | None = None
kv_cache: BaseKVCachePool = field(init=False)
# Per-request recurrent state for GatedDeltaNet layers; set by the engine for
diff --git a/python/freetoken/engine/cache_budget.py b/python/freetoken/engine/cache_budget.py
index ab7c0a9f..d522be49 100644
--- a/python/freetoken/engine/cache_budget.py
+++ b/python/freetoken/engine/cache_budget.py
@@ -108,16 +108,18 @@ def resolve_moe_cache_auto(
prefill_overlap: bool,
kv_reserve_tokens: int,
page_size: int,
- quant_format: str,
+ max_slots: int | None = None,
) -> tuple[int, int, bool]:
"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).
+ ``max_slots`` is the expert kernel's addressable slot limit; the plan never exceeds it.
+
Applies memory_ratio to the persisted pre-model baseline exactly once, then defers
the MoE-vs-KV split to plan_cache_budget. The (1-memory_ratio) remainder is the
CUDA-graph/activation headroom (not subtracted here).
"""
budget_bytes = net_cache_budget_bytes(memory_ratio, baseline_free, weights_bytes, fixed_cache_size)
- max_slots = 992 if quant_format == "nvfp4_marlin" else total_experts
+ max_slots = total_experts if max_slots is None else min(max_slots, total_experts)
kv_reserve_pages = div_ceil(kv_reserve_tokens, page_size)
return plan_cache_budget(
budget_bytes=budget_bytes,
diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.py
index bcbe6bcf..28d9336a 100644
--- a/python/freetoken/engine/config.py
+++ b/python/freetoken/engine/config.py
@@ -1,17 +1,20 @@
from __future__ import annotations
-from dataclasses import dataclass
+from dataclasses import dataclass, field, replace
from functools import cached_property
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, Any, List
import torch
from freetoken.distributed import DistributedInfo
from freetoken.models.register import _load_attr, get_model_spec
-from freetoken.utils import cached_load_hf_config
+from freetoken.utils import cached_load_hf_config, init_logger
+from freetoken.utils.hf import optional_hf_file
if TYPE_CHECKING:
from freetoken.models import ModelConfig
+logger = init_logger(__name__)
+
@dataclass(frozen=True)
class EngineConfig:
@@ -20,9 +23,11 @@ class EngineConfig:
dtype: torch.dtype
max_running_req: int = 4
attention_backend: str = "auto"
- moe_backend: str = "auto"
- # NVFP4 routed-expert GEMM backend (--nvfp4-backend): auto|marlin|flashinfer|triton.
- nvfp4_backend: str = "triton"
+ moe_strategy: str = "auto"
+ # old name of moe_strategy; __post_init__ folds it in
+ moe_backend: str | None = field(default=None, repr=False)
+ # --quant-backend: layer[.kind]=kernel entries, comma separated
+ quant_backend: str | None = None
# PLE table backend: "disk" (default) reads rows from the checkpoint files per fill, "pinned" preloads the table into page-locked host RAM.
ple_backend: str = "disk"
# Expert-bank host load (--expert-load): auto|serial|parallel. "auto" reads scattered
@@ -41,16 +46,16 @@ class EngineConfig:
# (cudaMemcpyBatchAsync); no-op unless moe_cache_size > 2 * num_experts.
moe_prefill_hit_d2d: bool = False
moe_collect_stats: bool = False # capture decode miss-rate counters into the cuda graph
- # CPU MoE backend (--moe-backend cpu): number of CPU worker threads computing
+ # CPU MoE backend (--moe-strategy cpu): number of CPU worker threads computing
# the decode experts. 0 = auto (physical cores). Ignored by other backends.
moe_cpu_threads: int = 0
- # Hybrid CPU/GPU decode (--moe-backend offload only): which MoE layers decode on
+ # Hybrid CPU/GPU decode (--moe-strategy offload only): which MoE layers decode on
# the CPU executor instead of the GPU offload/PCIe path. Spec is an explicit id
# list ("3,7,11"), a count ("8" -> 8 layers evenly strided across depth), or a
- # fraction ("0.5"). None/"" = all layers on GPU (plain offload). --moe-backend cpu
+ # fraction ("0.5"). None/"" = all layers on GPU (plain offload). --moe-strategy cpu
# already means all layers on CPU and ignores this.
moe_cpu_layers: str | None = None
- # Hybrid MoE backend (--moe-backend hybrid): max experts fetched over PCIe per
+ # Hybrid MoE backend (--moe-strategy hybrid): max experts fetched over PCIe per
# (layer, decode step); the rest of that step's misses are computed on the CPU.
# -1 (default) = auto: fetch the benched pcie_bw/cpu_bw fraction of each step's
# misses so the PCIe fetch and the CPU compute finish together (perfect overlap);
@@ -83,6 +88,15 @@ class EngineConfig:
# is final. Mutually exclusive with num_page_override.
num_token_override: int | None = None
+ def __post_init__(self):
+ if self.moe_backend is None:
+ return
+ if self.moe_strategy != "auto":
+ raise ValueError("moe_backend is the old name of moe_strategy; pass only moe_strategy")
+ logger.warning("EngineConfig.moe_backend is deprecated; use moe_strategy")
+ object.__setattr__(self, "moe_strategy", self.moe_backend)
+ object.__setattr__(self, "moe_backend", None)
+
@cached_property
def hf_config(self):
return cached_load_hf_config(self.model_path)
@@ -91,7 +105,8 @@ class EngineConfig:
def model_config(self) -> ModelConfig:
spec = get_model_spec(self.hf_config.architectures[0])
parse_config = _load_attr(spec.module, spec.parse_config)
- return parse_config(self.hf_config)
+ model_config = parse_config(self.hf_config)
+ return replace(model_config, quant=checkpoint_quant_config(self.model_path, self.hf_config, spec))
@property
def max_seq_len(self) -> int:
@@ -106,3 +121,25 @@ class EngineConfig:
@property
def distributed_addr(self) -> str:
return "tcp://127.0.0.1:2333"
+
+
+def checkpoint_quant_config(model_path: str, hf_config: Any, spec: Any):
+ """The checkpoint's QuantConfig under the family's naming, or None for GGUF, whose native-quant ops the shared parser does not model yet."""
+ from freetoken.layers.quantization import NameMap, QuantConfig
+
+ if spec.parse_config == "parse_gguf_config":
+ return None
+ # NOTE: ModelOpt exports before 0.41 keep the quantization config only in hf_quant_config.json, and the weight download fetches nothing but the safetensors shards, so this sidecar is fetched on its own.
+ hf_quant_config = None
+ sidecar = optional_hf_file(model_path, "hf_quant_config.json")
+ if sidecar is not None:
+ import json
+
+ with open(sidecar) as f:
+ hf_quant_config = json.load(f)
+ return QuantConfig.from_hf(
+ hf_config,
+ name_map=NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping),
+ unquantized=spec.unquantized_modules,
+ hf_quant_config=hf_quant_config,
+ )
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index b5a6fa3b..006c1908 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -12,9 +12,12 @@ from freetoken.core import Batch, Context, Req, set_global_ctx
from freetoken.distributed import destroy_distributed, enable_pynccl_distributed, set_tp_info
from freetoken.gpu_select import gpu_identity
from freetoken.layers import set_rope_device
+from freetoken.layers.quantization import LayerKind, QuantBackend, finalize_quant, set_quant_backend
+from freetoken.moe.offload_cache import iter_offload_moe_layers
from freetoken.models import create_model, load_weight
-from freetoken.moe import create_moe_backend, is_offload_moe_backend
+from freetoken.moe import is_offload_moe_strategy
from freetoken.moe.expert_banks import load_expert_banks
+from freetoken.moe.host_banks import PinFailed
from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache
from freetoken.utils import align_ceil, init_logger, is_sm90_family, is_sm100_family, mem_GB, torch_dtype
@@ -39,7 +42,7 @@ def _require_offload_cache_size(cache_size: int, num_experts: int) -> None:
f"moe_cache_size={cache_size} is too small: need at least num_experts={num_experts} "
f"slots. Pass --moe-cache-size/--moe-cache-rate, or use --moe-cache-auto "
f"(the default for offload/hybrid backends when no cache-sizing flag is given; "
- f"--moe-backend cpu always sizes its own fixed two-layer buffer and ignores "
+ f"--moe-strategy cpu always sizes its own fixed two-layer buffer and ignores "
f"cache-sizing flags)."
)
@@ -250,7 +253,7 @@ def _make_dummy_weight_state_dict(
if param.dtype in fp8_dtypes:
# torch.randn is not implemented for fp8; fill via a uint8 view with small
# codes (avoid NaN/inf fp8 encodings). Lets dummy-weight startup work for
- # block-fp8 models (the dense fp8 linears are fp8 regardless of moe_backend).
+ # block-fp8 models (the dense fp8 linears are fp8 regardless of moe_strategy).
t = torch.empty(param.shape, dtype=param.dtype, device=device)
t.view(torch.uint8).random_(0, 16)
state_dict[key] = t
@@ -294,6 +297,7 @@ class Engine:
def __init__(self, config: EngineConfig):
assert not torch.cuda.is_initialized()
set_tp_info(rank=config.tp_info.rank, size=config.tp_info.size)
+ set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
_ensure_expandable_segments() # before the first CUDA allocation below
from freetoken.gpu_select import bind_assigned_gpu
@@ -323,6 +327,7 @@ class Engine:
with torch.device("meta"), torch_dtype(config.dtype):
self.model = create_model(config.model_config)
self.model.load_state_dict(self._load_weight_state_dict(config))
+ finalize_quant(self.model)
post_weights_free = self._sync_get_memory()[0]
self._weights_bytes = self._baseline_free - post_weights_free
# Pool-budget baseline for the desktop cache sliders: free VRAM after the weights are
@@ -339,7 +344,7 @@ class Engine:
self._host_tables_bytes = 0
if hasattr(self.model, "load_host_tables"):
self._host_tables_bytes = int(self.model.load_host_tables(config) or 0)
- if is_offload_moe_backend(config.moe_backend):
+ if is_offload_moe_strategy(config.moe_strategy):
self._init_offload_moe_cache(config)
if hasattr(self.model, "prepare_for_runtime"):
self.model.prepare_for_runtime()
@@ -387,12 +392,10 @@ class Engine:
# through them belongs to the attention backend, built later in init_capture_graph.
self.kv_cache.attach_page_table(self.page_table)
- # ======================= Attention & MoE backend initialization ========================
+ # ======================= Attention backend initialization ========================
self.ctx.attn_backend = self.attn_backend = create_attention_backend(
config.attention_backend, config.model_config
)
- if config.model_config.is_moe:
- self.ctx.moe_backend = self.moe_backend = create_moe_backend(config.moe_backend)
# ======================= Sampler initialization ========================
self.sampler = Sampler(self.device, config.model_config.vocab_size)
@@ -470,12 +473,13 @@ class Engine:
load_weight(
config.model_path,
self.device,
- include_moe_experts=not is_offload_moe_backend(config.moe_backend),
+ include_moe_experts=not is_offload_moe_strategy(config.moe_strategy),
),
device=self.device,
)
- def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks) -> tuple[int, int, bool]:
+
+ def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None) -> tuple[int, int, bool]:
"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).
Pure glue over the Phase-1 budget policy; isolated here so it is unit-testable
@@ -499,63 +503,38 @@ class Engine:
prefill_overlap=config.moe_prefill_overlap,
kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve),
page_size=page_tokens,
- quant_format=banks.quant_format,
+ max_slots=method.slot_limit() if method is not None else None,
)
def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:
- # A model may fully own cache construction via make_offload_moe_cache.
- # Otherwise load_expert_banks gives the model module a setup hook first, then
- # falls back to per-quant providers, and the engine wires the banks into cache.
- cache_factory = getattr(self.model, "make_offload_moe_cache", None)
- if cache_factory is not None and config.moe_cache_auto:
- raise ValueError(
- "--moe-cache-auto is not supported for models with a custom "
- "make_offload_moe_cache; pass --moe-cache-size explicitly."
- )
- # decode_target picks the bank layout + the per-decode mechanism:
- # "hybrid" -> GPU-cache + CPU-overflow co-compute, every layer (--moe-backend hybrid);
- # "cpu" -> CPU executor for the cpu_layer_ids set (all layers under --moe-backend
- # cpu, the --moe-cpu-layers subset under offload);
- # "gpu" -> plain GPU offload.
- # cpu/hybrid both read experts on the CPU, so banks load in the native (CPU-readable)
- # layout; the GPU slot-cache GEMM reads those same native rows. decode_target also
- # gates the CPU executor build below.
- cpu_layer_ids = _resolve_cpu_layers(config, config.model_config.num_moe_layers)
- if (
- not cpu_layer_ids
- and config.moe_cpu_layers is None
- and config.moe_backend in ("offload", "hybrid")
- and _pin_budget_bytes(self._host_tables_bytes) is not None
- ):
- cpu_layer_ids = _auto_cpu_layers(
- config, config.model_config.num_moe_layers, reserved=self._host_tables_bytes
- )
- if config.moe_backend == "hybrid":
- decode_target = "hybrid"
- elif cpu_layer_ids:
- decode_target = "cpu"
- else:
+ method = shared_offload_method(self.model)
+ num_moe_layers = config.model_config.num_moe_layers
+ cpu_layer_ids = _resolve_cpu_layers(config, num_moe_layers, reserved=self._host_tables_bytes, method=method)
+ _check_pin_budget(config, reserved=self._host_tables_bytes, method=method)
+ # the kernels were picked for model_config.decode_target; --moe-cpu-layers auto may still find that every bank fits the pin budget
+ decode_target = config.model_config.decode_target
+ if decode_target == "cpu" and not cpu_layer_ids:
decode_target = "gpu"
# split residency: where pinning is quota-capped (_pin_budget_bytes), pin only the GPU layers' banks and mlock the CPU layers'
# uncapped hosts keep every bank pinned (CPU decode reads them the same; overlap prefill stays on)
- # not applied to plain --moe-backend cpu; all-locked under a cap = --moe-backend offload --moe-cpu-layers 1.0
+ # not applied to plain --moe-strategy cpu; all-locked under a cap = --moe-strategy offload --moe-cpu-layers 1.0
split_residency = (
bool(cpu_layer_ids)
- and config.moe_backend in ("offload", "hybrid")
+ and config.moe_strategy in ("offload", "hybrid")
and _pin_budget_bytes(self._host_tables_bytes) is not None
)
- if config.moe_backend == "cpu" and not split_residency:
+ if config.moe_strategy == "cpu" and not split_residency:
# cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything instead
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
budget = _pin_budget_bytes(self._host_tables_bytes)
bank_bytes = None
if budget is not None:
- bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)
+ bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)
if bank_bytes and bank_bytes > budget:
split_residency = True
logger.info_rank0(
- f"--moe-backend cpu: banks {bank_bytes / 2**30:.2f} GiB exceed the "
+ f"--moe-strategy cpu: banks {bank_bytes / 2**30:.2f} GiB exceed the "
f"pin budget; OS-locking all layers instead of pinning"
)
if split_residency and config.moe_prefill_overlap:
@@ -565,25 +544,26 @@ class Engine:
"(locked layers prefill via synchronous pageable copies)"
)
object.__setattr__(config, "moe_prefill_overlap", False)
- if cache_factory is None:
- # Fast path: an FTW checkpoint loads its repacked banks directly.
- # Slow path: load_expert_banks auto-picks parallel vs serial baseline by
- # expert-tensor granularity. Both pin-after-fill.
- # --expert-load: serial/parallel force the read; auto (None) lets load_expert_banks
- # pick (parallel for scattered experts, with a low-RAM fallback to serial).
- expert_parallel = {"serial": False, "parallel": True}.get(config.expert_load, None)
- requested_residency = None
- if split_residency:
- from freetoken.moe.host_banks import HostResidency
-
- requested_residency = [
- HostResidency.LOCKED.value if i in cpu_layer_ids
- else HostResidency.PINNED.value
- for i in range(config.model_config.num_moe_layers)
- ]
+ # Fast path: an FTW checkpoint loads its repacked banks directly.
+ # Slow path: load_expert_banks auto-picks parallel vs serial baseline by
+ # expert-tensor granularity. Both pin-after-fill.
+ # --expert-load: serial/parallel force the read; auto (None) lets load_expert_banks
+ # pick (parallel for scattered experts, with a low-RAM fallback to serial).
+ expert_parallel = {"serial": False, "parallel": True}.get(config.expert_load, None)
+ requested_residency = None
+ if split_residency:
+ from freetoken.moe.host_banks import HostResidency
+
+ requested_residency = [
+ HostResidency.LOCKED.value if i in cpu_layer_ids
+ else HostResidency.PINNED.value
+ for i in range(config.model_config.num_moe_layers)
+ ]
+ try:
banks = load_expert_banks(
config.model_path,
config.model_config,
+ method=method,
device=self.device,
dtype=self.dtype,
dummy=config.use_dummy_weight,
@@ -591,54 +571,60 @@ class Engine:
decode_target=("cpu" if decode_target in ("cpu", "hybrid") else "gpu"),
layer_residency=requested_residency,
)
- if config.moe_cache_auto:
- size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks)
- object.__setattr__(config, "moe_cache_size", size)
- object.__setattr__(config, "moe_prefill_overlap", overlap)
- if config.num_page_override is None:
- # Honor the plan's KV half too: MoE slots and KV pages were solved
- # against ONE budget (ratio x baseline - weights), so both must come
- # from it. Re-solving pages later from a fresh free-memory reading
- # double-counts everything allocated since the weights measurement
- # (this expert cache, the CPU-executor GPU buffers, allocator
- # slack) and goes negative whenever the expert fill is exact --
- # a greedy fill leaves no headroom for the measurement delta.
- object.__setattr__(config, "num_page_override", pages)
- logger.info_rank0(
- f"--moe-cache-auto resolved moe_cache_size={size} "
- f"num_pages={pages} (prefill_overlap={overlap})"
- )
- _require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)
- cache = OffloadMoeCache(
- # Models with leading dense layers (GLM-4) only have experts on the MoE
- # layers; num_moe_layers == num_layers when first_k_dense_replace == 0.
- num_layers=config.model_config.num_moe_layers,
- num_experts=config.model_config.num_experts,
- cache_size=config.moe_cache_size,
- device=self.device,
- cache_policy=config.moe_cache_policy,
- prefill_overlap=config.moe_prefill_overlap,
- prefill_hit_d2d=config.moe_prefill_hit_d2d,
- quant_format=banks.quant_format,
- decode_target=decode_target,
- hybrid_max_fetch=config.moe_hybrid_max_fetch,
+ except PinFailed as exc:
+ raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc
+ if config.moe_cache_auto:
+ size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method)
+ object.__setattr__(config, "moe_cache_size", size)
+ object.__setattr__(config, "moe_prefill_overlap", overlap)
+ if config.num_page_override is None:
+ # Honor the plan's KV half too: MoE slots and KV pages were solved
+ # against ONE budget (ratio x baseline - weights), so both must come
+ # from it. Re-solving pages later from a fresh free-memory reading
+ # double-counts everything allocated since the weights measurement
+ # (this expert cache, the CPU-executor GPU buffers, allocator
+ # slack) and goes negative whenever the expert fill is exact --
+ # a greedy fill leaves no headroom for the measurement delta.
+ object.__setattr__(config, "num_page_override", pages)
+ logger.info_rank0(
+ f"--moe-cache-auto resolved moe_cache_size={size} "
+ f"num_pages={pages} (prefill_overlap={overlap})"
)
- # before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
- cache.cpu_layer_ids = cpu_layer_ids
- cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)
- cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
- else:
- cache = cache_factory(config, self.device)
- cache.decode_target = decode_target
- cache.hybrid_max_fetch = config.moe_hybrid_max_fetch
- cache.cpu_layer_ids = cpu_layer_ids
+ _require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)
+ layout = max_slots = None
+ if method is not None:
+ if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):
+ raise ValueError(
+ f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "
+ f"binds {method.kind} / {method.kernel.name}; reconvert or select that kernel"
+ )
+ layout = method.layout()
+ max_slots = method.slot_limit()
+ cache = OffloadMoeCache(
+ # Models with leading dense layers (GLM-4) only have experts on the MoE
+ # layers; num_moe_layers == num_layers when first_k_dense_replace == 0.
+ num_layers=config.model_config.num_moe_layers,
+ num_experts=config.model_config.num_experts,
+ cache_size=config.moe_cache_size,
+ device=self.device,
+ cache_policy=config.moe_cache_policy,
+ prefill_overlap=config.moe_prefill_overlap,
+ prefill_hit_d2d=config.moe_prefill_hit_d2d,
+ quant_format=banks.quant_format,
+ decode_target=decode_target,
+ hybrid_max_fetch=config.moe_hybrid_max_fetch,
+ layout=layout,
+ max_slots=max_slots,
+ )
+ # before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
+ cache.cpu_layer_ids = cpu_layer_ids
+ cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)
+ cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
if decode_target == "hybrid":
self._resolve_hybrid_fetch(config, cache)
# Must be set before CUDA graph capture so the (device-side) accumulation ops are
# captured and re-run on every decode replay.
cache.collect_stats = config.moe_collect_stats
- # attach_offload_moe_cache walks for OffloadMoELayers, or defers to a model's
- # _iter_offload_moe_layers() hook when its MoE blocks are bespoke nn.Modules (DSV4).
layers = attach_offload_moe_cache(self.model, cache)
assert len(layers) == config.model_config.num_moe_layers
if cache.decode_target in ("cpu", "hybrid"):
@@ -698,7 +684,6 @@ class Engine:
# Decode batches never exceed max_running_req, but CUDA-graph padding can
# round a batch up to the largest captured size; cover both.
max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, 1)
- # gpt-oss mxfp4 carries clamped-swiglu scalars; other formats use the defaults.
executor = CpuMoeExecutor(
cache,
top_k=sample.top_k,
@@ -707,8 +692,10 @@ class Engine:
num_threads=config.moe_cpu_threads,
max_tokens=max_tokens,
device=self.device,
- swiglu_alpha=getattr(sample, "hidden_act_alpha", 1.702),
- swiglu_limit=getattr(sample, "swiglu_limit", None),
+ swiglu_alpha=float(sample.alpha),
+ swiglu_limit=sample.limit,
+ # FIXME: the None branch serves GGUF q4_0 banks, which have no quant method yet; drop it once GGUF joins the quant path
+ fmt=sample.quant_method.cpu_format if sample.quant_method is not None else None,
)
cache.set_cpu_executor(executor)
self.cpu_moe_executor = executor
@@ -1124,20 +1111,31 @@ def _parse_cpu_layers_spec(spec: str, num_moe_layers: int) -> frozenset[int]:
return frozenset(round(i * num_moe_layers / k) for i in range(k))
-def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int]:
+def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
"""MoE layer ids whose decode runs on the CPU executor.
- ``--moe-backend cpu`` -> every layer. ``--moe-backend offload`` + ``--moe-cpu-layers``
- -> the parsed subset (the rest stay on the GPU offload/PCIe path). Otherwise none.
+ ``--moe-strategy cpu`` -> every layer. ``--moe-strategy offload`` + ``--moe-cpu-layers``
+ -> the parsed subset, or the pin-budget pick for ``auto`` (the rest stay on the GPU offload/PCIe path). Otherwise none.
"""
- if config.moe_backend == "cpu":
+ if config.moe_strategy == "cpu":
return frozenset(range(num_moe_layers))
spec = config.moe_cpu_layers
- if not spec or not is_offload_moe_backend(config.moe_backend):
+ if not spec or not is_offload_moe_strategy(config.moe_strategy):
return frozenset()
+ if spec.strip() == "auto":
+ return _auto_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)
return _parse_cpu_layers_spec(spec, num_moe_layers)
+def _decode_target(config: EngineConfig) -> str:
+ """Where routed experts decode, from the flags alone: hybrid co-compute, the CPU executor for some or all layers, or the GPU slot cache."""
+ if config.moe_strategy == "hybrid":
+ return "hybrid"
+ if config.moe_strategy == "cpu" or (config.moe_cpu_layers and is_offload_moe_strategy(config.moe_strategy)):
+ return "cpu"
+ return "gpu"
+
+
# expert activations the CPU MoE executor supports (csrc ActKind)
_CPU_MOE_ACTS = (
"silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",
@@ -1179,25 +1177,50 @@ def _pin_budget_bytes(reserved: int = 0) -> int | None:
return max(0, cap - reserved)
-def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, reserved: int = 0) -> frozenset[int]:
- """Pick CPU (locked) MoE layers automatically when the banks exceed the pin budget.
-
- Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""
+def _bank_bytes(config: EngineConfig, method=None) -> int | None:
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
- bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)
+ return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)
+
+
+def _pin_hint(reserved: int) -> str:
+ if _pin_budget_bytes(reserved) is None:
+ return "the expert banks need more page-locked host RAM than this host has; free host RAM or serve a smaller model"
+ return (
+ "pass --moe-cpu-layers auto to lock the layers over the pin budget for CPU decode, "
+ "or --moe-cpu-layers <count|fraction|ids> to choose them yourself"
+ )
+
+
+def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None) -> None:
+ """Stop a plain offload boot whose banks exceed a known pin budget before any bank is read."""
+ if config.moe_cpu_layers or config.moe_strategy not in ("offload", "hybrid"):
+ return
+ budget = _pin_budget_bytes(reserved)
+ bank_bytes = _bank_bytes(config, method) if budget is not None else None
+ if bank_bytes and bank_bytes > budget:
+ raise ValueError(
+ f"expert banks need {bank_bytes / 2**30:.1f} GiB of pinned host RAM but the pin budget is "
+ f"{budget / 2**30:.1f} GiB (WSL caps CUDA pinning; FREETOKEN_PIN_BUDGET_GB overrides); {_pin_hint(reserved)}"
+ )
+
+
+def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
+ """Pick CPU (locked) MoE layers for ``--moe-cpu-layers auto``: none while the banks fit the pin budget.
+
+ Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""
+ bank_bytes = _bank_bytes(config, method)
if not bank_bytes:
return frozenset()
budget = _pin_budget_bytes(reserved)
if budget is None or bank_bytes <= budget:
return frozenset()
if not _cpu_moe_executor_viable(config.model_config):
- logger.info_rank0(
+ raise ValueError(
f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB exceed the "
f"pin budget {budget / 2**30:.2f} GiB, but the CPU MoE executor cannot "
- f"serve this model; keeping every layer pinned on the GPU offload path"
+ f"serve this model (see --moe-strategy cpu requirements)"
)
- return frozenset()
n = min(num_moe_layers, math.ceil(num_moe_layers * (1 - budget / bank_bytes)))
head = (n + 1) // 2
ids = frozenset(range(head)) | frozenset(range(num_moe_layers - (n - head), num_moe_layers))
@@ -1209,7 +1232,7 @@ def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, reserved: int =
return ids
-# MoE-only knobs and the value each resolves to on a dense model. moe_backend is handled
+# MoE-only knobs and the value each resolves to on a dense model. moe_strategy is handled
# separately (its dense value is 'fused', but 'auto' resolves there without a warning).
_DENSE_MOE_SETTINGS = {
"moe_cache_size": 0,
@@ -1224,6 +1247,62 @@ _DENSE_MOE_SETTINGS = {
}
+def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> QuantBackend:
+ """--quant-backend with an FTW checkpoint's packed expert kernel filled in where the flag leaves that table automatic.
+
+ An entry that names another kernel is refused here, before any bank is read."""
+ from freetoken.checkpoint.ftw import ftw_quant_format
+ from freetoken.moe.legacy_format import kind_kernel_for
+
+ fmt = ftw_quant_format(model_path) if model_path else None
+ if fmt is None:
+ return quant_backend
+ try:
+ kind, kernel = kind_kernel_for(fmt)
+ except KeyError:
+ return quant_backend
+ requested = quant_backend.select(LayerKind.MOE, kind)
+ if requested == kernel:
+ return quant_backend
+ if requested != "auto":
+ raise ValueError(
+ f"the FTW checkpoint's expert banks were packed for {kind} / {kernel} but --quant-backend asks for "
+ f"{requested}; drop the entry, or reconvert with ft checkpoint --quant-backend moe.{kind}={requested}"
+ )
+ return QuantBackend(quant_backend.items + (((LayerKind.MOE, kind), kernel),))
+
+
+def shared_offload_method(model):
+ """The expert method every offload MoE layer of ``model`` uses, or None for models whose MoE layers carry none (GGUF).
+
+ The offload cache holds one bank layout, so the layers must agree on (kind, kernel)."""
+ layers = [l for l in iter_offload_moe_layers(model) if getattr(l, "quant_method", None) is not None]
+ if not layers:
+ return None
+ keys = {(layer.quant_method.kind, layer.quant_method.kernel.name) for layer in layers}
+ if len(keys) != 1:
+ found = sorted(f"{kind} / {kernel}" for kind, kernel in keys)
+ raise ValueError(f"expert layers disagree on format / kernel: {found}; per-layer mixed expert formats are not supported")
+ kind, kernel = keys.pop()
+ logger.info_rank0(f"MoE experts: {kind} via {kernel}")
+ return layers[0].quant_method
+
+
+def offload_expert_method(config: EngineConfig):
+ """The offload expert method of ``config``'s model, from a meta-device build.
+
+ For tools that load expert banks without an engine (the FTW converter); they pack for GPU decode."""
+ from freetoken.models import create_model
+ from freetoken.utils.torch_utils import torch_dtype
+
+ set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
+ object.__setattr__(config.model_config, "moe_strategy", config.moe_strategy)
+ object.__setattr__(config.model_config, "decode_target", "gpu")
+ with torch.device("meta"), torch_dtype(config.dtype):
+ model = create_model(config.model_config)
+ return shared_offload_method(model)
+
+
def _adjust_config(config: EngineConfig):
def override(attr: str, value: Any): # this is dangerous, use with caution
object.__setattr__(config, attr, value)
@@ -1248,9 +1327,9 @@ def _adjust_config(config: EngineConfig):
for name, dense_value in _DENSE_MOE_SETTINGS.items()
if getattr(config, name, dense_value) != dense_value
]
- if config.moe_backend not in ("auto", "fused"):
- dropped.insert(0, f"moe_backend={config.moe_backend!r}")
- override("moe_backend", "fused")
+ if config.moe_strategy not in ("auto", "fused"):
+ dropped.insert(0, f"moe_strategy={config.moe_strategy!r}")
+ override("moe_strategy", "fused")
for name, dense_value in _DENSE_MOE_SETTINGS.items():
override(name, dense_value)
if dropped:
@@ -1351,12 +1430,12 @@ def _adjust_config(config: EngineConfig):
if (
is_moe
and not _cpu_moe_act_ok
- and (config.moe_backend in ("cpu", "hybrid") or config.moe_cpu_layers)
+ and (config.moe_strategy in ("cpu", "hybrid") or config.moe_cpu_layers)
):
asked = (
f"--moe-cpu-layers={config.moe_cpu_layers!r}"
- if config.moe_backend not in ("cpu", "hybrid")
- else f"--moe-backend {config.moe_backend!r}"
+ if config.moe_strategy not in ("cpu", "hybrid")
+ else f"--moe-strategy {config.moe_strategy!r}"
)
raise ValueError(
f"{asked}: the CPU MoE executor does not support this model's expert "
@@ -1364,7 +1443,7 @@ def _adjust_config(config: EngineConfig):
"and let every layer decode on the GPU offload path instead."
)
- if is_moe and config.moe_backend == "auto":
+ if is_moe and config.moe_strategy == "auto":
# A MoE model always defaults to the offload family: experts stream from pinned host
# banks into an auto-sized GPU slot cache, which is the only default that serves a model
# bigger than the GPU. The resident 'fused' path (bf16 / block-fp8 experts, the two
@@ -1409,18 +1488,18 @@ def _adjust_config(config: EngineConfig):
logger.info_rank0(
f"benchbw profile recommends hybrid for {bench_fmt!r} experts on this GPU"
)
- override("moe_backend", default_backend)
- logger.info_rank0(f"Auto-selected MoE backend: {config.moe_backend}")
+ override("moe_strategy", default_backend)
+ logger.info_rank0(f"Auto-selected MoE strategy: {config.moe_strategy}")
if (
- is_offload_moe_backend(config.moe_backend)
+ is_offload_moe_strategy(config.moe_strategy)
and config.moe_cache_size <= 0
and config.moe_cache_rate is None
and not getattr(config, "moe_cache_auto", False)
):
# args.py's "no sizing flag -> default --moe-cache-auto" only fires when the
# backend is already offload-family at *parse* time. A bare `ft serve <FTW MoE
- # checkpoint>` (no --moe-backend, no cache flags) still has moe_backend=="auto" at
+ # checkpoint>` (no --moe-strategy, no cache flags) still has moe_strategy=="auto" at
# parse time -- the auto -> offload/cpu/hybrid resolution above is the first point
# the concrete backend is known, so mirror the same default here: no sizing flag
# was given, so let the scheduler resolve the cache size from free VRAM instead of
@@ -1428,10 +1507,10 @@ def _adjust_config(config: EngineConfig):
override("moe_cache_auto", True)
logger.info_rank0(
"No MoE cache sizing flag given; defaulting to --moe-cache-auto for "
- f"auto-selected backend {config.moe_backend!r}"
+ f"auto-selected strategy {config.moe_strategy!r}"
)
- if is_moe and config.moe_backend == "fused":
+ if is_moe and config.moe_strategy == "fused":
# An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The
# sizing flags no longer redirect the backend, so ignore them here and say so -- the
# geometry the user asked for is what runs. Report the flag actually passed: --moe-cache-
@@ -1447,13 +1526,13 @@ def _adjust_config(config: EngineConfig):
if inert:
logger.warning_rank0(
f"MoE backend 'fused' keeps its experts resident; ignoring {inert} "
- "(use --moe-backend offload to serve experts from a slot cache)"
+ "(use --moe-strategy offload to serve experts from a slot cache)"
)
override("moe_cache_size", 0)
override("moe_cache_rate", None)
override("moe_cache_auto", False)
- if is_moe and config.moe_backend == "cpu":
+ if is_moe and config.moe_strategy == "cpu":
# CPU-compute decode keeps experts in host RAM and computes them on the CPU;
# the GPU only holds the two-layer prefill double buffer. So the slot cache is
# fixed at exactly two expert layers (prefill overlap requires >= 2*num_experts)
@@ -1471,23 +1550,27 @@ def _adjust_config(config: EngineConfig):
if (
is_moe
and expert_quant not in ("none", "fp8_block")
- and not is_offload_moe_backend(config.moe_backend)
+ and not is_offload_moe_strategy(config.moe_strategy)
):
raise ValueError(
- f"{expert_quant} experts require --moe-backend offload or cpu, "
- f"got {config.moe_backend!r}"
+ f"{expert_quant} experts require --moe-strategy offload or cpu, "
+ f"got {config.moe_strategy!r}"
)
- if is_moe and config.moe_cpu_layers and config.moe_backend not in ("offload", "hybrid"):
+ if is_moe and config.moe_cpu_layers and config.moe_strategy not in ("offload", "hybrid"):
# the layer split needs the offload host banks + slot cache; 'cpu' already runs every layer on CPU, 'fused' keeps experts resident on the GPU (no host banks)
raise ValueError(
- "--moe-cpu-layers requires --moe-backend offload or hybrid (got "
- f"{config.moe_backend!r}); use --moe-backend cpu to run all layers on CPU"
+ "--moe-cpu-layers requires --moe-strategy offload or hybrid (got "
+ f"{config.moe_strategy!r}); use --moe-strategy cpu to run all layers on CPU"
)
+ if is_moe and config.moe_cpu_layers and config.moe_cpu_layers.strip() != "auto":
+ if not _parse_cpu_layers_spec(config.moe_cpu_layers, model_config.num_moe_layers):
+ override("moe_cpu_layers", None)
+
if is_moe:
- object.__setattr__(model_config, "moe_backend", config.moe_backend)
- object.__setattr__(model_config, "nvfp4_backend", config.nvfp4_backend)
+ object.__setattr__(model_config, "moe_strategy", config.moe_strategy)
+ object.__setattr__(model_config, "decode_target", _decode_target(config))
# Must stay LAST: page_size is only final here (_adjust_dsv4_config sets P=128, the
# TRTLLM block sets 64). Also covers the programmatic LLM(...) path that bypasses parse_args.
@@ -1518,7 +1601,7 @@ def _adjust_config(config: EngineConfig):
)
# The startup ServerArgs dump is the *requested* config, printed in the frontend process
- # before any of the resolution above ran -- so "moe_backend='auto'" is all it can say. This
+ # before any of the resolution above ran -- so "moe_strategy='auto'" is all it can say. This
# is the one line that reports what actually runs, for every path (explicit backends never
# hit an "Auto-selected ..." log at all).
resolved = [
@@ -1527,5 +1610,5 @@ def _adjust_config(config: EngineConfig):
f"page_size={config.page_size}",
]
if is_moe:
- resolved.insert(0, f"moe_backend={config.moe_backend!r}")
+ resolved.insert(0, f"moe_strategy={config.moe_strategy!r}")
logger.info_rank0(f"Resolved config: {', '.join(resolved)}")
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 0ce249de..8d066d43 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -79,10 +79,10 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
"""
H, I = hidden_size, moe_intermediate_size
if fmt == "bf16":
- # models/loader.py stream_moe_expert_sources: gate_up [E, 2I, H], down [E, H, I], bf16
+ # bf16 expert banks: gate_up [E, 2I, H], down [E, H, I]
return {"gate_up": 2 * I * H * 2, "down": H * I * 2}
if fmt == "fp8_block":
- # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block
+ # block-fp8 expert banks: fp8 weights + bf16 128x128 block
# scales, trailing scale dim 16B-padded (same helper as the loader)
B = 128
return {
@@ -97,7 +97,7 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"):
# models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp16
# per-row globals; marlin/b12x repacks are byte-identical with the globals
- # folded into GPU-resident alphas (moe/nvfp4_backends.py), so no global banks.
+ # folded into GPU-resident alphas (layers/quantization/moe/nvfp4.py), so no global banks.
banks = {
"gate_up_packed": 2 * I * (H // 2),
"gate_up_scale": 2 * I * (H // 16),
@@ -109,7 +109,7 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
banks["down_global"] = H * 2
return banks
if fmt == "mxfp4_triton":
- # gpt_oss/weight.py _empty_mxfp4_triton_banks: transposed split-K blocks/scales + bf16 bias
+ # gpt-oss mxfp4 expert banks: transposed split-K blocks/scales + bf16 bias
return {
"gate_up_blocks": (H // 2) * (2 * I),
"gate_up_scales": (H // 32) * (2 * I),
@@ -285,7 +285,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
kv_groups=(),
top_k=8,
moe_intermediate_size=2048,
- expert_formats=_NVFP4_FORMATS,
+ expert_formats=(*_NVFP4_FORMATS, "fp8_block"),
aliases=("zai-org/GLM-5.3-Flash", "LibertAIDAI/GLM-5.3-Flash-NVFP4"),
),
AotModel(
diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.py
index fc42c49f..2a886cf7 100644
--- a/python/freetoken/kernel/backend.py
+++ b/python/freetoken/kernel/backend.py
@@ -10,6 +10,8 @@ from __future__ import annotations
import functools
import importlib.util
+import torch
+
def _importable(name: str) -> bool:
# find_spec normally returns None when a package is absent, but it can raise
@@ -31,6 +33,20 @@ def is_sgl_kernel_installed() -> bool:
return _importable("sgl_kernel")
+def is_vllm_installed() -> bool:
+ return _importable("vllm")
+
+
+def device_capability() -> tuple[int, int]:
+ """Compute capability of the current device as (major, minor); (0, 0) without CUDA."""
+ if not torch.cuda.is_available():
+ return (0, 0)
+ major, minor = torch.cuda.get_device_capability()
+ return (int(major), int(minor))
+
+
@functools.cache
def driver_cuda_version() -> int | None:
"""Max CUDA version the installed NVIDIA driver supports (``13000`` == CUDA 13.0),
diff --git a/python/freetoken/kernel/triton/fp8_block_linear.py b/python/freetoken/kernel/triton/fp8_block_linear.py
index 0999c56d..b79aa5b2 100644
--- a/python/freetoken/kernel/triton/fp8_block_linear.py
+++ b/python/freetoken/kernel/triton/fp8_block_linear.py
@@ -16,7 +16,6 @@ from __future__ import annotations
import torch
import triton
import triton.language as tl
-from freetoken.layers import BaseOP
from freetoken.kernel.triton.e4m3_compat import (
e4m3_act_dtype,
@@ -271,46 +270,8 @@ def dequant_block_fp8(weight: torch.Tensor, scale: torch.Tensor, block: int = _B
return (weight.to(torch.float32) * s).to(torch.bfloat16)
-# ======================================================================================
-# BaseOP linear layers (TP=1, replicated).
-# ======================================================================================
-class Fp8BlockLinear(BaseOP):
- """Replicated block-fp8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + bf16
- ``weight_scale_inv`` ``[out//128, in//128]``. Activations are quantized dynamically;
- the GEMM dequantizes per block, so activations keep bf16 precision."""
-
- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):
- assert in_features % _BLOCK == 0 and out_features % _BLOCK == 0, (in_features, out_features)
- self.in_features = in_features
- self.out_features = out_features
- self.weight = torch.empty(out_features, in_features, dtype=FP8)
- self.weight_scale_inv = torch.empty(
- out_features // _BLOCK, in_features // _BLOCK, dtype=torch.bfloat16
- )
- self.bias = torch.empty(out_features) if has_bias else None
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- return block_fp8_linear(x, self.weight, self.weight_scale_inv, self.bias)
-
-
-class Fp8BlockColMerged(Fp8BlockLinear):
- """Block-fp8 column-merged linear (drop-in for ``LinearColParallelMerged`` at TP=1).
-
- Holds one fp8 weight that is the concatenation of several projections along the output
- dim; the caller splits the bf16 output by ``output_sizes`` exactly as before. Each part's
- out dim must be a multiple of 128 so the concatenated ``weight_scale_inv`` blocks align."""
-
- def __init__(self, in_features: int, output_sizes: list[int], has_bias: bool = False):
- for o in output_sizes:
- assert o % _BLOCK == 0, (output_sizes, "each merged output size must be /128 for fp8")
- self.output_sizes = list(output_sizes)
- super().__init__(in_features, sum(output_sizes), has_bias)
-
-
__all__ = [
"FP8",
- "Fp8BlockLinear",
- "Fp8BlockColMerged",
"block_fp8_linear",
"block_fp8_matmul",
"per_token_group_quant_fp8",
diff --git a/python/freetoken/kernel/triton/fp8_blockscale_moe.py b/python/freetoken/kernel/triton/fp8_blockscale_moe.py
index 8ba5626b..74760516 100644
--- a/python/freetoken/kernel/triton/fp8_blockscale_moe.py
+++ b/python/freetoken/kernel/triton/fp8_blockscale_moe.py
@@ -98,14 +98,13 @@ def _decode_gemm(a, w, s, c, topk_weights, topk_ids, mul_routed_weight, a_row_is
def fused_experts_decode_fp8_blockscale(
hidden_states, gate_up, gate_up_scale, down, down_scale,
- topk_weights, topk_ids, activation="silu",
+ topk_weights, topk_ids, activation="silu", act_alpha=1.0, act_limit=float("inf"),
) -> torch.Tensor:
"""Decode (bs-1) inline-dequant block-fp8 MoE. ``topk_ids`` index rows of the expert
banks (resident: expert id; offload: cache slot)."""
from freetoken.kernel import moe_sum_reduce_triton
- from freetoken.layers import silu_and_mul
+ from freetoken.layers import gated_act_and_mul
- assert activation == "silu"
M, H = hidden_states.shape
top_k = topk_ids.shape[1]
two_i = gate_up.shape[1]
@@ -115,7 +114,7 @@ def fused_experts_decode_fp8_blockscale(
ic1 = torch.empty((M, top_k, two_i), device=dev, dtype=dt)
_decode_gemm(hidden_states, gate_up, gate_up_scale, ic1, topk_weights, topk_ids, False, False)
ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)
- silu_and_mul(ic1.view(-1, two_i), ic2)
+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)
ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)
_decode_gemm(ic2, down, down_scale, ic3, topk_weights, topk_ids, True, True)
out = torch.empty_like(hidden_states)
@@ -217,16 +216,15 @@ def _prefill_gemm(a_fp8, a_scale, w, s, c, tw, sorted_ids, expert_ids, ntpp, num
def fused_experts_fp8_blockscale(
hidden_states, gate_up, gate_up_scale, down, down_scale,
- topk_weights, topk_ids, num_experts, activation="silu",
+ topk_weights, topk_ids, num_experts, activation="silu", act_alpha=1.0, act_limit=float("inf"),
) -> torch.Tensor:
"""Prefill inline-dequant block-fp8 MoE. ``topk_ids`` index expert rows in [0, num_experts)
(materialized layer: position == expert id)."""
from freetoken.kernel import moe_sum_reduce_triton
- from freetoken.layers import silu_and_mul
+ from freetoken.layers import gated_act_and_mul
from freetoken.kernel.triton.fp8_block_linear import per_token_group_quant_fp8
from freetoken.moe.fused import moe_align_block_size
- assert activation == "silu"
M, H = hidden_states.shape
top_k = topk_ids.shape[1]
two_i = gate_up.shape[1]
@@ -246,7 +244,7 @@ def fused_experts_fp8_blockscale(
_prefill_gemm(a1_fp8, a1_scale, gate_up, gate_up_scale, ic1, tw, sorted_ids, expert_ids, ntpp,
num_valid, top_k, False, cfg)
ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)
- silu_and_mul(ic1.view(-1, two_i), ic2)
+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)
a2_fp8, a2_scale = per_token_group_quant_fp8(ic2, 128)
ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)
_prefill_gemm(a2_fp8, a2_scale, down, down_scale, ic3, tw, sorted_ids, expert_ids, ntpp,
diff --git a/python/freetoken/kernel/triton/fp8_pertensor_linear.py b/python/freetoken/kernel/triton/fp8_pertensor_linear.py
index d1d4de5b..5561a86a 100644
--- a/python/freetoken/kernel/triton/fp8_pertensor_linear.py
+++ b/python/freetoken/kernel/triton/fp8_pertensor_linear.py
@@ -20,14 +20,11 @@ accurate than the previous ``weight.to(bf16) * scale`` materialization, which it
from __future__ import annotations
import functools
-import os
import re
import torch
import triton
import triton.language as tl
-from freetoken.layers import BaseOP
-from freetoken.layers.base import _concat_prefix
from freetoken.kernel.triton.e4m3_compat import (
e4m3_kernel_view,
@@ -39,10 +36,6 @@ from freetoken.kernel.triton.e4m3_compat import (
FP8 = torch.float8_e4m3fn
_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}
-# Escape hatch: FREETOKEN_DEBUG_FP8_REF=1 swaps the triton kernels for a pure-torch dequant matmul
-# (numeric reference / A-B debugging). Evaluated once; the kernels are the default.
-_USE_REF = os.environ.get("FREETOKEN_DEBUG_FP8_REF") == "1"
-
# Row-wise _scaled_mm on sm_89 with torch < 2.12 launches its CUTLASS stream-K kernel off the
# current stream (pytorch/pytorch#177651, fixed by pytorch/pytorch@252bb4a; #182/#72/#220), and
@@ -55,11 +48,7 @@ def _torch_version() -> tuple[int, int]:
@functools.cache
def rowwise_scaled_mm_ok() -> bool:
"""Whether row-wise ``torch._scaled_mm`` may be issued from a side stream on this GPU.
- Decided once per process, at load (never under graph capture). ``FREETOKEN_FP8_ROWWISE_MM=0/1``
- forces the answer."""
- forced = os.environ.get("FREETOKEN_FP8_ROWWISE_MM")
- if forced in ("0", "1"):
- return forced == "1"
+ Decided once per process, at load (never under graph capture)."""
if not torch.cuda.is_available():
return True
from freetoken.gpu_select import assigned_visible_gpu
@@ -347,10 +336,7 @@ def fp8_pertensor_linear(
segments = scale_segments if scale_segments is not None else weight_scale_segments(weight_scale)
if not _segments_w8a8_ok(segments):
w8a8 = False # W8A16 below is exact for any per-row scale and never calls _scaled_mm
- if _USE_REF: # numeric-reference fallback (debug / A-B)
- w = weight.to(x.dtype) * weight_scale.to(x.dtype)[:, None]
- out = (x.reshape(-1, K) @ w.t()).reshape(*lead, N)
- elif w8a8:
+ if w8a8:
out = _scaled_mm(
x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype,
scale_segments=segments,
@@ -367,67 +353,4 @@ def fp8_pertensor_linear(
# ======================================================================================
-# BaseOP linear layers (TP=1, replicated). Buffers: fp8 ``weight`` + fp32 ``weight_scale``.
# ======================================================================================
-class Fp8PerTensorLinear(BaseOP):
- """Replicated per-tensor-FP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + per-row fp32
- ``weight_scale`` ``[out]`` (a genuine per-tensor weight stores the same scalar in every
- row; a fused projection stores each part's scalar across its own rows).
-
- ``input_scale`` (modelopt's calibrated per-tensor activation scale) is optional: when the
- checkpoint ships it, batched decode runs W8A8 through cuBLASLt; when it does not (models
- that quantize to fp8 at load, e.g. glm_moe_dsa), every batch size stays W8A16."""
-
- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):
- self.in_features = in_features
- self.out_features = out_features
- self.weight = torch.empty(out_features, in_features, dtype=FP8)
- self.weight_scale = torch.empty(out_features, dtype=torch.float32)
- self.bias = torch.empty(out_features) if has_bias else None
- # Set from the state dict when present. Left as None (not a tensor) so BaseOP's
- # reflective state_dict/load_state_dict skip it entirely on checkpoints without one.
- self.input_scale: torch.Tensor | None = None
- self._uniform_scale = False
- self._scale_segments: list[tuple[int, int]] | None = None
-
- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:
- # Taken out before BaseOP's reflective pass (so it is not an "unexpected key") and
- # put back after (so that pass does not try to pop it a second time on a reload).
- input_scale = state_dict.pop(_concat_prefix(prefix, "input_scale"), None)
- self.input_scale = None
- super().load_state_dict(state_dict, prefix=prefix, _internal=_internal)
- self.input_scale = input_scale
- # Tensor-wise scaling needs one scalar for the whole weight; a fused projection is
- # only piecewise-constant, so decide once here rather than syncing on every forward.
- scale = self.weight_scale
- self._uniform_scale = bool((scale == scale[0]).all().item())
- # Segments for the per-part path; decide row-wise safety now, not under graph capture.
- self._scale_segments = None if self._uniform_scale else weight_scale_segments(scale)
- if self.input_scale is not None and not self._uniform_scale:
- rowwise_scaled_mm_ok()
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- return fp8_pertensor_linear(
- x, self.weight, self.weight_scale, self.bias,
- self.input_scale, self._uniform_scale, scale_segments=self._scale_segments,
- )
-
-
-class Fp8PerTensorColMerged(Fp8PerTensorLinear):
- """Column-merged per-tensor-FP8 linear (drop-in for ``LinearColParallelMerged`` at TP=1):
- one fp8 weight concatenating several projections along the output dim; the caller splits
- the bf16 output by ``output_sizes`` as before."""
-
- def __init__(self, in_features: int, output_sizes: list[int], has_bias: bool = False):
- self.output_sizes = list(output_sizes)
- super().__init__(in_features, sum(output_sizes), has_bias)
-
-
-__all__ = [
- "FP8",
- "Fp8PerTensorLinear",
- "Fp8PerTensorColMerged",
- "fp8_pertensor_linear",
- "rowwise_scaled_mm_ok",
- "weight_scale_segments",
-]
diff --git a/python/freetoken/kernel/triton/mxfp8_linear.py b/python/freetoken/kernel/triton/mxfp8_linear.py
index 98a7001c..ebe78217 100644
--- a/python/freetoken/kernel/triton/mxfp8_linear.py
+++ b/python/freetoken/kernel/triton/mxfp8_linear.py
@@ -17,12 +17,10 @@ any other bf16 projection.
from __future__ import annotations
-import os
import torch
import triton
import triton.language as tl
-from freetoken.layers.base import BaseOP
from freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view, e4m3_native_cx, e4m3_u8_to_f32
@@ -30,17 +28,6 @@ FP8 = torch.float8_e4m3fn
MXFP8_BLOCK = 32
_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}
-# Escape hatch: FREETOKEN_DEBUG_MXFP8_REF=1 swaps the kernels for a pure-torch
-# dequant matmul (numeric reference / A-B debugging). Evaluated once, logged once.
-_USE_REF = os.environ.get("FREETOKEN_DEBUG_MXFP8_REF") == "1"
-if _USE_REF:
- from freetoken.utils import init_logger
-
- init_logger(__name__).info(
- "FREETOKEN_DEBUG_MXFP8_REF=1: MXFP8 linears serve the pure-torch fp32 "
- "dequant reference (slow; debugging only)."
- )
-
def mxfp8_dequant(weight: torch.Tensor, scale_codes: torch.Tensor,
dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
@@ -250,10 +237,7 @@ def mxfp8_linear(
N = weight.shape[0]
M = x.numel() // K
assert K % MXFP8_BLOCK == 0 and scale_codes.shape == (N, K // MXFP8_BLOCK)
- if _USE_REF: # numeric-reference fallback (debug / A-B)
- w = mxfp8_dequant(weight, scale_codes, dtype=torch.float32)
- out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, N)
- elif M <= _GEMV_MAX_M:
+ if M <= _GEMV_MAX_M:
w8 = e4m3_kernel_view(weight)
out = _gemv(x.reshape(M, K), w8, scale_codes, x.dtype).reshape(*lead, N)
else:
@@ -265,28 +249,4 @@ def mxfp8_linear(
return out
-# ======================================================================================
-# BaseOP linear layers (TP=1, replicated). Buffers: fp8 ``weight`` + uint8
-# ``weight_scale_inv`` (the checkpoint's e8m0 exponent codes, loaded verbatim).
-# ======================================================================================
-class Mxfp8Linear(BaseOP):
- """Replicated MXFP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + uint8 e8m0
- ``weight_scale_inv`` ``[out, in // 32]``. Fused projections concatenate several
- checkpoint projections along the output dim; the scales are per-output-row so the
- concatenation is exact."""
-
- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):
- assert in_features % MXFP8_BLOCK == 0
- self.in_features = in_features
- self.out_features = out_features
- self.weight = torch.empty(out_features, in_features, dtype=FP8)
- self.weight_scale_inv = torch.empty(
- out_features, in_features // MXFP8_BLOCK, dtype=torch.uint8
- )
- self.bias = torch.empty(out_features) if has_bias else None
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- return mxfp8_linear(x, self.weight, self.weight_scale_inv, self.bias)
-
-
-__all__ = ["FP8", "MXFP8_BLOCK", "Mxfp8Linear", "mxfp8_linear", "mxfp8_dequant"]
+__all__ = ["FP8", "MXFP8_BLOCK", "mxfp8_linear", "mxfp8_dequant"]
diff --git a/python/freetoken/kernel/triton/nvfp4_linear.py b/python/freetoken/kernel/triton/nvfp4_linear.py
index 9ed286f4..53abdedc 100644
--- a/python/freetoken/kernel/triton/nvfp4_linear.py
+++ b/python/freetoken/kernel/triton/nvfp4_linear.py
@@ -35,13 +35,10 @@ CUDA-graph safe on the decode paths: fixed shapes, no host sync.
from __future__ import annotations
-import os
import torch
import triton
import triton.language as tl
-from freetoken.layers import BaseOP
-from freetoken.layers.base import _concat_prefix
from freetoken.kernel.triton.e4m3_compat import (
e4m3_kernel_view,
@@ -53,10 +50,6 @@ from freetoken.kernel.triton.e4m3_compat import (
FP8 = torch.float8_e4m3fn
_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}
-# Escape hatch: FREETOKEN_DEBUG_DENSE_NVFP4_REF=1 swaps the triton kernels for a dequant_nvfp4 +
-# torch matmul reference (numeric A-B debugging). Evaluated once; the kernels are the default.
-_USE_REF = os.environ.get("FREETOKEN_DEBUG_DENSE_NVFP4_REF") == "1"
-
# Decode GEMV tiles (H100-tuned; the fp16-trick dequant is ALU-cheap so wider K tiles win).
# Two sets: the transposed K-major resident layout (coalesced along N; narrower K tiles win)
# vs the legacy row-major layout (coalesced along K).
@@ -764,19 +757,6 @@ def _gemm(a: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tensor,
return _gemm_scratch(a, packed_i32, scale, gscale, out_dtype, transposed)
-def _ref(a: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor,
- gscale: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor:
- """Numeric reference: dequant the FP4 weight to bf16 (well-tested ``dequant_nvfp4``)
- then a plain matmul. Used by FREETOKEN_DEBUG_DENSE_NVFP4_REF=1 for A-B validation."""
- from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4
-
- slots = torch.zeros(1, dtype=torch.int32, device=a.device)
- w = dequant_nvfp4(
- packed.unsqueeze(0), scale.unsqueeze(0), gscale.unsqueeze(0), slots, dtype=torch.bfloat16
- )[0] # [N, K] bf16
- return (a.reshape(-1, a.shape[-1]) @ w.t()).to(out_dtype)
-
-
def _linear_impl(
x: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tensor,
gscale: torch.Tensor, bias: torch.Tensor | None, out_dtype: torch.dtype,
@@ -802,10 +782,6 @@ def nvfp4_dense_linear(
) -> torch.Tensor:
"""``y = x @ dequant(weight)^T`` on the checkpoint-native row-major layout.
``weight`` [N, K//2] uint8, ``weight_scale`` [N, K//16] fp8-e4m3, ``weight_global`` [N] fp16."""
- if _USE_REF:
- *lead, K = x.shape
- out = _ref(x, weight, weight_scale, weight_global, x.dtype).reshape(*lead, weight.shape[0])
- return out + bias.to(out.dtype) if bias is not None else out
return _linear_impl(
x, weight.view(torch.int32), weight_scale, weight_global, bias, x.dtype, transposed=False
)
@@ -829,113 +805,14 @@ def nvfp4_dense_linear_t(
) -> torch.Tensor:
"""``y = x @ dequant(W)^T`` on the K-major resident layout from
:func:`nvfp4_transpose_resident` (``weight_t`` [K//8, N] int32, ``scale_t`` [K//16, N])."""
- if _USE_REF:
- out = _gemm_scratch(
- x.reshape(-1, x.shape[-1]).contiguous(), weight_t.t(), scale_t.t(),
- weight_global, x.dtype, transposed=True,
- ).reshape(*x.shape[:-1], weight_t.shape[1])
- return out + bias.to(out.dtype) if bias is not None else out
return _linear_impl(
x, weight_t.t(), scale_t.t(), weight_global, bias, x.dtype, transposed=True
)
-# ======================================================================================
-# BaseOP linear layers (TP=1, replicated). Buffers: uint8 packed ``weight`` + fp8 block
-# ``weight_scale`` + fp16 per-row ``weight_global``.
-# ======================================================================================
-class Nvfp4DenseLinear(BaseOP):
- """Replicated NVFP4 dense linear (W4A16). Drop-in for ``LinearReplicated`` /
- ``LinearRowParallel`` at TP=1 on the mixed-precision checkpoint's NVFP4 dense weights.
-
- Buffers are declared (and loaded) in the checkpoint's row-major layout; at load the
- packed weight + block scales are repacked to K-major (:func:`nvfp4_transpose_resident`)
- so the decode kernels' weight loads coalesce along N (~2x batched-decode throughput)."""
-
- def __init__(self, in_features: int, out_features: int, has_bias: bool = False):
- assert in_features % 16 == 0, f"NVFP4 in_features must be %16, got {in_features}"
- self.in_features = in_features
- self.out_features = out_features
- self.weight = torch.empty(out_features, in_features // 2, dtype=torch.uint8)
- self.weight_scale = torch.empty(out_features, in_features // 16, dtype=FP8)
- self.weight_global = torch.empty(out_features, dtype=torch.float16)
- self.bias = torch.empty(out_features) if has_bias else None
- self._transposed = False
-
- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:
- w = state_dict.pop(_concat_prefix(prefix, "weight"))
- s = state_dict.pop(_concat_prefix(prefix, "weight_scale"))
- assert w.shape == self.weight.shape and w.dtype == torch.uint8
- assert s.shape == self.weight_scale.shape
- self.weight, self.weight_scale = nvfp4_transpose_resident(w, s)
- self.weight_global = state_dict.pop(_concat_prefix(prefix, "weight_global"))
- if self.bias is not None:
- self.bias = state_dict.pop(_concat_prefix(prefix, "bias"))
- self._transposed = True
- if not _internal and state_dict:
- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- if self._transposed:
- return nvfp4_dense_linear_t(
- x, self.weight, self.weight_scale, self.weight_global, self.bias
- )
- return nvfp4_dense_linear(x, self.weight, self.weight_scale, self.weight_global, self.bias)
-
-
-class Nvfp4DenseColMerged(Nvfp4DenseLinear):
- """Column-merged NVFP4 dense linear (drop-in for ``LinearColParallelMerged`` at TP=1):
- one packed weight concatenating several projections on the output dim; each part keeps its
- own per-row ``weight_global`` (and block scales), so the fused weight is exact. The caller
- splits the output by ``output_sizes`` (e.g. shared-expert gate|up) as before."""
-
- def __init__(self, in_features: int, output_sizes: list[int], has_bias: bool = False):
- self.output_sizes = list(output_sizes)
- super().__init__(in_features, sum(output_sizes), has_bias)
-
-
-class Nvfp4LMHead(BaseOP):
- """NVFP4 (W4A16) LM head for the mixed checkpoint (TP=1, untied). Mirrors
- ``ParallelLMHead.forward`` at TP=1: slice to the last token per sequence at prefill, then
- the W4A16 GEMV/GEMM instead of a bf16 ``F.linear`` over the (here ~1 GB) bf16 weight."""
-
- def __init__(self, num_embeddings: int, embedding_dim: int):
- assert embedding_dim % 16 == 0
- self.num_embeddings = num_embeddings
- self.embedding_dim = embedding_dim
- self.weight = torch.empty(num_embeddings, embedding_dim // 2, dtype=torch.uint8)
- self.weight_scale = torch.empty(num_embeddings, embedding_dim // 16, dtype=FP8)
- self.weight_global = torch.empty(num_embeddings, dtype=torch.float16)
- self._transposed = False
-
- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:
- w = state_dict.pop(_concat_prefix(prefix, "weight"))
- s = state_dict.pop(_concat_prefix(prefix, "weight_scale"))
- assert w.shape == self.weight.shape and w.dtype == torch.uint8
- self.weight, self.weight_scale = nvfp4_transpose_resident(w, s)
- self.weight_global = state_dict.pop(_concat_prefix(prefix, "weight_global"))
- self._transposed = True
- if not _internal and state_dict:
- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- from freetoken.core import get_global_ctx
-
- batch = get_global_ctx().batch
- if batch.is_prefill:
- indices = batch.attn_metadata.get_last_indices(batch.size)
- x = x[indices].contiguous()
- if self._transposed:
- return nvfp4_dense_linear_t(x, self.weight, self.weight_scale, self.weight_global)
- return nvfp4_dense_linear(x, self.weight, self.weight_scale, self.weight_global)
-
-
__all__ = [
"FP8",
"nvfp4_dense_linear",
"nvfp4_dense_linear_t",
"nvfp4_transpose_resident",
- "Nvfp4DenseLinear",
- "Nvfp4DenseColMerged",
- "Nvfp4LMHead",
]
diff --git a/python/freetoken/layers/__init__.py b/python/freetoken/layers/__init__.py
index 31c19bf9..579dc6e4 100644
--- a/python/freetoken/layers/__init__.py
+++ b/python/freetoken/layers/__init__.py
@@ -1,4 +1,6 @@
from .activation import (
+ GATED_ACTIVATIONS,
+ gated_act_and_mul,
gelu_and_mul,
gelu_tanh_and_mul,
silu_and_mul,
@@ -16,6 +18,7 @@ from .linear import (
)
from .moe import MoELayer, OffloadMoELayer, make_moe_layer
from .norm import (
+ GatedRMSNorm,
GemmaPlusOneRMSNorm,
GemmaPlusOneRMSNormFused,
GemmaRMSNorm,
@@ -30,6 +33,8 @@ __all__ = [
"gelu_tanh_and_mul",
"swigluoai_and_mul",
"swiglu_clamp_and_mul",
+ "gated_act_and_mul",
+ "GATED_ACTIVATIONS",
"BaseOP",
"StateLessOP",
"OPList",
@@ -41,6 +46,7 @@ __all__ = [
"LinearQKVMerged",
"RMSNorm",
"RMSNormFused",
+ "GatedRMSNorm",
"GemmaRMSNorm",
"GemmaPlusOneRMSNorm",
"GemmaPlusOneRMSNormFused",
diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.py
index ae1d9881..cd7a402a 100644
--- a/python/freetoken/layers/activation.py
+++ b/python/freetoken/layers/activation.py
@@ -65,10 +65,30 @@ def swiglu_clamp_and_mul(
return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit)
+GATED_ACTIVATIONS = ("silu", "gelu", "gelu_tanh", "swigluoai", "swiglu_clamp")
+
+
+def gated_act_and_mul(activation: str, x, out, *, alpha: float = 1.0, limit: float = float("inf")):
+ """The epilogue between the two expert GEMMs: ``activation`` over uninterleaved [gate; up] halves into ``out``."""
+ if activation == "silu":
+ return silu_and_mul(x, out)
+ if activation == "gelu":
+ return gelu_and_mul(x, out)
+ if activation == "gelu_tanh":
+ return gelu_tanh_and_mul(x, out)
+ if activation == "swigluoai":
+ return swigluoai_and_mul(x, out, alpha=alpha, limit=limit)
+ if activation == "swiglu_clamp":
+ return swiglu_clamp_and_mul(x, out, alpha=alpha, limit=limit)
+ raise ValueError(f"no gated activation {activation!r}; known: {GATED_ACTIVATIONS}")
+
+
__all__ = [
"silu_and_mul",
"gelu_and_mul",
"gelu_tanh_and_mul",
"swigluoai_and_mul",
"swiglu_clamp_and_mul",
+ "gated_act_and_mul",
+ "GATED_ACTIVATIONS",
]
diff --git a/python/freetoken/layers/embedding.py b/python/freetoken/layers/embedding.py
index 76cd759b..a2221654 100644
--- a/python/freetoken/layers/embedding.py
+++ b/python/freetoken/layers/embedding.py
@@ -9,6 +9,7 @@ from freetoken.distributed import DistributedCommunicator, get_tp_info
from freetoken.utils import div_ceil, nvtx_annotate
from .base import BaseOP
+from .quantization import LayerKind, QuantConfig, quant_method_for
class VocabParallelEmbedding(BaseOP):
@@ -58,6 +59,11 @@ class VocabParallelEmbedding(BaseOP):
class ParallelLMHead(VocabParallelEmbedding):
+ """The head is a linear layer over the vocab shard: its weights come from ``quant_method``
+ unless they are tied to the input embedding."""
+
+ quant_layer_kind = LayerKind.LINEAR
+
def __init__(
self,
num_embeddings: int,
@@ -65,11 +71,27 @@ class ParallelLMHead(VocabParallelEmbedding):
bias: bool = False,
tie_word_embeddings: bool = False,
tied_embedding: VocabParallelEmbedding | None = None,
+ *,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
super().__init__(num_embeddings, embedding_dim)
- self.bias = torch.empty(self.num_embeddings_tp) if bias else None
+ self.has_bias = bias
+ self.prefix = prefix
self.tied_embedding = tied_embedding
assert (tied_embedding is not None) == tie_word_embeddings
+ self.in_features = embedding_dim
+ self.out_features = self.num_embeddings_tp
+ self.output_sizes = (self.num_embeddings_tp,)
+ self.quant_method = None
+ if tied_embedding is None:
+ self.quant_method = quant_method_for(quant_config, self, prefix)
+ self.quant_method.create_weights(self)
+ self.bias = torch.empty(self.num_embeddings_tp) if bias else None
+
+ def finalize(self) -> None:
+ if self.quant_method is not None:
+ self.quant_method.finalize(self)
def load_state_dict(
self,
@@ -109,8 +131,10 @@ class ParallelLMHead(VocabParallelEmbedding):
x = x[indices].contiguous()
del indices
- module = self.tied_embedding or self
- logits = F.linear(x, module.weight, self.bias)
+ if self.tied_embedding is not None:
+ logits = F.linear(x, self.tied_embedding.weight, self.bias)
+ else:
+ logits = self.quant_method.apply(self, x)
if self.tp_size == 1:
return logits
input_shape = logits.shape
diff --git a/python/freetoken/layers/linear.py b/python/freetoken/layers/linear.py
index f707e042..d8a3e861 100644
--- a/python/freetoken/layers/linear.py
+++ b/python/freetoken/layers/linear.py
@@ -3,15 +3,20 @@ from __future__ import annotations
from typing import List
import torch
-import torch.nn.functional as F
from freetoken.distributed import DistributedCommunicator, get_tp_info
from freetoken.utils import div_even
from .base import BaseOP
+from .quantization import LayerKind, QuantConfig, quant_method_for
class _LinearTPImpl(BaseOP):
- """Real implementation of a linear layer with tensor parallelism."""
+ """Real implementation of a linear layer with tensor parallelism.
+
+ The weights are declared and applied by ``quant_method``, picked from ``quant_config`` by
+ the layer's ``prefix``; without a config the layer is plain bf16."""
+
+ quant_layer_kind = LayerKind.LINEAR
def __init__(
self,
@@ -20,16 +25,30 @@ class _LinearTPImpl(BaseOP):
local_isize: int,
local_osize: int,
has_bias: bool,
+ *,
+ output_sizes: List[int] | None = None,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
self.full_input_size = full_isize
self.full_output_size = full_osize
self.local_input_size = local_isize
self.local_output_size = local_osize
- self.weight = torch.empty(local_osize, local_isize)
+ self.has_bias = has_bias
+ self.prefix = prefix
+ # the TP-local shape the quant method declares weights for; fused projections list their segments
+ self.in_features = local_isize
+ self.out_features = local_osize
+ self.output_sizes = tuple(output_sizes or (local_osize,))
+ self.quant_method = quant_method_for(quant_config, self, prefix)
+ self.quant_method.create_weights(self)
self.bias = torch.empty(local_osize) if has_bias else None
+ def finalize(self) -> None:
+ self.quant_method.finalize(self)
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
- return F.linear(x, self.weight, self.bias)
+ return self.quant_method.apply(self, x)
class LinearReplicated(_LinearTPImpl):
@@ -43,6 +62,9 @@ class LinearReplicated(_LinearTPImpl):
input_size: int,
output_size: int,
has_bias: bool,
+ *,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
super().__init__(
full_isize=input_size,
@@ -50,6 +72,8 @@ class LinearReplicated(_LinearTPImpl):
local_isize=input_size,
local_osize=output_size,
has_bias=has_bias,
+ quant_config=quant_config,
+ prefix=prefix,
)
@@ -59,13 +83,19 @@ class LinearColParallelMerged(_LinearTPImpl):
input_size: int,
output_sizes: List[int],
has_bias: bool,
+ *,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
# check that all output sizes are divisible by tp_size
tp_info = get_tp_info()
tp_output_sizes = [div_even(size, tp_info.size) for size in output_sizes]
output_size = sum(output_sizes)
tp_output_size = sum(tp_output_sizes)
- super().__init__(input_size, output_size, input_size, tp_output_size, has_bias)
+ super().__init__(
+ input_size, output_size, input_size, tp_output_size, has_bias,
+ output_sizes=tp_output_sizes, quant_config=quant_config, prefix=prefix,
+ )
class LinearQKVMerged(_LinearTPImpl):
@@ -76,6 +106,9 @@ class LinearQKVMerged(_LinearTPImpl):
num_qo_heads: int,
num_kv_heads: int,
has_bias: bool,
+ *,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
tp_info = get_tp_info()
@@ -85,11 +118,23 @@ class LinearQKVMerged(_LinearTPImpl):
full_osize = (num_qo_heads + 2 * num_kv_heads) * head_dim
local_isize = hidden_size
local_osize = (local_num_qo + 2 * local_num_kv) * head_dim
- super().__init__(full_isize, full_osize, local_isize, local_osize, has_bias)
+ super().__init__(
+ full_isize, full_osize, local_isize, local_osize, has_bias,
+ output_sizes=[local_num_qo * head_dim, local_num_kv * head_dim, local_num_kv * head_dim],
+ quant_config=quant_config, prefix=prefix,
+ )
class LinearOProj(_LinearTPImpl):
- def __init__(self, input_size: int, output_size: int, has_bias: bool):
+ def __init__(
+ self,
+ input_size: int,
+ output_size: int,
+ has_bias: bool,
+ *,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
+ ):
tp_info = get_tp_info()
full_isize = input_size
full_osize = output_size
@@ -97,10 +142,13 @@ class LinearOProj(_LinearTPImpl):
local_osize = output_size
self._comm = DistributedCommunicator()
self._tp_size = tp_info.size
- super().__init__(full_isize, full_osize, local_isize, local_osize, has_bias)
+ super().__init__(
+ full_isize, full_osize, local_isize, local_osize, has_bias,
+ quant_config=quant_config, prefix=prefix,
+ )
def forward(self, x: torch.Tensor) -> torch.Tensor:
- y = F.linear(x, self.weight, self.bias)
+ y = self.quant_method.apply(self, x)
if self._tp_size > 1:
y = self._comm.all_reduce(y)
return y
@@ -112,16 +160,22 @@ class LinearRowParallel(_LinearTPImpl):
input_size: int,
output_size: int,
has_bias: bool,
+ *,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
tp_info = get_tp_info()
local_input_size = div_even(input_size, tp_info.size)
local_output_size = output_size
self._comm = DistributedCommunicator()
self._tp_size = tp_info.size
- super().__init__(input_size, output_size, local_input_size, local_output_size, has_bias)
+ super().__init__(
+ input_size, output_size, local_input_size, local_output_size, has_bias,
+ quant_config=quant_config, prefix=prefix,
+ )
def forward(self, x: torch.Tensor) -> torch.Tensor:
- y = F.linear(x, self.weight, self.bias)
+ y = self.quant_method.apply(self, x)
if self._tp_size > 1:
y = self._comm.all_reduce(y)
return y
diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.py
index d68d8ded..42ad822b 100644
--- a/python/freetoken/layers/moe.py
+++ b/python/freetoken/layers/moe.py
@@ -4,20 +4,20 @@ from typing import TYPE_CHECKING, Tuple
import torch
from freetoken.core import get_global_ctx
from freetoken.distributed import DistributedCommunicator, get_tp_info
-from freetoken.moe import is_offload_moe_backend
-from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl, fused_topk
+from freetoken.moe import is_offload_moe_strategy
+from freetoken.moe.fused import fused_topk
from freetoken.moe.offload_cache import OffloadMoeCache
-from freetoken.utils import div_even
+
from .base import BaseOP
+from .quantization import ExpertView, LayerKind, QuantConfig, quant_method_for
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
# Router decision (topk_weights[float32], topk_ids[int32]) for models whose router
-# is computed outside the MoE layer. Such models call ``routed_forward`` (offload) or
-# ``_run_experts`` (dense) with a precomputed routing instead of going through the
-# generic softmax+top-k path.
+# is computed outside the MoE layer. Such models call ``routed_forward`` with a
+# precomputed routing instead of going through the generic softmax+top-k path.
TopK = Tuple[torch.Tensor, torch.Tensor]
# Hybrid decode overlaps the CPU overflow GEMV behind the GPU PCIe fetch + GEMM by
@@ -27,6 +27,15 @@ _HYBRID_OVERLAP = os.getenv("FREETOKEN_HYBRID_OVERLAP", "1") != "0"
class MoELayer(BaseOP):
+ """Resident routed experts.
+
+ The expert format comes from ``quant_method`` (declared by ``create_weights``, run by
+ ``apply``); without a ``quant_config`` the experts are plain bf16. The gated activation is
+ ``act(clamp(g, limit) * alpha) * (clamp(u) + beta)`` with ``interleaved`` gate|up rows
+ for gpt-oss."""
+
+ quant_layer_kind = LayerKind.MOE
+
def __init__(
self,
num_experts: int,
@@ -37,7 +46,17 @@ class MoELayer(BaseOP):
activation: str = "silu",
apply_router_weight_on_input: bool = False,
allocate_experts: bool = True,
- weight_format: str = "bf16",
+ *,
+ alpha: float = 1.0,
+ beta: float = 0.0,
+ limit: float | None = None,
+ interleaved: bool = False,
+ has_bias: bool = False,
+ layer_id: int | None = None,
+ strategy: str = "resident",
+ decode_target: str = "gpu",
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
super().__init__()
@@ -48,101 +67,47 @@ class MoELayer(BaseOP):
self._comm = DistributedCommunicator()
tp_info = get_tp_info()
+ self.tp_rank = tp_info.rank
self.tp_size = tp_size = tp_info.size
self.renormalize = renormalize
self.activation = activation
self.apply_router_weight_on_input = apply_router_weight_on_input
- self.weight_format = weight_format
- intermediate_size_per_partition = div_even(intermediate_size, tp_size)
- if allocate_experts:
- self._alloc_resident_experts(intermediate_size_per_partition)
-
- def _alloc_resident_experts(self, intermediate_size_per_partition: int) -> None:
- """Allocate the resident (in-GPU) expert weights for ``self.weight_format``.
-
- The resident sibling of the offload bank schemas: each format owns its
- tensor layout here and its kernel branch in ``_resident_gemm``.
- """
- if self.weight_format == "fp8_block":
- # Stacked block-fp8 experts + bf16 per-128x128-block inverse scales.
- # Full (unpartitioned) intermediate size: this layout is TP=1-only.
- from freetoken.kernel.triton.fp8_block_linear import FP8
-
- blk = 128
- n, i, h = self.num_experts, self.intermediate_size, self.hidden_size
- self.gate_up_proj = torch.empty(n, 2 * i, h, dtype=FP8)
- self.gate_up_scale_inv = torch.empty(
- n, 2 * i // blk, h // blk, dtype=torch.bfloat16
- )
- self.down_proj = torch.empty(n, h, i, dtype=FP8)
- self.down_scale_inv = torch.empty(n, h // blk, i // blk, dtype=torch.bfloat16)
- return
- assert self.weight_format == "bf16", (
- f"no resident expert allocation for weight_format {self.weight_format!r}"
- )
- self.gate_up_proj = torch.empty(
- self.num_experts,
- 2 * intermediate_size_per_partition,
- self.hidden_size,
- )
- self.down_proj = torch.empty(
- self.num_experts,
- self.hidden_size,
- intermediate_size_per_partition,
- )
+ self.alpha = alpha
+ self.beta = beta
+ self.limit = limit
+ self.interleaved = interleaved
+ self.has_bias = has_bias
+ self.layer_id = layer_id
+ self.strategy = strategy
+ self.decode_target = decode_target
+ self.prefix = prefix
+ # offload layers without a quant config stay on the format-tag banks (GGUF q4_0)
+ self.quant_method = None
+ if quant_config is not None or allocate_experts:
+ self.quant_method = quant_method_for(quant_config, self, prefix)
+ if allocate_experts:
+ self.quant_method.create_weights(self)
+
+ def finalize(self) -> None:
+ if self.quant_method is not None:
+ self.quant_method.finalize(self)
def _maybe_all_reduce(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self.tp_size > 1:
return self._comm.all_reduce(hidden_states)
return hidden_states
- def _run_experts(
- self,
- hidden_states: torch.Tensor,
- topk_weights: torch.Tensor,
- topk_ids: torch.Tensor,
- ) -> torch.Tensor:
- """Dense (in-GPU) expert compute for a precomputed routing decision."""
- return fused_experts_impl(
- hidden_states,
- self.gate_up_proj,
- self.down_proj,
- topk_weights,
- topk_ids,
- self.activation,
- apply_router_weight_on_input=self.apply_router_weight_on_input,
- )
-
def _resident_gemm(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
- """Kernel dispatch on ``self.weight_format`` -- the resident mirror of
- ``OffloadMoELayer._expert_gemm``'s ``cache.quant_format`` dispatch."""
- if self.weight_format == "fp8_block":
- # Prefill dequantizes the layer's experts to bf16 and runs the bf16
- # grouped GEMM; decode dequantizes only the routed rows.
- from freetoken.moe.fused_fp8_block import (
- fused_experts_decode_fp8_block,
- fused_experts_fp8_block,
- )
-
- if get_global_ctx().batch.is_prefill:
- return fused_experts_fp8_block(
- hidden_states, self.gate_up_proj, self.gate_up_scale_inv,
- self.down_proj, self.down_scale_inv,
- topk_weights, topk_ids, self.num_experts,
- )
- return fused_experts_decode_fp8_block(
- hidden_states, self.gate_up_proj, self.gate_up_scale_inv,
- self.down_proj, self.down_scale_inv, topk_weights, topk_ids,
- )
- assert self.weight_format == "bf16", (
- f"no resident expert kernel for weight_format {self.weight_format!r}"
+ assert self.quant_method is not None
+ return self.quant_method.apply(
+ hidden_states, topk_weights, topk_ids, self.quant_method.resident_view(self),
+ layer=self, is_prefill=get_global_ctx().batch.is_prefill,
)
- return self._run_experts(hidden_states, topk_weights, topk_ids)
def routed_forward(
self,
@@ -168,30 +133,13 @@ class MoELayer(BaseOP):
hidden_states: torch.Tensor,
router_logits: torch.Tensor | None = None,
):
- if self.weight_format != "bf16":
- # Quantized resident experts: generic softmax router + format kernel.
- # The bf16 path below stays on ctx.moe_backend byte-for-byte.
- topk_weights, topk_ids = fused_topk(
- hidden_states=hidden_states,
- gating_output=router_logits,
- topk=self.top_k,
- renormalize=self.renormalize,
- )
- return self._maybe_all_reduce(
- self._resident_gemm(hidden_states, topk_weights, topk_ids)
- )
- ctx = get_global_ctx()
- final_hidden_states = ctx.moe_backend.forward(
+ topk_weights, topk_ids = fused_topk(
hidden_states=hidden_states,
- w1=self.gate_up_proj,
- w2=self.down_proj,
gating_output=router_logits,
topk=self.top_k,
renormalize=self.renormalize,
- activation=self.activation,
- apply_router_weight_on_input=self.apply_router_weight_on_input,
)
- return self._maybe_all_reduce(final_hidden_states)
+ return self._maybe_all_reduce(self._resident_gemm(hidden_states, topk_weights, topk_ids))
class OffloadMoELayer(MoELayer):
@@ -205,6 +153,16 @@ class OffloadMoELayer(MoELayer):
renormalize: bool = True,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
+ *,
+ alpha: float = 1.0,
+ beta: float = 0.0,
+ limit: float | None = None,
+ interleaved: bool = False,
+ has_bias: bool = False,
+ strategy: str = "offload",
+ decode_target: str = "gpu",
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
super().__init__(
num_experts=num_experts,
@@ -215,8 +173,17 @@ class OffloadMoELayer(MoELayer):
activation=activation,
apply_router_weight_on_input=apply_router_weight_on_input,
allocate_experts=False,
+ alpha=alpha,
+ beta=beta,
+ limit=limit,
+ interleaved=interleaved,
+ has_bias=has_bias,
+ layer_id=layer_id,
+ strategy=strategy,
+ decode_target=decode_target,
+ quant_config=quant_config,
+ prefix=prefix,
)
- self.layer_id = layer_id
self.offload_cache: OffloadMoeCache | None = None
def forward(
@@ -423,9 +390,8 @@ class OffloadMoELayer(MoELayer):
return cache.wait_prefill_layer(self.layer_id)
# ------------------------------------------------------------------
- # Kernel dispatch -- pure routing on the cache's quant format. ``views``
- # are the bank tensors the movement step produced (in bank registration
- # order) and ``topk_ids`` already index their rows.
+ # Kernel dispatch: ``views`` are the bank tensors the movement step produced (in bank registration order) and ``topk_ids`` already index their rows.
+ # GGUF q4_0 experts still dispatch on the cache's format tag until they get a method.
# ------------------------------------------------------------------
def _expert_gemm(
@@ -440,88 +406,17 @@ class OffloadMoELayer(MoELayer):
alphas: tuple[torch.Tensor, torch.Tensor] | None,
is_prefill: bool,
) -> torch.Tensor:
- fmt = cache.quant_format
- if fmt in ("nvfp4_marlin", "nvfp4_b12x"):
- # Borrowed W4A16 fused MoE -- Marlin (vLLM, sm_80-99) or b12x
- # (flashinfer, sm_120) over their pre-tiled banks; one kernel serves
- # prefill and decode, with the movement-matched per-row global scales.
- from freetoken.moe.nvfp4_backends import b12x_fused_experts, marlin_fused_experts
-
- assert alphas is not None
- gate_up_packed, gate_up_scale, down_packed, down_scale = views
- fused = marlin_fused_experts if fmt == "nvfp4_marlin" else b12x_fused_experts
- return fused(
- hidden_states,
- gate_up_packed,
- gate_up_scale,
- alphas[0],
- down_packed,
- down_scale,
- alphas[1],
- topk_weights,
- topk_ids,
- self.activation,
- self.apply_router_weight_on_input,
- )
- if fmt == "nvfp4":
- # FreeToken's Triton inline-dequant kernels over the native ModelOpt
- # rows: the FP4 banks are read directly in the GEMM, no BF16 copy of
- # the experts is ever materialized. The swigluoai scalars (MiniMax-M3)
- # live on the layer via make_moe_layer's extra_attrs (gpt-oss precedent)
- # and are ignored by the plain *_and_mul activations.
- act_alpha = getattr(self, "hidden_act_alpha", 1.702)
- act_limit = getattr(self, "swiglu_limit", None)
- # None == "no clamp" everywhere else in the repo (mxfp4 maps it to +inf).
- act_limit = float("inf") if act_limit is None else act_limit
- if is_prefill:
- from freetoken.moe.fused_nvfp4 import fused_experts_nvfp4
-
- return fused_experts_nvfp4(
- hidden_states,
- *views,
- topk_weights,
- topk_ids,
- n,
- self.activation,
- self.apply_router_weight_on_input,
- act_alpha,
- act_limit,
- )
- # Marlin-style int32 wide-load GEMV (arithmetic dequant, no HW cvt).
- # Bit-identical to the byte-at-a-time path; lifts gate/up BW ~43%->51%
- # (I=512), ~41%->53% (I=768), 65%->72% (I=1536). CUDA-graph safe (fixed
- # shapes, no host sync).
- from freetoken.moe.fused_nvfp4 import fused_experts_decode_nvfp4_marlin
-
- return fused_experts_decode_nvfp4_marlin(
- hidden_states,
- *views,
- topk_weights,
- topk_ids,
- self.activation,
- self.apply_router_weight_on_input,
- act_alpha,
- act_limit,
- )
- if fmt == "fp8_block":
- # Block-fp8 experts: fused inline-dequant grouped GEMM reads the routed fp8 rows
- # directly (fp8 banks halve host/cache bytes; no bf16 materialization).
- from freetoken.moe.fused_fp8_block import (
- fused_experts_decode_fp8_block,
- fused_experts_fp8_block,
- )
+ if self.quant_method is not None:
+ from freetoken.moe.legacy_format import canonical_role # legacy_format imports this package
- gate_up, gate_up_scale, down, down_scale = views
- if is_prefill:
- return fused_experts_fp8_block(
- hidden_states, gate_up, gate_up_scale, down, down_scale,
- topk_weights, topk_ids, n, self.activation,
- self.apply_router_weight_on_input,
- )
- return fused_experts_decode_fp8_block(
- hidden_states, gate_up, gate_up_scale, down, down_scale,
- topk_weights, topk_ids, self.activation, self.apply_router_weight_on_input,
+ view = ExpertView(
+ {canonical_role(name): t for name, t in zip(cache.bank_schema, views)},
+ slots=None if n is not None else topk_ids, n=n, alphas=alphas,
+ )
+ return self.quant_method.apply(
+ hidden_states, topk_weights, topk_ids, view, layer=self, is_prefill=is_prefill
)
+ fmt = cache.quant_format
if fmt == "q4_0":
# Native GGUF Q4_0 experts: dequant-in-kernel grouped GEMV (MMVQ) over the
# streamed packed banks; topk_ids already index the cache slots / layer.
@@ -531,57 +426,7 @@ class OffloadMoELayer(MoELayer):
return fused_experts_gguf_q4_0(
hidden_states, gate_up, down, topk_weights, topk_ids, self.activation
)
- if fmt == "mxfp4_triton":
- # gpt-oss MXFP4 experts (biased, clamped swiglu): transposed split-K GEMV
- # decode + grouped `_t` prefill. The swiglu scalars live on the layer
- # (set at construction), not in the base signature.
- from freetoken.moe.fused_mxfp4 import (
- run_mxfp4_prefill_experts_t,
- run_mxfp4_splitk_decode_experts,
- )
-
- gu_blocks, gu_scales, gu_bias, dn_blocks, dn_scales, dn_bias = views
- run = run_mxfp4_prefill_experts_t if is_prefill else run_mxfp4_splitk_decode_experts
- return run(
- hidden_states, topk_weights, topk_ids,
- gu_blocks, gu_scales, gu_bias, dn_blocks, dn_scales, dn_bias,
- top_k=self.top_k,
- hidden_act_alpha=self.hidden_act_alpha,
- swiglu_limit=self.swiglu_limit,
- )
- if fmt == "ds_fp4":
- # DeepSeek-V4 FP4 experts: grouped inline-dequant GEMM for streaming
- # prefill chunks (n = bank rows to sort over); per-route dequant GEMV
- # for decode and the sparse small-chunk slot path (n is None there,
- # and sorting over the full slot cache would drown in padding).
- gate_up_packed, gate_up_scale, down_packed, down_scale = views
- if is_prefill and n is not None:
- from freetoken.moe.fused_ds_fp4 import routed_experts_fp4_prefill
-
- return routed_experts_fp4_prefill(
- hidden_states, topk_ids, topk_weights,
- gate_up_packed, gate_up_scale, down_packed, down_scale,
- self.swiglu_limit, n,
- )
- from freetoken.moe.fused_ds_fp4 import routed_experts_fp4
-
- return routed_experts_fp4(
- hidden_states, topk_ids, topk_weights,
- gate_up_packed, gate_up_scale, down_packed, down_scale,
- self.swiglu_limit,
- )
- assert fmt == "bf16", f"unknown quant_format {fmt!r}"
- gate_up, down = views
- impl = fused_experts_impl if is_prefill else fused_experts_decode_impl
- return impl(
- hidden_states,
- gate_up,
- down,
- topk_weights,
- topk_ids,
- self.activation,
- self.apply_router_weight_on_input,
- )
+ raise AssertionError(f"offload experts without a quant method only serve q4_0 banks, got {fmt!r}")
def make_moe_layer(
@@ -589,7 +434,6 @@ def make_moe_layer(
*,
layer_id: int | None = None,
activation: str = "silu",
- weight_format: str = "bf16",
renormalize: bool | None = None,
apply_router_weight_on_input: bool = False,
num_experts: int | None = None,
@@ -598,21 +442,23 @@ def make_moe_layer(
intermediate_size: int | None = None,
resident_cls: type[MoELayer] | None = None,
offload_cls: "type[OffloadMoELayer] | None" = None,
- extra_attrs: dict | None = None,
+ alpha: float = 1.0,
+ beta: float = 0.0,
+ limit: float | None = None,
+ interleaved: bool = False,
+ has_bias: bool = False,
+ quant_config: QuantConfig | None = None,
+ prefix: str = "",
) -> MoELayer:
- """Build the experts layer for ``config.moe_backend`` -- the one construction
+ """Build the experts layer for ``config.moe_strategy`` -- the one construction
seam between a model and the MoE strategy.
- Picks ``OffloadMoELayer`` for the offload family (offload/cpu/hybrid, which
- ignore ``weight_format``: the quant format comes from the offload cache) and
+ Picks ``OffloadMoELayer`` for the offload family (offload/cpu/hybrid) and
``MoELayer`` otherwise. Geometry defaults come from ``config``; pass overrides
- for models whose fields deviate. ``extra_attrs`` become instance attributes --
- the seam for per-format scalars the base signature does not carry (e.g.
- ``hidden_act_alpha``/``swiglu_limit``, read back via ``getattr`` by the engine
- and format kernels). ``resident_cls``/``offload_cls`` keep model-specific
+ for models whose fields deviate. ``resident_cls``/``offload_cls`` keep model-specific
subclasses constructible through the same seam.
"""
- offload = is_offload_moe_backend(config.moe_backend)
+ offload = is_offload_moe_strategy(config.moe_strategy)
layer_cls = (offload_cls or OffloadMoELayer) if offload else (resident_cls or MoELayer)
kwargs = dict(
num_experts=num_experts if num_experts is not None else config.num_experts,
@@ -624,13 +470,17 @@ def make_moe_layer(
renormalize=renormalize if renormalize is not None else config.norm_topk_prob,
activation=activation,
apply_router_weight_on_input=apply_router_weight_on_input,
+ alpha=alpha,
+ beta=beta,
+ limit=limit,
+ interleaved=interleaved,
+ has_bias=has_bias,
+ quant_config=quant_config,
+ prefix=prefix,
)
if offload:
assert layer_id is not None, "offload MoE backends need the layer_id"
kwargs["layer_id"] = layer_id
- else:
- kwargs["weight_format"] = weight_format
- layer = layer_cls(**kwargs)
- for name, value in (extra_attrs or {}).items():
- setattr(layer, name, value)
- return layer
+ kwargs["strategy"] = config.moe_strategy
+ kwargs["decode_target"] = config.decode_target
+ return layer_cls(**kwargs)
diff --git a/python/freetoken/layers/norm.py b/python/freetoken/layers/norm.py
index df248136..5eca789b 100644
--- a/python/freetoken/layers/norm.py
+++ b/python/freetoken/layers/norm.py
@@ -168,3 +168,25 @@ class RMSNormFused(BaseOP):
return self.rmsnorm(x, self.weight, self.eps), x
self.fused_add_rmsnorm(x, residual, self.weight, self.eps)
return x, residual
+
+
+_GATE_ACTIVATIONS = ("silu", "swish", "sigmoid")
+
+
+class GatedRMSNorm(BaseOP):
+ """``rmsnorm(x) * activation(z)`` in one fla kernel: the GDN / KDA output norm."""
+
+ def __init__(self, size: int, eps: float, activation: str = "silu") -> None:
+ # rms_norm_gated drops the gate entirely (no error) for a name it does not know
+ assert activation in _GATE_ACTIVATIONS, f"unsupported gate activation {activation!r}"
+ self.weight = torch.empty(size)
+ self.eps = eps
+ self.activation = activation
+
+ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.fla import rms_norm_gated
+
+ return rms_norm_gated(
+ x=x, weight=self.weight, bias=None, z=z, eps=self.eps,
+ is_rms_norm=True, norm_before_gate=True, activation=self.activation,
+ )
diff --git a/python/freetoken/layers/quantization/__init__.py b/python/freetoken/layers/quantization/__init__.py
new file mode 100644
index 00000000..99e3d71f
--- /dev/null
+++ b/python/freetoken/layers/quantization/__init__.py
@@ -0,0 +1,50 @@
+from .scheme import GroupShape, QuantKind, QuantScheme, WeightDesc
+from .registry import LayerKind, dialects, method_class, methods_for, register_dialect, register_method
+from .method import KernelSelectionError, QuantMethod, finalize_quant, select_kernel
+from .linear import (
+ Fp8BlockLinearMethod,
+ Fp8TensorLinearMethod,
+ LinearConfig,
+ LinearKernel,
+ LinearMethod,
+ Mxfp8LinearMethod,
+ Nvfp4LinearMethod,
+ UnquantizedLinearMethod,
+)
+from .moe import (
+ BankSpec,
+ ExpertView,
+ Fp8BlockMoEMethod,
+ MoEConfig,
+ MoEKernel,
+ MoEMethod,
+ Mxfp4MoEMethod,
+ Mxfp8MoEMethod,
+ Nvfp4MoEMethod,
+ UnquantizedMoEMethod,
+)
+from .quant_backend import QuantBackend, get_quant_backend, set_quant_backend
+from .names import NameMap
+from .configs import (
+ CompressedTensorsConfig,
+ Fp8BlockConfig,
+ ModelOptConfig,
+ Mxfp4Config,
+ NoQuantConfig,
+ QuantConfig,
+ quant_method_for,
+ quantization_config_of,
+)
+
+__all__ = [
+ "QuantKind", "GroupShape", "QuantScheme", "WeightDesc",
+ "LayerKind", "dialects", "method_class", "methods_for", "register_dialect", "register_method",
+ "KernelSelectionError", "select_kernel", "QuantMethod",
+ "LinearConfig", "LinearKernel", "LinearMethod",
+ "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",
+ "BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",
+ "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",
+ "QuantBackend", "set_quant_backend", "get_quant_backend", "NameMap",
+ "QuantConfig", "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",
+ "quant_method_for", "quantization_config_of", "finalize_quant",
+]
diff --git a/python/freetoken/layers/quantization/configs/__init__.py b/python/freetoken/layers/quantization/configs/__init__.py
new file mode 100644
index 00000000..6905b60d
--- /dev/null
+++ b/python/freetoken/layers/quantization/configs/__init__.py
@@ -0,0 +1,22 @@
+"""One QuantConfig per checkpoint dialect; importing the package registers them all."""
+
+from .base import NoQuantConfig, QuantConfig, quantization_config_of
+from . import compressed_tensors, fp8, modelopt, mxfp4
+from .compressed_tensors import CompressedTensorsConfig
+from .fp8 import Fp8BlockConfig
+from .modelopt import ModelOptConfig
+from .mxfp4 import Mxfp4Config
+
+_NO_QUANT = NoQuantConfig()
+
+
+def quant_method_for(quant_config, layer, prefix: str):
+ """The layer's Method from its config; a layer built without one is plain bf16."""
+ return (quant_config if quant_config is not None else _NO_QUANT).get_quant_method(layer, prefix)
+
+
+__all__ = [
+ "QuantConfig", "quantization_config_of", "quant_method_for",
+ "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",
+ "compressed_tensors", "fp8", "modelopt", "mxfp4",
+]
diff --git a/python/freetoken/layers/quantization/configs/base.py b/python/freetoken/layers/quantization/configs/base.py
new file mode 100644
index 00000000..d786a333
--- /dev/null
+++ b/python/freetoken/layers/quantization/configs/base.py
@@ -0,0 +1,110 @@
+"""QuantConfig: reads one quantization_config dialect and answers what scheme each module has."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, ClassVar
+
+from ..linear import LinearConfig
+from ..moe import MoEConfig
+from ..names import Matcher, NameMap, name_set
+from ..quant_backend import get_quant_backend
+from ..registry import LayerKind, dialects, method_class, register_dialect
+from ..scheme import QuantKind, QuantScheme
+
+LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig}
+
+
+def cfg_get(cfg: Any, key: str, default: Any = None) -> Any:
+ if cfg is None:
+ return default
+ if isinstance(cfg, dict):
+ return cfg.get(key, default)
+ return getattr(cfg, key, default)
+
+
+def quantization_config_of(hf_config: Any) -> dict[str, Any] | None:
+ q = cfg_get(hf_config, "quantization_config")
+ if q is None:
+ q = cfg_get(cfg_get(hf_config, "text_config"), "quantization_config")
+ if q is None:
+ return None
+ if isinstance(q, dict):
+ return q
+ if hasattr(q, "to_dict"):
+ return dict(q.to_dict())
+ return dict(vars(q))
+
+
+class QuantConfig(ABC):
+ dialect: ClassVar[str]
+
+ def __init__(self, name_map: NameMap | None = None, unquantized: tuple[str, ...] = ()):
+ self.name_map = name_map or NameMap()
+ # modules the family serves in bf16 regardless of the dialect (glob patterns on checkpoint names)
+ self.unquantized: Matcher = name_set(tuple(unquantized))
+ self._schemes: dict[str, QuantScheme | None] = {}
+
+ @classmethod
+ def claims(cls, q: dict[str, Any]) -> bool:
+ """Whether this dialect reads the given ``quantization_config``."""
+ return str(q.get("quant_method") or "").lower() == cls.dialect
+
+ @abstractmethod
+ def scheme_for_name(self, name: str) -> QuantScheme | None:
+ """Scheme of one module by its checkpoint name; None means unquantized."""
+
+ def scheme_for(self, prefix: str) -> QuantScheme | None:
+ if prefix in self._schemes:
+ return self._schemes[prefix]
+ names = self.name_map.to_checkpoint(prefix)
+ schemes = {None if self.unquantized(n) else self.scheme_for_name(n) for n in names}
+ if len(schemes) != 1:
+ raise ValueError(f"fused module {prefix!r} mixes quantization schemes across {names}: {schemes}")
+ scheme = schemes.pop()
+ self._schemes[prefix] = scheme
+ return scheme
+
+ def get_quant_method(self, layer: Any, prefix: str):
+ scheme = self.scheme_for(prefix)
+ layer_kind = layer.quant_layer_kind
+ kind = scheme.kind if scheme else QuantKind.NONE
+ cls = method_class(kind, layer_kind)
+ cfg = LAYER_CONFIGS[layer_kind].from_layer(layer, scheme)
+ return cls(cfg, get_quant_backend().select(layer_kind, kind))
+
+ @staticmethod
+ def from_hf(
+ hf_config: Any,
+ *,
+ name_map: NameMap | None = None,
+ unquantized: tuple[str, ...] = (),
+ hf_quant_config: dict[str, Any] | None = None,
+ ) -> "QuantConfig":
+ """``hf_quant_config`` is the parsed ``hf_quant_config.json`` of ModelOpt exports that keep
+ no ``quantization_config`` in config.json (modelopt < 0.41); ``unquantized`` lists the modules
+ the family keeps bf16 when the checkpoint's config does not (DeepSeek-V4's compressors)."""
+ q = quantization_config_of(hf_config)
+ if q is None and hf_quant_config and isinstance(hf_quant_config.get("quantization"), dict):
+ q = dict(hf_quant_config["quantization"])
+ q.setdefault("quant_method", "modelopt")
+ if q is None:
+ return NoQuantConfig(name_map, unquantized)
+ for cls in dialects():
+ if cls.claims(q):
+ return cls(q, hf_config, name_map=name_map, unquantized=unquantized)
+ raise NotImplementedError(f"quantization method {q.get('quant_method')!r} is not supported")
+
+
+@register_dialect
+class NoQuantConfig(QuantConfig):
+ """No ``quantization_config``: every module is bf16."""
+
+ dialect = "none"
+
+ @classmethod
+ def claims(cls, q: dict[str, Any]) -> bool:
+ return False
+
+ def scheme_for_name(self, name: str) -> QuantScheme | None:
+ return None
diff --git a/python/freetoken/layers/quantization/configs/compressed_tensors.py b/python/freetoken/layers/quantization/configs/compressed_tensors.py
new file mode 100644
index 00000000..d2b31dba
--- /dev/null
+++ b/python/freetoken/layers/quantization/configs/compressed_tensors.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from ..names import Matcher, ct_set
+from ..registry import register_dialect
+from ..scheme import QuantScheme
+from ..scheme import fp8_block_scheme, fp8_tensor_scheme, nvfp4_scheme
+from .base import QuantConfig
+
+
+@register_dialect
+class CompressedTensorsConfig(QuantConfig):
+ """llm-compressor exports: ordered ``config_groups`` with ``targets`` (class names, names or ``re:``) minus ``ignore``."""
+
+ dialect = "compressed-tensors"
+
+ SCHEMES: ClassVar[dict[str, QuantScheme]] = {
+ "NVFP4": nvfp4_scheme(input_scale=False),
+ "NVFP4_LOCAL": nvfp4_scheme(input_scale=True),
+ "FP8_TENSOR_STATIC": fp8_tensor_scheme("bf16", input_scale=True),
+ "FP8_TENSOR_DYNAMIC": fp8_tensor_scheme("bf16"),
+ "FP8_CHANNEL": fp8_tensor_scheme("bf16", per_row=True),
+ "FP8_BLOCK": fp8_block_scheme("float"),
+ }
+
+ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
+ super().__init__(name_map, unquantized)
+ self.ignore = ct_set(tuple(q.get("ignore") or ()), class_names=False)
+ groups = q.get("config_groups") or {}
+ self.groups: list[tuple[Matcher, QuantScheme]] = [
+ (ct_set(tuple(spec.get("targets") or ()), class_names=True), self._scheme_of(spec)) for spec in groups.values()
+ ]
+
+ def scheme_for_name(self, name: str) -> QuantScheme | None:
+ if self.ignore(name):
+ return None
+ for targets, scheme in self.groups:
+ if targets(name):
+ return scheme
+ return None
+
+ @classmethod
+ def _scheme_of(cls, spec: dict[str, Any]) -> QuantScheme:
+ w = spec.get("weights") or {}
+ act = spec.get("input_activations")
+ bits = int(w.get("num_bits") or 0)
+ wtype = str(w.get("type") or "").lower()
+ strategy = str(w.get("strategy") or "").lower()
+ group = int(w.get("group_size") or 0)
+ if bits == 4 and wtype == "float":
+ if strategy == "tensor_group" and group == 16:
+ return cls.SCHEMES["NVFP4_LOCAL" if act and act.get("dynamic") == "local" else "NVFP4"]
+ raise NotImplementedError(f"compressed-tensors 4-bit float scheme (strategy={strategy!r}, group_size={group}) is not supported; only NVFP4 is")
+ if bits == 8 and wtype == "float":
+ if strategy == "tensor":
+ return cls.SCHEMES["FP8_TENSOR_DYNAMIC" if act is None or act.get("dynamic") else "FP8_TENSOR_STATIC"]
+ if strategy == "channel":
+ return cls.SCHEMES["FP8_CHANNEL"]
+ if strategy == "block":
+ block = tuple(int(x) for x in (w.get("block_structure") or ()))
+ if block == (128, 128):
+ return cls.SCHEMES["FP8_BLOCK"]
+ raise NotImplementedError(f"compressed-tensors weight scheme {w} is not supported")
diff --git a/python/freetoken/layers/quantization/configs/fp8.py b/python/freetoken/layers/quantization/configs/fp8.py
new file mode 100644
index 00000000..11c8fbe2
--- /dev/null
+++ b/python/freetoken/layers/quantization/configs/fp8.py
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from ..names import is_routed_expert, name_set, substr_set
+from ..registry import register_dialect
+from ..scheme import QuantScheme
+from ..scheme import FP8_BLOCK, fp8_block_scheme, fp8_tensor_scheme, mxfp4_scheme
+from .base import QuantConfig, cfg_get
+
+
+@register_dialect
+class Fp8BlockConfig(QuantConfig):
+ """HF ``quant_method: fp8`` (DeepSeek-V3 style 128x128 block scales) plus the DeepSeek-V4 e8m0 / fp4-expert variant."""
+
+ dialect = "fp8"
+
+ SCHEMES: ClassVar[dict[str, QuantScheme]] = {
+ "BLOCK": fp8_block_scheme("float"),
+ "BLOCK_E8M0": fp8_block_scheme("e8m0"),
+ # HF ``modules_to_convert``: a table (Qwen3.8-Flash-Next PLE) stored e4m3 with one scalar scale
+ "TABLE": fp8_tensor_scheme("float"),
+ "EXPERT_MXFP4": mxfp4_scheme(),
+ }
+
+ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
+ super().__init__(name_map, unquantized)
+ block = tuple(int(x) for x in (q.get("weight_block_size") or ()))
+ if q.get("weight_per_tensor") or block != (FP8_BLOCK, FP8_BLOCK):
+ raise NotImplementedError(f"fp8 checkpoint with weight_block_size={block} per_tensor={q.get('weight_per_tensor')} is not supported; only 128x128 blocks are")
+ # transformers skips lm_head when the checkpoint gives no list
+ not_convert = tuple(q.get("modules_to_not_convert") or ("lm_head",))
+ self.not_convert = name_set(not_convert)
+ self.not_convert_substr = substr_set(not_convert)
+ self.convert_tables = name_set(tuple(q.get("modules_to_convert") or ()))
+ self.e8m0 = str(q.get("scale_fmt") or "").lower() == "ue8m0"
+ self.expert_fp4 = str(cfg_get(hf_config, "expert_dtype") or "").lower() == "fp4"
+
+ def scheme_for_name(self, name: str) -> QuantScheme | None:
+ if self.convert_tables(name):
+ return self.SCHEMES["TABLE"]
+ if self.not_convert(name) or self.not_convert_substr(name):
+ return None
+ if self.expert_fp4 and is_routed_expert(name):
+ return self.SCHEMES["EXPERT_MXFP4"]
+ return self.SCHEMES["BLOCK_E8M0" if self.e8m0 else "BLOCK"]
diff --git a/python/freetoken/layers/quantization/configs/modelopt.py b/python/freetoken/layers/quantization/configs/modelopt.py
new file mode 100644
index 00000000..813b0667
--- /dev/null
+++ b/python/freetoken/layers/quantization/configs/modelopt.py
@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from ..names import ancestors, name_set
+from ..registry import register_dialect
+from ..scheme import QuantScheme
+from ..scheme import fp8_block_scheme, fp8_tensor_scheme, mxfp8_scheme, nvfp4_scheme
+from .base import QuantConfig
+
+
+@register_dialect
+class ModelOptConfig(QuantConfig):
+ """NVIDIA ModelOpt exports: one ``quant_algo`` for every Linear minus ``ignore``, or
+ ``MIXED_PRECISION`` with a per-module ``quantized_layers`` allow-list."""
+
+ dialect = "modelopt"
+
+ SCHEMES: ClassVar[dict[str, QuantScheme]] = {
+ "NVFP4": nvfp4_scheme(input_scale=True),
+ "NVFP4_NO_INPUT": nvfp4_scheme(input_scale=False),
+ "FP8": fp8_tensor_scheme("fp32", input_scale=True),
+ "FP8_PER_CHANNEL_PER_TOKEN": fp8_tensor_scheme("fp32", per_row=True),
+ "FP8_PB_WO": fp8_block_scheme("fp32"),
+ "MXFP8": mxfp8_scheme(),
+ }
+
+ @classmethod
+ def claims(cls, q: dict[str, Any]) -> bool:
+ method = str(q.get("quant_method") or "").lower()
+ return method == "modelopt" or (not method and bool(q.get("quant_algo")))
+
+ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
+ super().__init__(name_map, unquantized)
+ self.algo = str(q.get("quant_algo") or "").upper()
+ self.ignore = name_set(tuple(q.get("ignore") or q.get("exclude_modules") or ()))
+ layers = q.get("quantized_layers") or {}
+ self.quantized_layers = {k: str((v or {}).get("quant_algo") or "").upper() for k, v in layers.items()} if isinstance(layers, dict) else {}
+ self.with_input_scale = bool(q.get("with_input_scale", True))
+ if self.algo == "MIXED_PRECISION" and not self.quantized_layers:
+ raise NotImplementedError("ModelOpt MIXED_PRECISION without quantized_layers in quantization_config")
+ if self.algo != "MIXED_PRECISION":
+ self._scheme_of(self.algo)
+
+ def scheme_for_name(self, name: str) -> QuantScheme | None:
+ if self.ignore(name):
+ return None
+ algo = self._module_algo(name)
+ return None if algo is None else self._scheme_of(algo)
+
+ def _module_algo(self, name: str) -> str | None:
+ if self.algo != "MIXED_PRECISION":
+ return self.algo
+ for a in ancestors(name):
+ algo = self.quantized_layers.get(a)
+ if algo is not None:
+ return algo
+ return None
+
+ def _scheme_of(self, algo: str) -> QuantScheme:
+ if algo in ("NVFP4", "W4A16_NVFP4"):
+ return self.SCHEMES["NVFP4" if self.with_input_scale else "NVFP4_NO_INPUT"]
+ try:
+ return self.SCHEMES[algo]
+ except KeyError:
+ raise NotImplementedError(f"ModelOpt quant_algo {algo!r} is not supported") from None
diff --git a/python/freetoken/layers/quantization/configs/mxfp4.py b/python/freetoken/layers/quantization/configs/mxfp4.py
new file mode 100644
index 00000000..e9458f03
--- /dev/null
+++ b/python/freetoken/layers/quantization/configs/mxfp4.py
@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from ..names import name_set
+from ..registry import register_dialect
+from ..scheme import QuantScheme
+from ..scheme import mxfp4_scheme
+from .base import QuantConfig
+
+
+@register_dialect
+class Mxfp4Config(QuantConfig):
+ """gpt-oss: the stacked ``experts`` containers are MXFP4 with biases, everything else bf16."""
+
+ dialect = "mxfp4"
+
+ SCHEME: ClassVar[QuantScheme] = mxfp4_scheme()
+
+ def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
+ super().__init__(name_map, unquantized)
+ self.not_convert = name_set(tuple(q.get("modules_to_not_convert") or ()))
+
+ def scheme_for_name(self, name: str) -> QuantScheme | None:
+ if not name.endswith(".experts") or self.not_convert(name):
+ return None
+ return self.SCHEME
diff --git a/python/freetoken/layers/quantization/linear/__init__.py b/python/freetoken/layers/quantization/linear/__init__.py
new file mode 100644
index 00000000..8236abe0
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/__init__.py
@@ -0,0 +1,13 @@
+from .base import LinearConfig, LinearKernel, LinearMethod
+from . import fp8_block, fp8_tensor, mxfp8, nvfp4, unquantized
+from .fp8_block import Fp8BlockLinearMethod
+from .fp8_tensor import Fp8TensorLinearMethod
+from .mxfp8 import Mxfp8LinearMethod
+from .nvfp4 import Nvfp4LinearMethod
+from .unquantized import UnquantizedLinearMethod
+
+__all__ = [
+ "LinearConfig", "LinearKernel", "LinearMethod",
+ "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",
+ "fp8_block", "fp8_tensor", "mxfp8", "nvfp4", "unquantized",
+]
diff --git a/python/freetoken/layers/quantization/linear/base.py b/python/freetoken/layers/quantization/linear/base.py
new file mode 100644
index 00000000..e8bbeff4
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/base.py
@@ -0,0 +1,64 @@
+"""The linear side: per-layer config, the kernel ABC and the Method base."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+import torch
+
+from ..method import QuantMethod
+from ..scheme import QuantScheme
+
+
+@dataclass(frozen=True)
+class LinearConfig:
+ """What a linear kernel may look at to accept or decline a layer: the TP-local shape and the weight scheme."""
+
+ in_features: int
+ out_features: int
+ # fused segments along N; a plain linear is one segment
+ output_sizes: tuple[int, ...] = ()
+ scheme: QuantScheme | None = None
+
+ def __post_init__(self) -> None:
+ if not self.output_sizes:
+ object.__setattr__(self, "output_sizes", (self.out_features,))
+ if sum(self.output_sizes) != self.out_features:
+ raise ValueError(f"output_sizes {self.output_sizes} do not sum to {self.out_features}")
+
+ @classmethod
+ def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "LinearConfig":
+ return cls(layer.in_features, layer.out_features, tuple(layer.output_sizes), scheme)
+
+
+class LinearKernel(ABC):
+ """One backend for one linear kind: says whether it can run a layer, then finalizes and applies it."""
+
+ name: ClassVar[str]
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ return None
+
+ def worth_it(self, cfg: LinearConfig) -> bool:
+ return True
+
+ def finalize(self, layer: Any) -> None:
+ pass
+
+ @abstractmethod
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor: ...
+
+
+class LinearMethod(QuantMethod):
+ """Declares a linear layer's weights for its kind; finalize and apply go to the kernel."""
+
+ @abstractmethod
+ def create_weights(self, layer: Any) -> None: ...
+
+ def finalize(self, layer: Any) -> None:
+ self.kernel.finalize(layer)
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ return self.kernel.apply(layer, x)
diff --git a/python/freetoken/layers/quantization/linear/fp8_block.py b/python/freetoken/layers/quantization/linear/fp8_block.py
new file mode 100644
index 00000000..3b78e977
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/fp8_block.py
@@ -0,0 +1,60 @@
+"""fp8 e4m3 weight with 128x128 block scales (DeepSeek-V3 style)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+
+from ..registry import LayerKind, register_method
+from ..scheme import FP8_BLOCK as BLOCK, QuantKind
+from .base import LinearConfig, LinearKernel, LinearMethod
+
+FP8 = torch.float8_e4m3fn
+E8M0 = torch.float8_e8m0fnu
+
+
+def _e8m0(cfg: LinearConfig) -> bool:
+ return cfg.scheme is not None and cfg.scheme.weight.scale == "e8m0"
+
+
+class Dsv4Fp8BlockLinearKernel(LinearKernel):
+ """DeepSeek-V4's reference path: activations quantized to fp8 with power-of-two block scales, e8m0 weight scales read as codes."""
+
+ name = "dsv4"
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ return None if _e8m0(cfg) else "serves e8m0 block scales only"
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.dsv4.fp8_linear import block_fp8_linear
+
+ return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)
+
+
+class TritonFp8BlockLinearKernel(LinearKernel):
+ """W8A16 GEMV at M=1, dynamic 1x128 W8A8 GEMM above."""
+
+ name = "triton"
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ return "reads float block scales; e8m0 codes go to the dsv4 kernel" if _e8m0(cfg) else None
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.fp8_block_linear import block_fp8_linear
+
+ return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)
+
+
+@register_method(QuantKind.FP8_BLOCK, LayerKind.LINEAR)
+class Fp8BlockLinearMethod(LinearMethod):
+ candidates = (Dsv4Fp8BlockLinearKernel, TritonFp8BlockLinearKernel)
+
+ def create_weights(self, layer: Any) -> None:
+ g = self.cfg
+ if g.in_features % BLOCK or any(o % BLOCK for o in g.output_sizes):
+ raise ValueError(f"block-fp8 needs in/out sizes divisible by {BLOCK}, got K={g.in_features} N={g.output_sizes}")
+ layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)
+ # e8m0 codes stay codes for the dsv4 kernel; float scales are bf16 as the readers push them today
+ scale_dtype = E8M0 if _e8m0(g) else torch.bfloat16
+ layer.weight_scale_inv = torch.empty(g.out_features // BLOCK, g.in_features // BLOCK, dtype=scale_dtype)
diff --git a/python/freetoken/layers/quantization/linear/fp8_tensor.py b/python/freetoken/layers/quantization/linear/fp8_tensor.py
new file mode 100644
index 00000000..9461e044
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/fp8_tensor.py
@@ -0,0 +1,84 @@
+"""fp8 e4m3 weight with one fp32 scale per output row (per-tensor scales are broadcast)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+
+from freetoken.kernel import backend
+
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind
+from .base import LinearConfig, LinearKernel, LinearMethod
+
+FP8 = torch.float8_e4m3fn
+
+
+class TorchFp8TensorLinearKernel(LinearKernel):
+ """Static W8A8 through torch._scaled_mm; needs the checkpoint's input_scale and fp8 tensor cores."""
+
+ name = "torch"
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ if cfg.scheme is None or not cfg.scheme.has("input_scale"):
+ return "the checkpoint has no static activation scale"
+ cc = backend.device_capability()
+ if cc < (8, 9):
+ return f"fp8 tensor cores need sm_89+, got sm_{cc[0]}{cc[1]}"
+ from freetoken.kernel.triton.e4m3_compat import e4m3_native
+
+ if not e4m3_native():
+ return "e4m3 emulation mode has no fp8 GEMM"
+ return None
+
+ def finalize(self, layer: Any) -> None:
+ from freetoken.kernel.triton.fp8_pertensor_linear import rowwise_scaled_mm_ok, weight_scale_segments
+
+ # a fused projection carries one scalar per part; decide the GEMM shape once, not under graph capture
+ scale = layer.weight_scale
+ uniform = bool((scale == scale[0]).all().item())
+ layer._fp8_uniform_scale = uniform
+ layer._fp8_scale_segments = None if uniform else weight_scale_segments(scale)
+ if not uniform:
+ rowwise_scaled_mm_ok()
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
+
+ return fp8_pertensor_linear(
+ x, layer.weight, layer.weight_scale, layer.bias,
+ layer.input_scale, layer._fp8_uniform_scale, scale_segments=layer._fp8_scale_segments,
+ )
+
+
+class TritonFp8TensorLinearKernel(LinearKernel):
+ """W8A16: split-K GEMV at M=1, GEMM above."""
+
+ name = "triton"
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
+
+ return fp8_pertensor_linear(x, layer.weight, layer.weight_scale, layer.bias)
+
+
+class EmulationFp8TensorLinearKernel(LinearKernel):
+ name = "emulation"
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ *lead, K = x.shape
+ w = layer.weight.to(x.dtype) * layer.weight_scale.to(x.dtype)[:, None]
+ out = (x.reshape(-1, K) @ w.t()).reshape(*lead, w.shape[0])
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+
+
+@register_method(QuantKind.FP8_TENSOR, LayerKind.LINEAR)
+class Fp8TensorLinearMethod(LinearMethod):
+ candidates = (TorchFp8TensorLinearKernel, TritonFp8TensorLinearKernel, EmulationFp8TensorLinearKernel)
+
+ def create_weights(self, layer: Any) -> None:
+ g = self.cfg
+ layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)
+ layer.weight_scale = torch.empty(g.out_features, dtype=torch.float32)
+ layer.input_scale = torch.empty((), dtype=torch.float32) if self.scheme.has("input_scale") else None
diff --git a/python/freetoken/layers/quantization/linear/mxfp8.py b/python/freetoken/layers/quantization/linear/mxfp8.py
new file mode 100644
index 00000000..3102727f
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/mxfp8.py
@@ -0,0 +1,48 @@
+"""fp8 e4m3 weight with e8m0 scales per 32 input elements (OCP MXFP8)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+
+from ..registry import LayerKind, register_method
+from ..scheme import MX_GROUP as BLOCK, QuantKind
+from .base import LinearKernel, LinearMethod
+
+FP8 = torch.float8_e4m3fn
+
+
+class TritonMxfp8LinearKernel(LinearKernel):
+ """W8A16: split-K GEMV up to M=256, bf16 dequant + cuBLAS above."""
+
+ name = "triton"
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_linear
+
+ return mxfp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)
+
+
+class EmulationMxfp8LinearKernel(LinearKernel):
+ name = "emulation"
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant
+
+ *lead, K = x.shape
+ w = mxfp8_dequant(layer.weight, layer.weight_scale_inv, dtype=torch.float32)
+ out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, w.shape[0])
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+
+
+@register_method(QuantKind.MXFP8, LayerKind.LINEAR)
+class Mxfp8LinearMethod(LinearMethod):
+ candidates = (TritonMxfp8LinearKernel, EmulationMxfp8LinearKernel)
+
+ def create_weights(self, layer: Any) -> None:
+ g = self.cfg
+ if g.in_features % BLOCK:
+ raise ValueError(f"mxfp8 needs in_features divisible by {BLOCK}, got {g.in_features}")
+ layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)
+ layer.weight_scale_inv = torch.empty(g.out_features, g.in_features // BLOCK, dtype=torch.uint8)
diff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.py
new file mode 100644
index 00000000..6644acdb
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/nvfp4.py
@@ -0,0 +1,123 @@
+"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp16 global per output row."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import torch
+
+from freetoken.kernel import backend
+
+from ..registry import LayerKind, register_method
+from ..scheme import NVFP4_GROUP as GROUP, QuantKind
+from .base import LinearConfig, LinearKernel, LinearMethod
+
+FP8 = torch.float8_e4m3fn
+
+
+class TritonNvfp4LinearKernel(LinearKernel):
+ """W4A16 on the K-major resident layout."""
+
+ name = "triton"
+
+ def finalize(self, layer: Any) -> None:
+ from freetoken.kernel.triton.nvfp4_linear import nvfp4_transpose_resident
+
+ layer.weight, layer.weight_scale = nvfp4_transpose_resident(layer.weight, layer.weight_scale)
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.nvfp4_linear import nvfp4_dense_linear_t
+
+ return nvfp4_dense_linear_t(x, layer.weight, layer.weight_scale, layer.weight_global, layer.bias)
+
+
+class MarlinNvfp4LinearKernel(LinearKernel):
+ """vLLM's fused FP4 Marlin GEMM (W4A16); needs one global scale for the whole weight."""
+
+ name = "marlin"
+
+ def unusable_reason(self, cfg: LinearConfig) -> str | None:
+ if not backend.is_vllm_installed():
+ return "vLLM is not installed"
+ if len(cfg.output_sizes) > 1:
+ return "fused projection carries one global scale per part"
+ try:
+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401
+ apply_fp4_marlin_linear,
+ is_fp4_marlin_supported,
+ prepare_fp4_layer_for_marlin,
+ )
+ except Exception as exc:
+ return f"vLLM Marlin FP4 symbols are unusable ({exc!r})"
+ if not is_fp4_marlin_supported():
+ return "vLLM reports FP4 Marlin unsupported on this device"
+ return None
+
+ def finalize(self, layer: Any) -> None:
+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import prepare_fp4_layer_for_marlin
+
+ # Marlin wants the per-tensor scalar; weight_global is that scalar broadcast to [N]
+ packed = SimpleNamespace(
+ weight=layer.weight,
+ weight_scale=layer.weight_scale,
+ weight_scale_2=layer.weight_global.reshape(-1)[0].to(torch.bfloat16),
+ output_size_per_partition=layer.out_features,
+ input_size_per_partition=layer.in_features,
+ params_dtype=torch.bfloat16,
+ )
+ prepare_fp4_layer_for_marlin(packed)
+ layer._marlin_weight = packed.weight
+ layer._marlin_scale = packed.weight_scale
+ layer._marlin_global = packed.weight_scale_2
+ layer._marlin_workspace = packed.workspace
+ dev = layer.weight.device
+ layer.weight = torch.empty(0, dtype=torch.uint8, device=dev)
+ layer.weight_scale = torch.empty(0, dtype=FP8, device=dev)
+ layer.weight_global = torch.empty(0, dtype=torch.float16, device=dev)
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import apply_fp4_marlin_linear
+
+ out = apply_fp4_marlin_linear(
+ input=x,
+ weight=layer._marlin_weight,
+ weight_scale=layer._marlin_scale,
+ weight_scale_2=layer._marlin_global,
+ workspace=layer._marlin_workspace,
+ size_n=layer.out_features,
+ size_k=layer.in_features,
+ )
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+
+
+class EmulationNvfp4LinearKernel(LinearKernel):
+ """Dequantize to bf16 with dequant_nvfp4, then a plain matmul; keeps the row-major layout."""
+
+ name = "emulation"
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4
+
+ *lead, K = x.shape
+ slots = torch.zeros(1, dtype=torch.int32, device=x.device)
+ w = dequant_nvfp4(
+ layer.weight.unsqueeze(0), layer.weight_scale.unsqueeze(0), layer.weight_global.unsqueeze(0),
+ slots, dtype=torch.bfloat16,
+ )[0]
+ out = (x.reshape(-1, K) @ w.t()).to(x.dtype).reshape(*lead, w.shape[0])
+ return out + layer.bias.to(out.dtype) if layer.bias is not None else out
+
+
+@register_method(QuantKind.NVFP4, LayerKind.LINEAR)
+class Nvfp4LinearMethod(LinearMethod):
+ candidates = (TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel)
+
+ def create_weights(self, layer: Any) -> None:
+ g = self.cfg
+ if g.in_features % GROUP:
+ raise ValueError(f"nvfp4 needs in_features divisible by {GROUP}, got {g.in_features}")
+ layer.weight = torch.empty(g.out_features, g.in_features // 2, dtype=torch.uint8)
+ layer.weight_scale = torch.empty(g.out_features, g.in_features // GROUP, dtype=FP8)
+ layer.weight_global = torch.empty(g.out_features, dtype=torch.float16)
+ # input_scale stays undeclared: the W4A16 kernels never read it and today's readers drop it
diff --git a/python/freetoken/layers/quantization/linear/unquantized.py b/python/freetoken/layers/quantization/linear/unquantized.py
new file mode 100644
index 00000000..6cd8de8a
--- /dev/null
+++ b/python/freetoken/layers/quantization/linear/unquantized.py
@@ -0,0 +1,33 @@
+"""bf16 Linear: one kernel (torch), no scheme."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+import torch.nn.functional as F
+
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind
+from .base import LinearKernel, LinearMethod
+
+
+class TorchLinearKernel(LinearKernel):
+ name = "torch"
+
+ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
+ # an fp32 activation stream (DeepSeek-V4's compressors) upcasts the bf16 weight on the fly, as the reference does
+ w, b = layer.weight, layer.bias
+ if w.dtype != x.dtype:
+ w = w.to(x.dtype)
+ b = b.to(x.dtype) if b is not None else None
+ return F.linear(x, w, b)
+
+
+@register_method(QuantKind.NONE, LayerKind.LINEAR)
+class UnquantizedLinearMethod(LinearMethod):
+ candidates = (TorchLinearKernel,)
+
+ def create_weights(self, layer: Any) -> None:
+ g = self.cfg
+ layer.weight = torch.empty(g.out_features, g.in_features)
diff --git a/python/freetoken/layers/quantization/method.py b/python/freetoken/layers/quantization/method.py
new file mode 100644
index 00000000..2a079539
--- /dev/null
+++ b/python/freetoken/layers/quantization/method.py
@@ -0,0 +1,89 @@
+"""QuantMethod: one (kind, layer kind) pair bound to one kernel object; the selector and the post-load hook."""
+
+from __future__ import annotations
+
+from abc import ABC
+from typing import Any, ClassVar, Sequence
+
+from freetoken.utils import init_logger
+
+from .registry import LayerKind
+from .scheme import QuantKind
+
+logger = init_logger(__name__)
+
+
+class KernelSelectionError(RuntimeError):
+ pass
+
+
+def select_kernel(candidates: Sequence[type], requested: str, cfg: Any):
+ """Pick one kernel from an ordered table: the requested name if usable, else the first usable one that is worth it."""
+ if not candidates:
+ raise KernelSelectionError("empty kernel table")
+ names = [c.name for c in candidates]
+ if requested != "auto" and requested in names:
+ kernel = candidates[names.index(requested)]()
+ reason = kernel.unusable_reason(cfg)
+ if reason:
+ raise KernelSelectionError(f"kernel {requested!r} was requested but cannot run here: {reason}")
+ return kernel
+ skipped: list[str] = []
+ fallback = None
+ for cls in candidates:
+ kernel = cls()
+ reason = kernel.unusable_reason(cfg)
+ if reason:
+ skipped.append(f"{kernel.name}: {reason}")
+ continue
+ if kernel.worth_it(cfg):
+ if skipped:
+ logger.info("kernel %s selected; skipped %s", kernel.name, "; ".join(skipped))
+ return kernel
+ if fallback is None:
+ fallback = kernel
+ if fallback is not None:
+ return fallback
+ raise KernelSelectionError("no usable kernel in table; " + "; ".join(skipped))
+
+
+class QuantMethod(ABC):
+ kind: ClassVar[QuantKind]
+ layer_kind: ClassVar[LayerKind]
+ candidates: ClassVar[tuple[type, ...]] = ()
+
+ def __init__(self, cfg: Any, requested: str = "auto"):
+ self.cfg = cfg
+ self.requested = requested
+ self.kernel = select_kernel(self.candidates, requested, cfg)
+
+ @property
+ def scheme(self):
+ return self.cfg.scheme
+
+def finalize_quant(root: Any) -> int:
+ """Call ``quant_method.finalize(layer)`` on every layer under ``root``; returns the count."""
+ from freetoken.layers.base import BaseOP
+
+ seen: set[int] = set()
+ count = 0
+
+ def walk(op: Any) -> None:
+ nonlocal count
+ if id(op) in seen:
+ return
+ seen.add(id(op))
+ method = getattr(op, "quant_method", None)
+ if method is not None:
+ method.finalize(op)
+ count += 1
+ for value in vars(op).values():
+ if isinstance(value, BaseOP):
+ walk(value)
+ elif isinstance(value, (list, tuple)):
+ for item in value:
+ if isinstance(item, BaseOP):
+ walk(item)
+
+ walk(root)
+ return count
diff --git a/python/freetoken/layers/quantization/moe/__init__.py b/python/freetoken/layers/quantization/moe/__init__.py
new file mode 100644
index 00000000..a5f2dc66
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/__init__.py
@@ -0,0 +1,13 @@
+from .base import BankSpec, ExpertView, MoEConfig, MoEKernel, MoEMethod
+from . import fp8_block, mxfp4, mxfp8, nvfp4, unquantized
+from .fp8_block import Fp8BlockMoEMethod
+from .mxfp4 import Mxfp4MoEMethod
+from .mxfp8 import Mxfp8MoEMethod
+from .nvfp4 import Nvfp4MoEMethod
+from .unquantized import UnquantizedMoEMethod
+
+__all__ = [
+ "BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",
+ "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",
+ "fp8_block", "mxfp4", "mxfp8", "nvfp4", "unquantized",
+]
diff --git a/python/freetoken/layers/quantization/moe/base.py b/python/freetoken/layers/quantization/moe/base.py
new file mode 100644
index 00000000..89f13ed0
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/base.py
@@ -0,0 +1,201 @@
+"""The MoE side: per-layer config, bank descriptions, the kernel ABC and the Method base."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+import torch
+
+from ..method import QuantMethod
+from ..scheme import QuantScheme
+
+
+@dataclass(frozen=True)
+class MoEConfig:
+ """Everything an expert kernel may look at to accept or decline a layer and to size its banks."""
+
+ num_experts: int
+ hidden: int
+ intermediate: int
+ top_k: int
+ tp_rank: int = 0
+ tp_size: int = 1
+ scheme: QuantScheme | None = None
+ activation: str = "silu"
+ alpha: float = 1.0
+ beta: float = 0.0
+ limit: float | None = None
+ interleaved: bool = False
+ has_bias: bool = False
+ apply_router_weight_on_input: bool = False
+ strategy: str = "resident"
+ decode_target: str = "gpu"
+
+ @classmethod
+ def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "MoEConfig":
+ return cls(
+ num_experts=layer.num_experts,
+ hidden=layer.hidden_size,
+ intermediate=layer.intermediate_size,
+ top_k=layer.top_k,
+ tp_rank=layer.tp_rank,
+ tp_size=layer.tp_size,
+ scheme=scheme,
+ activation=layer.activation,
+ alpha=float(layer.alpha),
+ beta=float(layer.beta),
+ limit=layer.limit,
+ interleaved=bool(layer.interleaved),
+ has_bias=bool(layer.has_bias),
+ apply_router_weight_on_input=bool(layer.apply_router_weight_on_input),
+ strategy=layer.strategy,
+ decode_target=layer.decode_target,
+ )
+
+ @property
+ def local_intermediate(self) -> int:
+ return self.intermediate // self.tp_size
+
+ @property
+ def plain_silu(self) -> bool:
+ return (self.activation, self.alpha, self.beta, self.limit) == ("silu", 1.0, 0.0, None)
+
+
+@dataclass(frozen=True)
+class BankSpec:
+ """One bank of an expert layout, per expert (the E dim is prepended by the allocator)."""
+
+ shape: tuple[int, ...]
+ dtype: torch.dtype
+ # GPU-resident per-expert vector (marlin / b12x alphas) instead of a host bank
+ resident: bool = False
+
+
+@dataclass
+class ExpertView:
+ """The weights an apply() call reads: layout roles -> [E, ...] (resident, materialized layer) or [S, ...] (slot cache)."""
+
+ tensors: dict[str, torch.Tensor]
+ slots: torch.Tensor | None = None
+ n: int | None = None
+ alphas: tuple[torch.Tensor, torch.Tensor] | None = None
+
+
+def fused_piece(pieces: dict[str, torch.Tensor], role: str) -> torch.Tensor:
+ """A gate_up-family piece, concatenating separate gate / up pieces along the row axis."""
+ if role in pieces:
+ return pieces[role]
+ suffix = role[len("gate_up"):]
+ return torch.cat([pieces["gate" + suffix], pieces["up" + suffix]], dim=1)
+
+
+def global_rows(piece: torch.Tensor, rows: int) -> torch.Tensor:
+ """Per-expert global scale as one fp16 value per output row (a scalar per expert is broadcast)."""
+ flat = piece.reshape(piece.shape[0], -1).to(torch.float16)
+ if flat.shape[1] == rows:
+ return flat
+ assert flat.shape[1] == 1, (flat.shape, rows)
+ return flat.expand(-1, rows)
+
+
+def fused_global(pieces: dict[str, torch.Tensor], half_rows: int) -> torch.Tensor:
+ if "gate_up_global" in pieces:
+ return global_rows(pieces["gate_up_global"], 2 * half_rows)
+ return torch.cat([global_rows(pieces["gate_global"], half_rows), global_rows(pieces["up_global"], half_rows)], dim=1)
+
+
+def limit_or_inf(layer: Any) -> float:
+ limit = getattr(layer, "limit", None)
+ return float("inf") if limit is None else float(limit)
+
+
+def gated_epilogue_reason(cfg: "MoEConfig") -> str | None:
+ """Why ``gated_act_and_mul`` cannot run ``cfg``'s activation quadruple, or None; the plain kinds have no alpha / limit and nothing takes beta or interleaved rows."""
+ from freetoken.layers import GATED_ACTIVATIONS
+
+ if cfg.activation not in GATED_ACTIVATIONS:
+ return f"no {cfg.activation!r} epilogue"
+ if cfg.interleaved:
+ return "the epilogue reads uninterleaved [gate; up] halves"
+ if cfg.beta != 0.0:
+ return f"the epilogue has no beta (got {cfg.beta})"
+ if cfg.activation in ("silu", "gelu", "gelu_tanh") and (cfg.alpha != 1.0 or cfg.limit is not None):
+ return f"{cfg.activation} takes no alpha / limit (got alpha={cfg.alpha}, limit={cfg.limit})"
+ return None
+
+
+def is_resident(layer: Any) -> bool:
+ return layer.quant_method.cfg.strategy == "resident"
+
+
+class MoEKernel(ABC):
+ """One backend for one expert kind: bank layout, packing from checkpoint pieces, and the fused forward."""
+
+ name: ClassVar[str]
+ cpu_format: ClassVar[str | None] = None
+ max_slots: ClassVar[int | None] = None
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ return None
+
+ def worth_it(self, cfg: MoEConfig) -> bool:
+ return True
+
+ def slot_limit(self, cfg: MoEConfig) -> int | None:
+ """Most GPU cache slots the kernel can address for ``cfg``; None for no limit."""
+ return self.max_slots
+
+ def _common_reject(self, cfg: MoEConfig, *, resident_ok: bool, tp_ok: bool, cpu_ok: bool, plain_silu_only: bool) -> str | None:
+ if not resident_ok and cfg.strategy == "resident":
+ return "not served resident; use --moe-strategy offload or cpu"
+ if not tp_ok and cfg.tp_size > 1:
+ return "TP > 1 is not supported for this expert format"
+ if not cpu_ok and cfg.decode_target != "gpu":
+ return "has no CPU executor format; decode must run on the GPU"
+ if cfg.has_bias:
+ return "kernel has no bias epilogue"
+ if plain_silu_only and not cfg.plain_silu:
+ return f"kernel is plain-silu only, experts use ({cfg.activation}, alpha={cfg.alpha}, beta={cfg.beta}, limit={cfg.limit})"
+ return None
+
+ @abstractmethod
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]: ...
+
+ @abstractmethod
+ def pack(self, pieces: dict[str, torch.Tensor], cfg: MoEConfig, out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
+ """Write one batch of expert pieces into ``out`` rows; returns the GPU-resident per-expert values, if any."""
+
+ @abstractmethod
+ def apply(self, layer: Any, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, is_prefill: bool) -> torch.Tensor: ...
+
+
+class MoEMethod(QuantMethod):
+ """Declares resident experts for its kind; layout, pack and apply go to the kernel."""
+
+ def layout(self) -> dict[str, BankSpec]:
+ return self.kernel.layout(self.cfg)
+
+ def pack(self, pieces: dict[str, torch.Tensor], out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
+ return self.kernel.pack(pieces, self.cfg, out)
+
+ def slot_limit(self) -> int | None:
+ return self.kernel.slot_limit(self.cfg)
+
+ def apply(self, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, layer: Any, is_prefill: bool) -> torch.Tensor:
+ return self.kernel.apply(layer, x, topk_weights, topk_ids, view, is_prefill=is_prefill)
+
+ @property
+ def cpu_format(self) -> str | None:
+ return self.kernel.cpu_format
+
+ @abstractmethod
+ def create_weights(self, layer: Any) -> None: ...
+
+ def finalize(self, layer: Any) -> None:
+ pass
+
+ @abstractmethod
+ def resident_view(self, layer: Any) -> ExpertView:
+ """Layout roles -> the resident layer's tensors, for apply()."""
diff --git a/python/freetoken/layers/quantization/moe/fp8_block.py b/python/freetoken/layers/quantization/moe/fp8_block.py
new file mode 100644
index 00000000..6c0167b7
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/fp8_block.py
@@ -0,0 +1,80 @@
+"""fp8 e4m3 experts with 128x128 block scales."""
+
+from __future__ import annotations
+
+import torch
+
+from ..registry import LayerKind, register_method
+from ..scheme import FP8_BLOCK as BLOCK, QuantKind
+from .base import BankSpec, ExpertView, fused_piece, gated_epilogue_reason, limit_or_inf, MoEConfig, MoEKernel, MoEMethod
+
+FP8 = torch.float8_e4m3fn
+
+
+def _pad_scale(rows: int, cols: int) -> int:
+ from freetoken.kernel.aot_models import fp8_block_scale_pad
+
+ return fp8_block_scale_pad(rows, cols)
+
+
+class TritonFp8BlockMoEKernel(MoEKernel):
+ name = "triton"
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ reason = self._common_reject(cfg, resident_ok=True, tp_ok=False, cpu_ok=False, plain_silu_only=False)
+ if reason:
+ return reason
+ reason = gated_epilogue_reason(cfg)
+ if reason:
+ return f"fp8 block MoE kernel: {reason}"
+ if cfg.apply_router_weight_on_input:
+ return "fp8 block MoE kernel cannot apply the router weight on the input"
+ return None
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ i, h, b = cfg.intermediate, cfg.hidden, BLOCK
+ return {
+ "gate_up": BankSpec((2 * i, h), FP8),
+ "gate_up_scale": BankSpec((2 * i // b, _pad_scale(2 * i // b, h // b)), torch.bfloat16),
+ "down": BankSpec((h, i), FP8),
+ "down_scale": BankSpec((h // b, _pad_scale(h // b, i // b)), torch.bfloat16),
+ }
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ out["gate_up"].copy_(fused_piece(pieces, "gate_up"))
+ out["down"].copy_(pieces["down"])
+ # the scale banks carry 16-byte padding on the last dim
+ gs = fused_piece(pieces, "gate_up_scale")
+ out["gate_up_scale"][:, :, : gs.shape[2]].copy_(gs)
+ ds = pieces["down_scale"]
+ out["down_scale"][:, :, : ds.shape[2]].copy_(ds)
+ return {}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ from freetoken.moe.fused_fp8_block import fused_experts_decode_fp8_block, fused_experts_fp8_block
+
+ t = view.tensors
+ alpha, limit = float(layer.alpha), limit_or_inf(layer)
+ if is_prefill:
+ n = view.n if view.n is not None else layer.num_experts
+ return fused_experts_fp8_block(x, t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"], topk_weights, topk_ids, n, layer.activation, layer.apply_router_weight_on_input, alpha, limit)
+ return fused_experts_decode_fp8_block(x, t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, alpha, limit)
+
+
+@register_method(QuantKind.FP8_BLOCK, LayerKind.MOE)
+class Fp8BlockMoEMethod(MoEMethod):
+ candidates = (TritonFp8BlockMoEKernel,)
+
+ def create_weights(self, layer) -> None:
+ g = self.cfg
+ e, i, h, b = g.num_experts, g.intermediate, g.hidden, BLOCK
+ layer.gate_up_proj = torch.empty(e, 2 * i, h, dtype=FP8)
+ layer.gate_up_scale_inv = torch.empty(e, 2 * i // b, h // b, dtype=torch.bfloat16)
+ layer.down_proj = torch.empty(e, h, i, dtype=FP8)
+ layer.down_scale_inv = torch.empty(e, h // b, i // b, dtype=torch.bfloat16)
+
+ def resident_view(self, layer) -> ExpertView:
+ return ExpertView({
+ "gate_up": layer.gate_up_proj, "gate_up_scale": layer.gate_up_scale_inv,
+ "down": layer.down_proj, "down_scale": layer.down_scale_inv,
+ })
diff --git a/python/freetoken/layers/quantization/moe/mxfp4.py b/python/freetoken/layers/quantization/moe/mxfp4.py
new file mode 100644
index 00000000..aa020ee3
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/mxfp4.py
@@ -0,0 +1,142 @@
+"""MXFP4 experts: the standard OCP form (DeepSeek-V4) and the gpt-oss variant with biases."""
+
+from __future__ import annotations
+
+import torch
+
+from ..registry import LayerKind, register_method
+from ..scheme import MX_GROUP as GROUP, QuantKind
+from .base import BankSpec, ExpertView, fused_piece, is_resident, limit_or_inf, MoEConfig, MoEKernel, MoEMethod
+
+E8M0 = torch.float8_e8m0fnu
+
+
+class TritonMxfp4MoEKernel(MoEKernel):
+ """Standard OCP MXFP4 experts (DeepSeek-V4 ds_fp4): e2m1 pairs + e8m0 scales, no bias."""
+
+ name = "triton"
+ cpu_format = "ds_fp4"
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ if cfg.interleaved:
+ return "standard MXFP4 kernel reads the concatenated gate|up row order"
+ if (cfg.alpha, cfg.beta) != (1.0, 0.0):
+ return "standard MXFP4 kernel has no alpha / beta in its swiglu"
+ return self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=True, plain_silu_only=False)
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ i, h = cfg.intermediate, cfg.hidden
+ return {
+ "gate_up": BankSpec((2 * i, h // 2), torch.uint8),
+ "gate_up_scale": BankSpec((2 * i, h // GROUP), E8M0),
+ "down": BankSpec((h, i // 2), torch.uint8),
+ "down_scale": BankSpec((h, i // GROUP), E8M0),
+ }
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ out["gate_up"].copy_(fused_piece(pieces, "gate_up").view(torch.uint8))
+ out["gate_up_scale"].copy_(fused_piece(pieces, "gate_up_scale").view(E8M0))
+ out["down"].copy_(pieces["down"].view(torch.uint8))
+ out["down_scale"].copy_(pieces["down_scale"].view(E8M0))
+ return {}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ t = view.tensors
+ banks = (t["gate_up"], t["gate_up_scale"], t["down"], t["down_scale"])
+ limit = limit_or_inf(layer)
+ if is_prefill and view.n is not None:
+ from freetoken.moe.fused_ds_fp4 import routed_experts_fp4_prefill
+
+ return routed_experts_fp4_prefill(x, topk_ids, topk_weights, *banks, limit, view.n)
+ from freetoken.moe.fused_ds_fp4 import routed_experts_fp4
+
+ return routed_experts_fp4(x, topk_ids, topk_weights, *banks, limit)
+
+
+class TritonGptossMxfp4MoEKernel(MoEKernel):
+ """gpt-oss MXFP4: bias, alpha / limit swiglu, interleaved gate|up rows, transposed split-K banks."""
+
+ name = "triton_gptoss"
+ cpu_format = "mxfp4_triton"
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ if not cfg.has_bias:
+ return "gpt-oss kernel needs the expert biases"
+ return None
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ i, h = cfg.local_intermediate, cfg.hidden
+ return {
+ "gate_up": BankSpec((h // 2, 2 * i), torch.uint8),
+ "gate_up_scale": BankSpec((h // GROUP, 2 * i), torch.uint8),
+ "gate_up_bias": BankSpec((2 * i,), torch.bfloat16),
+ "down": BankSpec((i // 2, h), torch.uint8),
+ "down_scale": BankSpec((i // GROUP, h), torch.uint8),
+ "down_bias": BankSpec((h,), torch.bfloat16),
+ }
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ # HF [e, N, K//32, 16] blocks / [e, N, K//32] scales -> split-K layout with N innermost
+ def blocks_t(t):
+ e, n = t.shape[0], t.shape[1]
+ return t.reshape(e, n, -1).permute(0, 2, 1)
+
+ out["gate_up"].copy_(blocks_t(fused_piece(pieces, "gate_up")))
+ out["gate_up_scale"].copy_(fused_piece(pieces, "gate_up_scale").permute(0, 2, 1))
+ out["gate_up_bias"].copy_(fused_piece(pieces, "gate_up_bias"))
+ out["down"].copy_(blocks_t(pieces["down"]))
+ out["down_scale"].copy_(pieces["down_scale"].permute(0, 2, 1))
+ out["down_bias"].copy_(pieces["down_bias"])
+ return {}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ from freetoken.moe.fused_mxfp4 import MXFP4_DECODE_MAX_TOKENS, run_mxfp4_prefill_experts_t, run_mxfp4_splitk_decode_experts
+
+ t = view.tensors
+ # the resident layer picks the split-K decode kernel by token count, not by phase
+ decode = x.shape[0] <= MXFP4_DECODE_MAX_TOKENS if is_resident(layer) else not is_prefill
+ run = run_mxfp4_splitk_decode_experts if decode else run_mxfp4_prefill_experts_t
+ return run(
+ x, topk_weights, topk_ids,
+ t["gate_up"], t["gate_up_scale"], t["gate_up_bias"], t["down"], t["down_scale"], t["down_bias"],
+ top_k=layer.top_k, hidden_act_alpha=float(layer.alpha), swiglu_limit=layer.limit,
+ )
+
+
+@register_method(QuantKind.MXFP4, LayerKind.MOE)
+class Mxfp4MoEMethod(MoEMethod):
+ candidates = (TritonMxfp4MoEKernel, TritonGptossMxfp4MoEKernel)
+
+ def create_weights(self, layer) -> None:
+ if not self.cfg.has_bias:
+ raise NotImplementedError("standard MXFP4 experts are served from the offload cache, not resident")
+ g = self.cfg
+ e, i, h = g.num_experts, g.local_intermediate, g.hidden
+ if h % GROUP:
+ raise ValueError(f"MXFP4 hidden size must be divisible by {GROUP}")
+ layer.gate_up_proj_blocks = torch.empty(e, 2 * i, h // GROUP, 16, dtype=torch.uint8)
+ layer.gate_up_proj_scales = torch.empty(e, 2 * i, h // GROUP, dtype=torch.uint8)
+ layer.gate_up_proj_bias = torch.empty(e, 2 * i, dtype=torch.bfloat16)
+ layer.down_proj_blocks = torch.empty(e, h, i // GROUP, 16, dtype=torch.uint8)
+ layer.down_proj_scales = torch.empty(e, h, i // GROUP, dtype=torch.uint8)
+ layer.down_proj_bias = torch.empty(e, h, dtype=torch.bfloat16)
+
+ def finalize(self, layer) -> None:
+ if getattr(layer, "gate_up_proj_blocks", None) is None:
+ return
+ from freetoken.moe.fused_mxfp4 import _transpose_mxfp4_for_decode
+
+ # one transposed copy serves prefill and decode; the HF blocks are freed so 120B fits
+ layer._gu_blocks_t, layer._gu_scales_t = _transpose_mxfp4_for_decode(layer.gate_up_proj_blocks, layer.gate_up_proj_scales)
+ layer._dn_blocks_t, layer._dn_scales_t = _transpose_mxfp4_for_decode(layer.down_proj_blocks, layer.down_proj_scales)
+ layer.gate_up_proj_blocks = None
+ layer.gate_up_proj_scales = None
+ layer.down_proj_blocks = None
+ layer.down_proj_scales = None
+ torch.cuda.empty_cache()
+
+ def resident_view(self, layer) -> ExpertView:
+ return ExpertView({
+ "gate_up": layer._gu_blocks_t, "gate_up_scale": layer._gu_scales_t, "gate_up_bias": layer.gate_up_proj_bias,
+ "down": layer._dn_blocks_t, "down_scale": layer._dn_scales_t, "down_bias": layer.down_proj_bias,
+ })
diff --git a/python/freetoken/layers/quantization/moe/mxfp8.py b/python/freetoken/layers/quantization/moe/mxfp8.py
new file mode 100644
index 00000000..f91d6dba
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/mxfp8.py
@@ -0,0 +1,21 @@
+"""MXFP8 experts: registered so the table is complete, not served yet."""
+
+from __future__ import annotations
+
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind
+from .base import ExpertView, MoEMethod
+
+
+@register_method(QuantKind.MXFP8, LayerKind.MOE)
+class Mxfp8MoEMethod(MoEMethod):
+ candidates = ()
+
+ def __init__(self, *args, **kwargs):
+ raise NotImplementedError("mxfp8 experts are not served yet")
+
+ def create_weights(self, layer) -> None:
+ raise NotImplementedError
+
+ def resident_view(self, layer) -> ExpertView:
+ raise NotImplementedError
diff --git a/python/freetoken/layers/quantization/moe/nvfp4.py b/python/freetoken/layers/quantization/moe/nvfp4.py
new file mode 100644
index 00000000..bb341cdf
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/nvfp4.py
@@ -0,0 +1,575 @@
+"""NVFP4 experts: served from the offload cache through the Triton inline-dequant kernels, vLLM's Marlin or flashinfer's b12x.
+
+FreeToken stores MoE experts as ModelOpt NVFP4 (packed e2m1 codes + fp8-e4m3 per-16
+block scales + per-tensor global scale) in pinned host banks and gathers the routed
+experts into a GPU slot cache. Rules every kernel here follows:
+
+* A kernel owns ``{load-time pack, forward}`` as a unit. Prefill and decode consume
+ the *same* bank layout -- there is never a second copy of the experts.
+* Every pack keeps the expert dimension outermost with per-expert blocks contiguous
+ and byte-identical in size to the native layout, so the banks are repacked *in
+ place* and the offload cache's slot gather (``copy_missing``) works unchanged.
+* The per-tensor global scales never enter the offload banks: they are tiny
+ ``[L*E]`` vectors kept resident on the GPU and gathered per forward call
+ (device-side, CUDA-graph safe).
+"""
+
+from __future__ import annotations
+
+import math
+
+import torch
+
+from freetoken.kernel import backend
+from freetoken.utils import init_logger
+
+from ..registry import LayerKind, register_method
+from ..scheme import NVFP4_GROUP as GROUP, QuantKind
+from .base import BankSpec, ExpertView, fused_global, fused_piece, gated_epilogue_reason, global_rows, limit_or_inf, MoEConfig, MoEKernel, MoEMethod
+
+logger = init_logger(__name__)
+
+FP8 = torch.float8_e4m3fn
+MARLIN_MAX_SLOTS = 992
+B12X_MIN_INTERMEDIATE = 1024
+
+
+class TritonNvfp4MoEKernel(MoEKernel):
+ """FreeToken's inline-dequant kernels over the native ModelOpt rows."""
+
+ name = "triton"
+ cpu_format = "nvfp4"
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ reason = self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=True, plain_silu_only=False)
+ if reason:
+ return reason
+ reason = gated_epilogue_reason(cfg)
+ return f"triton nvfp4 MoE kernel: {reason}" if reason else None
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ i, h = cfg.intermediate, cfg.hidden
+ return {
+ "gate_up": BankSpec((2 * i, h // 2), torch.uint8),
+ "gate_up_scale": BankSpec((2 * i, h // GROUP), FP8),
+ "gate_up_global": BankSpec((2 * i,), torch.float16),
+ "down": BankSpec((h, i // 2), torch.uint8),
+ "down_scale": BankSpec((h, i // GROUP), FP8),
+ "down_global": BankSpec((h,), torch.float16),
+ }
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ out["gate_up"].copy_(fused_piece(pieces, "gate_up"))
+ out["gate_up_scale"].copy_(fused_piece(pieces, "gate_up_scale"))
+ out["gate_up_global"].copy_(fused_global(pieces, cfg.intermediate))
+ out["down"].copy_(pieces["down"])
+ out["down_scale"].copy_(pieces["down_scale"])
+ out["down_global"].copy_(global_rows(pieces["down_global"], cfg.hidden))
+ return {}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ from freetoken.moe.fused_nvfp4 import fused_experts_decode_nvfp4_marlin, fused_experts_nvfp4
+
+ t = view.tensors
+ banks = (t["gate_up"], t["gate_up_scale"], t["gate_up_global"], t["down"], t["down_scale"], t["down_global"])
+ alpha, limit = float(layer.alpha), limit_or_inf(layer)
+ if is_prefill:
+ return fused_experts_nvfp4(x, *banks, topk_weights, topk_ids, view.n, layer.activation, layer.apply_router_weight_on_input, alpha, limit)
+ return fused_experts_decode_nvfp4_marlin(x, *banks, topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, alpha, limit)
+
+
+# ---------------------------------------------------------------------------
+# Marlin: vLLM's fused Marlin MoE (W4A16, dequant in kernel) over Marlin-tiled banks.
+# The Marlin kernels come from vLLM (not sgl-kernel) deliberately: vLLM ships the
+# NVFP4 instantiations in its AOT wheel together with the matching host-side layout
+# transforms, so the pair stays consistent by construction. sgl-kernel's AOT wheel
+# excludes the NVFP4 MoE Marlin kernels to cut wheel size (its python transforms pair
+# with sglang's *JIT* kernel tree, whose scale encoding has since diverged).
+#
+# The small layout-transform helpers are ported from sglang
+# (``srt/layers/quantization/marlin_utils{,_fp4}.py``, Apache-2.0); the kernels
+# themselves are imported, not vendored.
+# ---------------------------------------------------------------------------
+
+
+def _marlin_symbols_ok() -> bool:
+ """Probe the exact vLLM symbols the pack/forward paths below use.
+
+ ``find_spec`` only proves the top-level package exists; an incompatible donor
+ version would otherwise crash mid-init, after minutes of bank loading. Probing
+ at selection time degrades to triton before any bank is touched.
+ """
+ try:
+ from vllm import _custom_ops # noqa: F401
+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( # noqa: F401
+ fused_marlin_moe,
+ )
+ from vllm.model_executor.layers.quantization.utils.marlin_utils import ( # noqa: F401
+ marlin_permute_scales,
+ )
+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401
+ nvfp4_marlin_process_global_scale,
+ nvfp4_marlin_process_scales,
+ )
+ from vllm.scalar_type import scalar_types # noqa: F401
+ except Exception as exc:
+ logger.warning(
+ f"NVFP4 marlin backend is installed but unusable ({exc!r}); "
+ "falling back to the Triton inline-dequant backend"
+ )
+ return False
+ return True
+
+
+# Marlin pack: mirrors vLLM's prepare_moe_fp4_layer_for_marlin so the host prep always matches the AOT op.
+def _marlin_pack_proj(
+ packed: torch.Tensor,
+ scale: torch.Tensor,
+ row_global: torch.Tensor,
+ *,
+ size_k: int,
+ size_n: int,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Pack one expert projection (on GPU) into the Marlin layout.
+
+ ``packed`` ``[N, K//2]`` uint8 e2m1 pairs, ``scale`` ``[N, K//16]`` e4m3,
+ ``row_global`` ``[N]`` fp16 per-row global (rows of a merged gate_up may carry
+ w1's and w3's different globals). Returns ``(qweight [K//16, 2N] int32,
+ scales [K//16, N] e4m3-coded, global scalar bf16)``.
+
+ Marlin takes one global scalar per projection, so when the row globals differ
+ the ratio is folded into the fp16 block scales before they are re-encoded
+ (exact when all rows share one global, <=1 scale ulp otherwise).
+ """
+ from vllm import _custom_ops as ops
+ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
+ marlin_permute_scales,
+ )
+ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
+ nvfp4_marlin_process_global_scale,
+ nvfp4_marlin_process_scales,
+ )
+
+ assert size_n % 64 == 0, f"Marlin requires N % 64 == 0, got {size_n}"
+ qweight = ops.gptq_marlin_repack(
+ b_q_weight=packed.view(torch.int32).T.contiguous(),
+ perm=torch.empty(0, dtype=torch.int, device=packed.device),
+ size_k=size_k,
+ size_n=size_n,
+ num_bits=4,
+ )
+
+ g = row_global.to(torch.float32)
+ g_max = g.max()
+ s = scale.to(torch.bfloat16)
+ if not torch.all(g == g_max):
+ s = (s.float() * (g / g_max).unsqueeze(1)).to(torch.bfloat16)
+ s = marlin_permute_scales(s.T, size_k=size_k, size_n=size_n, group_size=16)
+ s = nvfp4_marlin_process_scales(s)
+ if isinstance(s, tuple): # newer vLLM returns (scales, scale_factor)
+ s, factor = s
+ g_max = g_max / factor
+ g_out = nvfp4_marlin_process_global_scale(g_max.to(torch.bfloat16).reshape(1))
+ return qweight, s, g_out
+
+
+def marlin_fused_experts(
+ hidden_states: torch.Tensor,
+ gate_up_q: torch.Tensor,
+ gate_up_s: torch.Tensor,
+ gate_up_alpha: torch.Tensor,
+ down_q: torch.Tensor,
+ down_s: torch.Tensor,
+ down_alpha: torch.Tensor,
+ topk_weights: torch.Tensor,
+ topk_ids: torch.Tensor,
+ activation: str,
+ apply_router_weight_on_input: bool,
+) -> torch.Tensor:
+ """Marlin W4A16 fused MoE (two grouped GEMMs + activation + reduce).
+
+ Two calling regimes share this entry: decode passes the full ``[S]`` slot cache
+ with ``topk_ids`` rewritten to slot ids; full-layer prefill passes banks whose
+ position == expert id (the materialized ``[:E]`` slot view or the overlap double
+ buffer views), so the raw routing ids arrive unmapped. The ``*_alpha`` vectors
+ are the matching per-row global scales in both regimes.
+ vLLM's implementation is device-side only (no host syncs), so the decode call is
+ CUDA-graph capturable.
+ """
+ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe
+ from vllm.scalar_type import scalar_types
+
+ assert activation == "silu", "Marlin NVFP4 backend supports gated silu only"
+ return fused_marlin_moe(
+ hidden_states,
+ gate_up_q,
+ down_q,
+ None, # bias1
+ None, # bias2
+ gate_up_s,
+ down_s,
+ gating_output=None,
+ topk_weights=topk_weights,
+ topk_ids=topk_ids,
+ quant_type_id=scalar_types.float4_e2m1f.id,
+ apply_router_weight_on_input=apply_router_weight_on_input,
+ global_num_experts=gate_up_q.size(0),
+ activation=activation,
+ global_scale1=gate_up_alpha,
+ global_scale2=down_alpha,
+ )
+
+
+class MarlinNvfp4MoEKernel(MoEKernel):
+ """vLLM's fused Marlin MoE over pre-tiled banks; the globals fold into GPU-resident alphas."""
+
+ name = "marlin"
+ max_slots = MARLIN_MAX_SLOTS
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ if not backend.is_vllm_installed():
+ return "vLLM is not installed"
+ reason = self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=False, plain_silu_only=True)
+ if reason:
+ return reason
+ if not _marlin_symbols_ok():
+ return "vLLM Marlin donor symbols are unusable"
+ return None
+
+ def worth_it(self, cfg: MoEConfig) -> bool:
+ return (8, 0) <= backend.device_capability() < (10, 0)
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ i, h = cfg.intermediate, cfg.hidden
+ return {
+ "gate_up": BankSpec((h // GROUP, 4 * i), torch.int32),
+ "gate_up_scale": BankSpec((h // GROUP, 2 * i), FP8),
+ "down": BankSpec((i // GROUP, 2 * h), torch.int32),
+ "down_scale": BankSpec((i // GROUP, h), FP8),
+ "gate_up_alpha": BankSpec((), torch.bfloat16, resident=True),
+ "down_alpha": BankSpec((), torch.bfloat16, resident=True),
+ }
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ i, h = cfg.intermediate, cfg.hidden
+ device = torch.device("cuda")
+ gu, gus, gug = fused_piece(pieces, "gate_up"), fused_piece(pieces, "gate_up_scale"), fused_global(pieces, i)
+ dn, dns, dng = pieces["down"], pieces["down_scale"], global_rows(pieces["down_global"], h)
+ e = gu.shape[0]
+ gate_up_alpha = torch.empty(e, dtype=torch.bfloat16, device=device)
+ down_alpha = torch.empty(e, dtype=torch.bfloat16, device=device)
+ for k in range(e):
+ qw, sc, al = _marlin_pack_proj(gu[k].to(device), gus[k].to(device), gug[k].to(device), size_k=h, size_n=2 * i)
+ out["gate_up"][k].copy_(qw)
+ out["gate_up_scale"][k].copy_(sc.view(FP8))
+ gate_up_alpha[k] = al[0]
+ qw, sc, al = _marlin_pack_proj(dn[k].to(device), dns[k].to(device), dng[k].to(device), size_k=i, size_n=h)
+ out["down"][k].copy_(qw)
+ out["down_scale"][k].copy_(sc.view(FP8))
+ down_alpha[k] = al[0]
+ torch.cuda.synchronize(device)
+ return {"gate_up_alpha": gate_up_alpha, "down_alpha": down_alpha}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ t = view.tensors
+ assert view.alphas is not None
+ return marlin_fused_experts(x, t["gate_up"], t["gate_up_scale"], view.alphas[0], t["down"], t["down_scale"], view.alphas[1], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input)
+
+
+# ---------------------------------------------------------------------------
+# b12x: flashinfer's SM12x CuTe-DSL fused MoE (W4A16) over b12x-packed banks.
+# The pack (``prepare_w4a16_packed_weights``) runs on any CUDA build, but the fused-MoE
+# kernel needs sm_120/121 AND a CUDA>=13 *driver* (it JIT-compiles PTX at runtime). The
+# kernel object gates on that, so the forward is only reached on hardware that can run it.
+#
+# Layout contract (verified against flashinfer 0.6.12 prepare_w4a16_packed_weights):
+# * prepare() takes the block scales already *swizzled* (it calls unswizzle_expert_
+# scales internally), so the native row-major fp8 scales are run through
+# nvfp4_block_scale_interleave first.
+# * prepare() reorders the w13 rows with reorder_w13_to_gate_up == cat([second, first]),
+# i.e. it expects the merged proj as [up, gate] and emits [gate, up]. FreeToken stores
+# [gate, up], so the halves are swapped before prepare() to come out [gate, up] again
+# (the kernel computes silu(first) * second == silu(gate) * up).
+# * modelopt globals are passed straight through (no 1/g inversion).
+# The fused forward reconstructs the W4A16PackedWeights from the banks and uses the
+# prepared-weights launch (_launch_sm120_w4a16_moe) directly, so weights are prepared
+# exactly once at load time (b12x_fused_moe would re-prepare + ptr-cache every call).
+# ---------------------------------------------------------------------------
+
+
+def _b12x_symbols_ok() -> bool:
+ """Probe the exact flashinfer symbols the pack/forward paths below use.
+
+ ``find_spec`` only proves the top-level package exists; an incompatible donor
+ version would otherwise crash mid-init, after minutes of bank loading. Probing
+ at selection time degrades to triton before any bank is touched.
+ """
+ try:
+ # Probe exactly what the pack/forward use: the row-major->swizzled scale
+ # transform, the W4A16 prepare, and the prepared-weights launch (which also
+ # transitively imports the CuTe-DSL kernel + cutlass).
+ from flashinfer import nvfp4_block_scale_interleave # noqa: F401
+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import ( # noqa: F401
+ _launch_sm120_w4a16_moe,
+ )
+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import ( # noqa: F401
+ W4A16PackedWeights,
+ _make_workspace,
+ prepare_w4a16_packed_weights,
+ )
+ except Exception as exc:
+ logger.warning(
+ f"NVFP4 b12x backend is installed but unusable ({exc!r}); "
+ "falling back to the Triton inline-dequant backend"
+ )
+ return False
+ return True
+
+
+def _flashinfer_cuda_major() -> int | None:
+ """flashinfer's own toolkit-CUDA major; None if it can't be determined. Only a
+ fallback proxy when the driver version is unavailable -- the b12x kernel is JIT
+ PTX-compiled through the driver, so the toolkit major is *not* what gates it."""
+ try:
+ from flashinfer.jit.cpp_ext import get_cuda_version
+
+ return int(get_cuda_version().major)
+ except Exception:
+ return None
+
+
+def _b12x_unusable_reason(cc: tuple[int, int]) -> str | None:
+ """None if the flashinfer b12x decode/prefill kernel can actually run on this device,
+ else a human-readable reason. The b12x *pack* works anywhere, but the SM12x CuTe-DSL
+ kernel itself requires sm_120+ and a CUDA>=13 *driver* (it JIT-compiles PTX through the
+ driver), so selection must check the runtime here -- not just that flashinfer imports --
+ or the model loads in the b12x layout and then crashes on the first decode."""
+ import importlib.util
+
+ if cc < (12, 0):
+ return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"
+ if importlib.util.find_spec("flashinfer") is None:
+ return "flashinfer is not installed"
+ from freetoken.kernel.backend import driver_cuda_version
+
+ drv = driver_cuda_version()
+ if drv is not None:
+ if drv < 13000:
+ return (
+ "b12x fused MoE requires a CUDA>=13 driver "
+ f"(driver supports CUDA {drv // 1000}.{(drv % 1000) // 10})"
+ )
+ else:
+ # Driver version undetermined: fall back to flashinfer's toolkit major as a
+ # conservative proxy rather than risk loading the b12x layout and crashing.
+ major = _flashinfer_cuda_major()
+ if major is not None and major < 13:
+ return (
+ "b12x fused MoE requires CUDA>=13 (driver undetermined; "
+ f"flashinfer toolkit is CUDA {major}.x)"
+ )
+ if not _b12x_symbols_ok():
+ return "flashinfer b12x donor symbols are unusable"
+ return None
+
+
+def _b12x_swizzle_block_scales(scale: torch.Tensor) -> torch.Tensor:
+ """Row-major per-16 fp8 block scales ``[n, N, K//16]`` -> flashinfer's expert-leading
+ *swizzled* storage, which is what :func:`prepare_w4a16_packed_weights` expects on input
+ (it unswizzles internally). The 128x4 interleave is per-expert; ``N`` is a multiple of
+ 128 and ``K//16`` a multiple of 4 for the supported models, so the shape is preserved."""
+ from flashinfer import nvfp4_block_scale_interleave
+
+ pieces = [
+ nvfp4_block_scale_interleave(scale[e].view(torch.uint8).contiguous())
+ for e in range(scale.shape[0])
+ ]
+ return torch.stack(pieces, dim=0).view(torch.float8_e4m3fn)
+
+
+# Per-device scratch (sized sms*4+2) that run_w4a16_moe reads off the prepared object;
+# tiny and shape-independent, so one per device is reused across all layers/decodes.
+_B12X_WORKSPACE: dict[torch.device, torch.Tensor] = {}
+
+
+def _b12x_small_workspace(device: torch.device) -> torch.Tensor:
+ ws = _B12X_WORKSPACE.get(device)
+ if ws is None:
+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (
+ _make_workspace,
+ )
+
+ ws = _make_workspace(device, max_blocks_per_sm=4)
+ _B12X_WORKSPACE[device] = ws
+ return ws
+
+
+def b12x_fused_experts(
+ hidden_states: torch.Tensor,
+ gate_up_q: torch.Tensor,
+ gate_up_s: torch.Tensor,
+ gate_up_alpha: torch.Tensor,
+ down_q: torch.Tensor,
+ down_s: torch.Tensor,
+ down_alpha: torch.Tensor,
+ topk_weights: torch.Tensor,
+ topk_ids: torch.Tensor,
+ activation: str,
+ apply_router_weight_on_input: bool,
+) -> torch.Tensor:
+ """SM12x W4A16 fused MoE over cache slots (same calling convention as
+ :func:`marlin_fused_experts`). Requires sm_120/121 + a CUDA>=13 driver; reached only
+ when the b12x kernel object has confirmed the runtime supports it.
+
+ The banks already hold flashinfer's prepared (tiled) layout from
+ the b12x kernel object's pack, so this wraps them back into a
+ ``W4A16PackedWeights`` and calls the prepared-weights launch directly -- the public
+ ``b12x_fused_moe`` would re-prepare the raw modelopt weights (and ptr-cache them) on
+ every call, which both costs a prepare per step and breaks for the offload cache,
+ whose slot contents move between calls.
+
+ CUDA-graph note: the launch resolves a (shape-keyed, module-cached) scratch workspace
+ on first use and flashinfer raises if that happens *during* capture, so the decode
+ path must be warmed once eagerly before graph capture (FreeToken already does)."""
+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import (
+ _launch_sm120_w4a16_moe,
+ )
+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (
+ W4A16PackedWeights,
+ )
+
+ assert activation == "silu", "b12x backend supports gated silu only"
+ assert not apply_router_weight_on_input
+ num_experts = gate_up_q.size(0)
+ hidden_size = hidden_states.size(-1)
+ # down bank is the prepared w2 == [E, K_tiles, ...] with K_tiles == intermediate//16.
+ intermediate_size = down_q.size(1) * 16
+ prepared = W4A16PackedWeights(
+ w13=gate_up_q,
+ w13_scale=gate_up_s,
+ w13_global_scale=gate_up_alpha,
+ w2=down_q,
+ w2_scale=down_s,
+ w2_global_scale=down_alpha,
+ workspace=_b12x_small_workspace(hidden_states.device),
+ hidden_size=hidden_size,
+ intermediate_size=intermediate_size,
+ num_experts=num_experts,
+ is_gated=True,
+ params_dtype=hidden_states.dtype,
+ source_format="modelopt",
+ )
+ out = torch.empty(
+ hidden_states.size(0), hidden_size, dtype=hidden_states.dtype, device=hidden_states.device
+ )
+ _launch_sm120_w4a16_moe(
+ a=hidden_states,
+ topk_ids=topk_ids,
+ topk_weights=topk_weights,
+ w1_weight=gate_up_q,
+ w1_weight_sf=gate_up_s,
+ w1_alpha=gate_up_alpha,
+ w2_weight=down_q,
+ w2_weight_sf=down_s,
+ w2_alpha=down_alpha,
+ num_experts=num_experts,
+ top_k=topk_ids.size(1),
+ num_local_experts=num_experts,
+ scatter_output=out,
+ activation="silu",
+ source_format="modelopt",
+ _prepared_weights=prepared,
+ )
+ return out
+
+
+class B12xNvfp4MoEKernel(MoEKernel):
+ """flashinfer's SM12x CuTe-DSL W4A16 MoE; the packed blocks reuse the native bank bytes."""
+
+ name = "b12x"
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ cc = backend.device_capability()
+ if cc < (12, 0):
+ return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"
+ if not backend.is_flashinfer_installed():
+ return "flashinfer is not installed"
+ reason = self._common_reject(cfg, resident_ok=False, tp_ok=False, cpu_ok=False, plain_silu_only=True)
+ if reason:
+ return reason
+ return _b12x_unusable_reason(cc)
+
+ def worth_it(self, cfg: MoEConfig) -> bool:
+ # NOTE: never auto-selected. flashinfer's cute launcher indexes each bank with int32 element offsets, so the GPU slot cache is capped at (2^31 - 1) / elements-per-slot (about 1000 slots for GLM-5.3-Flash's 12960 experts); until flashinfer lifts that, b12x stays behind triton in the table; the selection does not weigh the cap, cache-auto clamps to slot_limit() and OffloadMoeCache refuses a larger cache.
+ return cfg.intermediate >= B12X_MIN_INTERMEDIATE
+
+ def slot_limit(self, cfg: MoEConfig) -> int | None:
+ # the cute launcher indexes each bank with int32 element offsets
+ per_slot = max(math.prod(spec.shape) for spec in self.layout(cfg).values() if not spec.resident)
+ return (2**31 - 1) // per_slot
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ # flashinfer's prepared tiles: the Marlin shapes, byte-identical to the native rows
+ i, h = cfg.intermediate, cfg.hidden
+ return {
+ "gate_up": BankSpec((h // GROUP, 4 * i), torch.int32),
+ "gate_up_scale": BankSpec((h // GROUP, 2 * i), FP8),
+ "down": BankSpec((i // GROUP, 2 * h), torch.int32),
+ "down_scale": BankSpec((i // GROUP, h), FP8),
+ "gate_up_alpha": BankSpec((), torch.float32, resident=True),
+ "down_alpha": BankSpec((), torch.float32, resident=True),
+ }
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import prepare_w4a16_packed_weights
+
+ i, h = cfg.intermediate, cfg.hidden
+ device = torch.device("cuda")
+ gu = fused_piece(pieces, "gate_up").to(device)
+ gug = fused_global(pieces, i).to(device).float()
+ g_max = gug.max(dim=1, keepdim=True).values
+ gus = fused_piece(pieces, "gate_up_scale").to(device).to(torch.float16)
+ ratio = gug / g_max
+ if not torch.all(ratio == 1.0):
+ gus = (gus.float() * ratio.unsqueeze(-1)).to(torch.float16)
+ dng = global_rows(pieces["down_global"], h).to(device)
+ if not torch.all(dng == dng[:, :1]):
+ raise ValueError("b12x pack requires a row-constant per-expert down_proj global scale")
+ # b12x wants up|gate row order
+ gu = torch.cat([gu[:, i:], gu[:, :i]], dim=1).contiguous()
+ gus = torch.cat([gus[:, i:], gus[:, :i]], dim=1).contiguous()
+ prepared = prepare_w4a16_packed_weights(
+ gu,
+ _b12x_swizzle_block_scales(gus.to(FP8)),
+ g_max.squeeze(1),
+ pieces["down"].to(device),
+ _b12x_swizzle_block_scales(pieces["down_scale"].to(device)),
+ dng[:, 0].float(),
+ activation="silu",
+ params_dtype=torch.bfloat16,
+ source_format="modelopt",
+ )
+ e = gu.shape[0]
+ for role, t in (("gate_up", prepared.w13), ("gate_up_scale", prepared.w13_scale), ("down", prepared.w2), ("down_scale", prepared.w2_scale)):
+ bank = out[role]
+ assert t.shape[1:] == bank.shape[1:] and t.dtype == bank.dtype, (role, t.shape, t.dtype, bank.shape, bank.dtype)
+ bank.copy_(t[:e])
+ return {"gate_up_alpha": prepared.w13_global_scale[:e].float(), "down_alpha": prepared.w2_global_scale[:e].float()}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ t = view.tensors
+ assert view.alphas is not None
+ return b12x_fused_experts(x, t["gate_up"], t["gate_up_scale"], view.alphas[0], t["down"], t["down_scale"], view.alphas[1], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input)
+
+
+@register_method(QuantKind.NVFP4, LayerKind.MOE)
+class Nvfp4MoEMethod(MoEMethod):
+ candidates = (TritonNvfp4MoEKernel, MarlinNvfp4MoEKernel, B12xNvfp4MoEKernel)
+
+ def create_weights(self, layer) -> None:
+ raise NotImplementedError("NVFP4 experts are served from the offload cache, not resident")
+
+ def resident_view(self, layer) -> ExpertView:
+ raise NotImplementedError("NVFP4 experts are not resident")
diff --git a/python/freetoken/layers/quantization/moe/unquantized.py b/python/freetoken/layers/quantization/moe/unquantized.py
new file mode 100644
index 00000000..ef435686
--- /dev/null
+++ b/python/freetoken/layers/quantization/moe/unquantized.py
@@ -0,0 +1,48 @@
+"""bf16 experts: the fused Triton MoE over gate_up / down banks."""
+
+from __future__ import annotations
+
+import torch
+
+from ..registry import LayerKind, register_method
+from ..scheme import QuantKind
+from .base import BankSpec, ExpertView, fused_piece, gated_epilogue_reason, is_resident, limit_or_inf, MoEConfig, MoEKernel, MoEMethod
+
+
+class FusedMoEKernel(MoEKernel):
+ name = "fused"
+ cpu_format = "bf16"
+
+ def unusable_reason(self, cfg: MoEConfig) -> str | None:
+ reason = gated_epilogue_reason(cfg)
+ return f"bf16 fused MoE: {reason}" if reason else None
+
+ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
+ i = cfg.local_intermediate
+ return {"gate_up": BankSpec((2 * i, cfg.hidden), torch.bfloat16), "down": BankSpec((cfg.hidden, i), torch.bfloat16)}
+
+ def pack(self, pieces, cfg: MoEConfig, out):
+ out["gate_up"].copy_(fused_piece(pieces, "gate_up"))
+ out["down"].copy_(pieces["down"])
+ return {}
+
+ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
+ from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl
+
+ # the resident layer runs the in-place prefill kernel for both phases
+ impl = fused_experts_impl if is_prefill or is_resident(layer) else fused_experts_decode_impl
+ return impl(x, view.tensors["gate_up"], view.tensors["down"], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, float(layer.alpha), limit_or_inf(layer))
+
+
+@register_method(QuantKind.NONE, LayerKind.MOE)
+class UnquantizedMoEMethod(MoEMethod):
+ candidates = (FusedMoEKernel,)
+
+ def create_weights(self, layer) -> None:
+ g = self.cfg
+ i = g.local_intermediate
+ layer.gate_up_proj = torch.empty(g.num_experts, 2 * i, g.hidden)
+ layer.down_proj = torch.empty(g.num_experts, g.hidden, i)
+
+ def resident_view(self, layer) -> ExpertView:
+ return ExpertView({"gate_up": layer.gate_up_proj, "down": layer.down_proj})
diff --git a/python/freetoken/layers/quantization/names.py b/python/freetoken/layers/quantization/names.py
new file mode 100644
index 00000000..c7a79743
--- /dev/null
+++ b/python/freetoken/layers/quantization/names.py
@@ -0,0 +1,92 @@
+"""Module-name matching for the dialect configs and the attribute -> checkpoint name map."""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from dataclasses import dataclass
+from typing import Callable, Iterator
+
+Matcher = Callable[[str], bool]
+
+
+def ancestors(name: str) -> Iterator[str]:
+ while name:
+ yield name
+ name, _, _ = name.rpartition(".")
+
+
+_GLOB_CHARS = re.compile(r"[*?\[]")
+
+
+def name_set(patterns: tuple[str, ...]) -> Matcher:
+ """Membership of a module or any parent in a list of literal names and glob patterns."""
+ literal = frozenset(p for p in patterns if not _GLOB_CHARS.search(p))
+ globs = [p for p in patterns if _GLOB_CHARS.search(p)]
+ glob = re.compile("|".join(fnmatch.translate(p) for p in globs)) if globs else None
+
+ def hit(name: str) -> bool:
+ return any(a in literal or (glob is not None and glob.match(a)) for a in ancestors(name))
+
+ return hit
+
+
+def substr_set(patterns: tuple[str, ...]) -> Matcher:
+ """transformers' fp8 integration skips a module when ``entry + "."`` occurs in ``name + "."``."""
+ rx = re.compile("|".join(re.escape(p) + r"\." for p in patterns)) if patterns else None
+ return lambda name: rx is not None and rx.search(name + ".") is not None
+
+
+def ct_set(patterns: tuple[str, ...], *, class_names: bool) -> Matcher:
+ """compressed-tensors ``targets`` / ``ignore``: module names, ``re:`` regexes, or the class name Linear."""
+ names = name_set(tuple(p for p in patterns if not p.startswith("re:") and p != "Linear"))
+ regexes = [p[3:] for p in patterns if p.startswith("re:")]
+ rx = re.compile("|".join(f"(?:{r})" for r in regexes)) if regexes else None
+ any_linear = class_names and "Linear" in patterns
+ return lambda name: any_linear or names(name) or (rx is not None and rx.match(name) is not None)
+
+
+_ROUTED_EXPERT = re.compile(r"\.experts\.\d+(\.|$)")
+
+
+def is_routed_expert(name: str) -> bool:
+ return bool(_ROUTED_EXPERT.search(name)) or name.endswith(".experts")
+
+
+@dataclass(frozen=True)
+class NameMap:
+ """FreeToken attribute path -> checkpoint module name(s), applied to every scheme_for query.
+
+ ``roots`` rename a leading path (``model.layers`` -> ``model.language_model.layers``);
+ ``segments`` rename an inner path (``feed_forward.shared_mlp`` -> ``mlp``);
+ ``packed`` expands a fused leaf into its source leaves (``qkv_proj`` -> q/k/v_proj).
+ """
+
+ roots: tuple[tuple[str, str], ...] = ()
+ segments: tuple[tuple[str, str], ...] = ()
+ packed: tuple[tuple[str, tuple[str, ...]], ...] = ()
+
+ def to_checkpoint(self, prefix: str) -> tuple[str, ...]:
+ for attr_root, ckpt_root in sorted(self.roots, key=lambda r: -len(r[0])):
+ if prefix == attr_root or prefix.startswith(attr_root + "."):
+ prefix = ckpt_root + prefix[len(attr_root):]
+ break
+ for attr_seg, ckpt_seg in self.segments:
+ prefix = _replace_segment(prefix, attr_seg, ckpt_seg)
+ head, _, leaf = prefix.rpartition(".")
+ sources = dict(self.packed).get(leaf)
+ if not sources:
+ return (prefix,)
+ return tuple(f"{head}.{s}" if head else s for s in sources)
+
+
+def _replace_segment(prefix: str, attr_seg: str, ckpt_seg: str) -> str:
+ parts, seg, out, i = prefix.split("."), attr_seg.split("."), [], 0
+ while i < len(parts):
+ if parts[i : i + len(seg)] == seg:
+ out.extend(ckpt_seg.split(".") if ckpt_seg else [])
+ i += len(seg)
+ else:
+ out.append(parts[i])
+ i += 1
+ return ".".join(out)
diff --git a/python/freetoken/layers/quantization/quant_backend.py b/python/freetoken/layers/quantization/quant_backend.py
new file mode 100644
index 00000000..1eebada6
--- /dev/null
+++ b/python/freetoken/layers/quantization/quant_backend.py
@@ -0,0 +1,80 @@
+"""The --quant-backend value: the kernel name wanted for each table, ``layer=name`` for every kind of a layer type, ``layer.kind=name`` for one."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .registry import LayerKind, method_class, methods_for
+from .scheme import QuantKind
+
+Table = tuple[LayerKind, "QuantKind | None"]
+
+
+def _kernel_names(layer_kind: LayerKind, kind: QuantKind | None) -> set[str]:
+ if kind is not None:
+ try:
+ return {c.name for c in method_class(kind, layer_kind).candidates}
+ except NotImplementedError:
+ return set()
+ return {c.name for cls in methods_for(layer_kind).values() for c in cls.candidates}
+
+
+def _parse_table(key: str) -> Table | None:
+ layer, _, kind = key.partition(".")
+ try:
+ return LayerKind(layer.strip()), QuantKind(kind.strip()) if kind else None
+ except ValueError:
+ return None
+
+
+def _table_text(table: Table) -> str:
+ layer, kind = table
+ return f"{layer}.{kind}" if kind is not None else str(layer)
+
+
+@dataclass(frozen=True)
+class QuantBackend:
+ items: tuple[tuple[Table, str], ...] = ()
+
+ @classmethod
+ def parse(cls, text: str | None) -> "QuantBackend":
+ items: dict[Table, str] = {}
+ for raw in (text or "").split(","):
+ item = raw.strip()
+ if not item:
+ continue
+ key, sep, name = item.partition("=")
+ table = _parse_table(key)
+ name = name.strip().lower()
+ if not sep or not name or table is None:
+ layers = [l.value for l in LayerKind]
+ raise ValueError(f"bad --quant-backend item {item!r}; expected layer[.kind]=name with layer in {layers}")
+ known = _kernel_names(*table)
+ if name not in known:
+ raise ValueError(f"--quant-backend {key}: no kernel {name!r}; known: {sorted(known)}")
+ items[table] = name
+ return cls(tuple(sorted(items.items(), key=lambda kv: _table_text(kv[0]))))
+
+ def select(self, layer_kind: LayerKind, kind: QuantKind) -> str:
+ """The kernel name for one table: its own entry, else the layer-wide entry when the table lists that kernel, else ``"auto"``."""
+ entries = dict(self.items)
+ exact = entries.get((layer_kind, kind))
+ if exact:
+ return exact
+ broad = entries.get((layer_kind, None))
+ if broad and broad in _kernel_names(layer_kind, kind):
+ return broad
+ return "auto"
+
+
+_QUANT_BACKEND = QuantBackend()
+
+
+def set_quant_backend(backend: QuantBackend) -> None:
+ """Install the kernel requests every layer built afterwards consults; the engine sets them once before create_model."""
+ global _QUANT_BACKEND
+ _QUANT_BACKEND = backend
+
+
+def get_quant_backend() -> QuantBackend:
+ return _QUANT_BACKEND
diff --git a/python/freetoken/layers/quantization/registry.py b/python/freetoken/layers/quantization/registry.py
new file mode 100644
index 00000000..2d13cf50
--- /dev/null
+++ b/python/freetoken/layers/quantization/registry.py
@@ -0,0 +1,59 @@
+"""Two registries: checkpoint dialects, and (kind, layer kind) -> Method."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from .scheme import QuantKind
+
+
+class LayerKind(Enum):
+ LINEAR = "linear"
+ MOE = "moe"
+
+ def __str__(self) -> str:
+ return self.value
+
+
+_DIALECTS: dict[str, type] = {}
+_METHODS: dict[tuple[QuantKind, LayerKind], type] = {}
+
+
+def register_dialect(cls: type) -> type:
+ name = cls.dialect
+ if name in _DIALECTS:
+ raise ValueError(f"quant dialect {name!r} already registered: {_DIALECTS[name].__name__}")
+ _DIALECTS[name] = cls
+ return cls
+
+
+def dialects() -> tuple[type, ...]:
+ return tuple(_DIALECTS.values())
+
+
+def register_method(kind: QuantKind, layer_kind: LayerKind):
+ if not isinstance(kind, QuantKind) or not isinstance(layer_kind, LayerKind):
+ raise TypeError(f"cannot register quant method for {layer_kind!r}.{kind!r}")
+
+ def deco(cls: type) -> type:
+ key = (kind, layer_kind)
+ if key in _METHODS:
+ raise ValueError(f"quant method for {layer_kind}.{kind} already registered: {_METHODS[key].__name__}")
+ cls.kind = kind
+ cls.layer_kind = layer_kind
+ _METHODS[key] = cls
+ return cls
+
+ return deco
+
+
+def method_class(kind: QuantKind, layer_kind: LayerKind) -> type[Any]:
+ try:
+ return _METHODS[(kind, layer_kind)]
+ except KeyError:
+ raise NotImplementedError(f"no quant method for {layer_kind}.{kind}") from None
+
+
+def methods_for(layer_kind: LayerKind) -> dict[QuantKind, type]:
+ return {kind: cls for (kind, lk), cls in _METHODS.items() if lk == layer_kind}
diff --git a/python/freetoken/layers/quantization/scheme.py b/python/freetoken/layers/quantization/scheme.py
new file mode 100644
index 00000000..668eb6fb
--- /dev/null
+++ b/python/freetoken/layers/quantization/scheme.py
@@ -0,0 +1,97 @@
+"""QuantScheme: the storage description and role set of one quantized weight, plus the per-kind builders."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Iterable
+
+import torch
+
+# -1 selects the whole dimension: (-1, -1) one scale per tensor, (1, -1) one per output row, (1, 16) one per 16 input elements, (128, 128) a 2-D block
+GroupShape = tuple[int, int]
+
+class QuantKind(Enum):
+ """The quantization kinds; one Method per (kind, layer kind)."""
+
+ NONE = "none"
+ FP8_TENSOR = "fp8_tensor"
+ FP8_BLOCK = "fp8_block"
+ MXFP8 = "mxfp8"
+ NVFP4 = "nvfp4"
+ MXFP4 = "mxfp4"
+
+ def __str__(self) -> str:
+ return self.value
+
+
+FP8_BLOCK = 128
+NVFP4_GROUP = 16
+MX_GROUP = 32
+
+
+@dataclass(frozen=True)
+class WeightDesc:
+ elem: str
+ group: GroupShape | None
+ scale: str | None
+
+
+
+
+@dataclass(frozen=True, eq=False)
+class QuantScheme:
+ """``roles`` are the canonical tensor names the kind's Method declares; which checkpoint tensors feed them is the dialect Config's business."""
+
+ kind: QuantKind
+ weight: WeightDesc
+ roles: frozenset[str]
+
+ def __init__(self, kind: QuantKind, weight: WeightDesc, roles: Iterable[str]):
+ if not isinstance(kind, QuantKind):
+ raise TypeError(f"kind must be a QuantKind, got {kind!r}")
+ object.__setattr__(self, "kind", kind)
+ object.__setattr__(self, "weight", weight)
+ object.__setattr__(self, "roles", frozenset(roles))
+
+ def _key(self):
+ return (self.kind, self.weight, self.roles)
+
+ def __eq__(self, other: object) -> bool:
+ return isinstance(other, QuantScheme) and self._key() == other._key()
+
+ def __hash__(self) -> int:
+ return hash(self._key())
+
+ def has(self, role: str) -> bool:
+ return role in self.roles
+
+ def __repr__(self) -> str:
+ return f"QuantScheme({self.kind}, {self.weight}, roles={sorted(self.roles)})"
+
+
+# ---------------------------------------------------------------------------
+# Builders: one per kind; the dialect configs pick the variant their checkpoints carry
+# ---------------------------------------------------------------------------
+
+
+def fp8_tensor_scheme(scale: str, *, per_row: bool = False, input_scale: bool = False) -> QuantScheme:
+ roles = {"weight", "weight_scale"} | ({"input_scale"} if input_scale else set())
+ return QuantScheme(QuantKind.FP8_TENSOR, WeightDesc("e4m3", (1, -1) if per_row else (-1, -1), scale), roles)
+
+
+def fp8_block_scheme(scale: str) -> QuantScheme:
+ return QuantScheme(QuantKind.FP8_BLOCK, WeightDesc("e4m3", (FP8_BLOCK, FP8_BLOCK), scale), {"weight", "weight_scale_inv"})
+
+
+def mxfp8_scheme() -> QuantScheme:
+ return QuantScheme(QuantKind.MXFP8, WeightDesc("e4m3", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale_inv"})
+
+
+def nvfp4_scheme(*, input_scale: bool) -> QuantScheme:
+ roles = {"weight", "weight_scale", "weight_global"} | ({"input_scale"} if input_scale else set())
+ return QuantScheme(QuantKind.NVFP4, WeightDesc("e2m1", (1, NVFP4_GROUP), "e4m3"), roles)
+
+
+def mxfp4_scheme() -> QuantScheme:
+ return QuantScheme(QuantKind.MXFP4, WeightDesc("e2m1", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale"})
diff --git a/python/freetoken/models/__init__.py b/python/freetoken/models/__init__.py
index 2167d089..532b1371 100644
--- a/python/freetoken/models/__init__.py
+++ b/python/freetoken/models/__init__.py
@@ -11,7 +11,7 @@ from .config import (
SWAAttentionGroupConfig,
)
from .register import get_model_class
-from .weight import load_moe_expert_sources, load_weight
+from .weight import load_weight
def create_model(model_config: ModelConfig) -> BaseLLMModel:
@@ -22,7 +22,6 @@ __all__ = [
"BaseLLMModel",
"create_model",
"load_weight",
- "load_moe_expert_sources",
"AttentionGroupConfig",
"BaseAttentionGroupConfig",
"DSV4AttentionGroupConfig",
diff --git a/python/freetoken/models/blocks.py b/python/freetoken/models/blocks.py
index e811ca80..0fa5fe4d 100644
--- a/python/freetoken/models/blocks.py
+++ b/python/freetoken/models/blocks.py
@@ -18,6 +18,7 @@ if TYPE_CHECKING:
import torch
from freetoken.core import Batch
+ from freetoken.layers.quantization import QuantConfig
from .config import ModelConfig
@@ -33,11 +34,13 @@ class BaseLLMModel(ABC, BaseOP):
class GatedMLP(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, quant_config: QuantConfig | None = None, prefix: str = ""):
self.gate_up_proj = LinearColParallelMerged(
config.hidden_size,
[config.intermediate_size, config.intermediate_size],
has_bias=False,
+ quant_config=quant_config,
+ prefix=f"{prefix}.gate_up_proj",
)
fn_map = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
@@ -49,6 +52,8 @@ class GatedMLP(BaseOP):
config.intermediate_size,
config.hidden_size,
has_bias=False,
+ quant_config=quant_config,
+ prefix=f"{prefix}.down_proj",
)
@nvtx_annotate("MLP")
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py
index 1bce039c..07a16df5 100644
--- a/python/freetoken/models/config.py
+++ b/python/freetoken/models/config.py
@@ -260,7 +260,11 @@ class ModelConfig:
norm_topk_prob: bool
model_type: str
architectures: list[str]
- moe_backend: str = "fused"
+ moe_strategy: str = "fused"
+ # where routed experts decode (gpu / cpu / hybrid); set by the engine from the flags, it gates which expert kernels can serve
+ decode_target: str = "gpu"
+ # The QuantConfig the engine builds from the checkpoint; layers ask it for their method.
+ quant: Any | None = None
# ----- optional, model-specific extensions (default keeps other models intact) -----
moe_enabled: bool = False
# Weight quantization of the MoE experts only. "none" keeps the default BF16
@@ -268,26 +272,13 @@ class ModelConfig:
# "fp8_block" is DeepSeek-V3-style 128x128 block-fp8 (weight fp8-e4m3 +
# weight_scale_inv per block), also applied to the dense projections.
expert_quant: str = "none"
- # NVFP4 routed-expert GEMM backend (--nvfp4-backend); injected from EngineConfig.
- nvfp4_backend: str = "triton"
# Block size (out, in) for block-wise weight quantization (fp8_block: (128, 128)).
weight_block_size: tuple[int, int] | None = None
- # Weight quantization of the *dense* attention / GatedDeltaNet projections (separate
- # from the routed experts above). "fp8_pertensor" keeps them fp8-e4m3 + a per-output-row
- # scale and runs a W8A16 kernel (modelopt MIXED_PRECISION); "none" leaves them bf16
- # (dequant-at-load for any other dense quant, e.g. NVFP4 shared_expert/lm_head).
+ # the checkpoint's quant kind for the dense attention / GatedDeltaNet projections, detected by the family's parse_config for its reader
attn_quant: str = "none"
- # Weight quantization of the *dense* NVFP4 MLP projections -- the shared expert, and dense
- # (non-MoE) MLP layers -- which NVFP4 checkpoints store as packed FP4 like the routed
- # experts. "nvfp4" keeps them packed and runs the W4A16 dense kernels (quartering their
- # decode weight traffic); "none" dequantizes them to bf16 at load. Set independently of the
- # routed experts and lm_head: e.g. pure-NVFP4 Qwen3.5 has bf16 attn + bf16 lm_head but FP4
- # shared experts, so this is "nvfp4" while attn_quant / lm_head_quant are "none".
+ # the checkpoint's quant kind for the dense MLP projections (shared expert, dense layers), detected the same way
dense_quant: str = "none"
- # Weight quantization of the lm_head. "nvfp4" keeps the (untied) FP4 head native (W4A16) --
- # the bf16 dequant of this ~1 GB matrix was the single largest decode kernel; "none" leaves
- # it bf16. Separate from dense_quant because only some NVFP4 checkpoints quantize lm_head
- # (modelopt MIXED_PRECISION does; pure NVFP4 leaves it bf16).
+ # the checkpoint's quant kind for the lm_head, detected the same way (only some NVFP4 exports quantize it)
lm_head_quant: str = "none"
shared_expert_intermediate_size: int = 0
use_qk_norm: bool = False
diff --git a/python/freetoken/models/deepseek_v4/__init__.py b/python/freetoken/models/deepseek_v4/__init__.py
index b3cb41db..37f5076b 100644
--- a/python/freetoken/models/deepseek_v4/__init__.py
+++ b/python/freetoken/models/deepseek_v4/__init__.py
@@ -16,7 +16,7 @@ addressed by page tables, and sparse attention is a physical-slot gather (see
from .args import DeepseekV4Args, load_args
from .config import parse_config
from .model import DeepseekV4ForCausalLM
-from .weight import iter_weights, load_dsfp4_expert_sources
+from .weight import iter_expert_pieces, iter_weights
__all__ = [
"DeepseekV4Args",
@@ -24,5 +24,5 @@ __all__ = [
"parse_config",
"DeepseekV4ForCausalLM",
"iter_weights",
- "load_dsfp4_expert_sources",
+ "iter_expert_pieces",
]
diff --git a/python/freetoken/models/deepseek_v4/attention.py b/python/freetoken/models/deepseek_v4/attention.py
index 357585ee..cb0dd315 100644
--- a/python/freetoken/models/deepseek_v4/attention.py
+++ b/python/freetoken/models/deepseek_v4/attention.py
@@ -5,19 +5,19 @@ from __future__ import annotations
import torch
import torch.nn.functional as F
-from torch import nn
from freetoken.core import get_global_ctx
from freetoken.kernel.triton.dsv4.fp8_linear import act_quant_fp8_inplace
from freetoken.kernel.triton.dsv4.norm import rms_norm
+from freetoken.layers import BaseOP, LinearColParallelMerged, LinearReplicated, LinearRowParallel, RMSNorm
from .args import DeepseekV4Args
from .compress import Compressor, Indexer
-from .layers import Linear, RMSNorm, get_compress_topk_idxs, get_window_topk_idxs
+from .layers import get_compress_topk_idxs, get_window_topk_idxs
from .ops import apply_rotary_emb, apply_rotary_emb_decode, get_freqs_cis
-class Attention(nn.Module):
+class Attention(BaseOP):
"""Multi-head Latent Attention with sliding window + optional KV compression.
KV lives in the paged DSV4 pools (window / compressed / indexer), resolved per token via
@@ -26,8 +26,7 @@ class Attention(nn.Module):
``window_pool[L]`` / ``cmp_pool[L]`` inside the kernel -- no per-forward staging slab. Both
prefill and decode use the same paged kernel."""
- def __init__(self, layer_id: int, args: DeepseekV4Args):
- super().__init__()
+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, quant_config=None, prefix: str = ""):
self.layer_id = layer_id
self.dim = args.dim
self.n_heads = args.n_heads
@@ -40,22 +39,23 @@ class Attention(nn.Module):
self.compress_ratio = args.compress_ratios[layer_id]
self.eps = args.norm_eps
- self.attn_sink = nn.Parameter(torch.empty(self.n_heads, dtype=torch.float32), requires_grad=False)
- self.wq_a = Linear(self.dim, self.q_lora_rank, kind="fp8")
+ self.attn_sink = torch.empty(self.n_heads, dtype=torch.float32)
+ # the latent projections are replicated; wq_b shards over heads, wo_b over the output groups
+ self.wq_a = LinearReplicated(self.dim, self.q_lora_rank, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wq_a")
self.q_norm = RMSNorm(self.q_lora_rank, self.eps)
- self.wq_b = Linear(self.q_lora_rank, self.n_heads * self.head_dim, kind="fp8")
- self.wkv = Linear(self.dim, self.head_dim, kind="fp8")
+ self.wq_b = LinearColParallelMerged(self.q_lora_rank, [self.n_heads * self.head_dim], has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wq_b")
+ self.wkv = LinearReplicated(self.dim, self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wkv")
self.kv_norm = RMSNorm(self.head_dim, self.eps)
- # wo_a: dequantized to bf16 (reference runs a bf16 grouped-output einsum).
+ # wo_a is one [o_lora_rank, K] matrix per output group, stacked on N and applied as a bmm; the reference dequantizes it to bf16 and so does the reader. Under TP it shards on N by group like wo_b shards on K.
wo_a_rows = self.n_groups * args.o_lora_rank
wo_a_k = self.n_heads * self.head_dim // self.n_groups
- self.wo_a = nn.Parameter(torch.empty(wo_a_rows, wo_a_k, dtype=torch.bfloat16), requires_grad=False)
- self.wo_b = Linear(self.n_groups * args.o_lora_rank, self.dim, kind="fp8")
+ self.wo_a = torch.empty(wo_a_rows, wo_a_k, dtype=torch.bfloat16)
+ self.wo_b = LinearRowParallel(self.n_groups * args.o_lora_rank, self.dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wo_b")
self.softmax_scale = self.head_dim ** -0.5
if self.compress_ratio:
- self.compressor = Compressor(args, self.compress_ratio, self.head_dim)
- self.indexer = Indexer(args, self.compress_ratio) if self.compress_ratio == 4 else None
+ self.compressor = Compressor(args, self.compress_ratio, self.head_dim, quant_config=quant_config, prefix=f"{prefix}.compressor")
+ self.indexer = Indexer(args, self.compress_ratio, quant_config=quant_config, prefix=f"{prefix}.indexer") if self.compress_ratio == 4 else None
else:
self.compressor = None
self.indexer = None
@@ -69,7 +69,7 @@ class Attention(nn.Module):
rope_theta, args.rope_factor, args.beta_fast, args.beta_slow,
)
# Bound on first forward (freqs/arange only; pool buffers are read off the LIVE pool).
- self.freqs_cis: torch.Tensor | None = None
+ self._freqs_cis: torch.Tensor | None = None
# KV addressing lives in the attention backend (ctx.attn_backend), pool buffers on the live
# pool (ctx.kv_cache) -- both read per access, so the model holds NO state a runtime rebuild
@@ -85,11 +85,11 @@ class Attention(nn.Module):
L = self.layer_id
win = self.window_size
self.P = pool.P
- self.freqs_cis = get_freqs_cis(*self._freqs_params, device)
+ self._freqs_cis = get_freqs_cis(*self._freqs_params, device)
if self.compress_ratio:
- self.compressor.bind_paged(pool, L, self.freqs_cis, device, tier="attn")
+ self.compressor.bind_paged(pool, L, self._freqs_cis, device, tier="attn")
if self.indexer is not None:
- self.indexer.bind_paged(pool, L, self.freqs_cis, device)
+ self.indexer.bind_paged(pool, L, self._freqs_cis, device)
def reset(self) -> None:
if self.compressor is not None:
@@ -102,7 +102,7 @@ class Attention(nn.Module):
o = o.reshape(bsz, seqlen, self.n_groups, -1)
wo_a = self.wo_a.view(self.n_groups, self.o_lora_rank, -1)
o = torch.einsum("bsgd,grd->bsgr", o, wo_a).flatten(2)
- return self.wo_b(o)
+ return self.wo_b.forward(o)
def _prefill_segment(self, x_seg, qr_seg, kv_seg, ti: int, start_pos: int, n: int):
"""One request's prefill work inside a (possibly ragged) batch: persist its window KV,
@@ -144,17 +144,17 @@ class Attention(nn.Module):
if start_pos == 0:
self.reset() # re-seed the compressor/indexer carry from scratch
blocks = (
- self.indexer(x_seg, qr_seg, 0, 0, slots, ti) if self.indexer is not None
+ self.indexer.forward(x_seg, qr_seg, 0, 0, slots, ti) if self.indexer is not None
else get_compress_topk_idxs(ratio, 1, n, 0, 0).to(device)
)
- self.compressor(x_seg, 0, slots, ti=ti)
+ self.compressor.forward(x_seg, 0, slots, ti=ti)
else:
blocks = (
self.indexer.extend(x_seg, qr_seg, start_pos, 0, slots, tail_ws, ti)
if self.indexer is not None
else self._compress_topk_extend(n, start_pos, end, 0, device, 1)
)
- self.compressor(x_seg, start_pos, slots, tail_window_slot=tail_ws, ti=ti)
+ self.compressor.forward(x_seg, start_pos, slots, tail_window_slot=tail_ws, ti=ti)
return win_global, self.attn.blocks_to_global(blocks, ratio, ti=ti)
def forward_ragged(self, x, segments, flat_positions):
@@ -178,17 +178,17 @@ class Attention(nn.Module):
_, T, _ = x.size()
if len(segments) == 1:
# single contiguous segment: a free slice view instead of a per-layer gather
- freqs = self.freqs_cis[segments[0][3]:segments[0][3] + T]
+ freqs = self._freqs_cis[segments[0][3]:segments[0][3] + T]
else:
- freqs = self.freqs_cis.index_select(0, flat_positions) # [T, rd//2] per-token rope
+ freqs = self._freqs_cis.index_select(0, flat_positions) # [T, rd//2] per-token rope
- qr = q = self.q_norm(self.wq_a(x))
- q = self.wq_b(q).unflatten(-1, (self.n_heads, self.head_dim))
+ qr = q = self.q_norm.forward(self.wq_a.forward(x))
+ q = self.wq_b.forward(q).unflatten(-1, (self.n_heads, self.head_dim))
q = rms_norm(q, None, self.eps)
apply_rotary_emb(q[..., -rd:], freqs)
- kv = self.wkv(x)
- kv = self.kv_norm(kv)
+ kv = self.wkv.forward(x)
+ kv = self.kv_norm.forward(kv)
apply_rotary_emb(kv[..., -rd:], freqs)
act_quant_fp8_inplace(kv[..., :-rd], 64)
@@ -273,15 +273,15 @@ class Attention(nn.Module):
if wctx is None:
wctx = get_global_ctx().batch.attn_metadata.window_ctx(pos, rows)
window_slots, prev_window_slots, window_slots_topk = wctx
- freqs_t = self.freqs_cis.index_select(0, pos) # [B, rd_pairs] (per-layer rope)
+ freqs_t = self._freqs_cis.index_select(0, pos) # [B, rd_pairs] (per-layer rope)
- qr = q = self.q_norm(self.wq_a(x))
- q = self.wq_b(q).unflatten(-1, (self.n_heads, self.head_dim))
+ qr = q = self.q_norm.forward(self.wq_a.forward(x))
+ q = self.wq_b.forward(q).unflatten(-1, (self.n_heads, self.head_dim))
q = rms_norm(q, None, self.eps)
apply_rotary_emb_decode(q[..., -rd:], freqs_t) # per-row position freqs
- kv = self.wkv(x)
- kv = self.kv_norm(kv)
+ kv = self.wkv.forward(x)
+ kv = self.kv_norm.forward(kv)
apply_rotary_emb_decode(kv[..., -rd:], freqs_t)
act_quant_fp8_inplace(kv[..., :-rd], 64)
# Persistent window write: each row's new token to its own window slot (swa_ratio=1).
diff --git a/python/freetoken/models/deepseek_v4/compress.py b/python/freetoken/models/deepseek_v4/compress.py
index 4d578f36..3295976f 100644
--- a/python/freetoken/models/deepseek_v4/compress.py
+++ b/python/freetoken/models/deepseek_v4/compress.py
@@ -3,25 +3,23 @@
from __future__ import annotations
import torch
-from torch import nn
from freetoken.core import get_global_ctx
from freetoken.kernel.triton.dsv4.bf16_linear import bf16_linear_fp32
from freetoken.kernel.triton.dsv4.compress import gated_pool
from freetoken.kernel.triton.dsv4.fp8_linear import act_quant_fp8_inplace, fp4_act_quant_inplace
from freetoken.kernel.triton.dsv4.hadamard import hadamard_transform
+from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
from .args import DeepseekV4Args
-from .layers import Linear, RMSNorm
from .ops import apply_rotary_emb, apply_rotary_emb_decode
-class Compressor(nn.Module):
+class Compressor(BaseOP):
"""Learned gated pooling of KV over ``compress_ratio`` tokens (CSA overlap when
ratio==4). Stateful across decode steps. KV / state buffers are bound from the pool."""
- def __init__(self, args: DeepseekV4Args, compress_ratio: int, head_dim: int, rotate: bool = False):
- super().__init__()
+ def __init__(self, args: DeepseekV4Args, compress_ratio: int, head_dim: int, rotate: bool = False, *, quant_config=None, prefix: str = ""):
self.dim = args.dim
self.head_dim = head_dim
self.rope_head_dim = args.rope_head_dim
@@ -31,10 +29,10 @@ class Compressor(nn.Module):
self.max_batch_size = args.max_batch_size
coff = 1 + self.overlap
- self.ape = nn.Parameter(torch.empty(compress_ratio, coff * self.head_dim, dtype=torch.float32), requires_grad=False)
+ self.ape = torch.empty(compress_ratio, coff * self.head_dim, dtype=torch.float32)
# checkpoint stores these bf16; matmul upcasts on-chip (halves footprint + read).
- self.wkv = Linear(self.dim, coff * self.head_dim, kind="bf16")
- self.wgate = Linear(self.dim, coff * self.head_dim, kind="bf16")
+ self.wkv = LinearReplicated(self.dim, coff * self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wkv")
+ self.wgate = LinearReplicated(self.dim, coff * self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wgate")
self.norm = RMSNorm(self.head_dim, args.norm_eps)
# Paged-KV binding (set on first forward). All addressing -- the compressed-KV pool view,
# the per-window-page compress-state ring, the decode snapshot -- is reached through the
@@ -46,7 +44,7 @@ class Compressor(nn.Module):
self.coff = coff
self.item_size = coff * self.head_dim # the ring row's kv|score split point
self.P: int = 128 # window-page size (set in bind)
- self.freqs_cis: torch.Tensor | None = None
+ self._freqs_cis: torch.Tensor | None = None
# Rolling state register held in fp32 (the carry); seeded by prefill, advanced by
# decode. At bs=1 it mirrors the page's ring block; persisted to the ring on write.
self._kv_state: torch.Tensor | None = None
@@ -70,7 +68,7 @@ class Compressor(nn.Module):
self.layer_id = layer_id
self.tier = tier
self.P = pool.P
- self.freqs_cis = freqs_cis
+ self._freqs_cis = freqs_cis
self._device = device
coff = self.coff
self._kv_state = torch.zeros(
@@ -140,8 +138,8 @@ class Compressor(nn.Module):
return self.extend(x, start_pos, window_slots, int(tail_window_slot), ti)
dtype = x.dtype
x = x.float()
- kv = self.wkv(x)
- score = self.wgate(x)
+ kv = self.wkv.forward(x)
+ score = self.wgate.forward(x)
kv_full, score_full = kv, score # pre-split (for the page-boundary carry write-through)
ks, ss = self._kv_state, self._score_state
should_compress = seqlen >= ratio
@@ -171,8 +169,8 @@ class Compressor(nn.Module):
self._write_through_carry(int(window_slots[-1].item()))
if not should_compress:
return None
- kv = self.norm(kv.to(dtype))
- freqs_cis = self.freqs_cis[:cutoff:ratio]
+ kv = self.norm.forward(kv.to(dtype))
+ freqs_cis = self._freqs_cis[:cutoff:ratio]
apply_rotary_emb(kv[..., -rd:], freqs_cis)
if self.rotate:
kv = hadamard_transform(kv)
@@ -210,8 +208,8 @@ class Compressor(nn.Module):
self._seed_carry_from_ring(tail_window_slot)
x = x.float()
- kv = self.wkv(x)
- score = self.wgate(x)
+ kv = self.wkv.forward(x)
+ score = self.wgate.forward(x)
kv_full, score_full = kv, score
ks, ss = self._kv_state, self._score_state
should_compress = (start_pos + seqlen) // ratio > start_pos // ratio
@@ -268,13 +266,13 @@ class Compressor(nn.Module):
self._write_through_carry(int(window_slots[-1].item()))
if not should_compress:
return None
- reduced = self.norm(reduced.to(dtype))
+ reduced = self.norm.forward(reduced.to(dtype))
# freqs per compressed block: position b*ratio for absolute block b. start_pos is ratio-
# aligned, so the new FULL blocks span [start_pos, start_pos+cutoff) at stride ratio. Use
# the block-aligned `cutoff` end (NOT start_pos+seqlen): a stepped slice over an unaligned
# span ceils to one extra element when seqlen % ratio != 0 (mirrors the from-scratch
# prefill's freqs_cis[:cutoff:ratio]).
- freqs_cis = self.freqs_cis[start_pos:start_pos + cutoff:ratio]
+ freqs_cis = self._freqs_cis[start_pos:start_pos + cutoff:ratio]
assert freqs_cis.size(0) == reduced.size(1), f"{freqs_cis.shape=} {reduced.shape=}"
apply_rotary_emb(reduced[..., -rd:], freqs_cis)
if self.rotate:
@@ -342,8 +340,8 @@ class Compressor(nn.Module):
self.layer_id, self.tier, window_slots, self.ring_size, torch.cat([ks, ss], dim=-1)
)
- compressed = self.norm(compressed)
- freqs_t = self.freqs_cis.index_select(0, (pos + 1 - ratio).clamp_min(0)) # [B, rd//2]
+ compressed = self.norm.forward(compressed)
+ freqs_t = self._freqs_cis.index_select(0, (pos + 1 - ratio).clamp_min(0)) # [B, rd//2]
apply_rotary_emb_decode(compressed[..., -rd:], freqs_t)
if self.rotate:
compressed = hadamard_transform(compressed)
@@ -363,27 +361,26 @@ class Compressor(nn.Module):
self.attn.scatter_compressed(self.layer_id, self.tier, cmp_dst, compressed.view(B, -1))
-class Indexer(nn.Module):
+class Indexer(BaseOP):
"""Lightning Indexer: scores compressed KV and returns top-k positions to attend."""
- def __init__(self, args: DeepseekV4Args, compress_ratio: int):
- super().__init__()
+ def __init__(self, args: DeepseekV4Args, compress_ratio: int, *, quant_config=None, prefix: str = ""):
self.dim = args.dim
self.n_heads = args.index_n_heads
self.head_dim = args.index_head_dim
self.rope_head_dim = args.rope_head_dim
self.index_topk = args.index_topk
self.q_lora_rank = args.q_lora_rank
- self.wq_b = Linear(self.q_lora_rank, self.n_heads * self.head_dim, kind="fp8")
- self.weights_proj = Linear(self.dim, self.n_heads, kind="bf16")
+ self.wq_b = LinearReplicated(self.q_lora_rank, self.n_heads * self.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wq_b")
+ self.weights_proj = LinearReplicated(self.dim, self.n_heads, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.weights_proj")
self.softmax_scale = self.head_dim ** -0.5
self.compress_ratio = compress_ratio
- self.compressor = Compressor(args, compress_ratio, self.head_dim, rotate=True)
+ self.compressor = Compressor(args, compress_ratio, self.head_dim, rotate=True, quant_config=quant_config, prefix=f"{prefix}.compressor")
# The indexer's own compressor writes its compressed keys into the idx tier; scoring
# gathers them by block index. Both go through the LIVE backend (self.attn ->
# ctx.attn_backend) per access, so a runtime rebuild needs no model unbind.
self.layer_id: int | None = None
- self.freqs_cis: torch.Tensor | None = None
+ self._freqs_cis: torch.Tensor | None = None
@property
def attn(self):
@@ -398,7 +395,7 @@ class Indexer(nn.Module):
# arithmetic rows (full_loc // ratio); its OWN compress-state ring is pool.indexer_state_ring[L]
# (distinct from the attention compressor's, so the two never collide on per-page state slots).
self.layer_id = layer_id
- self.freqs_cis = freqs_cis
+ self._freqs_cis = freqs_cis
self._device = device
self.compressor.bind_paged(pool, layer_id, freqs_cis, device, tier="idx")
@@ -407,17 +404,17 @@ class Indexer(nn.Module):
def forward(self, x: torch.Tensor, qr: torch.Tensor, start_pos: int, offset: int, window_slots: torch.Tensor, ti: int = 0):
bsz, seqlen, _ = x.size()
- freqs_cis = self.freqs_cis[start_pos:start_pos + seqlen]
+ freqs_cis = self._freqs_cis[start_pos:start_pos + seqlen]
ratio = self.compress_ratio
rd = self.rope_head_dim
end_pos = start_pos + seqlen
- q = self.wq_b(qr)
+ q = self.wq_b.forward(qr)
q = q.unflatten(-1, (self.n_heads, self.head_dim))
apply_rotary_emb(q[..., -rd:], freqs_cis)
q = hadamard_transform(q)
fp4_act_quant_inplace(q, 32)
- self.compressor(x, start_pos, window_slots, ti=ti) # scatters indexer keys to idx_pool
- weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads ** -0.5)
+ self.compressor.forward(x, start_pos, window_slots, ti=ti) # scatters indexer keys to idx_pool
+ weights = self.weights_proj.forward(x) * (self.softmax_scale * self.n_heads ** -0.5)
keys = self.attn.indexer_keys(ti, end_pos // ratio, ratio, self.layer_id, bsz)
scores = self.attn.indexer_prefill_logits(q, keys, weights)
return self.attn.indexer_select_prefill(
@@ -431,14 +428,14 @@ class Indexer(nn.Module):
blocks [0, end//ratio) with a causal mask, returns top-k block indices offset by ``offset``."""
bsz, seqlen, _ = x.size()
end = start_pos + seqlen
- freqs_cis = self.freqs_cis[start_pos:end]
+ freqs_cis = self._freqs_cis[start_pos:end]
ratio, rd = self.compress_ratio, self.rope_head_dim
- q = self.wq_b(qr).unflatten(-1, (self.n_heads, self.head_dim))
+ q = self.wq_b.forward(qr).unflatten(-1, (self.n_heads, self.head_dim))
apply_rotary_emb(q[..., -rd:], freqs_cis)
q = hadamard_transform(q)
fp4_act_quant_inplace(q, 32)
- self.compressor(x, start_pos, window_slots, tail_window_slot=tail_window_slot, ti=ti) # writes idx_pool
- weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads ** -0.5)
+ self.compressor.forward(x, start_pos, window_slots, tail_window_slot=tail_window_slot, ti=ti) # writes idx_pool
+ weights = self.weights_proj.forward(x) * (self.softmax_scale * self.n_heads ** -0.5)
keys = self.attn.indexer_keys(ti, end // ratio, ratio, self.layer_id, bsz)
scores = self.attn.indexer_prefill_logits(q, keys, weights)
return self.attn.indexer_select_prefill(
@@ -460,13 +457,13 @@ class Indexer(nn.Module):
fixed staging width (== max valid count in eager, a static capture width under graph)."""
B = x.size(0)
ratio, rd = self.compress_ratio, self.rope_head_dim
- q = self.wq_b(qr).unflatten(-1, (self.n_heads, self.head_dim))
- apply_rotary_emb_decode(q[..., -rd:], self.freqs_cis.index_select(0, pos)) # per-row freqs
+ q = self.wq_b.forward(qr).unflatten(-1, (self.n_heads, self.head_dim))
+ apply_rotary_emb_decode(q[..., -rd:], self._freqs_cis.index_select(0, pos)) # per-row freqs
q = hadamard_transform(q)
fp4_act_quant_inplace(q, 32)
# Advance the indexer's own compressor (writes this token's idx key to idx_pool per row).
self.compressor.decode_step(x, pos, prev_window_slots, window_slots, rows)
- weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads ** -0.5)
+ weights = self.weights_proj.forward(x) * (self.softmax_scale * self.n_heads ** -0.5)
valid = (pos + 1) // ratio # [B] per-row valid block count
# Head-reduced scores in one pass: the kernel gathers each block's key off the full-loc
# SNAPSHOT (not the live map -> overlap-safe replay) and reads its column bound from
diff --git a/python/freetoken/models/deepseek_v4/layers.py b/python/freetoken/models/deepseek_v4/layers.py
index c3a4e35f..83bc7689 100644
--- a/python/freetoken/models/deepseek_v4/layers.py
+++ b/python/freetoken/models/deepseek_v4/layers.py
@@ -1,59 +1,9 @@
-"""Shared DSV4 primitives: weight-only linears, RMSNorm, and the static window /
-compressed top-k index builders."""
+"""Static window / compressed top-k index builders."""
from __future__ import annotations
import torch
import torch.nn.functional as F
-from torch import nn
-
-from freetoken.kernel.triton.dsv4.fp8_linear import block_fp8_linear
-from freetoken.kernel.triton.dsv4.norm import rms_norm
-
-FP8 = torch.float8_e4m3fn
-
-
-class Linear(nn.Module):
- """BF16 / FP32 / block-scaled-FP8 linear (weight-only).
-
- ``kind="fp8"``: weight float8_e4m3fn + 128x128 e8m0 ``scale`` (dequant in the
- Triton GEMM). ``kind="fp32"``/``"bf16"``: plain ``F.linear``.
- """
-
- def __init__(self, in_features, out_features, bias=False, kind="fp8"):
- super().__init__()
- self.in_features = in_features
- self.out_features = out_features
- self.kind = kind
- if kind == "fp8":
- self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=FP8), requires_grad=False)
- self.scale = nn.Parameter(
- torch.empty(out_features // 128, in_features // 128, dtype=torch.float8_e8m0fnu),
- requires_grad=False,
- )
- else:
- dtype = torch.float32 if kind == "fp32" else torch.bfloat16
- self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype), requires_grad=False)
- self.register_parameter("scale", None)
- if bias:
- self.bias = nn.Parameter(torch.empty(out_features), requires_grad=False)
- else:
- self.register_parameter("bias", None)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- if self.kind == "fp8":
- return block_fp8_linear(x, self.weight, self.scale, self.bias)
- return F.linear(x, self.weight.to(x.dtype), self.bias)
-
-
-class RMSNorm(nn.Module):
- def __init__(self, dim: int, eps: float = 1e-6):
- super().__init__()
- self.eps = eps
- self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32), requires_grad=False)
-
- def forward(self, x: torch.Tensor):
- return rms_norm(x, self.weight, self.eps)
def get_window_topk_idxs(window_size, bsz, seqlen, start_pos):
diff --git a/python/freetoken/models/deepseek_v4/model.py b/python/freetoken/models/deepseek_v4/model.py
index c00d20cc..a2d4387a 100644
--- a/python/freetoken/models/deepseek_v4/model.py
+++ b/python/freetoken/models/deepseek_v4/model.py
@@ -12,8 +12,9 @@ sqrtsoftplus / hash MoE), wired onto FreeToken's shared paged engine:
- The model is a registered :class:`BaseLLMModel` built by ``create_model``; it is driven
by ``ctx.batch`` (positions / input_ids), captured by the engine ``GraphRunner`` for
decode, and freed by dropping page-table indices (no separate runner / python cursor).
- - Routed FP4 experts are served from :class:`OffloadMoeCache` (on-demand) -- the
- framework's core acceleration.
+ - Routed MXFP4 experts are served from :class:`OffloadMoeCache` (on-demand) -- the
+ framework's core acceleration; their format and the dense fp8 block linears come from
+ ``config.quant``.
Heavy ops are FreeToken Triton kernels. Precision matches the reference (FP8/FP4
activation quant + Hadamard rotation re-introduced; see ``ops.py`` / the dsv4 kernels).
@@ -23,11 +24,11 @@ from __future__ import annotations
import torch
import torch.nn.functional as F
-from torch import nn
from freetoken.core import get_global_ctx
from freetoken.kernel.triton.dsv4.hc import hc_post_combine, hc_pre_combine
from freetoken.kernel.triton.dsv4.sinkhorn import hc_split_sinkhorn
+from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNorm, VocabParallelEmbedding
from freetoken.models.blocks import BaseLLMModel
from .args import DeepseekV4Args
@@ -37,26 +38,19 @@ from .moe import MoE
# Re-exports: keep every class/helper previously defined here importable from .model
# (external import stability; the moved definitions live in their own modules).
from .compress import Compressor, Indexer # noqa: F401
-from .layers import ( # noqa: F401
- FP8,
- Linear,
- RMSNorm,
- get_compress_topk_idxs,
- get_window_topk_idxs,
-)
+from .layers import get_compress_topk_idxs, get_window_topk_idxs # noqa: F401
from .moe import Expert, Gate # noqa: F401
-class Block(nn.Module):
+class Block(BaseOP):
"""Decoder block with manifold-constrained Hyper-Connections (4 residual streams)."""
- def __init__(self, layer_id: int, args: DeepseekV4Args):
- super().__init__()
+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, strategy: str = "offload", decode_target: str = "gpu", quant_config=None, prefix: str = ""):
self.layer_id = layer_id
self.norm_eps = args.norm_eps
self.dim = args.dim
- self.attn = Attention(layer_id, args)
- self.ffn = MoE(layer_id, args)
+ self.attn = Attention(layer_id, args, quant_config=quant_config, prefix=f"{prefix}.attn")
+ self.ffn = MoE(layer_id, args, strategy=strategy, decode_target=decode_target, quant_config=quant_config, prefix=f"{prefix}.ffn")
self.attn_norm = RMSNorm(args.dim, self.norm_eps)
self.ffn_norm = RMSNorm(args.dim, self.norm_eps)
self.hc_mult = hc_mult = args.hc_mult
@@ -64,12 +58,12 @@ class Block(nn.Module):
self.hc_eps = args.hc_eps
mix_hc = (2 + hc_mult) * hc_mult
hc_dim = hc_mult * args.dim
- self.hc_attn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim, dtype=torch.float32), requires_grad=False)
- self.hc_ffn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim, dtype=torch.float32), requires_grad=False)
- self.hc_attn_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32), requires_grad=False)
- self.hc_ffn_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32), requires_grad=False)
- self.hc_attn_scale = nn.Parameter(torch.empty(3, dtype=torch.float32), requires_grad=False)
- self.hc_ffn_scale = nn.Parameter(torch.empty(3, dtype=torch.float32), requires_grad=False)
+ self.hc_attn_fn = torch.empty(mix_hc, hc_dim, dtype=torch.float32)
+ self.hc_ffn_fn = torch.empty(mix_hc, hc_dim, dtype=torch.float32)
+ self.hc_attn_base = torch.empty(mix_hc, dtype=torch.float32)
+ self.hc_ffn_base = torch.empty(mix_hc, dtype=torch.float32)
+ self.hc_attn_scale = torch.empty(3, dtype=torch.float32)
+ self.hc_ffn_scale = torch.empty(3, dtype=torch.float32)
def hc_pre(self, x, hc_fn, hc_scale, hc_base):
shape, dtype = x.size(), x.dtype
@@ -101,51 +95,49 @@ class Block(nn.Module):
# metadata; ``flat_positions`` [T] = per-token ABSOLUTE position (batch.positions).
residual = x
x, post, comb = self.hc_pre(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)
- x = self.attn_norm(x)
+ x = self.attn_norm.forward(x)
x = self.attn.forward_ragged(x, segments, flat_positions)
x = self.hc_post(x, residual, post, comb)
residual = x
x, post, comb = self.hc_pre(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)
- x = self.ffn_norm(x)
- x = self.ffn(x, input_ids)
+ x = self.ffn_norm.forward(x)
+ x = self.ffn.forward(x, input_ids)
x = self.hc_post(x, residual, post, comb)
return x
def decode_step(self, x, pos, rows, cmp_stage_cap, input_ids, wctx=None):
residual = x
x, post, comb = self.hc_pre(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)
- x = self.attn_norm(x)
+ x = self.attn_norm.forward(x)
x = self.attn.decode_step(x, pos, rows, cmp_stage_cap, wctx)
x = self.hc_post(x, residual, post, comb)
residual = x
x, post, comb = self.hc_pre(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)
- x = self.ffn_norm(x)
- x = self.ffn(x, input_ids)
+ x = self.ffn_norm.forward(x)
+ x = self.ffn.forward(x, input_ids)
x = self.hc_post(x, residual, post, comb)
return x
-class Transformer(nn.Module):
- def __init__(self, args: DeepseekV4Args):
- super().__init__()
+class Transformer(BaseOP):
+ def __init__(self, args: DeepseekV4Args, quant_config=None, *, strategy: str = "offload", decode_target: str = "gpu", prefix: str = ""):
self.args = args
self.norm_eps = args.norm_eps
self.hc_eps = args.hc_eps
self.hc_mult = hc_mult = args.hc_mult
- self.embed = nn.Embedding(args.vocab_size, args.dim)
- self.embed.weight.requires_grad_(False)
- self.layers = nn.ModuleList([Block(i, args) for i in range(args.n_layers)])
+ self.embed = VocabParallelEmbedding(args.vocab_size, args.dim)
+ self.layers = OPList([Block(i, args, strategy=strategy, decode_target=decode_target, quant_config=quant_config, prefix=f"{prefix}.layers.{i}") for i in range(args.n_layers)])
self.norm = RMSNorm(args.dim, self.norm_eps)
- self.head = nn.Parameter(torch.empty(args.vocab_size, args.dim, dtype=torch.bfloat16), requires_grad=False)
+ self.head = ParallelLMHead(args.vocab_size, args.dim, quant_config=quant_config, prefix=f"{prefix}.head")
hc_dim = hc_mult * args.dim
- self.hc_head_fn = nn.Parameter(torch.empty(hc_mult, hc_dim, dtype=torch.float32), requires_grad=False)
- self.hc_head_base = nn.Parameter(torch.empty(hc_mult, dtype=torch.float32), requires_grad=False)
- self.hc_head_scale = nn.Parameter(torch.empty(1, dtype=torch.float32), requires_grad=False)
+ self.hc_head_fn = torch.empty(hc_mult, hc_dim, dtype=torch.float32)
+ self.hc_head_base = torch.empty(hc_mult, dtype=torch.float32)
+ self.hc_head_scale = torch.empty(1, dtype=torch.float32)
def bind(self, pool, device: torch.device) -> None:
- for layer in self.layers:
+ for layer in self.layers.op_list:
layer.attn.bind(pool, device)
def hc_head(self, x):
@@ -160,7 +152,6 @@ class Transformer(nn.Module):
def prefill_batched(
self, input_ids: torch.Tensor, segments, flat_positions: torch.Tensor,
- last_indices: torch.Tensor,
) -> torch.Tensor:
# Ragged batched prefill (bs >= 1). ``input_ids`` is [1, T] -- the requests' NEW tokens
# concatenated (cu_seqlens, no padding); each request starts at its own cached_len
@@ -170,15 +161,15 @@ class Transformer(nn.Module):
# allocate_paged, so the per-request attention never reads another request's KV/carry.
# ``segments`` [(offset, extend_len, table_idx, start_pos)] comes off the attention
# metadata; ``flat_positions`` [T] is the scheduler-staged batch.positions (per-token
- # ABSOLUTE position); ``last_indices`` [B] is the flattened index of each request's
- # final token -> its next-token logits row.
- h = self.embed(input_ids)
+ # ABSOLUTE position); the head picks each request's final token off the attention
+ # metadata -> its next-token logits row.
+ h = self.embed.forward(input_ids.view(-1)).view(1, -1, self.args.dim)
h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)
- for layer in self.layers:
+ for layer in self.layers.op_list:
h = layer.prefill_batched(h, input_ids, segments, flat_positions)
h = self.hc_head(h)
- h = self.norm(h)
- return F.linear(h[0, last_indices], self.head) # [B, vocab]
+ h = self.norm.forward(h)
+ return self.head.forward(h[0]) # [B, vocab]
def decode(
self, input_ids: torch.Tensor, pos: torch.Tensor, cmp_stage_cap: int
@@ -194,7 +185,7 @@ class Transformer(nn.Module):
# in-flight replay (it mutates only the live map).
B = input_ids.size(0)
rows = torch.arange(B, device=input_ids.device)
- h = self.embed(input_ids)
+ h = self.embed.forward(input_ids.view(-1)).view(B, 1, self.args.dim)
h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)
# Hoist the layer-invariant per-step decode tensors (shared window-ring global slots):
# resolved ONCE off the attention metadata (recomputed per call, so a capture records
@@ -202,32 +193,30 @@ class Transformer(nn.Module):
# into every layer. They read only the shared snapshot / positions, so they are identical
# across layers.
wctx = get_global_ctx().batch.attn_metadata.window_ctx(pos, rows)
- for layer in self.layers:
+ for layer in self.layers.op_list:
h = layer.decode_step(h, pos, rows, cmp_stage_cap, input_ids, wctx)
h = self.hc_head(h)
- h = self.norm(h)
- return F.linear(h[:, -1], self.head)
+ h = self.norm.forward(h)
+ return self.head.forward(h[:, -1])
class DeepseekV4ForCausalLM(BaseLLMModel):
"""Engine adapter: a registered :class:`BaseLLMModel` wrapping the DSV4 transformer.
- Built on the engine ``meta`` device then loaded via :meth:`load_state_dict` (which
- casts each incoming tensor to its parameter dtype and assigns it). KV pools, rope
- constants and compressor state are bound on the first forward.
+ KV pools, rope constants and compressor state are bound on the first forward.
"""
def __init__(self, config):
self._config = config
self._args: DeepseekV4Args = config.dsv4_args
- self._transformer = Transformer(self._args)
+ self.model = Transformer(self._args, config.quant, strategy=config.moe_strategy, decode_target=config.decode_target, prefix="model")
self._bound = False
def _ensure_bound(self) -> None:
if self._bound:
return
pool = get_global_ctx().kv_cache
- self._transformer.bind(pool, pool.device)
+ self.model.bind(pool, pool.device)
self._bound = True
def mark_for_rebind(self) -> None:
@@ -237,30 +226,6 @@ class DeepseekV4ForCausalLM(BaseLLMModel):
block count, freqs) depends on the new pool's geometry, so re-derive it via _ensure_bound."""
self._bound = False
- def _iter_offload_moe_layers(self):
- # Hook for moe.offload_cache.iter_offload_moe_layers: DSV4's MoE block (block.ffn)
- # is a bespoke nn.Module (not OffloadMoELayer), so yield its inner routed-expert
- # layer instead, whose .offload_cache the generic attach sets.
- for block in self._transformer.layers:
- yield block.ffn.experts
-
- def state_dict(self, *, prefix: str = "", result=None):
- result = {} if result is None else result
- for name, p in self._transformer.named_parameters():
- result[f"{prefix}.{name}" if prefix else name] = p
- return result
-
- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:
- casted = {}
- for name, p in self._transformer.named_parameters():
- key = f"{prefix}.{name}" if prefix else name
- if key not in state_dict:
- raise RuntimeError(f"Missing weight for DeepSeek-V4 parameter: {key}")
- casted[name] = state_dict.pop(key).to(p.dtype)
- if state_dict and not _internal:
- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
- self._transformer.load_state_dict(casted, assign=True, strict=False)
-
def forward(self) -> torch.Tensor:
self._ensure_bound()
batch = get_global_ctx().batch
@@ -273,9 +238,8 @@ class DeepseekV4ForCausalLM(BaseLLMModel):
# FROM THE RING. Per-token ops (embed / HC / norm / MoE) run batched over the
# concatenated tokens; attention runs per segment so the carry / slot maps never
# cross requests.
- return self._transformer.prefill_batched(
+ return self.model.prefill_batched(
input_ids.view(1, -1), md.segments, batch.positions.long(),
- md.last_indices.long(),
)
# DECODE (bs>=1): per-row position (GPU int tensor -> no host syncs / graph safe). The
# compressed staging cap is the max position any row reaches (eager); a static max_seq-1
@@ -290,7 +254,7 @@ class DeepseekV4ForCausalLM(BaseLLMModel):
cmp_stage_cap = md.stage_width - 1
else:
cmp_stage_cap = int(pos.max().item())
- return self._transformer.decode(input_ids.view(B, 1), pos, cmp_stage_cap)
+ return self.model.decode(input_ids.view(B, 1), pos, cmp_stage_cap)
__all__ = ["Transformer", "DeepseekV4ForCausalLM"]
diff --git a/python/freetoken/models/deepseek_v4/moe.py b/python/freetoken/models/deepseek_v4/moe.py
index d7e555a9..6b624567 100644
--- a/python/freetoken/models/deepseek_v4/moe.py
+++ b/python/freetoken/models/deepseek_v4/moe.py
@@ -1,37 +1,32 @@
-"""DSV4 MoE: sqrtsoftplus/hash router, shared SwiGLU expert, offloaded FP4 routed
+"""DSV4 MoE: sqrtsoftplus/hash router, shared SwiGLU expert, offloaded MXFP4 routed
experts (GPU slot-cache / cpu / hybrid decode paths)."""
from __future__ import annotations
import torch
import torch.nn.functional as F
-from torch import nn
from freetoken.kernel.triton.dsv4.bf16_linear import bf16_linear_fp32
from freetoken.kernel.triton.dsv4.swiglu import fused_swiglu
-from freetoken.layers import OffloadMoELayer
+from freetoken.layers import BaseOP, LinearColParallelMerged, LinearRowParallel, OffloadMoELayer
from .args import DeepseekV4Args
-from .layers import Linear
-class Gate(nn.Module):
+class Gate(BaseOP):
"""MoE router: sqrtsoftplus scoring + hash routing (first ``n_hash_layers``)."""
def __init__(self, layer_id: int, args: DeepseekV4Args):
- super().__init__()
self.topk = args.n_activated_experts
self.score_func = args.score_func
self.route_scale = args.route_scale
self.hash = layer_id < args.n_hash_layers
- self.weight = nn.Parameter(torch.empty(args.n_routed_experts, args.dim, dtype=torch.bfloat16), requires_grad=False)
+ self.weight = torch.empty(args.n_routed_experts, args.dim, dtype=torch.bfloat16)
if self.hash:
- self.tid2eid = nn.Parameter(
- torch.empty(args.vocab_size, args.n_activated_experts, dtype=torch.int64), requires_grad=False
- )
- self.register_parameter("bias", None)
+ self.tid2eid = torch.empty(args.vocab_size, args.n_activated_experts, dtype=torch.int64)
+ self.bias = None
else:
- self.bias = nn.Parameter(torch.empty(args.n_routed_experts, dtype=torch.float32), requires_grad=False)
+ self.bias = torch.empty(args.n_routed_experts, dtype=torch.float32)
def forward(self, x: torch.Tensor, input_ids: torch.Tensor):
scores = bf16_linear_fp32(x, self.weight)
@@ -55,28 +50,27 @@ class Gate(nn.Module):
return weights, indices
-class Expert(nn.Module):
+class Expert(BaseOP):
"""Dense SwiGLU expert (the shared expert; routed experts are offloaded FP4)."""
- def __init__(self, dim: int, inter_dim: int, swiglu_limit: float):
- super().__init__()
- self.w1 = Linear(dim, inter_dim, kind="fp8")
- self.w2 = Linear(inter_dim, dim, kind="fp8")
- self.w3 = Linear(dim, inter_dim, kind="fp8")
+ def __init__(self, dim: int, inter_dim: int, swiglu_limit: float, *, quant_config=None, prefix: str = ""):
+ self.w1 = LinearColParallelMerged(dim, [inter_dim], has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w1")
+ self.w2 = LinearRowParallel(inter_dim, dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w2")
+ self.w3 = LinearColParallelMerged(dim, [inter_dim], has_bias=False, quant_config=quant_config, prefix=f"{prefix}.w3")
self.swiglu_limit = swiglu_limit
def forward(self, x: torch.Tensor) -> torch.Tensor:
- h = fused_swiglu(self.w1(x), self.w3(x), self.swiglu_limit, x.dtype)
- return self.w2(h)
+ h = fused_swiglu(self.w1.forward(x), self.w3.forward(x), self.swiglu_limit, x.dtype)
+ return self.w2.forward(h)
class DSV4OffloadMoELayer(OffloadMoELayer):
- """Routed FP4 experts on the shared offload cache: the base whole-layer
+ """Routed MXFP4 experts on the shared offload cache: the base whole-layer
streaming prefill (grouped inline-dequant GEMM for dense chunks, GEMV
below the route crossover) and slot-cache / cpu / hybrid decode paths
(per-route dequant GEMV)."""
- def __init__(self, layer_id: int, args: DeepseekV4Args):
+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, strategy: str = "offload", decode_target: str = "gpu", quant_config=None, prefix: str = ""):
super().__init__(
layer_id=layer_id,
num_experts=args.n_routed_experts,
@@ -85,8 +79,12 @@ class DSV4OffloadMoELayer(OffloadMoELayer):
intermediate_size=args.moe_inter_dim,
renormalize=True,
activation="silu",
+ limit=args.swiglu_limit,
+ strategy=strategy,
+ decode_target=decode_target,
+ quant_config=quant_config,
+ prefix=prefix,
)
- self.swiglu_limit = args.swiglu_limit
def _prefill_routed(
self,
@@ -124,24 +122,23 @@ class DSV4OffloadMoELayer(OffloadMoELayer):
)
-class MoE(nn.Module):
- """Sparse MoE: hash/score router -> offloaded FP4 routed experts + shared expert."""
+class MoE(BaseOP):
+ """Sparse MoE: hash/score router -> offloaded MXFP4 routed experts + shared expert."""
- def __init__(self, layer_id: int, args: DeepseekV4Args):
- super().__init__()
+ def __init__(self, layer_id: int, args: DeepseekV4Args, *, strategy: str = "offload", decode_target: str = "gpu", quant_config=None, prefix: str = ""):
self.dim = args.dim
self.gate = Gate(layer_id, args)
- self.shared_experts = Expert(args.dim, args.moe_inter_dim, args.swiglu_limit)
- self.experts = DSV4OffloadMoELayer(layer_id, args)
+ self.shared_experts = Expert(args.dim, args.moe_inter_dim, args.swiglu_limit, quant_config=quant_config, prefix=f"{prefix}.shared_experts")
+ self.experts = DSV4OffloadMoELayer(layer_id, args, strategy=strategy, decode_target=decode_target, quant_config=quant_config, prefix=f"{prefix}.experts")
def forward(self, x: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:
shape = x.size()
x = x.view(-1, self.dim)
- weights, indices = self.gate(x, input_ids.flatten())
+ weights, indices = self.gate.forward(x, input_ids.flatten())
# Shared expert enqueued before routed_forward: hybrid decode blocks on the
# CPU pool inside routed_forward, so this GEMM must already be on the stream
# to overlap the CPU overflow compute.
- shared = self.shared_experts(x)
+ shared = self.shared_experts.forward(x)
# routed_forward may mutate the ids in place (offload decode slot remap);
# indices.to(int32) always copies (int64 source), so no clone needed here.
routed = self.experts.routed_forward(
diff --git a/python/freetoken/models/deepseek_v4/weight.py b/python/freetoken/models/deepseek_v4/weight.py
index f5a3fd17..4728f7f9 100644
--- a/python/freetoken/models/deepseek_v4/weight.py
+++ b/python/freetoken/models/deepseek_v4/weight.py
@@ -1,24 +1,24 @@
"""Weight loading for DeepSeek-V4-Flash (engine path).
- - :func:`iter_weights` streams resident (non-expert) tensors keyed to engine param
- names; model's ``load_state_dict`` casts each. ``wo_a`` dequantized to bf16 to match
+ - :func:`iter_weights` streams resident (non-expert) tensors keyed by the model's
+ attribute paths (``model.`` + the checkpoint name). ``wo_a`` dequantized to bf16 to match
the reference bf16 einsum.
- - :func:`load_dsfp4_expert_sources` packs routed FP4 experts into pinned CPU banks for
- the offload cache. DeepSeek FP4: e8m0 per-32 block scale, no global scale.
+ - :func:`iter_expert_pieces` streams the routed MXFP4 experts (e2m1 pairs + e8m0 per-32
+ scales, no global) as per-expert pieces for the expert quant method's banks.
"""
from __future__ import annotations
-import collections
import json
import os
import re
-from typing import Iterator
import safetensors
import torch
from tqdm import tqdm
+from freetoken.layers.quantization import QuantKind
+from freetoken.distributed import get_tp_info
from freetoken.models.loader import drop_page_cache
from .args import DeepseekV4Args, load_args
@@ -81,14 +81,14 @@ def iter_weights(
):
"""Stream resident (non-expert) weights as ``(name, tensor)`` keyed to engine params.
- Routed FP4 experts come from the offload cache, so ``include_moe_experts`` must be
- False (DeepSeek-V4 only runs ``--moe-backend offload``). Tensors yielded in checkpoint
+ Routed MXFP4 experts come from the offload cache, so ``include_moe_experts`` must be
+ False (DeepSeek-V4 only runs ``--moe-strategy offload``). Tensors yielded in checkpoint
dtype (fp8 + e8m0 preserved); ``wo_a`` dequantized to bf16 to match the reference einsum.
"""
if include_moe_experts:
raise ValueError(
"DeepSeek-V4 routed experts are served from the offload cache; "
- "run with --moe-backend offload (include_moe_experts must be False)."
+ "run with --moe-strategy offload (include_moe_experts must be False)."
)
if not include_non_moe:
return
@@ -99,248 +99,117 @@ def iter_weights(
def get(name: str) -> torch.Tensor:
return reader.get(name)
- def linear(prefix: str):
- yield f"{prefix}.weight", get(f"{prefix}.weight")
- if reader.has(f"{prefix}.scale"):
- yield f"{prefix}.scale", get(f"{prefix}.scale")
+ def linear(src: str, dst: str):
+ yield f"{dst}.weight", get(f"{src}.weight")
+ # fp8 linears declare the e8m0 block scale under the quant method's role name
+ if reader.has(f"{src}.scale"):
+ yield f"{dst}.weight_scale_inv", get(f"{src}.scale")
try:
- yield "embed.weight", get("embed.weight")
- yield "norm.weight", get("norm.weight")
- yield "head", get("head.weight")
+ yield "model.embed.weight", get("embed.weight")
+ yield "model.norm.weight", get("norm.weight")
+ yield "model.head.weight", get("head.weight")
for nm in ("hc_head_fn", "hc_head_base", "hc_head_scale"):
- yield nm, get(nm)
+ yield f"model.{nm}", get(nm)
for L in range(args.n_layers):
a = f"layers.{L}.attn"
- yield from linear(f"{a}.wq_a")
- yield f"{a}.q_norm.weight", get(f"{a}.q_norm.weight")
- yield from linear(f"{a}.wq_b")
- yield from linear(f"{a}.wkv")
- yield f"{a}.kv_norm.weight", get(f"{a}.kv_norm.weight")
+ m = f"model.{a}"
+ yield from linear(f"{a}.wq_a", f"{m}.wq_a")
+ yield f"{m}.q_norm.weight", get(f"{a}.q_norm.weight")
+ yield from linear(f"{a}.wq_b", f"{m}.wq_b")
+ yield from linear(f"{a}.wkv", f"{m}.wkv")
+ yield f"{m}.kv_norm.weight", get(f"{a}.kv_norm.weight")
# wo_a: FP8 in the checkpoint, dequantized to bf16 (reference bf16 einsum).
- yield f"{a}.wo_a", _dequant_fp8_block(
+ yield f"{m}.wo_a", _dequant_fp8_block(
get(f"{a}.wo_a.weight"), get(f"{a}.wo_a.scale")
)
- yield from linear(f"{a}.wo_b")
- yield f"{a}.attn_sink", get(f"{a}.attn_sink")
+ yield from linear(f"{a}.wo_b", f"{m}.wo_b")
+ yield f"{m}.attn_sink", get(f"{a}.attn_sink")
ratio = args.compress_ratios[L]
if ratio:
c = f"{a}.compressor"
- yield f"{c}.ape", get(f"{c}.ape")
- yield f"{c}.wkv.weight", get(f"{c}.wkv.weight")
- yield f"{c}.wgate.weight", get(f"{c}.wgate.weight")
- yield f"{c}.norm.weight", get(f"{c}.norm.weight")
+ for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):
+ yield f"model.{c}.{nm}", get(f"{c}.{nm}")
if ratio == 4:
idx = f"{a}.indexer"
- yield from linear(f"{idx}.wq_b")
- yield f"{idx}.weights_proj.weight", get(f"{idx}.weights_proj.weight")
+ yield from linear(f"{idx}.wq_b", f"model.{idx}.wq_b")
+ yield f"model.{idx}.weights_proj.weight", get(f"{idx}.weights_proj.weight")
ic = f"{idx}.compressor"
- yield f"{ic}.ape", get(f"{ic}.ape")
- yield f"{ic}.wkv.weight", get(f"{ic}.wkv.weight")
- yield f"{ic}.wgate.weight", get(f"{ic}.wgate.weight")
- yield f"{ic}.norm.weight", get(f"{ic}.norm.weight")
+ for nm in ("ape", "wkv.weight", "wgate.weight", "norm.weight"):
+ yield f"model.{ic}.{nm}", get(f"{ic}.{nm}")
- yield f"layers.{L}.attn_norm.weight", get(f"layers.{L}.attn_norm.weight")
- yield f"layers.{L}.ffn_norm.weight", get(f"layers.{L}.ffn_norm.weight")
+ yield f"model.layers.{L}.attn_norm.weight", get(f"layers.{L}.attn_norm.weight")
+ yield f"model.layers.{L}.ffn_norm.weight", get(f"layers.{L}.ffn_norm.weight")
g = f"layers.{L}.ffn.gate"
- yield f"{g}.weight", get(f"{g}.weight")
+ yield f"model.{g}.weight", get(f"{g}.weight")
if L < args.n_hash_layers:
- yield f"{g}.tid2eid", get(f"{g}.tid2eid")
+ yield f"model.{g}.tid2eid", get(f"{g}.tid2eid")
else:
- yield f"{g}.bias", get(f"{g}.bias")
+ yield f"model.{g}.bias", get(f"{g}.bias")
for proj in ("w1", "w2", "w3"):
- yield from linear(f"layers.{L}.ffn.shared_experts.{proj}")
+ src = f"layers.{L}.ffn.shared_experts.{proj}"
+ yield from linear(src, f"model.{src}")
for nm in (
"hc_attn_fn", "hc_ffn_fn", "hc_attn_base",
"hc_ffn_base", "hc_attn_scale", "hc_ffn_scale",
):
- yield f"layers.{L}.{nm}", get(f"layers.{L}.{nm}")
+ yield f"model.layers.{L}.{nm}", get(f"layers.{L}.{nm}")
finally:
reader.close()
# --------------------------------------------------------------------------------------
-# Routed FP4 expert pinned banks.
+# Routed MXFP4 expert pieces.
# --------------------------------------------------------------------------------------
_EXPERT_RE = re.compile(
r"^layers\.(?P<layer>\d+)\.ffn\.experts\.(?P<expert>\d+)\."
r"(?P<proj>w1|w2|w3)\.(?P<kind>weight|scale)$"
)
+_PROJ_ROLE = {"w1": "gate", "w3": "up", "w2": "down"}
+_KIND_SUFFIX = {"weight": "", "scale": "_scale"}
-def load_dsfp4_expert_sources(
- model_path: str, args: DeepseekV4Args, *, layer_sink=None
-) -> dict[str, list[torch.Tensor]]:
- """Build pinned CPU DeepSeek-FP4 banks for the routed experts.
-
- 4 banks, one tensor per layer (independent allocations): ``gate_up_packed/scale``
- ``[E, 2I, H//2]`` uint8 / ``[..., H//32]`` e8m0, ``down_packed/scale`` ``[E, H, I//2]``
- / ``[..., I//32]``.
-
- ``layer_sink=None`` (serving): pin each layer as its writes complete, via an
- internally-owned :class:`PinPipeline`. ``layer_sink`` given (converter): the
- completion tracker fires into it instead -- nothing here is pinned, and the sink
- may release banks it has written out, so the returned tensors are only valid
- until then (the caller owns that tradeoff).
- """
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks
-
- folder = model_path
- weight_map = _weight_map(folder)
- L, E = args.n_layers, args.n_routed_experts
- H, I = args.dim, args.moe_inter_dim
-
- for shard in sorted(set(weight_map.values())):
- drop_page_cache(os.path.join(folder, shard))
-
- shards: dict[str, list[tuple[str, re.Match]]] = collections.defaultdict(list)
- for name, shard in weight_map.items():
- m = _EXPERT_RE.match(name)
- if m is None:
- continue
- if int(m.group("layer")) >= L: # skip the MTP layer (index L)
- continue
- shards[shard].append((name, m))
-
- e8m0 = torch.float8_e8m0fnu
- specs = { # alloc UNPINNED, fill, then pin-after-fill (skips slow cudaHostAlloc zero-fill)
- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),
- "gate_up_scale": ((E, 2 * I, H // 32), e8m0),
- "down_packed": ((E, H, I // 2), torch.uint8),
- "down_scale": ((E, H, I // 32), e8m0),
- }
- hb = alloc_layer_banks(specs, L)
- banks = {name: [b.tensor for b in hb[name]] for name in specs}
-
- def _load(sink) -> int:
- tracker = LayerCompletionTracker(E * 6, hb, sink) # {w1,w2,w3} x {weight,scale} x experts
- placed = 0
- for shard in tqdm(sorted(shards), desc="Loading DSV4 FP4 experts"):
- path = os.path.join(folder, shard)
- with safetensors.safe_open(path, framework="pt", device="cpu") as f:
- for name, m in shards[shard]:
- layer = _place_dsfp4(banks, name, f.get_tensor(name), I)
- tracker.note(layer)
- placed += 1
- drop_page_cache(path)
- return placed
-
- if layer_sink is not None:
- placed = _load(layer_sink)
- else:
- with PinPipeline() as pins:
- placed = _load(pins)
-
- expected = L * E * 6 # {w1,w2,w3} x {weight, scale}
- assert placed == expected, f"loaded {placed} expert tensors, expected {expected}"
- return banks
-
-
-def dummy_dsfp4_expert_sources(args: DeepseekV4Args) -> dict[str, list[torch.Tensor]]:
- """Fabricate the 4 ds_fp4 banks for --dummy-weight (no checkpoint on disk)."""
- from freetoken.moe.host_banks import alloc_layer_banks, pin_banks
-
- L, E = args.n_layers, args.n_routed_experts
- H, I = args.dim, args.moe_inter_dim
- e8m0 = torch.float8_e8m0fnu
- specs = {
- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),
- "gate_up_scale": ((E, 2 * I, H // 32), e8m0),
- "down_packed": ((E, H, I // 2), torch.uint8),
- "down_scale": ((E, H, I // 32), e8m0),
- }
- hb = alloc_layer_banks(specs, L)
- banks = {name: [b.tensor for b in hb[name]] for name in specs}
- for t in banks["gate_up_packed"]: # packed e2m1; scales stay 0 (valid e8m0)
- t.random_(0, 256)
- for t in banks["down_packed"]:
- t.random_(0, 256)
- pin_banks(hb)
- return banks
-
-
-def is_expert_tensor(name: str) -> bool:
- """Predicate for the common parallel reader: is this a routed-expert tensor?"""
- return _EXPERT_RE.match(name) is not None
-
-
-def _place_dsfp4(banks: dict, name: str, t: torch.Tensor, I: int) -> int:
- """Copy one expert tensor into its layer/expert slot (shared by serial + parallel
- readers): w1->gate_up[:,:I], w3->gate_up[:,I:], w2->down. Returns the layer index."""
- m = _EXPERT_RE.match(name)
- layer, expert = int(m.group("layer")), int(m.group("expert"))
- proj, kind = m.group("proj"), m.group("kind")
- if kind == "weight":
- t = t.view(torch.uint8)
- if proj == "w1":
- banks["gate_up_packed"][layer][expert, :I] = t
- elif proj == "w3":
- banks["gate_up_packed"][layer][expert, I:] = t
- else: # w2 -> down
- banks["down_packed"][layer][expert] = t
- else: # scale (e8m0)
- if proj == "w1":
- banks["gate_up_scale"][layer][expert, :I] = t
- elif proj == "w3":
- banks["gate_up_scale"][layer][expert, I:] = t
- else:
- banks["down_scale"][layer][expert] = t
- return layer
-
-
-def load_dsfp4_expert_sources_parallel(
- model_path: str, args: DeepseekV4Args, *, workers: int = 8, chunk: int = 8 << 20,
- layer_sink=None,
-) -> dict[str, list[torch.Tensor]]:
- """parallel path: same banks as load_dsfp4_expert_sources, filled from the common
- chunked multi-threaded O_DIRECT reader instead of serial per-shard safe_open.
- ``layer_sink``: see :func:`load_dsfp4_expert_sources`."""
+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
+ """Routed experts, one piece per expert: ``{gate, up, down}`` e2m1 pairs and their e8m0
+ ``_scale`` companions (``w1`` / ``w3`` / ``w2``). The MTP layer's experts are skipped."""
+ if kind is not QuantKind.MXFP4:
+ return None
+ if get_tp_info().size > 1:
+ raise NotImplementedError("DeepSeek-V4 expert banks support TP=1 only")
from freetoken.models.weight import iter_expert_tensors_parallel
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks
+ from freetoken.moe.expert_pieces import per_expert_pieces
+ args = load_args(model_path, max_batch_size=1)
L, E = args.n_layers, args.n_routed_experts
- H, I = args.dim, args.moe_inter_dim
- e8m0 = torch.float8_e8m0fnu
- specs = {
- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),
- "gate_up_scale": ((E, 2 * I, H // 32), e8m0),
- "down_packed": ((E, H, I // 2), torch.uint8),
- "down_scale": ((E, H, I // 32), e8m0),
- }
- hb = alloc_layer_banks(specs, L) # lazy host banks (unpinned)
- banks = {name: [b.tensor for b in hb[name]] for name in specs}
-
- def _is_expert(name: str) -> bool:
- m = _EXPERT_RE.match(name)
- return m is not None and int(m.group("layer")) < L # skip the MTP layer (index L)
-
- def _load(sink) -> int:
- tracker = LayerCompletionTracker(E * 6, hb, sink)
- placed = 0
- for name, t in iter_expert_tensors_parallel(model_path, _is_expert, workers=workers, chunk=chunk):
- layer = _place_dsfp4(banks, name, t, I)
- tracker.note(layer)
- placed += 1
- return placed
-
- if layer_sink is not None:
- placed = _load(layer_sink)
- else:
- with PinPipeline() as pins:
- placed = _load(pins)
-
- expected = L * E * 6
- assert placed == expected, f"loaded {placed} expert tensors, expected {expected}"
- return banks
-
-
-__all__ = [
- "iter_weights",
- "load_dsfp4_expert_sources",
- "load_dsfp4_expert_sources_parallel",
- "is_expert_tensor",
-]
+
+ def locate(raw_name: str):
+ m = _EXPERT_RE.match(raw_name)
+ if m is None or int(m["layer"]) >= L:
+ return None
+ return int(m["layer"]), int(m["expert"]), _PROJ_ROLE[m["proj"]] + _KIND_SUFFIX[m["kind"]]
+
+ if parallel:
+ tensors = iter_expert_tensors_parallel(model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk)
+ return per_expert_pieces(tensors, locate, tensors_per_expert=6)
+
+ def _serial():
+ reader = _ShardReader(model_path, _weight_map(model_path), torch.device("cpu"))
+ try:
+ for li in tqdm(range(L), desc="Loading DSV4 experts (serial)", disable=not get_tp_info().is_primary()):
+ for e in range(E):
+ base = f"layers.{li}.ffn.experts.{e}"
+ for proj in ("w1", "w3", "w2"):
+ for kind_ in ("weight", "scale"):
+ name = f"{base}.{proj}.{kind_}"
+ yield name, reader.get(name)
+ finally:
+ reader.close()
+
+ return per_expert_pieces(_serial(), locate, tensors_per_expert=6)
+
+
+__all__ = ["iter_weights", "iter_expert_pieces"]
diff --git a/python/freetoken/models/gemma4/__init__.py b/python/freetoken/models/gemma4/__init__.py
index 37a8dcb3..a6b285f2 100644
--- a/python/freetoken/models/gemma4/__init__.py
+++ b/python/freetoken/models/gemma4/__init__.py
@@ -10,13 +10,13 @@ from .model import Gemma4ForCausalLM
from .moe import Gemma4Router
from .vision import Gemma4MultimodalEmbedder, Gemma4VisionModel
from .weight import (
+ nvfp4_expert_spec,
iter_weights,
iter_weights_parallel,
- load_nvfp4_expert_sources,
- load_nvfp4_expert_sources_parallel,
)
__all__ = [
+ "nvfp4_expert_spec",
"Gemma4Attention",
"Gemma4ForCausalLM",
"Gemma4MultimodalEmbedder",
@@ -25,8 +25,6 @@ __all__ = [
"parse_config",
"iter_weights",
"iter_weights_parallel",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
"parse_gguf_config",
"iter_gguf_weights",
"load_q4_0_expert_sources",
diff --git a/python/freetoken/models/gemma4/attention.py b/python/freetoken/models/gemma4/attention.py
index 9103bf34..4f85255f 100644
--- a/python/freetoken/models/gemma4/attention.py
+++ b/python/freetoken/models/gemma4/attention.py
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
class Gemma4Attention(BaseOP):
"""Gemma 4 attention for one full-context or SWA layer."""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.layer_id = layer_id
group = config.attention_group_for_layer(layer_id)
self.is_swa = isinstance(group, SWAAttentionGroupConfig)
@@ -37,8 +37,13 @@ class Gemma4Attention(BaseOP):
self.num_qo_heads,
self.num_kv_heads,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
+ )
+ self.o_proj = LinearReplicated(
+ self.q_dim, config.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.o_proj",
)
- self.o_proj = LinearReplicated(self.q_dim, config.hidden_size, has_bias=False)
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.v_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps, with_scale=False)
diff --git a/python/freetoken/models/gemma4/model.py b/python/freetoken/models/gemma4/model.py
index 66fcf412..83e49c4d 100644
--- a/python/freetoken/models/gemma4/model.py
+++ b/python/freetoken/models/gemma4/model.py
@@ -28,11 +28,13 @@ class Gemma4DecoderLayer(BaseOP):
per-layer ``layer_scalar``. The feed-forward is the dual (shared MLP || routed MoE)
branch for MoE checkpoints, or a single dense MLP branch for dense checkpoints."""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self._layer_id = layer_id
- self.self_attn = Gemma4Attention(config, layer_id)
+ self.self_attn = Gemma4Attention(config, layer_id, prefix=f"{prefix}.self_attn")
self.feed_forward = (
- Gemma4MLP(config, layer_id) if config.is_moe else Gemma4DenseMLP(config)
+ Gemma4MLP(config, layer_id, prefix=f"{prefix}.feed_forward")
+ if config.is_moe
+ else Gemma4DenseMLP(config, prefix=f"{prefix}.feed_forward")
)
eps = config.rms_norm_eps
@@ -53,14 +55,17 @@ class Gemma4DecoderLayer(BaseOP):
class Gemma4Model(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
embed_scale=config.embedding_scale,
)
self.layers = OPList(
- [Gemma4DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ Gemma4DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self._image_token_id = config.image_token_id
@@ -101,6 +106,8 @@ class Gemma4ForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
self._final_logit_softcapping = config.final_logit_softcapping
if config.is_multimodal:
diff --git a/python/freetoken/models/gemma4/moe.py b/python/freetoken/models/gemma4/moe.py
index 0185561b..f5c8ca22 100644
--- a/python/freetoken/models/gemma4/moe.py
+++ b/python/freetoken/models/gemma4/moe.py
@@ -39,12 +39,14 @@ class Gemma4Router(BaseOP):
class Gemma4MLP(BaseOP):
"""Gemma 4 feed-forward sandwich: shared MLP plus routed MoE branch."""
- def __init__(self, config: ModelConfig, layer_id: int):
- self.shared_mlp = GatedMLP(config)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.shared_mlp = GatedMLP(config, quant_config=config.quant, prefix=f"{prefix}.shared_mlp")
self.experts = make_moe_layer(
config,
layer_id=layer_id,
activation="gelu_tanh",
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
)
self.router = Gemma4Router(config)
@@ -80,29 +82,6 @@ class Gemma4MLP(BaseOP):
)
-class _Nvfp4GatedMLP(BaseOP):
- """W4A16 NVFP4 gated MLP (fused gate_up + down, gelu-tanh gated). Drop-in for the bf16
- GatedMLP inside Gemma4DenseMLP on modelopt-NVFP4 dense checkpoints; same attribute names
- (gate_up_proj / down_proj) so the loader targets it identically."""
-
- def __init__(self, config: ModelConfig):
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear
- from freetoken.layers import gelu_tanh_and_mul
-
- self.gate_up_proj = Nvfp4DenseColMerged(
- config.hidden_size,
- [config.intermediate_size, config.intermediate_size],
- has_bias=False,
- )
- self.down_proj = Nvfp4DenseLinear(
- config.intermediate_size, config.hidden_size, has_bias=False
- )
- self._act = gelu_tanh_and_mul
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- return self.down_proj.forward(self._act(self.gate_up_proj.forward(x)))
-
-
class Gemma4DenseMLP(BaseOP):
"""Gemma 4 dense feed-forward: a single gated MLP branch (no routed experts),
RMS-normed then residual-added, scaled by the per-layer ``layer_scalar``.
@@ -111,15 +90,10 @@ class Gemma4DenseMLP(BaseOP):
``out = (x + post_feedforward_layernorm(shared_mlp(pre_ff))) * layer_scalar``.
Attribute names match the MoE Gemma4MLP so weight.py's rename machinery
(``mlp.* -> feed_forward.shared_mlp.*``, bare ``layer_scalar`` /
- ``post_feedforward_layernorm.``) resolves the dense checkpoint unchanged. The gated
- MLP is bf16, or W4A16 NVFP4 when the checkpoint quantizes the dense MLP."""
+ ``post_feedforward_layernorm.``) resolves the dense checkpoint unchanged."""
- def __init__(self, config: ModelConfig):
- self.shared_mlp = (
- _Nvfp4GatedMLP(config)
- if getattr(config, "dense_quant", "none") == "nvfp4"
- else GatedMLP(config)
- )
+ def __init__(self, config: ModelConfig, *, prefix: str = ""):
+ self.shared_mlp = GatedMLP(config, quant_config=config.quant, prefix=f"{prefix}.shared_mlp")
self.post_feedforward_layernorm = GemmaRMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.py
index 43c2355d..71f5531b 100644
--- a/python/freetoken/models/gemma4/weight.py
+++ b/python/freetoken/models/gemma4/weight.py
@@ -14,7 +14,6 @@ from freetoken.models.loader import (
)
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
- load_nvfp4_expert_source_banks,
)
from freetoken.utils import cached_load_hf_config
from tqdm import tqdm
@@ -178,7 +177,7 @@ def iter_weights(
if name is None:
continue
- # Per-expert NVFP4 tensors go to the offload cache (load_nvfp4_expert_sources),
+ # Per-expert NVFP4 tensors go to the offload cache (expert pieces),
# not this dense pass; fused bf16/q4_0 experts lack ".experts.<int>." so are unaffected.
if _NVFP4_EXPERT_RE.search(raw_name):
continue
@@ -255,7 +254,7 @@ def iter_weights_parallel(
(``feed_forward.experts.{gate_up_proj,down_proj}``), so no merge needed; same key
rename as iter_weights, read via the common chunked O_DIRECT reader."""
assert include_moe_experts and not include_non_moe, (
- "gemma4 parallel reader is experts-only (used by load_moe_expert_sources)"
+ "gemma4 parallel reader is experts-only (used by the expert piece reader)"
)
from freetoken.models.weight import iter_expert_tensors_parallel
@@ -275,41 +274,12 @@ def iter_weights_parallel(
yield _expert_name(raw_name), tensor
-def load_nvfp4_expert_sources(
- model_path: str, config, *, layer_sink=None
-) -> dict[str, list[torch.Tensor]]:
- """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""
- return load_nvfp4_expert_source_banks(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- layer_sink=layer_sink,
- )
-
-
-def load_nvfp4_expert_sources_parallel(
- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
-):
- """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""
- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
-
- return load_nvfp4_expert_source_banks_parallel(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- workers=workers,
- chunk=chunk,
- layer_sink=layer_sink,
- )
+def nvfp4_expert_spec(model_path: str, config):
+ return _NVFP4_SOURCE_SPEC
__all__ = [
+ "nvfp4_expert_spec",
"iter_weights",
"iter_weights_parallel",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
]
diff --git a/python/freetoken/models/glm4_moe/__init__.py b/python/freetoken/models/glm4_moe/__init__.py
index a07134c3..b8748b1d 100644
--- a/python/freetoken/models/glm4_moe/__init__.py
+++ b/python/freetoken/models/glm4_moe/__init__.py
@@ -1,11 +1,10 @@
from .config import parse_config
from .model import Glm4MoeForCausalLM
-from .weight import iter_weights, load_nvfp4_expert_sources, load_nvfp4_expert_sources_parallel
+from .weight import iter_weights, nvfp4_expert_spec
__all__ = [
+ "nvfp4_expert_spec",
"Glm4MoeForCausalLM",
"parse_config",
"iter_weights",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
]
diff --git a/python/freetoken/models/glm4_moe/mlp.py b/python/freetoken/models/glm4_moe/mlp.py
index f8fb8bef..13fb6ee1 100644
--- a/python/freetoken/models/glm4_moe/mlp.py
+++ b/python/freetoken/models/glm4_moe/mlp.py
@@ -3,29 +3,26 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import torch.nn.functional as F
-from freetoken.layers import BaseOP
+from freetoken.layers import BaseOP, LinearReplicated
from freetoken.utils import nvtx_annotate
-from .nvfp4_linear import LinearNVFP4
-
if TYPE_CHECKING:
import torch
+ from freetoken.layers.quantization import QuantConfig
+
class GlmGatedMLP(BaseOP):
- """SwiGLU MLP whose projections keep the checkpoint's native NVFP4 weights.
-
- Used for both the leading dense layers (``intermediate_size``) and each MoE layer's
- always-on shared expert (``moe_intermediate_size``). GLM-4 ships these in NVFP4; we keep
- them NVFP4 and dequantize in the forward (:class:`LinearNVFP4`) -- identical math to the
- routed experts (faithful to the checkpoint) and the smallest footprint, which matters
- because GLM activates 89*8 experts per token. Activations stay bf16.
- """
-
- def __init__(self, hidden_size: int, intermediate_size: int):
- self.gate_proj = LinearNVFP4(hidden_size, intermediate_size)
- self.up_proj = LinearNVFP4(hidden_size, intermediate_size)
- self.down_proj = LinearNVFP4(intermediate_size, hidden_size)
+ """SwiGLU MLP for the leading dense layers (``intermediate_size``) and each MoE layer's
+ always-on shared expert (``moe_intermediate_size``); the projections take whatever the
+ checkpoint stores for their prefix (NVFP4 in the GLM-4 releases)."""
+
+ def __init__(
+ self, hidden_size: int, intermediate_size: int, *, quant_config: QuantConfig | None = None, prefix: str = ""
+ ):
+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")
+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")
+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")
@nvtx_annotate("MLP")
def forward(self, x: torch.Tensor) -> torch.Tensor:
diff --git a/python/freetoken/models/glm4_moe/model.py b/python/freetoken/models/glm4_moe/model.py
index 17f4f0aa..b5c647e1 100644
--- a/python/freetoken/models/glm4_moe/model.py
+++ b/python/freetoken/models/glm4_moe/model.py
@@ -18,12 +18,14 @@ if TYPE_CHECKING:
class Glm4MoeDecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.self_attn = Glm4MoeAttention(config, layer_id)
if layer_id >= config.first_k_dense_replace:
- self.mlp: BaseOP = Glm4MoeSparseBlock(config, layer_id)
+ self.mlp: BaseOP = Glm4MoeSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")
else:
- self.mlp = GlmGatedMLP(config.hidden_size, config.intermediate_size)
+ self.mlp = GlmGatedMLP(
+ config.hidden_size, config.intermediate_size, quant_config=config.quant, prefix=f"{prefix}.mlp"
+ )
self.input_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNormFused(
size=config.hidden_size, eps=config.rms_norm_eps
@@ -42,13 +44,16 @@ class Glm4MoeDecoderLayer(BaseOP):
class Glm4MoeModel(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = EmbeddingDF11(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [Glm4MoeDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ Glm4MoeDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
@@ -68,6 +73,8 @@ class Glm4MoeForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/glm4_moe/moe.py b/python/freetoken/models/glm4_moe/moe.py
index 757af0f5..e1b0f18b 100644
--- a/python/freetoken/models/glm4_moe/moe.py
+++ b/python/freetoken/models/glm4_moe/moe.py
@@ -23,7 +23,7 @@ class Glm4MoeSparseBlock(BaseOP):
``routed_scaling_factor``. The shared expert is applied to the same input and added.
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.top_k = config.num_experts_per_tok
self.num_experts = config.num_experts
self.norm_topk_prob = config.norm_topk_prob
@@ -44,10 +44,14 @@ class Glm4MoeSparseBlock(BaseOP):
config,
layer_id=layer_id - config.first_k_dense_replace,
renormalize=config.norm_topk_prob,
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
)
self.shared_experts = GlmGatedMLP(
config.hidden_size,
config.moe_intermediate_size * max(1, config.n_shared_experts),
+ quant_config=config.quant,
+ prefix=f"{prefix}.shared_experts",
)
def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:
diff --git a/python/freetoken/models/glm4_moe/nvfp4_linear.py b/python/freetoken/models/glm4_moe/nvfp4_linear.py
deleted file mode 100644
index 1eff394b..00000000
--- a/python/freetoken/models/glm4_moe/nvfp4_linear.py
+++ /dev/null
@@ -1,114 +0,0 @@
-from __future__ import annotations
-
-import os
-
-import torch
-from freetoken.layers import BaseOP
-from freetoken.layers.base import _concat_prefix
-from freetoken.kernel.triton.nvfp4_linear import (
- nvfp4_dense_linear_t,
- nvfp4_transpose_resident,
-)
-
-# Resident NVFP4 linears default to vLLM's fused FP4 Marlin GEMM (W4A16) on Marlin-capable GPUs
-# (cc in [8.0, 10.0), which lack native FP4 tensor cores): ~2-7x faster per-linear at bs=1 and it
-# frees the bf16 dequant scratch, with numerically-faithful output (validated under the model's
-# sampling regime). When Marlin is unavailable (e.g. sm_120, or vLLM not installed) the
-# forward runs FreeToken's own shared
-# W4A16 kernel (freetoken.kernel.triton.nvfp4_linear) -- reading the FP4 weight directly (fp32
-# accumulation, no per-forward bf16 dequant + scratch), which is faster *and* strictly more
-# accurate than the old dequant fallback. Set GLM_NVFP4_NOMARLIN=1 to force the shared kernel.
-# (Marlin's ~bf16-level difference can perturb *greedy* decoding, but GLM's recommended decoding
-# is sampling, where it is correctness-equivalent.)
-_DISABLE_MARLIN = os.environ.get("GLM_NVFP4_NOMARLIN", "0") == "1"
-
-
-class LinearNVFP4(BaseOP):
- """Replicated (TP=1) linear that keeps the checkpoint's native NVFP4 weight as-is.
-
- Used for GLM-4's always-resident dense MLPs and shared experts (W4A16: bf16 activation,
- FP4 weight). By default the weight is repacked to Marlin layout at load and the forward is a
- single fused dequant+GEMM (vLLM Marlin). With GLM_NVFP4_NOMARLIN=1 (or on non-Marlin GPUs /
- when vLLM is absent) it runs the shared FreeToken W4A16 kernel directly on the packed FP4
- weight (decode int32 GEMV / prefill GEMM, fp32 accumulation) -- no bf16 dequant scratch.
-
- Keeping NVFP4 (4 bits) is faithful to the checkpoint (identical math to the routed experts)
- and the smallest option -- re-quantizing to fp8 would add ~2.6% error *and* use twice the
- memory, and the budget is tight once attention is lossless DF11.
- """
-
- def __init__(self, in_features: int, out_features: int):
- assert in_features % 16 == 0
- self.in_features = in_features
- self.out_features = out_features
- self.weight = torch.empty(out_features, in_features // 2, dtype=torch.uint8)
- self.weight_scale = torch.empty(out_features, in_features // 16, dtype=torch.float8_e4m3fn)
- self.weight_global = torch.empty(out_features, dtype=torch.float16)
- self._marlin = False
-
- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:
- weight = state_dict.pop(_concat_prefix(prefix, "weight"))
- weight_scale = state_dict.pop(_concat_prefix(prefix, "weight_scale"))
- weight_global = state_dict.pop(_concat_prefix(prefix, "weight_global"))
-
- if not _DISABLE_MARLIN:
- try:
- from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
- apply_fp4_marlin_linear,
- is_fp4_marlin_supported,
- prepare_fp4_layer_for_marlin,
- )
-
- marlin_supported = is_fp4_marlin_supported()
- except Exception:
- marlin_supported = False
-
- if marlin_supported:
- from types import SimpleNamespace
-
- # weight_global is the per-tensor weight_scale_2 broadcast to [out]; Marlin
- # wants the scalar.
- layer = SimpleNamespace(
- weight=weight,
- weight_scale=weight_scale,
- weight_scale_2=weight_global.reshape(-1)[0].to(torch.bfloat16),
- output_size_per_partition=self.out_features,
- input_size_per_partition=self.in_features,
- params_dtype=torch.bfloat16,
- )
- prepare_fp4_layer_for_marlin(layer) # repacks weight + processes scales in-place
- self._mweight = layer.weight
- self._mscale = layer.weight_scale
- self._mglobal = layer.weight_scale_2
- self._mworkspace = layer.workspace
- self._apply = apply_fp4_marlin_linear
- self._marlin = True
- dev = weight.device
- self.weight = torch.empty(0, dtype=torch.uint8, device=dev)
- self.weight_scale = torch.empty(0, dtype=torch.float8_e4m3fn, device=dev)
- self.weight_global = torch.empty(0, dtype=torch.float16, device=dev)
-
- if not self._marlin:
- # K-major resident repack: coalesced weight loads in the shared W4A16 kernels.
- self.weight, self.weight_scale = nvfp4_transpose_resident(weight, weight_scale)
- self.weight_global = weight_global
-
- if not _internal and state_dict:
- raise RuntimeError(f"Unexpected keys in state_dict: {list(state_dict.keys())}")
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- if self._marlin:
- return self._apply(
- input=x,
- weight=self._mweight,
- weight_scale=self._mscale,
- weight_scale_2=self._mglobal,
- workspace=self._mworkspace,
- size_n=self.out_features,
- size_k=self.in_features,
- )
- # Shared FreeToken W4A16 kernel: read the packed FP4 weight directly (no bf16 scratch).
- return nvfp4_dense_linear_t(x, self.weight, self.weight_scale, self.weight_global)
-
-
-__all__ = ["LinearNVFP4"]
diff --git a/python/freetoken/models/glm4_moe/weight.py b/python/freetoken/models/glm4_moe/weight.py
index 7efc6e6b..5d4f9fce 100644
--- a/python/freetoken/models/glm4_moe/weight.py
+++ b/python/freetoken/models/glm4_moe/weight.py
@@ -11,7 +11,6 @@ from freetoken.distributed import get_tp_info
from freetoken.models.loader import drop_page_cache
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
- load_nvfp4_expert_source_banks,
)
from freetoken.utils import cached_load_hf_config, download_hf_weight
from tqdm import tqdm
@@ -79,8 +78,8 @@ def _iter_nvfp4_resident(
"""Yield native NVFP4 buffers for an always-resident Linear.
``weight`` (packed fp4) + ``weight_scale`` (fp8 block scale) verbatim; per-tensor
- ``weight_scale_2`` broadcast to per-row fp16 ``weight_global`` as LinearNVFP4 /
- dequant_nvfp4 expect. lossless vs checkpoint; same dequant math as routed experts.
+ ``weight_scale_2`` broadcast to per-row fp16 ``weight_global`` as the NVFP4 linear
+ method / dequant_nvfp4 expect. lossless vs checkpoint; same dequant math as routed experts.
"""
packed = reader.get(f"{src_prefix}.weight") # [OUT, IN//2] uint8
scale = reader.get(f"{src_prefix}.weight_scale") # [OUT, IN//16] fp8-e4m3
@@ -122,7 +121,7 @@ def iter_weights(
"""
assert not include_moe_experts, (
"GLM-4 MoE stores experts as NVFP4 and only supports the offload backend; experts "
- "are loaded into the offload cache via load_nvfp4_expert_sources()."
+ "are loaded into the offload cache from their NVFP4 pieces."
)
assert include_non_moe
config = parse_config(cached_load_hf_config(model_path))
@@ -191,39 +190,8 @@ def _iter_resident_weights(reader, config, primary) -> Iterator[tuple[str, torch
# --------------------------------------------------------------------------------------
# Routed expert host banks (NVFP4) for the offload cache.
# --------------------------------------------------------------------------------------
-def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None) -> dict[str, torch.Tensor]:
- """Build the pinned CPU NVFP4 banks for GLM-4's routed experts.
-
- experts exist only for layers [first_k_dense_replace, num_layers) and pack by MoE layer
- index (layer - first_k_dense_replace) so banks have no holes for leading dense layers.
- MTP layer excluded.
- """
- return load_nvfp4_expert_source_banks(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- layer_sink=layer_sink,
- )
-
-
-def load_nvfp4_expert_sources_parallel(
- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
-):
- """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""
- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
-
- return load_nvfp4_expert_source_banks_parallel(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- workers=workers,
- chunk=chunk,
- layer_sink=layer_sink,
- )
+def nvfp4_expert_spec(model_path: str, config):
+ return _NVFP4_SOURCE_SPEC
-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]
+__all__ = ["iter_weights", "nvfp4_expert_spec"]
diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.py
index c1c05140..7e949097 100644
--- a/python/freetoken/models/glm5_next/__init__.py
+++ b/python/freetoken/models/glm5_next/__init__.py
@@ -1,10 +1,11 @@
from .config import parse_config
from .model import Glm5NextForCausalLM
-from .weight import iter_weights, load_nvfp4_expert_sources
+from .weight import iter_expert_pieces, iter_weights, nvfp4_expert_spec
__all__ = [
+ "nvfp4_expert_spec",
"Glm5NextForCausalLM",
"parse_config",
"iter_weights",
- "load_nvfp4_expert_sources",
+ "iter_expert_pieces",
]
diff --git a/python/freetoken/models/glm5_next/attention.py b/python/freetoken/models/glm5_next/attention.py
index 1ab6c340..0814a687 100644
--- a/python/freetoken/models/glm5_next/attention.py
+++ b/python/freetoken/models/glm5_next/attention.py
@@ -31,10 +31,11 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import torch
+from freetoken.layers.quantization import QuantKind
from freetoken.core import get_global_ctx
from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
# Shared with GLM-5.2 (weight.py imports privately from the same package).
-from freetoken.models.glm_moe_dsa.attention import _IdxLayerNorm, _make_proj
+from freetoken.models.glm_moe_dsa.attention import _IdxLayerNorm
from freetoken.utils import nvtx_annotate
if TYPE_CHECKING:
@@ -46,17 +47,17 @@ class Glm5NextIndexer(BaseOP):
checkpoint's NoPE geometry leaves ``qk_rope_head_dim == 0`` so position enters
only via the compression APE).
- Kept bf16 in every quant mode: small (~17 MB/layer) and the top-k boundary is
- precision-sensitive (same reasoning as glm_moe_dsa's indexer).
+ ``wq_b`` follows the checkpoint's quant config; ``wk`` and ``weights_proj`` stay bf16 (same reasoning as glm_moe_dsa's indexer).
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.glm5_args
self.n_heads = args.index_n_heads
self.head_dim = args.index_head_dim
self.kpool = args.index_kpool
self.wq_b = LinearReplicated(
- args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False
+ args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.wq_b",
)
self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False)
self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6)
@@ -91,10 +92,10 @@ class Glm5NextIndexer(BaseOP):
class Glm5NextAttention(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.glm5_args
self.layer_id = layer_id
- self.indexer = Glm5NextIndexer(config, layer_id)
+ self.indexer = Glm5NextIndexer(config, layer_id, prefix=f"{prefix}.indexer")
self.num_heads = args.num_heads
self.qk_nope_head_dim = args.qk_nope_head_dim
self.qk_rope_head_dim = args.qk_rope_head_dim # 0 (NoPE)
@@ -106,25 +107,32 @@ class Glm5NextAttention(BaseOP):
"would need the glm_moe_dsa rope plumbing back"
)
- quant = config.attn_quant
- self.q_a_proj = _make_proj(quant, args.hidden_size, args.q_lora_rank)
+ self.q_a_proj = LinearReplicated(
+ args.hidden_size, args.q_lora_rank, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.q_a_proj",
+ )
self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps)
- self.q_b_proj = _make_proj(
- quant, args.q_lora_rank, self.num_heads * self.qk_head_dim
+ self.q_b_proj = LinearReplicated(
+ args.q_lora_rank, self.num_heads * self.qk_head_dim, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.q_b_proj",
)
# NoPE: kv_a projects to bare ckv (no +qk_rope_head_dim rows).
- self.kv_a_proj_with_mqa = _make_proj(
- quant, args.hidden_size, self.kv_lora_rank
+ self.kv_a_proj_with_mqa = LinearReplicated(
+ args.hidden_size, self.kv_lora_rank, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.kv_a_proj_with_mqa",
)
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps)
- # kv_b stays bf16 in every mode (bmm absorption operand, not a Linear).
+ # the MLA absorption reads kv_b_proj.weight as bmm operands, so only an unquantized scheme is served
self.kv_b_proj = LinearReplicated(
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.kv_b_proj",
)
- self.o_proj = _make_proj(
- quant, self.num_heads * self.v_head_dim, args.hidden_size
+ assert self.kv_b_proj.quant_method.kind is QuantKind.NONE, f"{prefix}.kv_b_proj: a quantized kv_b_proj needs a dequantized copy for the MLA absorption"
+ self.o_proj = LinearReplicated(
+ self.num_heads * self.v_head_dim, args.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.o_proj",
)
self._w_uk: torch.Tensor | None = None
self._w_uv: torch.Tensor | None = None
diff --git a/python/freetoken/models/glm5_next/config.py b/python/freetoken/models/glm5_next/config.py
index 294ef579..1cc6d798 100644
--- a/python/freetoken/models/glm5_next/config.py
+++ b/python/freetoken/models/glm5_next/config.py
@@ -17,15 +17,7 @@ scaling 2.5) with 288 routed experts and a clamped SwiGLU
(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in
``glm5_args`` (``Glm5NextArgs``).
-Resident-weight quantization (attn/dense/lm_head fp8-at-load, the GLM-5.2
-bandwidth trick) is OFF by default: the NVFP4 exports deliberately quantize only
-the routed experts (attention / shared expert / lm_head ship bf16 behind the
-quantization_config ignore list), and a serving engine must not override the
-checkpoint author's precision decision silently -- the checkpoint is served
-as-is, like vLLM/sglang. FREETOKEN_GLM5_ATTN_FP8=1 / FREETOKEN_GLM5_MLP_FP8=1
-opt into the W8A16 requantization (measured decode 36 -> 45.5 tok/s on the
-hybrid reference setup; the win needs CUDA graphs -- launch-bound eager decode
-gets slower).
+The checkpoint is served in the precision its quantization_config declares.
"""
from __future__ import annotations
@@ -43,12 +35,6 @@ from freetoken.models.config import (
from .args import load_args
-# Load-time W8A16 fp8 for resident weights: default OFF, env opt-in (see module
-# docstring for the rationale and measured numbers). attn covers the KDA
-# in_proj_qkv/o_proj + DSA projections (the precision-sensitive b|f_a|g_a gate
-# slice stays bf16, see kda.py); mlp covers dense MLPs, shared expert, lm_head.
-_ATTN_FP8 = os.getenv("FREETOKEN_GLM5_ATTN_FP8", "0") != "0"
-_MLP_FP8 = os.getenv("FREETOKEN_GLM5_MLP_FP8", "0") != "0"
def _dsa_on(args, dsa_layer_ids) -> bool:
@@ -142,6 +128,11 @@ def parse_config(hf_config: Any) -> ModelConfig:
t == "sparse" for t in mlp_types[first_dense:]
), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}"
+ from freetoken.models.qwen3_5_moe.config import _fp8_block_quant # shared until the legacy quant fields go
+
+ expert_quant, weight_block_size = _fp8_block_quant(hf_config)
+ if expert_quant == "none":
+ expert_quant = detect_expert_quant(hf_config)
return ModelConfig(
num_layers=num_layers,
num_qo_heads=args.num_heads,
@@ -177,7 +168,8 @@ def parse_config(hf_config: Any) -> ModelConfig:
hf_config, "architectures", ["Glm5NextForConditionalGeneration"]
),
moe_enabled=True,
- expert_quant=detect_expert_quant(hf_config),
+ expert_quant=expert_quant,
+ weight_block_size=weight_block_size,
first_k_dense_replace=first_dense,
n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0),
routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
@@ -186,9 +178,6 @@ def parse_config(hf_config: Any) -> ModelConfig:
attn_sm_scale=args.qk_head_dim**-0.5,
has_attn_bias=bool(getattr(text, "attention_bias", False)),
swiglu_limit=args.swiglu_limit,
- attn_quant="fp8_pertensor" if _ATTN_FP8 else "none",
- dense_quant="fp8_pertensor" if _MLP_FP8 else "none",
- lm_head_quant="fp8_pertensor" if _MLP_FP8 else "none",
vision_config=None, # text-only milestone; model.visual.* weights are dropped
image_token_id=getattr(hf_config, "image_token_id", None),
glm5_args=args,
diff --git a/python/freetoken/models/glm5_next/kda.py b/python/freetoken/models/glm5_next/kda.py
index 1d44e3a4..ba722ae2 100644
--- a/python/freetoken/models/glm5_next/kda.py
+++ b/python/freetoken/models/glm5_next/kda.py
@@ -28,7 +28,7 @@ from typing import TYPE_CHECKING
import torch
from freetoken.core import get_global_ctx
from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen
-from freetoken.layers import BaseOP, LinearColParallelMerged, LinearReplicated
+from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicated
from freetoken.utils import nvtx_annotate
if TYPE_CHECKING:
@@ -43,29 +43,13 @@ class _DepthwiseConv1d(BaseOP):
self.weight = torch.empty(conv_dim, 1, kernel)
-class _GatedRMSNormSigmoid(BaseOP):
- """RMSNorm(x) * sigmoid(z), fused (KDA's o_norm; GDN's variant gates with silu)."""
-
- def __init__(self, dim: int, eps: float):
- self.weight = torch.empty(dim)
- self.eps = eps
-
- def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
- from freetoken.kernel.fla import rms_norm_gated
-
- return rms_norm_gated(
- x=x, weight=self.weight, bias=None, z=z, eps=self.eps,
- is_rms_norm=True, norm_before_gate=True, activation="sigmoid",
- )
-
-
class Glm5NextKDA(BaseOP):
"""KDA op; state is held in ``ctx.linear_state_pool`` keyed by the request's
linear slot (``FLAMetadata.cache_indices``). Parameter names follow the
checkpoint modulo two load-time fusions (see weight.py): ``in_proj`` is
q|k|v|b|f_a|g_a concatenated, ``conv1d`` is q|k|v conv concatenated."""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.glm5_args
self.layer_id = layer_id
self.num_heads = args.linear_num_heads
@@ -77,41 +61,22 @@ class Glm5NextKDA(BaseOP):
self.scale = self.head_dim**-0.5
p, h, d = self.proj_size, self.num_heads, self.head_dim
- # q|k|v dominate the resident read (3 * 8192 x 4096 = 201 MB/layer bf16 --
- # the single biggest dense-weight stream in the model); under the fp8
- # resident mode they split into their own W8A16 GEMM while the small,
- # precision-sensitive gate projections (b|f_a|g_a) stay bf16 (the GDN
- # qkvz/ba split precedent). BF16 mode keeps the single fused GEMM.
- self._fp8 = config.attn_quant == "fp8_pertensor"
- self._bfg_split = [h, d, d]
- if self._fp8:
- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
-
- self.in_proj_qkv = Fp8PerTensorColMerged(
- args.hidden_size, [p, p, p], has_bias=False
- )
- self.in_proj_bfg = LinearColParallelMerged(
- args.hidden_size, self._bfg_split, has_bias=False
- )
- else:
- self._in_proj_split = [p, p, p, h, d, d]
- self.in_proj = LinearColParallelMerged(
- args.hidden_size, self._in_proj_split, has_bias=False
- )
+ # one fused input GEMM over q|k|v|b|f_a|g_a
+ self._in_proj_split = [p, p, p, h, d, d]
+ self.in_proj = LinearColParallelMerged(
+ args.hidden_size, self._in_proj_split, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.in_proj",
+ )
# Low-rank gate up-projections (128 -> 8192): forget gate and output gate.
- self.f_b_proj = LinearReplicated(d, p, has_bias=False)
- self.g_b_proj = LinearReplicated(d, p, has_bias=False)
+ self.f_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.f_b_proj")
+ self.g_b_proj = LinearReplicated(d, p, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.g_b_proj")
self.conv1d = _DepthwiseConv1d(self.conv_dim, self.conv_kernel_size)
# Gate params stay fp32 (exp/sigmoid precision; the kernels read fp32).
# models/weight.py exempts *.A_log / *.dt_bias from the model-dtype downcast.
self.A_log = torch.empty(h, dtype=torch.float32)
self.dt_bias = torch.empty(p, dtype=torch.float32)
- self.o_norm = _GatedRMSNormSigmoid(d, eps=args.norm_eps)
- # o_proj follows the resident quant mode (rationale: the split comment
- # at _fp8 above); f_b/g_b stay bf16 alongside the gate slice.
- from .attention import _make_proj
-
- self.o_proj = _make_proj(config.attn_quant, p, args.hidden_size)
+ self.o_norm = GatedRMSNorm(d, eps=args.norm_eps, activation="sigmoid")
+ self.o_proj = LinearReplicated(p, args.hidden_size, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.o_proj")
def _conv_weight(self) -> torch.Tensor:
return self.conv1d.weight.squeeze(1) # [conv_dim, kernel]
@@ -143,16 +108,10 @@ class Glm5NextKDA(BaseOP):
fla = build_fla_metadata(batch, hidden_states.device)
batch.fla_metadata = fla
- if self._fp8:
- conv_in = self.in_proj_qkv.forward(hidden_states)
- b, f_a, g_a = torch.split(
- self.in_proj_bfg.forward(hidden_states), self._bfg_split, dim=-1
- )
- else:
- proj = self.in_proj.forward(hidden_states)
- conv_in, b, f_a, g_a = torch.split(
- proj, [self.conv_dim, h, d, d], dim=-1
- )
+ proj = self.in_proj.forward(hidden_states)
+ conv_in, b, f_a, g_a = torch.split(
+ proj, [self.conv_dim, h, d, d], dim=-1
+ )
g1 = self.f_b_proj.forward(f_a) # raw forget-gate logits [T, H*D]
g2 = self.g_b_proj.forward(g_a) # output-gate logits [T, H*D]
li = pool.local_index(self.layer_id)
diff --git a/python/freetoken/models/glm5_next/mlp.py b/python/freetoken/models/glm5_next/mlp.py
index 1ee508bf..5ec3c66a 100644
--- a/python/freetoken/models/glm5_next/mlp.py
+++ b/python/freetoken/models/glm5_next/mlp.py
@@ -1,7 +1,6 @@
"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers and shared experts.
-Same shape as glm_moe_dsa's GlmDsaGatedMLP (bf16 in the NVFP4 checkpoint;
-optional W8A16 fp8-at-load via ``ModelConfig.dense_quant``), but the activation
+Same shape as glm_moe_dsa's GlmDsaGatedMLP (every projection built from the QuantConfig), but the activation
is the GLM-5.3 clamped SwiGLU (``swiglu_limit``):
``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``.
"""
@@ -13,7 +12,7 @@ import torch
from freetoken.layers import BaseOP, swiglu_clamp_and_mul
from freetoken.utils import nvtx_annotate
-from .attention import _make_proj
+from freetoken.layers import LinearReplicated
class Glm5NextGatedMLP(BaseOP):
@@ -21,12 +20,14 @@ class Glm5NextGatedMLP(BaseOP):
self,
hidden_size: int,
intermediate_size: int,
- quant: str = "none",
swiglu_limit: float | None = None,
+ *,
+ quant_config=None,
+ prefix: str = "",
):
- self.gate_proj = _make_proj(quant, hidden_size, intermediate_size)
- self.up_proj = _make_proj(quant, hidden_size, intermediate_size)
- self.down_proj = _make_proj(quant, intermediate_size, hidden_size)
+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")
+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")
+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")
self.swiglu_limit = swiglu_limit
@nvtx_annotate("MLP")
diff --git a/python/freetoken/models/glm5_next/model.py b/python/freetoken/models/glm5_next/model.py
index 4c011dd6..ae287215 100644
--- a/python/freetoken/models/glm5_next/model.py
+++ b/python/freetoken/models/glm5_next/model.py
@@ -41,28 +41,8 @@ if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
-class Glm5Fp8LMHead(ParallelLMHead):
- """W8A16 lm_head (fp8-e4m3 weight + per-row scale, quantized at load); the
- full-vocab decode GEMV reads the whole head every step -- fp8 halves it.
- Same contract as glm_moe_dsa's GlmFp8LMHead."""
-
- def __init__(self, num_embeddings: int, embedding_dim: int):
- super().__init__(num_embeddings, embedding_dim, tie_word_embeddings=False)
- self.weight = torch.empty(num_embeddings, embedding_dim, dtype=torch.float8_e4m3fn)
- self.weight_scale = torch.empty(num_embeddings, dtype=torch.float32)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
-
- batch = get_global_ctx().batch
- if batch.is_prefill:
- indices = batch.attn_metadata.get_last_indices(batch.size)
- x = x[indices].contiguous()
- return fp8_pertensor_linear(x, self.weight, self.weight_scale)
-
-
class Glm5NextDecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.glm5_args
self._layer_id = layer_id
self._is_last = layer_id == config.num_layers - 1
@@ -74,15 +54,15 @@ class Glm5NextDecoderLayer(BaseOP):
self._sinkhorn = args.mhc_sinkhorn_iterations
if args.is_kda_layer(layer_id):
- self.self_attn: BaseOP = Glm5NextKDA(config, layer_id)
+ self.self_attn: BaseOP = Glm5NextKDA(config, layer_id, prefix=f"{prefix}.self_attn")
else:
- self.self_attn = Glm5NextAttention(config, layer_id)
+ self.self_attn = Glm5NextAttention(config, layer_id, prefix=f"{prefix}.self_attn")
if layer_id >= config.first_k_dense_replace:
- self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id)
+ self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")
else:
self.mlp = Glm5NextGatedMLP(
- config.hidden_size, config.intermediate_size,
- quant=config.dense_quant, swiglu_limit=config.swiglu_limit,
+ config.hidden_size, config.intermediate_size, swiglu_limit=config.swiglu_limit,
+ quant_config=config.quant, prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)
self.post_attention_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps)
@@ -156,13 +136,13 @@ class Glm5NextDecoderLayer(BaseOP):
class Glm5NextModel(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [Glm5NextDecoderLayer(config, i) for i in range(config.num_layers)]
+ [Glm5NextDecoderLayer(config, i, prefix=f"{prefix}.layers.{i}") for i in range(config.num_layers)]
)
self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps)
@@ -178,17 +158,14 @@ class Glm5NextForCausalLM(BaseLLMModel):
def __init__(self, config: ModelConfig):
self._config = config
self.model = Glm5NextModel(config)
- if config.lm_head_quant == "fp8_pertensor" and not config.tie_word_embeddings:
- self.lm_head: BaseOP = Glm5Fp8LMHead(
- num_embeddings=config.vocab_size, embedding_dim=config.hidden_size
- )
- else:
- self.lm_head = ParallelLMHead(
- num_embeddings=config.vocab_size,
- embedding_dim=config.hidden_size,
- tie_word_embeddings=config.tie_word_embeddings,
- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
- )
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
+ )
def prepare_for_runtime(self) -> None:
"""Post-load, pre-KV-sizing hook: materialize the DSA layers' bmm-ready
diff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.py
index 038ffc0d..1cf27ca6 100644
--- a/python/freetoken/models/glm5_next/moe.py
+++ b/python/freetoken/models/glm5_next/moe.py
@@ -4,7 +4,7 @@ Routing is identical to glm_moe_dsa (sigmoid scores + selection-only
``e_score_correction_bias``, optional group-limited top-k, gather unbiased
scores, renormalize, scale by ``routed_scaling_factor``); the deltas are the
expert count (288, top-8) and the clamped-SwiGLU activation (``swiglu_limit`` =
-10), which rides ``make_moe_layer``'s ``extra_attrs`` into the offload kernels
+10), carried as the experts layer's ``limit`` into the offload kernels
(triton NVFP4 in-GPU, generic-epilogue CPU GEMV). The marlin/b12x borrowed
kernels hard-code silu; the backend selector already falls back to triton for
non-silu experts.
@@ -27,7 +27,7 @@ TopK = Tuple[torch.Tensor, torch.Tensor]
class Glm5NextSparseBlock(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.top_k = config.num_experts_per_tok
self.num_experts = config.num_experts
self.norm_topk_prob = config.norm_topk_prob
@@ -44,16 +44,16 @@ class Glm5NextSparseBlock(BaseOP):
layer_id=layer_id - config.first_k_dense_replace,
activation="swiglu_clamp" if config.swiglu_limit is not None else "silu",
renormalize=config.norm_topk_prob,
- extra_attrs={
- "swiglu_limit": config.swiglu_limit,
- "hidden_act_alpha": 1.0, # plain sigmoid inside the clamped swiglu
- },
+ limit=config.swiglu_limit,
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
)
self.shared_experts = Glm5NextGatedMLP(
config.hidden_size,
config.moe_intermediate_size * max(1, config.n_shared_experts),
- quant=config.dense_quant,
swiglu_limit=config.swiglu_limit,
+ quant_config=config.quant,
+ prefix=f"{prefix}.shared_experts",
)
def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:
diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.py
index fda85225..03e2a7a5 100644
--- a/python/freetoken/models/glm5_next/weight.py
+++ b/python/freetoken/models/glm5_next/weight.py
@@ -1,13 +1,14 @@
"""Weight loading for GLM-5.3-Flash (``glm5_next``).
-Supported checkpoints: NVFP4 exports of GLM-5.3-Flash in the multimodal-wrapper
-layout (``model.language_model.*``) -- ModelOpt tensor kinds (LibertAIDAI) or
-compressed-tensors kinds (RedHatAI), selected by ``quantization_config``. Not
-supported: bf16-expert originals (zai-org), text-only key layouts, TP > 1.
+Supported checkpoints, all in the multimodal-wrapper layout (``model.language_model.*``):
+NVFP4 exports (ModelOpt tensor kinds from LibertAIDAI, compressed-tensors kinds from
+RedHatAI, selected by ``quantization_config``) and the zai-org block-fp8 release. Not
+supported: text-only key layouts, TP > 1, resident routed experts.
-Routed experts go to the offload cache via ``load_nvfp4_expert_sources``;
-everything else loads bf16 with keys renamed ``model.language_model.X`` ->
-``model.X``. ``model.visual.*`` and the trailing MTP layer are never read.
+Routed experts go to the offload cache from their NVFP4 or block-fp8 pieces; every
+other projection loads as stored (bf16, or fp8 codes with their block scales) with keys
+renamed ``model.language_model.X`` -> ``model.X``. ``model.visual.*`` and the trailing
+MTP layer are never read.
Load-time fusions (must mirror the module split orders):
@@ -15,9 +16,7 @@ Load-time fusions (must mirror the module split orders):
* KDA ``conv1d`` = q|k|v depthwise conv weights concatenated on the channel axis
fp32-kept tensors: ``A_log`` / ``dt_bias``, the mHC ``hc_*`` tensors, the indexer
-APE, and the router ``e_score_correction_bias``. Optional W8A16 fp8-at-load
-follows ``ModelConfig.attn_quant`` / ``dense_quant`` / ``lm_head_quant``
-(defaults and env opt-ins: see config.py).
+APE, and the router ``e_score_correction_bias``.
"""
from __future__ import annotations
@@ -29,11 +28,11 @@ from typing import Iterator
import torch
from freetoken.distributed import get_tp_info
-from freetoken.models.glm_moe_dsa.weight import _ShardReader, _quant_fp8_per_row
+from freetoken.layers.quantization import QuantKind
+from freetoken.models.glm_moe_dsa.weight import _ShardReader
from freetoken.models.loader import drop_page_cache
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
- load_nvfp4_expert_source_banks,
)
from freetoken.utils import cached_load_hf_config, download_hf_weight
from tqdm import tqdm
@@ -94,73 +93,54 @@ def _select_expert_source_spec(model_path: str) -> Nvfp4ExpertSourceSpec:
# KDA in_proj fusion order; MUST match Glm5NextKDA._in_proj_split.
_KDA_IN_PROJ = ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")
-
-def load_nvfp4_expert_sources(model_path: str, config, layer_sink=None):
- return load_nvfp4_expert_source_banks(
- model_path,
- config,
- _select_expert_source_spec(model_path),
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- layer_sink=layer_sink,
- )
+# zai-org fp8 release: fp8 codes + fp32 128x128 block scales per expert projection
+_FP8_EXPERT_RE = re.compile(
+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."
+ r"(?P<proj>gate|up|down)_proj\.(?P<kind>weight|weight_scale_inv)$"
+)
-def _maybe_fp8(key: str, w: torch.Tensor, fp8: bool):
- if fp8:
- q, scale = _quant_fp8_per_row(w)
- yield f"{key}.weight", q
- yield f"{key}.weight_scale", scale
+def _proj(reader, src: str, dst: str) -> Iterator[tuple[str, torch.Tensor]]:
+ """One projection as the checkpoint stores it: bf16, or fp8 codes with their block scales."""
+ w = reader.get(f"{src}.weight")
+ if w.dtype == torch.float8_e4m3fn:
+ yield f"{dst}.weight", w
+ yield f"{dst}.weight_scale_inv", reader.get(f"{src}.weight_scale_inv")
else:
- yield f"{key}.weight", w.to(torch.bfloat16)
+ yield f"{dst}.weight", w.to(torch.bfloat16)
-def _iter_kda_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:
+def nvfp4_expert_spec(model_path: str, config) -> Nvfp4ExpertSourceSpec:
+ return _select_expert_source_spec(model_path)
+
+
+def _iter_kda_layer(reader, layer: int) -> Iterator[tuple[str, torch.Tensor]]:
src = f"{_CKPT}.layers.{layer}.self_attn"
dst = f"{_MODEL}.layers.{layer}.self_attn"
- if attn_fp8:
- # fp8 resident: q|k|v (the 201 MB/layer read) as one W8A16 GEMM with
- # per-row scales; the small gate projections b|f_a|g_a stay bf16.
- qkv = torch.cat(
- [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("q_proj", "k_proj", "v_proj")],
- dim=0,
- )
- q, scale = _quant_fp8_per_row(qkv)
- yield f"{dst}.in_proj_qkv.weight", q
- yield f"{dst}.in_proj_qkv.weight_scale", scale
- bfg = torch.cat(
- [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("b_proj", "f_a_proj", "g_a_proj")],
- dim=0,
- )
- yield f"{dst}.in_proj_bfg.weight", bfg
- else:
- # One fused input GEMM: q|k|v|b|f_a|g_a (output-axis concat).
- fused = torch.cat(
- [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in _KDA_IN_PROJ], dim=0
- )
- yield f"{dst}.in_proj.weight", fused
+ # One fused input GEMM: q|k|v|b|f_a|g_a (output-axis concat).
+ parts = [reader.get(f"{src}.{p}.weight") for p in _KDA_IN_PROJ]
+ if any(p.dtype == torch.float8_e4m3fn for p in parts):
+ raise NotImplementedError("fp8 KDA input projections are not fused by this reader")
+ yield f"{dst}.in_proj.weight", torch.cat([p.to(torch.bfloat16) for p in parts], dim=0)
# One merged depthwise conv over the q|k|v stream (channel-axis concat).
conv = torch.cat(
[reader.get(f"{src}.{p}_conv1d.weight").to(torch.bfloat16) for p in ("q", "k", "v")],
dim=0,
)
yield f"{dst}.conv1d.weight", conv
- for p in ("f_b_proj", "g_b_proj"):
- yield f"{dst}.{p}.weight", reader.get(f"{src}.{p}.weight").to(torch.bfloat16)
- yield from _maybe_fp8(f"{dst}.o_proj", reader.get(f"{src}.o_proj.weight"), attn_fp8)
+ for p in ("f_b_proj", "g_b_proj", "o_proj"):
+ yield from _proj(reader, f"{src}.{p}", f"{dst}.{p}")
# Gate params stay fp32 (the recurrent kernels read them as fp32).
yield f"{dst}.A_log", reader.get(f"{src}.A_log").to(torch.float32)
yield f"{dst}.dt_bias", reader.get(f"{src}.dt_bias").to(torch.float32)
yield f"{dst}.o_norm.weight", reader.get(f"{src}.o_norm.weight").to(torch.bfloat16)
-def _iter_dsa_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]:
+def _iter_dsa_layer(reader, layer: int) -> Iterator[tuple[str, torch.Tensor]]:
src = f"{_CKPT}.layers.{layer}.self_attn"
dst = f"{_MODEL}.layers.{layer}.self_attn"
- fp8_projs = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "o_proj") if attn_fp8 else ()
for proj in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"):
- w = reader.get(f"{src}.{proj}.weight")
- yield from _maybe_fp8(f"{dst}.{proj}", w, proj in fp8_projs)
+ yield from _proj(reader, f"{src}.{proj}", f"{dst}.{proj}")
for norm in ("q_a_layernorm", "kv_a_layernorm"):
yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to(torch.bfloat16)
# kpool indexer (every DSA layer owns one). Kept bf16; the APE is fp32.
@@ -185,8 +165,7 @@ def iter_weights(
include_non_moe: bool,
) -> Iterator[tuple[str, torch.Tensor]]:
assert not include_moe_experts, (
- "GLM-5.3 stores routed experts as NVFP4 and only supports the offload backend; "
- "experts are loaded into the offload cache via load_nvfp4_expert_sources()."
+ "GLM-5.3 routed experts only serve from the offload cache; they are loaded from their expert pieces."
)
assert include_non_moe
if get_tp_info().size > 1:
@@ -201,17 +180,6 @@ def iter_weights(
weight_map = json.load(f)["weight_map"]
reader = _ShardReader(folder, weight_map, device)
primary = get_tp_info().is_primary()
- attn_fp8 = config.attn_quant == "fp8_pertensor"
- mlp_fp8 = config.dense_quant == "fp8_pertensor"
- head_fp8 = config.lm_head_quant == "fp8_pertensor"
- if primary:
- from freetoken.utils import init_logger
-
- init_logger(__name__).info(
- f"GLM-5.3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "
- f"lm_head={config.lm_head_quant} (FREETOKEN_GLM5_ATTN_FP8/FREETOKEN_GLM5_MLP_FP8; "
- "an FTW conversion records these choices implicitly -- serve with the same flags)"
- )
try:
for layer in tqdm(
range(config.num_layers),
@@ -221,9 +189,9 @@ def iter_weights(
src = f"{_CKPT}.layers.{layer}"
dst = f"{_MODEL}.layers.{layer}"
if args.is_kda_layer(layer):
- yield from _iter_kda_layer(reader, layer, attn_fp8)
+ yield from _iter_kda_layer(reader, layer)
else:
- yield from _iter_dsa_layer(reader, layer, attn_fp8)
+ yield from _iter_dsa_layer(reader, layer)
# mHC mixing tensors, fp32 on every layer.
for hc in ("hc_attn_fn", "hc_attn_base", "hc_attn_scale",
@@ -237,9 +205,7 @@ def iter_weights(
if layer < config.first_k_dense_replace:
for proj in ("gate_proj", "up_proj", "down_proj"):
- yield from _maybe_fp8(
- f"{dst}.mlp.{proj}", reader.get(f"{src}.mlp.{proj}.weight"), mlp_fp8
- )
+ yield from _proj(reader, f"{src}.mlp.{proj}", f"{dst}.mlp.{proj}")
else:
yield f"{dst}.mlp.gate.weight", reader.get(f"{src}.mlp.gate.weight").to(
torch.bfloat16
@@ -251,25 +217,63 @@ def iter_weights(
reader.get(f"{src}.mlp.gate.e_score_correction_bias").to(torch.float32),
)
for proj in ("gate_proj", "up_proj", "down_proj"):
- yield from _maybe_fp8(
- f"{dst}.mlp.shared_experts.{proj}",
- reader.get(f"{src}.mlp.shared_experts.{proj}.weight"),
- mlp_fp8,
- )
+ yield from _proj(reader, f"{src}.mlp.shared_experts.{proj}", f"{dst}.mlp.shared_experts.{proj}")
yield f"{_MODEL}.embed_tokens.weight", reader.get(
f"{_CKPT}.embed_tokens.weight"
).to(torch.bfloat16)
yield f"{_MODEL}.norm.weight", reader.get(f"{_CKPT}.norm.weight").to(torch.bfloat16)
- head = reader.get("lm_head.weight")
- if head_fp8 and not config.tie_word_embeddings:
- q, scale = _quant_fp8_per_row(head)
- yield "lm_head.weight", q
- yield "lm_head.weight_scale", scale
- else:
- yield "lm_head.weight", head.to(torch.bfloat16)
+ yield "lm_head.weight", reader.get("lm_head.weight").to(torch.bfloat16)
finally:
reader.close()
-__all__ = ["iter_weights", "load_nvfp4_expert_sources"]
+def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
+ """Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their ``_scale`` companions; other kinds use the generic readers."""
+ if kind is not QuantKind.FP8_BLOCK:
+ return None
+ if get_tp_info().size > 1:
+ raise NotImplementedError("glm5_next fp8 expert banks support TP=1 only")
+ from freetoken.models.weight import experts_scattered, iter_expert_tensors_parallel
+ from freetoken.moe.expert_pieces import per_expert_pieces
+
+ suffix = {"weight": "", "weight_scale_inv": "_scale"}
+
+ def locate(raw_name: str):
+ m = _FP8_EXPERT_RE.match(raw_name)
+ if m is None:
+ return None
+ bank = _layer_to_bank(int(m["layer"]), config)
+ if bank is None:
+ return None
+ return bank, int(m["expert"]), m["proj"] + suffix[m["kind"]]
+
+ if parallel is None:
+ parallel = experts_scattered(model_path)
+ if parallel:
+ tensors = iter_expert_tensors_parallel(
+ model_path, lambda n: locate(n) is not None, workers=workers, chunk=chunk
+ )
+ return per_expert_pieces(tensors, locate, tensors_per_expert=6)
+
+ def _serial():
+ folder = download_hf_weight(model_path)
+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:
+ weight_map = json.load(f)["weight_map"]
+ reader = _ShardReader(folder, weight_map, torch.device("cpu"))
+ try:
+ layers = range(config.first_k_dense_replace, config.num_layers)
+ for layer in tqdm(layers, desc="Loading GLM-5.3 fp8 experts (serial)", disable=not get_tp_info().is_primary()):
+ for e in range(config.num_experts):
+ base = f"{_CKPT}.layers.{layer}.mlp.experts.{e}"
+ for proj in ("gate", "up", "down"):
+ for kind_name in suffix:
+ name = f"{base}.{proj}_proj.{kind_name}"
+ yield name, reader.get(name)
+ finally:
+ reader.close()
+
+ return per_expert_pieces(_serial(), locate, tensors_per_expert=6)
+
+
+__all__ = ["iter_weights", "iter_expert_pieces", "nvfp4_expert_spec"]
diff --git a/python/freetoken/models/glm_moe_dsa/__init__.py b/python/freetoken/models/glm_moe_dsa/__init__.py
index 1f2c189a..a0c05545 100644
--- a/python/freetoken/models/glm_moe_dsa/__init__.py
+++ b/python/freetoken/models/glm_moe_dsa/__init__.py
@@ -1,15 +1,13 @@
from .config import parse_config
from .model import GlmMoeDsaForCausalLM
from .weight import (
+ nvfp4_expert_spec,
iter_weights,
- load_nvfp4_expert_sources,
- load_nvfp4_expert_sources_parallel,
)
__all__ = [
+ "nvfp4_expert_spec",
"GlmMoeDsaForCausalLM",
"parse_config",
"iter_weights",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
]
diff --git a/python/freetoken/models/glm_moe_dsa/attention.py b/python/freetoken/models/glm_moe_dsa/attention.py
index e25d961f..4381de5b 100644
--- a/python/freetoken/models/glm_moe_dsa/attention.py
+++ b/python/freetoken/models/glm_moe_dsa/attention.py
@@ -27,6 +27,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import torch
+from freetoken.layers.quantization import QuantKind
from freetoken.core import get_global_ctx
from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
from freetoken.layers.rotary import get_rope
@@ -36,16 +37,6 @@ if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
-def _make_proj(quant: str, in_features: int, out_features: int) -> BaseOP:
- """A resident projection in the model's resolved quant mode: ``"fp8_pertensor"``
- (W8A16, per-row scale, quantized at load -- see weight.py) or bf16."""
- if quant == "fp8_pertensor":
- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear
-
- return Fp8PerTensorLinear(in_features, out_features, has_bias=False)
- return LinearReplicated(in_features, out_features, has_bias=False)
-
-
class _IdxLayerNorm(BaseOP):
"""LayerNorm with bias (the indexer's ``k_norm``; RMSNorm everywhere else)."""
@@ -63,17 +54,18 @@ class _IdxLayerNorm(BaseOP):
class GlmDsaIndexer(BaseOP):
"""DSA lightning indexer ("full" layers only; "shared" layers carry no weights).
- Kept bf16 in every quant mode: the projections are small (~17 MB/layer) and the
- top-k boundary is precision-sensitive. The HF reference itself is the bf16
- equivalent of DeepSeek's fp8 scoring kernel, so bf16 scoring is faithful.
+ ``wq_b`` follows the checkpoint's quant config; ``wk`` and ``weights_proj`` stay bf16 as in vLLM (the top-k boundary is precision-sensitive, and they are ~17 MB/layer).
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.glm_dsa_args
self.n_heads = args.index_n_heads
self.head_dim = args.index_head_dim
self.softmax_scale = args.index_head_dim**-0.5
- self.wq_b = LinearReplicated(args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False)
+ self.wq_b = LinearReplicated(
+ args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.wq_b",
+ )
self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False)
self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6)
self.weights_proj = LinearReplicated(args.hidden_size, self.n_heads, has_bias=False)
@@ -106,14 +98,14 @@ class GlmDsaIndexer(BaseOP):
class GlmMoeDsaAttention(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.glm_dsa_args
self.layer_id = layer_id
# IndexShare: "full" layers own an indexer; "shared" layers reuse the most
# recent full layer's top-k selection (resolved in the backend).
idx_types = args.indexer_types
self.indexer = (
- GlmDsaIndexer(config, layer_id)
+ GlmDsaIndexer(config, layer_id, prefix=f"{prefix}.indexer")
if idx_types and idx_types[layer_id] == "full"
else None
)
@@ -137,22 +129,32 @@ class GlmMoeDsaAttention(BaseOP):
is_neox=not args.rope_interleave, # GLM-5.2: interleaved (config-driven)
)
- quant = config.attn_quant
- self.q_a_proj = _make_proj(quant, args.hidden_size, args.q_lora_rank)
+ self.q_a_proj = LinearReplicated(
+ args.hidden_size, args.q_lora_rank, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.q_a_proj",
+ )
self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps)
- self.q_b_proj = _make_proj(quant, args.q_lora_rank, self.num_heads * self.qk_head_dim)
- self.kv_a_proj_with_mqa = _make_proj(
- quant, args.hidden_size, self.kv_lora_rank + self.qk_rope_head_dim
+ self.q_b_proj = LinearReplicated(
+ args.q_lora_rank, self.num_heads * self.qk_head_dim, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.q_b_proj",
+ )
+ self.kv_a_proj_with_mqa = LinearReplicated(
+ args.hidden_size, self.kv_lora_rank + self.qk_rope_head_dim, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.kv_a_proj_with_mqa",
)
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps)
- # kv_b stays bf16 in every mode: it is consumed as bmm operands by the MLA
- # absorption (below), not through a Linear forward.
+ # the MLA absorption reads kv_b_proj.weight as bmm operands, so only an unquantized scheme is served
self.kv_b_proj = LinearReplicated(
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.kv_b_proj",
+ )
+ assert self.kv_b_proj.quant_method.kind is QuantKind.NONE, f"{prefix}.kv_b_proj: a quantized kv_b_proj needs a dequantized copy for the MLA absorption"
+ self.o_proj = LinearReplicated(
+ self.num_heads * self.v_head_dim, args.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.o_proj",
)
- self.o_proj = _make_proj(quant, self.num_heads * self.v_head_dim, args.hidden_size)
# Contiguous per-head kv_b split, cached on first forward (see _kv_b). Absorbing
# kv_b into Q/O runs as a per-head bf16 bmm on these instead of re-slicing +
# re-upcasting the bf16 weight per token (bf16 einsum with tensor-core fp32
diff --git a/python/freetoken/models/glm_moe_dsa/config.py b/python/freetoken/models/glm_moe_dsa/config.py
index c0ba25c5..615bdc7f 100644
--- a/python/freetoken/models/glm_moe_dsa/config.py
+++ b/python/freetoken/models/glm_moe_dsa/config.py
@@ -9,11 +9,7 @@ kernel, identity selection for the dense regimes).
MoE routing knobs mirror the glm4_moe path (sigmoid ``noaux_tc`` router, shared
expert, routed scaling). The DSA indexer geometry rides in ``glm_dsa_args``.
-Resident-weight quantization is resolved HERE (from the FREETOKEN_GLM_*_FP8 env
-switches, default on) into the standard ``ModelConfig`` fields -- ``attn_quant`` /
-``dense_quant`` / ``lm_head_quant`` = ``"fp8_pertensor"`` or ``"none"`` -- and every
-consumer (the module constructors and the weight loader) reads those fields, so the
-resolved config is the single record of what the served weights actually are.
+Every resident projection is built from the QuantConfig, so the served precision is whatever the checkpoint's quantization_config declares.
"""
from __future__ import annotations
@@ -30,13 +26,6 @@ from freetoken.models.config import (
from .args import load_args
-# W8A16 fp8 (per-row scale, quantized at load) for the resident weights; decode is
-# weight-bandwidth bound and the freed VRAM densifies the expert cache. =0 restores
-# checkpoint-faithful bf16. NB: these resolve into ModelConfig at parse time and change
-# what iter_weights yields -- an FTW checkpoint converted under one setting must be
-# served under the same one (see weight.py).
-_ATTN_FP8 = os.getenv("FREETOKEN_GLM_ATTN_FP8", "1") != "0"
-_MLP_FP8 = os.getenv("FREETOKEN_GLM_MLP_FP8", "1") != "0"
def _dsa_on(args, num_layers: int) -> bool:
@@ -124,12 +113,6 @@ def parse_config(hf_config: Any) -> ModelConfig:
topk_group=int(getattr(hf_config, "topk_group", 1)),
attn_sm_scale=args.qk_head_dim**-0.5,
has_attn_bias=bool(getattr(hf_config, "attention_bias", False)),
- # Resident-weight quant modes (see the module docstring): recorded here so the
- # resolved config describes the served weights; the constructors and
- # iter_weights read these fields, never the env directly.
- attn_quant="fp8_pertensor" if _ATTN_FP8 else "none",
- dense_quant="fp8_pertensor" if _MLP_FP8 else "none",
- lm_head_quant="fp8_pertensor" if _MLP_FP8 else "none",
glm_dsa_args=args,
)
diff --git a/python/freetoken/models/glm_moe_dsa/mlp.py b/python/freetoken/models/glm_moe_dsa/mlp.py
index be9430c5..32c50ea7 100644
--- a/python/freetoken/models/glm_moe_dsa/mlp.py
+++ b/python/freetoken/models/glm_moe_dsa/mlp.py
@@ -1,11 +1,6 @@
"""SwiGLU MLP for GLM-5.2's leading dense layers and per-layer shared experts.
-The checkpoint ships these bf16 (NVIDIA's NVFP4 recipe leaves them unquantized; only
-the routed experts are FP4). Serving default is W8A16 fp8 with per-row scales,
-quantized at load (``ModelConfig.dense_quant == "fp8_pertensor"``, from
-FREETOKEN_GLM_MLP_FP8): decode reads every shared expert each token, so this halves
-~5.6 GiB/token of weight traffic and frees the same VRAM for expert-cache slots.
-``FREETOKEN_GLM_MLP_FP8=0`` restores the checkpoint-faithful bf16 weights.
+Every projection is built from the QuantConfig, so it serves whatever precision the checkpoint stores (bf16 under NVIDIA's NVFP4 recipe, which only quantizes the routed experts).
"""
from __future__ import annotations
@@ -16,17 +11,17 @@ import torch.nn.functional as F
from freetoken.layers import BaseOP
from freetoken.utils import nvtx_annotate
-from .attention import _make_proj
+from freetoken.layers import LinearReplicated
if TYPE_CHECKING:
import torch
class GlmDsaGatedMLP(BaseOP):
- def __init__(self, hidden_size: int, intermediate_size: int, quant: str = "none"):
- self.gate_proj = _make_proj(quant, hidden_size, intermediate_size)
- self.up_proj = _make_proj(quant, hidden_size, intermediate_size)
- self.down_proj = _make_proj(quant, intermediate_size, hidden_size)
+ def __init__(self, hidden_size: int, intermediate_size: int, *, quant_config=None, prefix: str = ""):
+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")
+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")
+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")
@nvtx_annotate("MLP")
def forward(self, x: torch.Tensor) -> torch.Tensor:
diff --git a/python/freetoken/models/glm_moe_dsa/model.py b/python/freetoken/models/glm_moe_dsa/model.py
index 57644932..b6e21a37 100644
--- a/python/freetoken/models/glm_moe_dsa/model.py
+++ b/python/freetoken/models/glm_moe_dsa/model.py
@@ -16,38 +16,15 @@ if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
-class GlmFp8LMHead(ParallelLMHead):
- """W8A16 lm_head (fp8-e4m3 weight + per-row scale, quantized at load).
-
- The full-vocab logits GEMV reads the whole ~1.9 GiB bf16 head every decode step;
- fp8 halves that. Selected by ``ModelConfig.lm_head_quant == "fp8_pertensor"``
- (weight.py quantizes at load off the same field). GLM-5.2 does not tie embeddings,
- so the fp8 weight is head-only.
- """
-
- def __init__(self, num_embeddings: int, embedding_dim: int):
- super().__init__(num_embeddings, embedding_dim, tie_word_embeddings=False)
- self.weight = torch.empty(num_embeddings, embedding_dim, dtype=torch.float8_e4m3fn)
- self.weight_scale = torch.empty(num_embeddings, dtype=torch.float32)
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear
-
- batch = get_global_ctx().batch
- if batch.is_prefill:
- indices = batch.attn_metadata.get_last_indices(batch.size)
- x = x[indices].contiguous()
- return fp8_pertensor_linear(x, self.weight, self.weight_scale)
-
-
class GlmMoeDsaDecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = GlmMoeDsaAttention(config, layer_id)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = GlmMoeDsaAttention(config, layer_id, prefix=f"{prefix}.self_attn")
if layer_id >= config.first_k_dense_replace:
- self.mlp: BaseOP = GlmMoeDsaSparseBlock(config, layer_id)
+ self.mlp: BaseOP = GlmMoeDsaSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")
else:
self.mlp = GlmDsaGatedMLP(
- config.hidden_size, config.intermediate_size, quant=config.dense_quant
+ config.hidden_size, config.intermediate_size,
+ quant_config=config.quant, prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNormFused(
@@ -67,13 +44,16 @@ class GlmMoeDsaDecoderLayer(BaseOP):
class GlmMoeDsaModel(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [GlmMoeDsaDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ GlmMoeDsaDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
@@ -88,18 +68,14 @@ class GlmMoeDsaModel(BaseOP):
class GlmMoeDsaForCausalLM(BaseLLMModel):
def __init__(self, config: ModelConfig):
self.model = GlmMoeDsaModel(config)
- if config.lm_head_quant == "fp8_pertensor" and not config.tie_word_embeddings:
- self.lm_head: BaseOP = GlmFp8LMHead(
- num_embeddings=config.vocab_size,
- embedding_dim=config.hidden_size,
- )
- else:
- self.lm_head = ParallelLMHead(
- num_embeddings=config.vocab_size,
- embedding_dim=config.hidden_size,
- tie_word_embeddings=config.tie_word_embeddings,
- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
- )
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
+ )
super().__init__()
def prepare_for_runtime(self) -> None:
diff --git a/python/freetoken/models/glm_moe_dsa/moe.py b/python/freetoken/models/glm_moe_dsa/moe.py
index cec4f0e3..0e17c9be 100644
--- a/python/freetoken/models/glm_moe_dsa/moe.py
+++ b/python/freetoken/models/glm_moe_dsa/moe.py
@@ -25,7 +25,7 @@ TopK = Tuple[torch.Tensor, torch.Tensor]
class GlmMoeDsaSparseBlock(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.top_k = config.num_experts_per_tok
self.num_experts = config.num_experts
self.norm_topk_prob = config.norm_topk_prob
@@ -44,11 +44,14 @@ class GlmMoeDsaSparseBlock(BaseOP):
config,
layer_id=layer_id - config.first_k_dense_replace,
renormalize=config.norm_topk_prob,
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
)
self.shared_experts = GlmDsaGatedMLP(
config.hidden_size,
config.moe_intermediate_size * max(1, config.n_shared_experts),
- quant=config.dense_quant,
+ quant_config=config.quant,
+ prefix=f"{prefix}.shared_experts",
)
def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor:
diff --git a/python/freetoken/models/glm_moe_dsa/weight.py b/python/freetoken/models/glm_moe_dsa/weight.py
index df22053f..900d71af 100644
--- a/python/freetoken/models/glm_moe_dsa/weight.py
+++ b/python/freetoken/models/glm_moe_dsa/weight.py
@@ -1,21 +1,11 @@
"""Weight loading for GLM-5.2 (``glm_moe_dsa``).
-Resident (non routed-expert) weights are bf16 in the checkpoint. What this loader
-yields follows the quant modes RESOLVED IN ``parse_config`` (``ModelConfig.attn_quant``
-/ ``dense_quant`` / ``lm_head_quant``, from the FREETOKEN_GLM_*_FP8 switches, default
-on): in the default fp8 mode the big projections are requantized at load to W8A16
-fp8-e4m3 with per-output-row scales (an extra ``*.weight_scale`` tensor per
-projection); with the switches off everything streams through verbatim as bf16. The
+Resident (non routed-expert) weights stream through verbatim in the checkpoint's precision (bf16). The
router selection bias is remapped ``mlp.gate.e_score_correction_bias ->
mlp.e_score_correction_bias``; the DSA indexer tensors load bf16 on "full" indexer
layers (serving runs faithful DSA top-k sparse attention; see attention.py); only the
trailing MTP layer is skipped. Routed experts are NVFP4
and go to the offload cache via the shared glm4_moe loader (identical key layout).
-
-FTW caveat: an FTW checkpoint stores whatever iter_weights yielded at CONVERSION time,
-and the model is built from the env at SERVE time -- the two must agree (a mismatch
-fails loudly in load_state_dict on the ``*.weight_scale`` keys). The active modes are
-logged at load so conversion logs record the choice.
"""
from __future__ import annotations
@@ -28,8 +18,7 @@ import safetensors
import torch
from freetoken.distributed import get_tp_info
from freetoken.models.glm4_moe.weight import (
- load_nvfp4_expert_sources,
- load_nvfp4_expert_sources_parallel,
+ nvfp4_expert_spec,
)
from freetoken.models.loader import drop_page_cache
from freetoken.utils import cached_load_hf_config, download_hf_weight
@@ -38,17 +27,6 @@ from tqdm import tqdm
from .config import parse_config
# fp8-e4m3 dynamic range for the per-row W8A16 quantization of the big MLA projections.
-_FP8_MAX = 448.0
-
-
-def _quant_fp8_per_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
- """Per-output-row fp8-e4m3 quantization: ``w ~= weight_fp8 * scale[:, None]``."""
- wf = w.float()
- scale = (wf.abs().amax(dim=1) / _FP8_MAX).clamp(min=1e-12)
- q = (wf / scale[:, None]).clamp(-_FP8_MAX, _FP8_MAX).to(torch.float8_e4m3fn)
- return q, scale.to(torch.float32)
-
-
class _ShardReader:
def __init__(self, folder: str, weight_map: dict, device: torch.device):
self._folder = folder
@@ -88,7 +66,7 @@ def iter_weights(
) -> Iterator[tuple[str, torch.Tensor]]:
assert not include_moe_experts, (
"GLM-5.2 stores routed experts as NVFP4 and only supports the offload backend; "
- "experts are loaded into the offload cache via load_nvfp4_expert_sources()."
+ "experts are loaded into the offload cache from their NVFP4 pieces."
)
assert include_non_moe
config = parse_config(cached_load_hf_config(model_path))
@@ -98,17 +76,6 @@ def iter_weights(
reader = _ShardReader(folder, weight_map, device)
primary = get_tp_info().is_primary()
dense = config.first_k_dense_replace
- attn_fp8 = config.attn_quant == "fp8_pertensor"
- mlp_fp8 = config.dense_quant == "fp8_pertensor"
- head_fp8 = config.lm_head_quant == "fp8_pertensor"
- if primary:
- from freetoken.utils import init_logger
-
- init_logger(__name__).info(
- f"GLM-5.2 resident quant: attn={config.attn_quant} dense={config.dense_quant} "
- f"lm_head={config.lm_head_quant} (FREETOKEN_GLM_ATTN_FP8/FREETOKEN_GLM_MLP_FP8; "
- "an FTW conversion records these choices implicitly -- serve with the same flags)"
- )
try:
for layer in tqdm(
range(config.num_layers),
@@ -116,17 +83,8 @@ def iter_weights(
disable=not primary,
):
a = f"model.layers.{layer}.self_attn"
- fp8_projs = (
- ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "o_proj") if attn_fp8 else ()
- )
for proj in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"):
- w = reader.get(f"{a}.{proj}.weight")
- if proj in fp8_projs:
- q, scale = _quant_fp8_per_row(w)
- yield f"{a}.{proj}.weight", q
- yield f"{a}.{proj}.weight_scale", scale
- else:
- yield f"{a}.{proj}.weight", w
+ yield f"{a}.{proj}.weight", reader.get(f"{a}.{proj}.weight")
for norm in ("q_a_layernorm", "kv_a_layernorm"):
yield f"{a}.{norm}.weight", reader.get(f"{a}.{norm}.weight")
# DSA lightning indexer ("full" layers only; "shared" layers reuse their
@@ -145,18 +103,9 @@ def iter_weights(
m = f"model.layers.{layer}.mlp"
- def _mlp_weight(key: str):
- w = reader.get(f"{key}.weight")
- if mlp_fp8:
- q, scale = _quant_fp8_per_row(w)
- yield f"{key}.weight", q
- yield f"{key}.weight_scale", scale
- else:
- yield f"{key}.weight", w
-
if layer < dense:
for proj in ("gate_proj", "up_proj", "down_proj"):
- yield from _mlp_weight(f"{m}.{proj}")
+ yield f"{m}.{proj}.weight", reader.get(f"{m}.{proj}.weight")
else:
yield f"{m}.gate.weight", reader.get(f"{m}.gate.weight")
yield (
@@ -164,19 +113,13 @@ def iter_weights(
reader.get(f"{m}.gate.e_score_correction_bias").to(torch.bfloat16),
)
for proj in ("gate_proj", "up_proj", "down_proj"):
- yield from _mlp_weight(f"{m}.shared_experts.{proj}")
+ yield f"{m}.shared_experts.{proj}.weight", reader.get(f"{m}.shared_experts.{proj}.weight")
yield "model.embed_tokens.weight", reader.get("model.embed_tokens.weight")
yield "model.norm.weight", reader.get("model.norm.weight")
- head = reader.get("lm_head.weight")
- if head_fp8 and not config.tie_word_embeddings:
- q, scale = _quant_fp8_per_row(head)
- yield "lm_head.weight", q
- yield "lm_head.weight_scale", scale
- else:
- yield "lm_head.weight", head
+ yield "lm_head.weight", reader.get("lm_head.weight")
finally:
reader.close()
-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]
+__all__ = ["iter_weights", "nvfp4_expert_spec"]
diff --git a/python/freetoken/models/gpt_oss/__init__.py b/python/freetoken/models/gpt_oss/__init__.py
index 92637487..45ef7d37 100644
--- a/python/freetoken/models/gpt_oss/__init__.py
+++ b/python/freetoken/models/gpt_oss/__init__.py
@@ -1,5 +1,5 @@
from .config import parse_config
from .model import GptOssForCausalLM
-from .weight import iter_weights, setup_offload_expert_banks
+from .weight import iter_expert_pieces, iter_weights
-__all__ = ["GptOssForCausalLM", "parse_config", "iter_weights", "setup_offload_expert_banks"]
+__all__ = ["GptOssForCausalLM", "parse_config", "iter_weights", "iter_expert_pieces"]
diff --git a/python/freetoken/models/gpt_oss/attention.py b/python/freetoken/models/gpt_oss/attention.py
index 8a67a465..259e54d9 100644
--- a/python/freetoken/models/gpt_oss/attention.py
+++ b/python/freetoken/models/gpt_oss/attention.py
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
class GptOssAttention(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
import torch
self.layer_id = layer_id
@@ -42,12 +42,16 @@ class GptOssAttention(BaseOP):
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=config.has_attn_bias,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
)
self.sinks = torch.empty(self.num_qo_heads)
self.o_proj = LinearOProj(
input_size=config.num_qo_heads * self.head_dim,
output_size=config.hidden_size,
has_bias=config.has_attn_bias,
+ quant_config=config.quant,
+ prefix=f"{prefix}.o_proj",
)
rotary_config = group.rotary_config
diff --git a/python/freetoken/models/gpt_oss/model.py b/python/freetoken/models/gpt_oss/model.py
index c0caa3a8..8398fb8b 100644
--- a/python/freetoken/models/gpt_oss/model.py
+++ b/python/freetoken/models/gpt_oss/model.py
@@ -22,9 +22,9 @@ if TYPE_CHECKING:
class GptOssDecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = GptOssAttention(config, layer_id)
- self.mlp = GptOssMLP(config, layer_id)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = GptOssAttention(config, layer_id, prefix=f"{prefix}.self_attn")
+ self.mlp = GptOssMLP(config, layer_id, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
@@ -47,13 +47,16 @@ class GptOssDecoderLayer(BaseOP):
class GptOssModel(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [GptOssDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ GptOssDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(
size=config.hidden_size,
@@ -80,6 +83,8 @@ class GptOssForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
self.config = config
super().__init__()
diff --git a/python/freetoken/models/gpt_oss/moe.py b/python/freetoken/models/gpt_oss/moe.py
index 8e4bd0c6..3b58946b 100644
--- a/python/freetoken/models/gpt_oss/moe.py
+++ b/python/freetoken/models/gpt_oss/moe.py
@@ -3,197 +3,39 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import torch
-import freetoken.layers.moe as moe_layers
-from freetoken.layers import BaseOP, LinearReplicated, MoELayer, OffloadMoELayer
-from freetoken.moe import is_offload_moe_backend
-from freetoken.moe.fused_mxfp4 import (
- MXFP4_DECODE_MAX_TOKENS,
- _transpose_mxfp4_for_decode,
- run_mxfp4_prefill_experts_t,
- run_mxfp4_splitk_decode_experts,
-)
+from freetoken.layers import BaseOP, LinearReplicated, MoELayer, OffloadMoELayer, make_moe_layer
from freetoken.utils import nvtx_annotate
-from .weight import local_mxfp4_intermediate_range
-
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
-class GptOssMxfp4TritonMoELayer(MoELayer):
- def __init__(self, config: ModelConfig):
- tp_info = moe_layers.get_tp_info()
- _, _, local_intermediate = local_mxfp4_intermediate_range(
- config.moe_intermediate_size,
- rank=tp_info.rank,
- world_size=tp_info.size,
- )
- if config.hidden_size % 32 != 0:
- raise ValueError("GPT-OSS MXFP4 hidden size must be divisible by 32")
-
- super().__init__(
- num_experts=config.num_experts,
- top_k=config.num_experts_per_tok,
- hidden_size=config.hidden_size,
- intermediate_size=config.moe_intermediate_size,
- renormalize=config.norm_topk_prob,
- activation="gpt_oss_swiglu",
- allocate_experts=False,
- # Descriptive only: this subclass owns its allocation and forward, but the
- # field must not claim bf16 for mxfp4 tensors.
- weight_format="mxfp4_triton",
- )
- self.local_intermediate_size = local_intermediate
- self.hidden_act_alpha = config.hidden_act_alpha
- self.swiglu_limit = config.swiglu_limit
+def _route(router_logits: torch.Tensor, top_k: int) -> tuple[torch.Tensor, torch.Tensor]:
+ from freetoken.kernel import gpt_oss_fused_routing
- hidden_blocks = config.hidden_size // 32
- intermediate_blocks = local_intermediate // 32
- self.gate_up_proj_blocks = torch.empty(
- config.num_experts,
- 2 * local_intermediate,
- hidden_blocks,
- 16,
- dtype=torch.uint8,
- )
- self.gate_up_proj_scales = torch.empty(
- config.num_experts,
- 2 * local_intermediate,
- hidden_blocks,
- dtype=torch.uint8,
- )
- self.gate_up_proj_bias = torch.empty(
- config.num_experts,
- 2 * local_intermediate,
- dtype=torch.bfloat16,
- )
- self.down_proj_blocks = torch.empty(
- config.num_experts,
- config.hidden_size,
- intermediate_blocks,
- 16,
- dtype=torch.uint8,
- )
- self.down_proj_scales = torch.empty(
- config.num_experts,
- config.hidden_size,
- intermediate_blocks,
- dtype=torch.uint8,
- )
- self.down_proj_bias = torch.empty(
- config.num_experts,
- config.hidden_size,
- dtype=torch.bfloat16,
- )
- # Transposed split-K decode weights, built lazily in _ensure_decode_weights.
- self._gu_blocks_t = None
- self._gu_scales_t = None
- self._dn_blocks_t = None
- self._dn_scales_t = None
-
- def _topk(self, router_logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
- from freetoken.kernel import gpt_oss_fused_routing
-
- return gpt_oss_fused_routing(router_logits, self.top_k)
-
- def _ensure_decode_weights(self) -> None:
- """Build the transposed split-K/prefill weights once and free the HF blocks/
- scales (single transposed layout ~61GB so 120B fits). Both prefill and decode
- read the transposed weights. Must run before CUDA graph capture."""
- if self._gu_blocks_t is not None:
- return
- self._gu_blocks_t, self._gu_scales_t = _transpose_mxfp4_for_decode(
- self.gate_up_proj_blocks, self.gate_up_proj_scales
- )
- self._dn_blocks_t, self._dn_scales_t = _transpose_mxfp4_for_decode(
- self.down_proj_blocks, self.down_proj_scales
- )
- self.gate_up_proj_blocks = None
- self.gate_up_proj_scales = None
- self.down_proj_blocks = None
- self.down_proj_scales = None
- torch.cuda.empty_cache()
+ return gpt_oss_fused_routing(router_logits.contiguous(), top_k)
- def prepare_for_runtime(self) -> None:
- self._ensure_decode_weights()
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- router_logits: torch.Tensor,
- ) -> torch.Tensor:
- if not hidden_states.is_cuda:
- raise RuntimeError("GPT-OSS MXFP4 MoE requires the Triton CUDA kernel")
- if not hidden_states.is_contiguous():
- hidden_states = hidden_states.contiguous()
- if not router_logits.is_contiguous():
- router_logits = router_logits.contiguous()
-
- topk_weights, topk_ids = self._topk(router_logits)
- self._ensure_decode_weights()
- if hidden_states.shape[0] <= MXFP4_DECODE_MAX_TOKENS:
- output = run_mxfp4_splitk_decode_experts(
- hidden_states, topk_weights, topk_ids,
- self._gu_blocks_t, self._gu_scales_t, self.gate_up_proj_bias,
- self._dn_blocks_t, self._dn_scales_t, self.down_proj_bias,
- top_k=self.top_k,
- hidden_act_alpha=self.hidden_act_alpha,
- swiglu_limit=self.swiglu_limit,
- )
- else:
- output = run_mxfp4_prefill_experts_t(
- hidden_states, topk_weights, topk_ids,
- self._gu_blocks_t, self._gu_scales_t, self.gate_up_proj_bias,
- self._dn_blocks_t, self._dn_scales_t, self.down_proj_bias,
- top_k=self.top_k,
- hidden_act_alpha=self.hidden_act_alpha,
- swiglu_limit=self.swiglu_limit,
- )
- return self._maybe_all_reduce(output)
+class GptOssMoELayer(MoELayer):
+ """Resident experts routed by the fused softmax-over-top-k kernel."""
-class GptOssMxfp4OffloadMoELayer(OffloadMoELayer):
- def __init__(self, config: ModelConfig, layer_id: int):
- tp_info = moe_layers.get_tp_info()
- _, _, local_intermediate = local_mxfp4_intermediate_range(
- config.moe_intermediate_size,
- rank=tp_info.rank,
- world_size=tp_info.size,
- )
- super().__init__(
- layer_id=layer_id,
- num_experts=config.num_experts,
- top_k=config.num_experts_per_tok,
- hidden_size=config.hidden_size,
- intermediate_size=config.moe_intermediate_size,
- renormalize=config.norm_topk_prob,
- activation="gpt_oss_swiglu",
- )
- self.local_intermediate_size = local_intermediate
- self.hidden_act_alpha = config.hidden_act_alpha
- self.swiglu_limit = config.swiglu_limit
+ def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor | None = None):
+ assert router_logits is not None
+ topk_weights, topk_ids = _route(router_logits, self.top_k)
+ return self.routed_forward(hidden_states.contiguous(), topk_weights, topk_ids)
- def _topk(self, router_logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
- from freetoken.kernel import gpt_oss_fused_routing
- return gpt_oss_fused_routing(router_logits, self.top_k)
+class GptOssOffloadMoELayer(OffloadMoELayer):
+ """Offloaded experts routed by the fused softmax-over-top-k kernel."""
- def forward(
- self,
- hidden_states: torch.Tensor,
- router_logits: torch.Tensor | None = None,
- ) -> torch.Tensor:
+ def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor | None = None):
assert router_logits is not None
- if not router_logits.is_contiguous():
- router_logits = router_logits.contiguous()
- topk_weights, topk_ids = self._topk(router_logits)
- # routed_forward dispatches prefill/decode movement and all-reduces once;
- # topk_ids is cloned because decode rewrites it in place into slot ids.
- return self.routed_forward(hidden_states, topk_weights, topk_ids.clone())
+ topk_weights, topk_ids = _route(router_logits, self.top_k)
+ return self.routed_forward(hidden_states, topk_weights, topk_ids)
class GptOssMLP(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int | None = None):
+ def __init__(self, config: ModelConfig, layer_id: int | None = None, *, prefix: str = ""):
self.router = LinearReplicated(
config.hidden_size,
config.num_experts,
@@ -204,11 +46,19 @@ class GptOssMLP(BaseOP):
f"gpt-oss supports only mxfp4 expert weights, got "
f"moe_weight_format={config.moe_weight_format!r}"
)
- if is_offload_moe_backend(config.moe_backend):
- assert layer_id is not None
- self.experts = GptOssMxfp4OffloadMoELayer(config, layer_id)
- else:
- self.experts = GptOssMxfp4TritonMoELayer(config)
+ self.experts = make_moe_layer(
+ config,
+ layer_id=layer_id,
+ activation="gpt_oss_swiglu",
+ alpha=config.hidden_act_alpha,
+ limit=config.swiglu_limit,
+ interleaved=True,
+ has_bias=True,
+ resident_cls=GptOssMoELayer,
+ offload_cls=GptOssOffloadMoELayer,
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
+ )
self._layer_id = layer_id
@nvtx_annotate("MoE")
@@ -226,6 +76,6 @@ class GptOssMLP(BaseOP):
__all__ = [
"GptOssMLP",
- "GptOssMxfp4OffloadMoELayer",
- "GptOssMxfp4TritonMoELayer",
+ "GptOssMoELayer",
+ "GptOssOffloadMoELayer",
]
diff --git a/python/freetoken/models/gpt_oss/weight.py b/python/freetoken/models/gpt_oss/weight.py
index 38f16d8a..686f0246 100644
--- a/python/freetoken/models/gpt_oss/weight.py
+++ b/python/freetoken/models/gpt_oss/weight.py
@@ -4,6 +4,7 @@ from typing import Iterator
import safetensors
import torch
+from freetoken.layers.quantization import QuantKind
from freetoken.distributed import get_tp_info
from freetoken.models.loader import (
MergeRule,
@@ -253,71 +254,6 @@ def _expert_layer_and_name(key: str) -> tuple[int, str] | None:
return layer_id, parts[5]
-def _empty_mxfp4_triton_banks(
- config,
- *,
- dtype: torch.dtype,
- tp_info=None,
-):
- """returns ``(banks, host_banks)``, unpinned. ``banks[name]`` is one ``[E, ...]``
- tensor per layer (independent allocations). host banks are lazy anon mmaps (already
- zero, so no ``zero_`` pass); caller fills then pins (pin-after-fill skips cudaHostAlloc's
- slow commit) -- per-layer via ``PinPipeline`` for the real streaming loaders, or
- ``pin_banks`` for the dummy path. unwritten padding stays zero."""
- if tp_info is None:
- tp_info = get_tp_info()
- _, _, local_intermediate = local_mxfp4_intermediate_range(
- config.moe_intermediate_size,
- rank=tp_info.rank,
- world_size=tp_info.size,
- )
- num_layers, E = config.num_layers, config.num_experts
- hidden_blocks = config.hidden_size // 32
- intermediate_blocks = local_intermediate // 32
- specs = {
- "gate_up_blocks": ((E, config.hidden_size // 2, 2 * local_intermediate), torch.uint8),
- "gate_up_scales": ((E, hidden_blocks, 2 * local_intermediate), torch.uint8),
- "gate_up_bias": ((E, 2 * local_intermediate), dtype),
- "down_blocks": ((E, local_intermediate // 2, config.hidden_size), torch.uint8),
- "down_scales": ((E, intermediate_blocks, config.hidden_size), torch.uint8),
- "down_bias": ((E, config.hidden_size), dtype),
- }
- if torch.cuda.is_available():
- from freetoken.moe.host_banks import alloc_layer_banks
-
- hb = alloc_layer_banks(specs, num_layers) # lazy anon mmap -> zero-initialized, unpinned
- return {name: [b.tensor for b in hb[name]] for name in specs}, hb
- return {
- name: [torch.zeros(shape, dtype=dt) for _ in range(num_layers)]
- for name, (shape, dt) in specs.items()
- }, {}
-
-
-def _make_dummy_mxfp4_triton_banks(
- config,
- *,
- dtype: torch.dtype,
- tp_info=None,
-) -> dict[str, list[torch.Tensor]]:
- from freetoken.moe.host_banks import pin_banks
-
- banks, _hb = _empty_mxfp4_triton_banks(config, dtype=dtype, tp_info=tp_info)
- for t in banks["gate_up_blocks"]:
- t.random_(0, 256)
- for t in banks["gate_up_scales"]:
- t.fill_(127)
- for t in banks["gate_up_bias"]:
- t.normal_()
- for t in banks["down_blocks"]:
- t.random_(0, 256)
- for t in banks["down_scales"]:
- t.fill_(127)
- for t in banks["down_bias"]:
- t.normal_()
- pin_banks(_hb) # pin-after-fill
- return banks
-
-
def _read_safetensor_slice(
f,
name: str,
@@ -328,286 +264,87 @@ def _read_safetensor_slice(
return f.get_slice(name)[slices]
-def _copy_transposed_mxfp4_blocks(dst: torch.Tensor, raw: torch.Tensor) -> None:
- # HF [E, N, K//32, 16] -> split-K [E, K//2, N].
- if raw.numel() == 0:
- return
- source = raw.reshape(raw.shape[0], raw.shape[1], raw.shape[2] * raw.shape[3]).permute(
- 0,
- 2,
- 1,
- )
- dst[:, : source.shape[1], : source.shape[2]].copy_(source)
-
-
-def _copy_transposed_mxfp4_scales(dst: torch.Tensor, raw: torch.Tensor) -> None:
- # HF [E, N, K//32] -> split-K [E, K//32, N].
- if raw.numel() == 0:
- return
- source = raw.permute(0, 2, 1)
- dst[:, : source.shape[1], : source.shape[2]].copy_(source)
+_EXPERT_SOURCES = (
+ "gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias",
+ "down_proj_blocks", "down_proj_scales", "down_proj_bias",
+)
-def _copy_prefix(dst: torch.Tensor, raw: torch.Tensor) -> None:
- dst[:, : raw.shape[1]].copy_(raw)
+def _source_slices(config, tp_info) -> dict[str, tuple[str, tuple[slice, ...]]]:
+ """HF expert tensor -> (piece role, this rank's slice of the intermediate axis)."""
+ tp_start, tp_end, _ = local_mxfp4_intermediate_range(
+ config.moe_intermediate_size, rank=tp_info.rank, world_size=tp_info.size
+ )
+ tp_end = max(tp_start, tp_end)
+ rows = slice(2 * tp_start, 2 * tp_end)
+ cols = slice(tp_start // 32, tp_end // 32)
+ return {
+ "gate_up_proj_blocks": ("gate_up", (slice(None), rows, slice(None), slice(None))),
+ "gate_up_proj_scales": ("gate_up_scale", (slice(None), rows, slice(None))),
+ "gate_up_proj_bias": ("gate_up_bias", (slice(None), rows)),
+ "down_proj_blocks": ("down", (slice(None), slice(None), cols, slice(None))),
+ "down_proj_scales": ("down_scale", (slice(None), slice(None), cols)),
+ "down_proj_bias": ("down_bias", (slice(None), slice(None))),
+ }
-def load_mxfp4_triton_banks_streaming(
- model_path: str,
- model_config=None,
- *,
- dtype: torch.dtype,
- tp_info=None,
- layer_sink=None,
-) -> dict[str, list[torch.Tensor]]:
- """``layer_sink=None`` (serving): pin each layer as its writes complete, via an
- internally-owned :class:`PinPipeline` (or no pinning at all in the CPU-only fallback,
- where ``_hb`` is empty). ``layer_sink`` given (converter): the completion tracker
- fires into it instead -- nothing here is pinned, and the sink may release banks it
- has written out, so the returned tensors are only valid until then (the caller owns
- that tradeoff)."""
- config = model_config if model_config is not None else parse_config(cached_load_hf_config(model_path))
+def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20):
+ """gpt-oss experts as HF ships them, one piece per layer: stacked ``[E, ...]`` mxfp4
+ ``_blocks`` / ``_scales`` / ``_bias`` tensors sliced to this rank's intermediate range."""
+ if kind is not QuantKind.MXFP4:
+ return None
if config.moe_weight_format != "mxfp4":
raise ValueError("GPT-OSS offload requires MXFP4 expert weights")
+ tp_info = get_tp_info()
+ slices = _source_slices(config, tp_info)
+ num_layers, num_experts = config.num_layers, config.num_experts
- if tp_info is None:
- tp_info = get_tp_info()
- tp_start, tp_end, local_intermediate = local_mxfp4_intermediate_range(
- config.moe_intermediate_size,
- rank=tp_info.rank,
- world_size=tp_info.size,
- )
- tp_end = max(tp_start, tp_end)
- banks, _hb = _empty_mxfp4_triton_banks(config, dtype=dtype, tp_info=tp_info)
- seen: set[tuple[int, str]] = set()
- all_expert_sources = (
- "gate_up_proj_blocks",
- "gate_up_proj_scales",
- "gate_up_proj_bias",
- "down_proj_blocks",
- "down_proj_scales",
- "down_proj_bias",
- )
- copy_plan = {
- "gate_up_proj_blocks": (
- "gate_up_blocks",
- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None), slice(None)),
- _copy_transposed_mxfp4_blocks,
- ),
- "gate_up_proj_scales": (
- "gate_up_scales",
- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None)),
- _copy_transposed_mxfp4_scales,
- ),
- "gate_up_proj_bias": (
- "gate_up_bias",
- (slice(None), slice(2 * tp_start, 2 * tp_end)),
- _copy_prefix,
- ),
- "down_proj_blocks": (
- "down_blocks",
- (
- slice(None),
- slice(None),
- slice(tp_start // 32, tp_end // 32),
- slice(None),
- ),
- _copy_transposed_mxfp4_blocks,
- ),
- "down_proj_scales": (
- "down_scales",
- (slice(None), slice(None), slice(tp_start // 32, tp_end // 32)),
- _copy_transposed_mxfp4_scales,
- ),
- }
+ def _is_expert(name: str) -> bool:
+ info = _expert_layer_and_name(name)
+ return info is not None and 0 <= info[0] < num_layers
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline
+ def _tensors():
+ if parallel:
+ from freetoken.models.weight import iter_expert_tensors_parallel
- def _load(sink) -> None:
- # _hb is empty in the CPU-only fallback (no banks to pin/stream): no-op tracker.
- tracker = LayerCompletionTracker(len(all_expert_sources), _hb, sink) if _hb else None
+ for name, whole in iter_expert_tensors_parallel(model_path, _is_expert, workers=workers, chunk=chunk):
+ _, source = _expert_layer_and_name(name)
+ yield name, whole[slices[source][1]]
+ return
for file in iter_root_safetensor_files_from_index(model_path):
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
for name in f.keys():
info = _expert_layer_and_name(name)
if info is None:
continue
- layer_id, source_name = info
- if layer_id < 0 or layer_id >= config.num_layers:
+ layer_id, source = info
+ if not 0 <= layer_id < num_layers:
raise ValueError(f"Unexpected GPT-OSS expert layer in checkpoint: {name}")
- seen.add((layer_id, source_name))
-
- if source_name == "down_proj_bias":
- if tp_info.rank == 0:
- raw = f.get_tensor(name)
- banks["down_bias"][layer_id].copy_(raw)
- if tracker is not None:
- tracker.note(layer_id)
- continue
-
- plan = copy_plan.get(source_name)
- if plan is None:
+ if source not in slices:
raise ValueError(f"Unexpected GPT-OSS expert source: {name}")
- bank_name, source_slice, copy_fn = plan
- raw = _read_safetensor_slice(f, name, source_slice)
- copy_fn(banks[bank_name][layer_id], raw)
- if tracker is not None:
- tracker.note(layer_id)
-
- if layer_sink is not None:
- _load(layer_sink)
- elif _hb:
- with PinPipeline() as pins:
- _load(pins)
- else:
- _load(None)
-
- expected = {
- (layer_id, source_name)
- for layer_id in range(config.num_layers)
- for source_name in all_expert_sources
- }
- missing = expected - seen
- if missing:
- raise ValueError(f"Missing GPT-OSS expert tensors: {sorted(missing)[:8]}")
- assert all(
- len(per_layer) == config.num_layers
- and all(t.is_contiguous() and t.size(0) == config.num_experts for t in per_layer)
- for per_layer in banks.values()
- )
- return banks
-
-
-def load_mxfp4_triton_banks_streaming_parallel(
- model_path: str,
- model_config=None,
- *,
- dtype: torch.dtype,
- tp_info=None,
- workers: int = 8,
- chunk: int = 8 << 20,
- layer_sink=None,
-) -> dict[str, list[torch.Tensor]]:
- """parallel counterpart of load_mxfp4_triton_banks_streaming: same banks + copy plan,
- but bulk expert tensors come from the common chunked multi-threaded O_DIRECT reader.
- source_slice applied to the whole tensor == the serial result, so correct for any TP.
- ``layer_sink``: see :func:`load_mxfp4_triton_banks_streaming`."""
- from freetoken.models.weight import iter_expert_tensors_parallel
-
- config = model_config if model_config is not None else parse_config(cached_load_hf_config(model_path))
- if config.moe_weight_format != "mxfp4":
- raise ValueError("GPT-OSS offload requires MXFP4 expert weights")
- if tp_info is None:
- tp_info = get_tp_info()
- tp_start, tp_end, _local = local_mxfp4_intermediate_range(
- config.moe_intermediate_size, rank=tp_info.rank, world_size=tp_info.size
- )
- tp_end = max(tp_start, tp_end)
- banks, _hb = _empty_mxfp4_triton_banks(config, dtype=dtype, tp_info=tp_info)
- all_expert_sources = (
- "gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias",
- "down_proj_blocks", "down_proj_scales", "down_proj_bias",
- )
- copy_plan = {
- "gate_up_proj_blocks": ("gate_up_blocks",
- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None), slice(None)),
- _copy_transposed_mxfp4_blocks),
- "gate_up_proj_scales": ("gate_up_scales",
- (slice(None), slice(2 * tp_start, 2 * tp_end), slice(None)),
- _copy_transposed_mxfp4_scales),
- "gate_up_proj_bias": ("gate_up_bias",
- (slice(None), slice(2 * tp_start, 2 * tp_end)), _copy_prefix),
- "down_proj_blocks": ("down_blocks",
- (slice(None), slice(None), slice(tp_start // 32, tp_end // 32), slice(None)),
- _copy_transposed_mxfp4_blocks),
- "down_proj_scales": ("down_scales",
- (slice(None), slice(None), slice(tp_start // 32, tp_end // 32)),
- _copy_transposed_mxfp4_scales),
- }
-
- def _is_expert(name: str) -> bool:
- info = _expert_layer_and_name(name)
- return info is not None and 0 <= info[0] < config.num_layers
-
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline
-
- seen: set[tuple[int, str]] = set()
-
- def _load(sink) -> None:
- # _hb is empty in the CPU-only fallback (no banks to pin/stream): no-op tracker.
- tracker = LayerCompletionTracker(len(all_expert_sources), _hb, sink) if _hb else None
- for name, whole in iter_expert_tensors_parallel(model_path, _is_expert, workers=workers, chunk=chunk):
- layer_id, source_name = _expert_layer_and_name(name)
- seen.add((layer_id, source_name))
- if source_name == "down_proj_bias":
- if tp_info.rank == 0:
- banks["down_bias"][layer_id].copy_(whole)
- if tracker is not None:
- tracker.note(layer_id)
- continue
- bank_name, source_slice, copy_fn = copy_plan[source_name]
- copy_fn(banks[bank_name][layer_id], whole[source_slice])
- if tracker is not None:
- tracker.note(layer_id)
-
- if layer_sink is not None:
- _load(layer_sink)
- elif _hb:
- with PinPipeline() as pins:
- _load(pins)
- else:
- _load(None)
-
- expected = {(layer_id, src) for layer_id in range(config.num_layers) for src in all_expert_sources}
- missing = expected - seen
- if missing:
- raise ValueError(f"Missing GPT-OSS expert tensors: {sorted(missing)[:8]}")
- assert all(
- len(per_layer) == config.num_layers
- and all(t.is_contiguous() and t.size(0) == config.num_experts for t in per_layer)
- for per_layer in banks.values()
- )
- return banks
-
-
-def setup_offload_expert_banks(
- model_path: str,
- model_config,
- *,
- device: torch.device,
- dtype: torch.dtype,
- dummy: bool = False,
- tp_info=None,
- parallel: bool = False,
- workers: int = 8,
- chunk: int = 8 << 20,
- layer_sink=None,
-):
- from freetoken.moe.expert_banks import ExpertBanks
-
- if model_config.moe_weight_format != "mxfp4":
- raise ValueError("GPT-OSS offload currently supports MXFP4 expert weights")
- if dummy:
- sources = _make_dummy_mxfp4_triton_banks(model_config, dtype=dtype, tp_info=tp_info)
- streamed = False
- elif parallel: # parallel: common chunked multi-threaded O_DIRECT reader
- sources = load_mxfp4_triton_banks_streaming_parallel(
- model_path, model_config, dtype=dtype, tp_info=tp_info, workers=workers, chunk=chunk,
- layer_sink=layer_sink,
- )
- streamed = layer_sink is not None
- else:
- sources = load_mxfp4_triton_banks_streaming(
- model_path, model_config, dtype=dtype, tp_info=tp_info, layer_sink=layer_sink
- )
- streamed = layer_sink is not None
- return ExpertBanks("mxfp4_triton", sources, streamed=streamed)
+ yield name, _read_safetensor_slice(f, name, slices[source][1])
+
+ def _pieces():
+ pending: dict[int, dict[str, torch.Tensor]] = {}
+ for name, tensor in _tensors():
+ layer_id, source = _expert_layer_and_name(name)
+ role = slices[source][0]
+ if role == "down_bias" and tp_info.rank != 0:
+ tensor = torch.zeros_like(tensor) # the TP reduction adds the bias once, on rank 0
+ piece = pending.setdefault(layer_id, {})
+ piece[role] = tensor
+ if len(piece) == len(_EXPERT_SOURCES):
+ del pending[layer_id]
+ yield layer_id, 0, num_experts, piece
+ if pending:
+ raise ValueError(f"Missing GPT-OSS expert tensors for layers {sorted(pending)[:8]}")
+
+ return _pieces()
__all__ = [
"iter_weights",
+ "iter_expert_pieces",
"local_mxfp4_intermediate_range",
- "load_mxfp4_triton_banks_streaming",
- "setup_offload_expert_banks",
"shard_gpt_oss_tensor",
]
diff --git a/python/freetoken/models/llama/attention.py b/python/freetoken/models/llama/attention.py
index 49516910..52b7e969 100644
--- a/python/freetoken/models/llama/attention.py
+++ b/python/freetoken/models/llama/attention.py
@@ -22,6 +22,7 @@ class LlamaAttention(BaseOP):
*,
has_attn_bias: bool = False,
has_qk_norm: bool = False,
+ prefix: str = "",
):
head_dim = config.head_dim
self.layer_id = layer_id
@@ -37,6 +38,8 @@ class LlamaAttention(BaseOP):
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=has_attn_bias,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
)
if has_qk_norm:
self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
@@ -59,6 +62,8 @@ class LlamaAttention(BaseOP):
head_dim * config.num_qo_heads,
config.hidden_size,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.o_proj",
)
@nvtx_annotate("MHA")
diff --git a/python/freetoken/models/llama/model.py b/python/freetoken/models/llama/model.py
index a3d80171..eb7ff28f 100644
--- a/python/freetoken/models/llama/model.py
+++ b/python/freetoken/models/llama/model.py
@@ -16,9 +16,9 @@ if TYPE_CHECKING:
class LlamaDecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = LlamaAttn(config, layer_id)
- self.mlp = LlamaMLP(config)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = LlamaAttn(config, layer_id, prefix=f"{prefix}.self_attn")
+ self.mlp = LlamaMLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
@@ -44,13 +44,16 @@ class LlamaDecoderLayer(BaseOP):
class LlamaModel(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [LlamaDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ LlamaDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(
size=config.hidden_size,
@@ -73,6 +76,8 @@ class LlamaForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/loader.py b/python/freetoken/models/loader.py
index 49419364..0a2b69ae 100644
--- a/python/freetoken/models/loader.py
+++ b/python/freetoken/models/loader.py
@@ -289,146 +289,6 @@ def iter_stacked_experts(
)
-def _packed_expert_source_info(key: str) -> tuple[int, str] | None:
- parts = key.split(".")
- if len(parts) < 5 or parts[0] != "model" or parts[1] != "layers":
- return None
- if parts[-2] != "experts" or parts[-1] not in {"gate_up_proj", "down_proj"}:
- return None
- try:
- return int(parts[2]), parts[-1]
- except ValueError:
- return None
-
-
-class _PlainBank:
- """CPU-only fallback bank: a plain unpinned tensor with a no-op pin (no CUDA)."""
-
- __slots__ = ("tensor",)
-
- def __init__(self, tensor: torch.Tensor):
- self.tensor = tensor
-
- def pin(self) -> None:
- pass
-
-
-def _alloc_expert_bank(shape: tuple[int, ...], *, dtype: torch.dtype):
- """Allocate an UNPINNED bank (lazy host mmap), to be filled then pinned at the end --
- pin-after-fill. Registering already-resident pages skips cudaHostAlloc's slow commit
- (~2.8 GiB/s) / zero-fill. Returns a bank object exposing ``.tensor`` and ``.pin()``."""
- if torch.cuda.is_available():
- from freetoken.moe.host_banks import HostBank
-
- return HostBank(tuple(shape), dtype)
- return _PlainBank(torch.empty(shape, dtype=dtype))
-
-
-def _copy_expert_layer_into_bank(
- banks: dict[str, list],
- row_shape: dict[str, tuple[int, ...]],
- seen_layers: dict[str, set[int]],
- *,
- bank_name: str,
- tensor: torch.Tensor,
- layer: int,
- config,
- dtype: torch.dtype,
-) -> None:
- if layer < 0 or layer >= config.num_layers:
- raise ValueError(
- f"Unexpected MoE expert layer {layer}; expected [0, {config.num_layers})"
- )
- if tensor.size(0) != config.num_experts:
- raise ValueError(
- f"Unexpected {bank_name} expert count {tensor.size(0)}; "
- f"expected {config.num_experts}"
- )
- expected_shape = row_shape.setdefault(bank_name, tuple(tensor.shape[1:]))
- if tuple(tensor.shape[1:]) != expected_shape:
- raise ValueError(
- f"Inconsistent {bank_name} expert shape {tuple(tensor.shape[1:])}; "
- f"expected {expected_shape}"
- )
-
- bank = banks[bank_name][layer]
- if bank is None:
- banks[bank_name][layer] = bank = _alloc_expert_bank(
- (config.num_experts, *tensor.shape[1:]), dtype=dtype
- )
- bank.tensor.copy_(tensor) # whole-layer arrival; pinned later, after fully resident
- seen_layers[bank_name].add(layer)
-
-
-def stream_moe_expert_sources(
- tensors: Iterable[tuple[str, torch.Tensor]],
- config,
- *,
- dtype: torch.dtype,
- layer_sink=None,
-) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
- """Stream packed per-layer BF16 expert tensors into final offload banks.
-
- Model adapters normalize ordinary MoE expert weights to
- ``...experts.gate_up_proj`` and ``...experts.down_proj`` with shape
- ``[num_experts, ...]``. Each arrives whole-layer, so it's written directly into
- its own ``[num_experts, ...]`` per-layer bank (independent allocation).
-
- ``layer_sink=None`` (serving): pin each layer's banks as its writes complete,
- via an internally-owned :class:`PinPipeline`. ``layer_sink`` given (converter):
- the tracker fires into it instead -- nothing is pinned, and the sink may release
- banks it has written out, so the returned tensors are only valid until then (the
- caller owns that tradeoff).
- """
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline
-
- banks: dict[str, list] = { # name -> per-layer [bank obj (HostBank/_PlainBank) or None]
- "gate_up": [None] * config.num_layers,
- "down": [None] * config.num_layers,
- }
- row_shape: dict[str, tuple[int, ...]] = {}
- seen_layers: dict[str, set[int]] = {"gate_up": set(), "down": set()}
-
- def _load(sink) -> None:
- tracker = LayerCompletionTracker(2, banks, sink) # gate_up + down per layer
- for name, tensor in tensors:
- expert_info = _packed_expert_source_info(name)
- if expert_info is None:
- raise ValueError(f"Unexpected expert weight key: {name}")
- layer, packed_name = expert_info
- bank_name = "gate_up" if packed_name == "gate_up_proj" else "down"
- _copy_expert_layer_into_bank(
- banks,
- row_shape,
- seen_layers,
- bank_name=bank_name,
- tensor=tensor,
- layer=layer,
- config=config,
- dtype=dtype,
- )
- tracker.note(layer)
-
- expected_layers = set(range(config.num_layers))
- missing = {
- name: sorted(expected_layers - seen)
- for name, seen in seen_layers.items()
- if seen != expected_layers
- }
- if missing:
- raise ValueError(f"Missing MoE expert source layers: {missing}")
-
- if layer_sink is not None:
- _load(layer_sink)
- else:
- with PinPipeline() as pins:
- _load(pins)
- return (
- [bank.tensor for bank in banks["gate_up"]],
- [bank.tensor for bank in banks["down"]],
- )
-
-
__all__ = [
"MergeRule",
"iter_root_safetensor_files_from_index",
@@ -436,5 +296,4 @@ __all__ = [
"iter_merged_tensors",
"iter_stacked_experts",
"shard_tensor",
- "stream_moe_expert_sources",
]
diff --git a/python/freetoken/models/minimax_m2/__init__.py b/python/freetoken/models/minimax_m2/__init__.py
index e4e1211c..ad276f84 100644
--- a/python/freetoken/models/minimax_m2/__init__.py
+++ b/python/freetoken/models/minimax_m2/__init__.py
@@ -1,11 +1,10 @@
from .config import parse_config
from .model import MiniMaxM2ForCausalLM
-from .weight import iter_weights, load_nvfp4_expert_sources, load_nvfp4_expert_sources_parallel
+from .weight import iter_weights, nvfp4_expert_spec
__all__ = [
+ "nvfp4_expert_spec",
"MiniMaxM2ForCausalLM",
"parse_config",
"iter_weights",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
]
diff --git a/python/freetoken/models/minimax_m2/attention.py b/python/freetoken/models/minimax_m2/attention.py
index fffe3603..320ec9ae 100644
--- a/python/freetoken/models/minimax_m2/attention.py
+++ b/python/freetoken/models/minimax_m2/attention.py
@@ -26,7 +26,7 @@ class MiniMaxM2Attention(BaseOP):
of each head are rotated.
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
head_dim = config.head_dim
self.layer_id = layer_id
tp_size = get_tp_info().size
@@ -41,6 +41,8 @@ class MiniMaxM2Attention(BaseOP):
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
)
if config.use_qk_norm:
self.q_norm = RMSNorm(self.qo_attn_dim, eps=config.rms_norm_eps)
@@ -63,6 +65,8 @@ class MiniMaxM2Attention(BaseOP):
head_dim * config.num_qo_heads,
config.hidden_size,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.o_proj",
)
@nvtx_annotate("MHA")
diff --git a/python/freetoken/models/minimax_m2/model.py b/python/freetoken/models/minimax_m2/model.py
index 671ea361..457ab565 100644
--- a/python/freetoken/models/minimax_m2/model.py
+++ b/python/freetoken/models/minimax_m2/model.py
@@ -16,9 +16,9 @@ if TYPE_CHECKING:
class MiniMaxM2DecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = MiniMaxM2Attention(config, layer_id)
- self.block_sparse_moe = MiniMaxM2SparseMoeBlock(config, layer_id)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = MiniMaxM2Attention(config, layer_id, prefix=f"{prefix}.self_attn")
+ self.block_sparse_moe = MiniMaxM2SparseMoeBlock(config, layer_id, prefix=f"{prefix}.block_sparse_moe")
self.input_layernorm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNormFused(
size=config.hidden_size, eps=config.rms_norm_eps
@@ -37,13 +37,16 @@ class MiniMaxM2DecoderLayer(BaseOP):
class MiniMaxM2Model(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [MiniMaxM2DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ MiniMaxM2DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
@@ -63,6 +66,8 @@ class MiniMaxM2ForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/minimax_m2/moe.py b/python/freetoken/models/minimax_m2/moe.py
index 5d319802..1410f388 100644
--- a/python/freetoken/models/minimax_m2/moe.py
+++ b/python/freetoken/models/minimax_m2/moe.py
@@ -16,7 +16,7 @@ class MiniMaxM2SparseMoeBlock(BaseOP):
NVFP4 expert banks in the unified offload cache (the expert GEMM backend follows
``cache.quant_format``; see :meth:`OffloadMoELayer.routed_forward`)."""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.top_k = config.num_experts_per_tok
self.num_experts = config.num_experts
self.norm_topk_prob = config.norm_topk_prob
@@ -27,6 +27,8 @@ class MiniMaxM2SparseMoeBlock(BaseOP):
config,
layer_id=layer_id,
renormalize=config.norm_topk_prob,
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
)
def _route(self, router_logits: torch.Tensor) -> TopK:
diff --git a/python/freetoken/models/minimax_m2/weight.py b/python/freetoken/models/minimax_m2/weight.py
index 103f8ec5..1b91ead8 100644
--- a/python/freetoken/models/minimax_m2/weight.py
+++ b/python/freetoken/models/minimax_m2/weight.py
@@ -15,7 +15,6 @@ from freetoken.models.loader import (
)
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
- load_nvfp4_expert_source_banks,
)
from freetoken.utils import cached_load_hf_config
from tqdm import tqdm
@@ -49,7 +48,7 @@ def iter_weights(
include_moe_experts: bool,
include_non_moe: bool,
) -> Iterator[tuple[str, torch.Tensor]]:
- """yield dense BF16 weights only; NVFP4 experts go to the offload cache via load_nvfp4_expert_sources."""
+ """yield dense BF16 weights only; NVFP4 experts go to the offload cache from their pieces."""
assert not include_moe_experts, (
"MiniMax-M2 stores experts as NVFP4 and only supports the offload MoE backend; "
"experts are loaded into the offload cache, not the dense model."
@@ -88,39 +87,8 @@ def iter_weights(
yield from iter_merged_tensors(raw(), _MERGE_RULES, model_name="minimax_m2")
-def load_nvfp4_expert_sources(
- model_path: str,
- config,
- *,
- layer_sink=None,
-) -> dict[str, torch.Tensor]:
- """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""
- return load_nvfp4_expert_source_banks(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- layer_sink=layer_sink,
- )
-
-
-def load_nvfp4_expert_sources_parallel(
- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
-):
- """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""
- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
-
- return load_nvfp4_expert_source_banks_parallel(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- workers=workers,
- chunk=chunk,
- layer_sink=layer_sink,
- )
+def nvfp4_expert_spec(model_path: str, config):
+ return _NVFP4_SOURCE_SPEC
-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]
+__all__ = ["iter_weights", "nvfp4_expert_spec"]
diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.py
index 71c0bb51..cfcebb6f 100644
--- a/python/freetoken/models/minimax_m3/__init__.py
+++ b/python/freetoken/models/minimax_m3/__init__.py
@@ -1,15 +1,13 @@
from .config import parse_config
from .model import MiniMaxM3ForCausalLM
from .weight import (
+ nvfp4_expert_spec,
iter_weights,
- load_nvfp4_expert_sources,
- load_nvfp4_expert_sources_parallel,
)
__all__ = [
+ "nvfp4_expert_spec",
"MiniMaxM3ForCausalLM",
"parse_config",
"iter_weights",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
]
diff --git a/python/freetoken/models/minimax_m3/attention.py b/python/freetoken/models/minimax_m3/attention.py
index fdf090b7..0d55c072 100644
--- a/python/freetoken/models/minimax_m3/attention.py
+++ b/python/freetoken/models/minimax_m3/attention.py
@@ -22,12 +22,10 @@ from typing import TYPE_CHECKING
from freetoken.core import get_global_ctx
from freetoken.distributed import get_tp_info
-from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm
+from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm, LinearReplicated
from freetoken.layers.rotary import get_rope
from freetoken.utils import nvtx_annotate
-from .mlp import make_proj
-
if TYPE_CHECKING:
import torch
@@ -35,7 +33,7 @@ if TYPE_CHECKING:
class MiniMaxM3Attention(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.m3_args
assert get_tp_info().size == 1, "MiniMax-M3 currently supports TP=1 only"
self.layer_id = layer_id
@@ -45,11 +43,14 @@ class MiniMaxM3Attention(BaseOP):
self.qo_attn_dim = self.num_qo_heads * self.head_dim
self.kv_attn_dim = self.num_kv_heads * self.head_dim
- quant = config.attn_quant
- self.qkv_proj = make_proj(
- quant, args.hidden_size, self.qo_attn_dim + 2 * self.kv_attn_dim
+ self.qkv_proj = LinearReplicated(
+ args.hidden_size, self.qo_attn_dim + 2 * self.kv_attn_dim, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.qkv_proj",
+ )
+ self.o_proj = LinearReplicated(
+ self.qo_attn_dim, args.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.o_proj",
)
- self.o_proj = make_proj(quant, self.qo_attn_dim, args.hidden_size)
# Per-head Gemma (1+w) q/k norms (qk_norm_type == "per_head").
self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=args.norm_eps)
@@ -70,8 +71,9 @@ class MiniMaxM3Attention(BaseOP):
self.index_dim = args.index_dim
self.index_q_dim = self.num_index_heads * self.index_dim
# Merged [index_q | index_k] projection (one shared index KEY head).
- self.index_qk_proj = make_proj(
- quant, args.hidden_size, self.index_q_dim + self.index_dim
+ self.index_qk_proj = LinearReplicated(
+ args.hidden_size, self.index_q_dim + self.index_dim, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.index_qk_proj",
)
self.index_q_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)
self.index_k_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)
diff --git a/python/freetoken/models/minimax_m3/config.py b/python/freetoken/models/minimax_m3/config.py
index 906870e3..7df81d79 100644
--- a/python/freetoken/models/minimax_m3/config.py
+++ b/python/freetoken/models/minimax_m3/config.py
@@ -16,14 +16,12 @@ ablation: plain MHAKVCache, every layer attends its whole history through a gene
FULL backend). ``page_size`` is pinned to the 128-token sparse block by the backend's
``page_sizes`` declaration, so one KV page == one sparse block.
-Resident-weight quantization is resolved HERE (from the FREETOKEN_M3_*_MXFP8 env
-switches, default on) into the standard ``ModelConfig`` fields -- ``attn_quant`` /
-``dense_quant`` = ``"mxfp8"`` or ``"none"`` -- and every consumer (the module
-constructors and the weight loader) reads those fields, so the resolved config is the
-single record of what the served weights actually are. The routed experts are NVFP4
+The resident projections are served in the precision the checkpoint stores (MXFP8 for the
+official export); ``attn_quant`` / ``dense_quant`` record what was detected for the weight
+loader. The routed experts are NVFP4
(same ModelOpt layout as MiniMax-M2 / GLM) and always live in the offload cache;
their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernels
-(``select_nvfp4_backend``). lm_head / embeddings are BF16 in the checkpoint.
+(the NVFP4 MoE kernel objects). lm_head / embeddings are BF16 in the checkpoint.
"""
from __future__ import annotations
@@ -85,22 +83,8 @@ def parse_config(hf_config: Any) -> ModelConfig:
)
args = load_args(text, num_layers, sparse_enabled=sparse_enabled)
- # W8A16 MXFP8 (the checkpoint's native dense quantization) for the resident
- # weights; decode is weight-bandwidth bound and the freed VRAM densifies the
- # expert cache. =0 dequantizes to bf16 at load (bring-up / ablation). Read at
- # PARSE time so the resolved config is the single record of the served weights
- # -- an FTW checkpoint converted under one setting must be served under the
- # same one (see weight.py) -- and logged HERE so the FTW serve path (which
- # never runs iter_weights) still leaves a serve-time record of the modes.
- attn_mxfp8 = os.getenv("FREETOKEN_M3_ATTN_MXFP8", "1") != "0"
- mlp_mxfp8 = os.getenv("FREETOKEN_M3_MLP_MXFP8", "1") != "0"
- _log_mode_once(
- f"quant={attn_mxfp8}/{mlp_mxfp8}",
- f"MiniMax-M3 resident quant: attn={'mxfp8' if attn_mxfp8 else 'none'} "
- f"dense={'mxfp8' if mlp_mxfp8 else 'none'} lm_head=none "
- "(FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; an FTW checkpoint "
- "converted under one setting must be served under the same one).",
- )
+ # the quantized release stores the dense projections MXFP8; the reader keeps them native and the layers take their scheme from the QuantConfig
+ dense_mxfp8 = getattr(hf_config, "quantization_config", None) is not None
# Leading dense-FFN layers: M3's moe_layer_freq is a contiguous 0-prefix; the
# offload cache and the generic num_moe_layers arithmetic rely on that shape.
@@ -191,8 +175,8 @@ def parse_config(hf_config: Any) -> ModelConfig:
# resolved config describes the served weights; the constructors and
# iter_weights read these fields, never the env directly. lm_head stays bf16
# (the checkpoint excludes it from quantization).
- attn_quant="mxfp8" if attn_mxfp8 else "none",
- dense_quant="mxfp8" if mlp_mxfp8 else "none",
+ attn_quant="mxfp8" if dense_mxfp8 else "none",
+ dense_quant="mxfp8" if dense_mxfp8 else "none",
lm_head_quant="none",
m3_args=args,
)
diff --git a/python/freetoken/models/minimax_m3/mlp.py b/python/freetoken/models/minimax_m3/mlp.py
index b08f6935..d5332eed 100644
--- a/python/freetoken/models/minimax_m3/mlp.py
+++ b/python/freetoken/models/minimax_m3/mlp.py
@@ -1,11 +1,9 @@
"""SwiGLU-OAI MLP for MiniMax-M3's leading dense layers and per-layer shared experts.
The checkpoint ships these MXFP8 (fp8-e4m3 weight + uint8 e8m0 block-32
-``weight_scale_inv``); serving default keeps them native W8A16
-(``ModelConfig.dense_quant == "mxfp8"``, from FREETOKEN_M3_MLP_MXFP8) -- decode reads
-every shared expert each token, so this halves the resident-MLP weight traffic and
-frees the same VRAM for expert-cache slots. ``FREETOKEN_M3_MLP_MXFP8=0`` dequantizes
-to bf16 at load (bring-up / ablation).
+``weight_scale_inv``) and they are served native W8A16 through the checkpoint's
+QuantConfig -- decode reads every shared expert each token, so this halves the
+resident-MLP weight traffic and frees the same VRAM for expert-cache slots.
gate/up are stored merged (``gate_up_proj``, [gate; up] halves -- the loader
concatenates the two checkpoint projections output-wise; MXFP8 scales are per-output-
@@ -25,28 +23,25 @@ if TYPE_CHECKING:
import torch
-def make_proj(quant: str, in_features: int, out_features: int) -> BaseOP:
- """A resident projection in the model's resolved quant mode: ``"mxfp8"`` (W8A16,
- the checkpoint's native block-32 e8m0 format -- see weight.py) or bf16."""
- if quant == "mxfp8":
- from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear
-
- return Mxfp8Linear(in_features, out_features, has_bias=False)
- return LinearReplicated(in_features, out_features, has_bias=False)
-
-
class MiniMaxM3MLP(BaseOP):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
*,
- quant: str,
alpha: float,
limit: float,
+ quant_config=None,
+ prefix: str = "",
):
- self.gate_up_proj = make_proj(quant, hidden_size, 2 * intermediate_size)
- self.down_proj = make_proj(quant, intermediate_size, hidden_size)
+ self.gate_up_proj = LinearReplicated(
+ hidden_size, 2 * intermediate_size, has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.gate_up_proj",
+ )
+ self.down_proj = LinearReplicated(
+ intermediate_size, hidden_size, has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.down_proj",
+ )
self._alpha = alpha
self._limit = limit
diff --git a/python/freetoken/models/minimax_m3/model.py b/python/freetoken/models/minimax_m3/model.py
index 86c47328..e8b6e5d0 100644
--- a/python/freetoken/models/minimax_m3/model.py
+++ b/python/freetoken/models/minimax_m3/model.py
@@ -27,19 +27,22 @@ class MiniMaxM3DecoderLayer(BaseOP):
``block_sparse_moe`` -- matching the checkpoint's naming. All layernorms are
Gemma-style (1+w) RMSNorm (``use_gemma_norm``)."""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.m3_args
- self.self_attn = MiniMaxM3Attention(config, layer_id)
+ self.self_attn = MiniMaxM3Attention(config, layer_id, prefix=f"{prefix}.self_attn")
self.is_moe_layer = layer_id in args.moe_layer_ids
if self.is_moe_layer:
- self.block_sparse_moe: BaseOP = MiniMaxM3SparseMoeBlock(config, layer_id)
+ self.block_sparse_moe: BaseOP = MiniMaxM3SparseMoeBlock(
+ config, layer_id, prefix=f"{prefix}.block_sparse_moe"
+ )
else:
self.mlp = MiniMaxM3MLP(
config.hidden_size,
args.dense_intermediate_size,
- quant=config.dense_quant,
alpha=args.swiglu_alpha,
limit=args.swiglu_limit,
+ quant_config=config.quant,
+ prefix=f"{prefix}.mlp",
)
self.input_layernorm = GemmaPlusOneRMSNormFused(
size=config.hidden_size, eps=config.rms_norm_eps
@@ -62,13 +65,16 @@ class MiniMaxM3DecoderLayer(BaseOP):
class MiniMaxM3Model(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [MiniMaxM3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ MiniMaxM3DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = GemmaPlusOneRMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
@@ -89,6 +95,8 @@ class MiniMaxM3ForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/minimax_m3/moe.py b/python/freetoken/models/minimax_m3/moe.py
index 9f54c156..fe1139b8 100644
--- a/python/freetoken/models/minimax_m3/moe.py
+++ b/python/freetoken/models/minimax_m3/moe.py
@@ -4,8 +4,8 @@ Routing follows the DeepSeek/GLM family (``GlmMoeDsaSparseBlock``): fp32 sigmoid
scores + a selection-only ``e_score_correction_bias``, gather the unbiased scores,
renormalize, scale by ``routed_scaling_factor`` (2.0). No group-limited top-k
(n_group == 1). The routed experts (NVFP4, swigluoai) go through ``make_moe_layer``
-(the offload family; the swigluoai alpha/limit ride the layer via ``extra_attrs``,
-gpt-oss precedent) and the always-on shared expert (MXFP8, swigluoai) is added.
+(the offload family; the swigluoai alpha/limit are the layer's ``alpha`` /
+``limit``) and the always-on shared expert (MXFP8, swigluoai) is added.
The checkpoint keeps ``e_score_correction_bias`` fp32 and the reference applies it
fp32; the bias sits on the top-k boundary, so it is stored fp32 here too (unlike
@@ -29,7 +29,7 @@ TopK = Tuple[torch.Tensor, torch.Tensor]
class MiniMaxM3SparseMoeBlock(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
args = config.m3_args
self.top_k = config.num_experts_per_tok
self.num_experts = config.num_experts
@@ -55,17 +55,18 @@ class MiniMaxM3SparseMoeBlock(BaseOP):
layer_id=layer_id - config.first_k_dense_replace,
renormalize=config.norm_topk_prob,
activation="swigluoai",
- extra_attrs={
- "hidden_act_alpha": args.swiglu_alpha,
- "swiglu_limit": args.swiglu_limit,
- },
+ alpha=args.swiglu_alpha,
+ limit=args.swiglu_limit,
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
)
self.shared_experts = MiniMaxM3MLP(
config.hidden_size,
config.shared_expert_intermediate_size * max(1, config.n_shared_experts),
- quant=config.dense_quant,
alpha=args.swiglu_alpha,
limit=args.swiglu_limit,
+ quant_config=config.quant,
+ prefix=f"{prefix}.shared_experts",
)
def _route(self, hidden_states: torch.Tensor) -> TopK:
diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.py
index e10a7237..3aea2347 100644
--- a/python/freetoken/models/minimax_m3/weight.py
+++ b/python/freetoken/models/minimax_m3/weight.py
@@ -6,22 +6,14 @@ The checkpoint is the multimodal wrapper layout: every text-tower tensor carries
``multi_modal_projector.`` / ``patch_merge_mlp.``) is never read (text-only serving).
Resident (non routed-expert) dense projections are MXFP8 in the checkpoint
-(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``). What this loader
-yields follows the quant modes RESOLVED IN ``parse_config`` (``ModelConfig.attn_quant``
-/ ``dense_quant``, from the FREETOKEN_M3_*_MXFP8 switches, default on): in the default
-mode the fp8 weight and its scale codes stream through verbatim (merged output-wise for
-the fused qkv / index-qk / gate-up projections -- the scales are per-output-row, so
-fusion is exact); with a switch off the projections are dequantized to bf16 at load.
+(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``): the fp8 weight and
+its scale codes stream through verbatim (merged output-wise for the fused qkv /
+index-qk / gate-up projections -- the scales are per-output-row, so fusion is exact).
Norms, the router gate, ``e_score_correction_bias`` (kept fp32), embeddings and
lm_head are unquantized and stream through verbatim. Routed experts are NVFP4
(``w1/w3/w2`` = gate/up/down, same ModelOpt layout as MiniMax-M2) and go to the
-offload cache via ``load_nvfp4_expert_sources``; expert ``input_scale`` calibration
+offload cache from their NVFP4 pieces; expert ``input_scale`` calibration
tensors are unused (W4A16) and never match the bank spec.
-
-FTW caveat: an FTW checkpoint stores whatever iter_weights yielded at CONVERSION time,
-and the model is built from the env at SERVE time -- the two must agree (a mismatch
-fails loudly in load_state_dict on the ``*.weight_scale_inv`` keys). The active modes
-are logged at load so conversion logs record the choice.
"""
from __future__ import annotations
@@ -37,7 +29,6 @@ from freetoken.distributed import get_tp_info
from freetoken.models.loader import drop_page_cache
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
- load_nvfp4_expert_source_banks,
)
from freetoken.utils import cached_load_hf_config, download_hf_weight
from tqdm import tqdm
@@ -153,7 +144,7 @@ def iter_weights(
) -> Iterator[tuple[str, torch.Tensor]]:
assert not include_moe_experts, (
"MiniMax-M3 stores routed experts as NVFP4 and only supports the offload MoE "
- "backend; experts are loaded into the offload cache via load_nvfp4_expert_sources()."
+ "backend; experts are loaded into the offload cache from their NVFP4 pieces."
)
assert include_non_moe
config = parse_config(cached_load_hf_config(model_path))
@@ -170,8 +161,7 @@ def iter_weights(
init_logger(__name__).info(
f"MiniMax-M3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "
- f"lm_head={config.lm_head_quant} (FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; "
- "an FTW conversion records these choices implicitly -- serve with the same flags)"
+ f"lm_head={config.lm_head_quant}"
)
try:
for layer in tqdm(
@@ -248,36 +238,8 @@ def iter_weights(
reader.close()
-def load_nvfp4_expert_sources(
- model_path: str, config, *, layer_sink=None
-) -> dict[str, list[torch.Tensor]]:
- """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""
- return load_nvfp4_expert_source_banks(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- layer_sink=layer_sink,
- )
-
-
-def load_nvfp4_expert_sources_parallel(
- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
-):
- """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""
- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
-
- return load_nvfp4_expert_source_banks_parallel(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- workers=workers,
- chunk=chunk,
- layer_sink=layer_sink,
- )
+def nvfp4_expert_spec(model_path: str, config):
+ return _NVFP4_SOURCE_SPEC
-__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]
+__all__ = ["iter_weights", "nvfp4_expert_spec"]
diff --git a/python/freetoken/models/mistral/attention.py b/python/freetoken/models/mistral/attention.py
index 14ac92f7..01a48388 100644
--- a/python/freetoken/models/mistral/attention.py
+++ b/python/freetoken/models/mistral/attention.py
@@ -22,6 +22,7 @@ class MistralAttention(BaseOP):
*,
has_attn_bias: bool = False,
has_qk_norm: bool = False,
+ prefix: str = "",
):
head_dim = config.head_dim
self.layer_id = layer_id
@@ -37,6 +38,8 @@ class MistralAttention(BaseOP):
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=has_attn_bias,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
)
if has_qk_norm:
self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
@@ -59,6 +62,8 @@ class MistralAttention(BaseOP):
head_dim * config.num_qo_heads,
config.hidden_size,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.o_proj",
)
@nvtx_annotate("MHA")
diff --git a/python/freetoken/models/mistral/model.py b/python/freetoken/models/mistral/model.py
index 5d90ec22..8d258bcf 100644
--- a/python/freetoken/models/mistral/model.py
+++ b/python/freetoken/models/mistral/model.py
@@ -16,9 +16,9 @@ if TYPE_CHECKING:
class MistralDecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = MistralAttn(config, layer_id)
- self.mlp = MistralMLP(config)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = MistralAttn(config, layer_id, prefix=f"{prefix}.self_attn")
+ self.mlp = MistralMLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
@@ -44,13 +44,16 @@ class MistralDecoderLayer(BaseOP):
class MistralModel(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [MistralDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ MistralDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(
size=config.hidden_size,
@@ -73,6 +76,8 @@ class MistralForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/muse_glimmer/attention.py b/python/freetoken/models/muse_glimmer/attention.py
index 29764dcf..7c814d4c 100644
--- a/python/freetoken/models/muse_glimmer/attention.py
+++ b/python/freetoken/models/muse_glimmer/attention.py
@@ -5,12 +5,11 @@ from typing import TYPE_CHECKING
import torch
from freetoken.attention import AttentionSpec
from freetoken.core import get_global_ctx
-from freetoken.layers import BaseOP, GemmaRMSNorm
+from freetoken.layers import BaseOP, GemmaRMSNorm, LinearColParallelMerged, LinearReplicated
from freetoken.layers.rotary import get_rope
from freetoken.models.config import SWAAttentionGroupConfig
from freetoken.utils import nvtx_annotate
-from freetoken.models.quant_linear import make_col_merged, make_replicated
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
@@ -31,7 +30,7 @@ class MuseGlimmerAttention(BaseOP):
the same normed layer input as q/k/v, so it rides the fused projection.
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self.layer_id = layer_id
group = config.attention_group_for_layer(layer_id)
self.is_swa = isinstance(group, SWAAttentionGroupConfig)
@@ -43,10 +42,14 @@ class MuseGlimmerAttention(BaseOP):
self.kv_attn_dim = self.num_kv * head_dim
self._qkvg_split = [self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim, self.qo_attn_dim]
- self.qkvg_proj = make_col_merged(
- config, config.hidden_size, self._qkvg_split, has_bias=False
+ self.qkvg_proj = LinearColParallelMerged(
+ config.hidden_size, self._qkvg_split, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.qkvg_proj",
+ )
+ self.o_proj = LinearReplicated(
+ self.qo_attn_dim, config.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.o_proj",
)
- self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False)
# Weightless: the checkpoint carries no q/k norm weights; the runtime ones vector
# is intentionally not part of state_dict.
self.qk_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps, with_scale=False)
diff --git a/python/freetoken/models/muse_glimmer/model.py b/python/freetoken/models/muse_glimmer/model.py
index ba632c0b..1e1072ec 100644
--- a/python/freetoken/models/muse_glimmer/model.py
+++ b/python/freetoken/models/muse_glimmer/model.py
@@ -9,6 +9,8 @@ from freetoken.layers import (
GemmaPlusOneRMSNorm,
GemmaPlusOneRMSNormFused,
GemmaRMSNorm,
+ LinearColParallelMerged,
+ LinearReplicated,
OPList,
ParallelLMHead,
RMSNorm,
@@ -19,24 +21,25 @@ from freetoken.models.blocks import BaseLLMModel
from freetoken.utils import nvtx_annotate
from .attention import MuseGlimmerAttention
-from freetoken.models.quant_linear import make_col_merged, make_replicated
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
class MuseGlimmerMLP(BaseOP):
- """SwiGLU MLP (gate|up fused), NVFP4 (W4A16) on the quantized checkpoint else bf16."""
+ """SwiGLU MLP (gate|up fused); the projections take the checkpoint's scheme for their prefix."""
- def __init__(self, config: ModelConfig):
- self.gate_up_proj = make_col_merged(
- config,
+ def __init__(self, config: ModelConfig, *, prefix: str = ""):
+ self.gate_up_proj = LinearColParallelMerged(
config.hidden_size,
[config.intermediate_size, config.intermediate_size],
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.gate_up_proj",
)
- self.down_proj = make_replicated(
- config, config.intermediate_size, config.hidden_size, has_bias=False
+ self.down_proj = LinearReplicated(
+ config.intermediate_size, config.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.down_proj",
)
@nvtx_annotate("MLP")
@@ -56,10 +59,10 @@ class MuseGlimmerDecoderLayer(BaseOP):
exists to keep.
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self._layer_id = layer_id
- self.self_attn = MuseGlimmerAttention(config, layer_id)
- self.mlp = MuseGlimmerMLP(config)
+ self.self_attn = MuseGlimmerAttention(config, layer_id, prefix=f"{prefix}.self_attn")
+ self.mlp = MuseGlimmerMLP(config, prefix=f"{prefix}.mlp")
H = config.hidden_size
post_eps = config.post_norm_eps if config.post_norm_eps is not None else config.rms_norm_eps
self.input_layernorm = GemmaPlusOneRMSNorm(H, eps=config.rms_norm_eps)
@@ -81,7 +84,7 @@ class MuseGlimmerDecoderLayer(BaseOP):
class MuseGlimmerModel(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
@@ -93,7 +96,10 @@ class MuseGlimmerModel(BaseOP):
config.hidden_size, eps=config.rms_norm_eps, with_scale=False
)
self.layers = OPList(
- [MuseGlimmerDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ MuseGlimmerDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
# Final norm scales by the raw checkpoint weight (plain RMSNorm, not the
# centered (1+w) form the decoder norms use).
@@ -114,6 +120,8 @@ class MuseGlimmerForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
self._output_multiplier = config.output_multiplier
self._final_logit_softcapping = config.final_logit_softcapping
diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.py
index 6b933ff1..1d1b4285 100644
--- a/python/freetoken/models/nvfp4_banks.py
+++ b/python/freetoken/models/nvfp4_banks.py
@@ -60,284 +60,80 @@ def _bank_layer(spec: Nvfp4ExpertSourceSpec, layer: int, config) -> int | None:
return bank_layer
-def _alloc_nvfp4_host_banks(num_layers: int, E: int, H: int, I: int):
- """6 NVFP4 source banks, one ``[E, ...]`` tensor per layer (independent allocations),
- unpinned (pin-after-fill): register only after fill to skip cudaHostAlloc's slow
- commit. Caller fills each layer's ``.tensor`` then pins it (per-layer, via
- ``PinPipeline``, as its writes complete)."""
- from freetoken.moe.host_banks import alloc_layer_banks
+def _kind_suffix(kind: str) -> str:
+ return {"weight": "", "weight_scale": "_scale", "weight_scale_2": "_global"}[kind]
- fp8 = torch.float8_e4m3fn
- return alloc_layer_banks({
- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),
- "gate_up_scale": ((E, 2 * I, H // 16), fp8),
- "gate_up_global": ((E, 2 * I), torch.float16),
- "down_packed": ((E, H, I // 2), torch.uint8),
- "down_scale": ((E, H, I // 16), fp8),
- "down_global": ((E, H), torch.float16),
- }, num_layers)
-
-def load_nvfp4_expert_source_banks(
+def iter_nvfp4_expert_pieces(
model_path: str,
config,
spec: Nvfp4ExpertSourceSpec,
*,
- drop_page_cache: DropPageCache,
- primary: bool,
- layer_sink=None,
-) -> dict[str, list[torch.Tensor]]:
- """Build the 6 native NVFP4 source banks by streaming checkpoint shards (serial per-shard read).
-
- ModelOpt row layout: gate/up fused on the output-row axis, down separate; the per-tensor
- global scale (weight_scale_2) is kept as a separate per-output-row FP16 bank (``*_global``),
- so dequant is ``fp4 * block_scale * global``. Each bank is one ``[E, ...]`` tensor per
- layer, indexed by ``[bank_layer][expert]``. (The marlin/b12x backends repack these and
- fold the global into per-expert alphas; see moe/nvfp4_backends.py.)
-
- ``layer_sink=None`` (serving): pin each bank layer as its writes complete, via an
- internally-owned :class:`PinPipeline`. ``layer_sink`` given (converter; for
- marlin/b12x the provider wraps it in a per-layer repacking sink first): the
- completion tracker fires into it instead -- nothing here is pinned, and the sink
- may release banks it has written out, so the returned tensors are only valid
- until then (the caller owns that tradeoff).
+ parallel: bool = False,
+ workers: int = 8,
+ chunk: int = 8 << 20,
+ drop_page_cache: DropPageCache | None = None,
+ primary: bool = True,
+):
+ """One piece per routed expert: ``gate`` / ``up`` / ``down`` codes plus their ``_scale``
+ (fp8 block scales) and ``_global`` (the per-tensor scale, reciprocal for quant-side dialects,
+ fp16) companions, straight from the safetensors shards.
+
+ Serial reads walk the shards in order; ``parallel`` uses the chunked O_DIRECT reader. Either
+ way tensors of one expert may span shards, so they are grouped by (layer, expert) as they land.
"""
+ from freetoken.models.loader import drop_page_cache as _drop
+ from freetoken.moe.expert_pieces import per_expert_pieces
+
+ drop = drop_page_cache or _drop
folder = download_hf_weight(model_path)
- index_path = os.path.join(folder, "model.safetensors.index.json")
- with open(index_path, encoding="utf-8") as f:
+ with open(os.path.join(folder, "model.safetensors.index.json"), encoding="utf-8") as f:
weight_map = json.load(f)["weight_map"]
- E = config.num_experts
- H = config.hidden_size
- I = config.moe_intermediate_size
- num_layers = _num_moe_layers(config)
-
- for shard in sorted(set(weight_map.values())):
- drop_page_cache(os.path.join(folder, shard))
-
- weight_shards: dict[str, list[tuple[str, re.Match[str], int]]] = collections.defaultdict(list)
- global_shards: dict[str, list[tuple[str, re.Match[str], int]]] = collections.defaultdict(list)
- for name, shard in weight_map.items():
+ wanted: dict[str, tuple[int, int, str]] = {}
+ for name in weight_map:
match = spec.key_pattern.match(name)
if match is None:
continue
- layer = int(match.group("layer"))
- bank_layer = _bank_layer(spec, layer, config)
+ bank_layer = _bank_layer(spec, int(match.group("layer")), config)
if bank_layer is None:
continue
proj = match.group("proj")
if proj not in spec.proj_to_role:
raise ValueError(f"{spec.desc}: unknown NVFP4 expert projection {proj!r}")
kind = _canon_kind(spec, match.group("kind"))
- if kind == "weight_scale_2":
- global_shards[shard].append((name, match, bank_layer))
- elif kind in {"weight", "weight_scale"}:
- weight_shards[shard].append((name, match, bank_layer))
- else:
+ if kind not in ("weight", "weight_scale", "weight_scale_2"):
raise ValueError(f"{spec.desc}: unknown NVFP4 expert tensor kind {kind!r}")
-
- globals_map: dict[tuple[int, int, str], torch.Tensor] = {}
- for shard in sorted(global_shards):
- path = os.path.join(folder, shard)
- with safetensors.safe_open(path, framework="pt", device="cpu") as f:
- for name, match, _bank_layer_id in global_shards[shard]:
- key = (
- int(match.group("layer")),
- int(match.group("expert")),
- match.group("proj"),
- )
- globals_map[key] = _ingest_global(spec, f.get_tensor(name))
- drop_page_cache(path)
-
- _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill
- gate_up_packed = [b.tensor for b in _hb["gate_up_packed"]]
- gate_up_scale = [b.tensor for b in _hb["gate_up_scale"]]
- gate_up_global = [b.tensor for b in _hb["gate_up_global"]]
- down_packed = [b.tensor for b in _hb["down_packed"]]
- down_scale = [b.tensor for b in _hb["down_scale"]]
- down_global = [b.tensor for b in _hb["down_global"]]
-
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline
-
- def _load(sink) -> int:
- tracker = LayerCompletionTracker(E * 6, _hb, sink)
- placed = 0
- for shard in tqdm(sorted(weight_shards), desc=f"Loading {spec.desc}", disable=not primary):
+ wanted[name] = (bank_layer, int(match.group("expert")), spec.proj_to_role[proj] + _kind_suffix(kind))
+ expected = _num_moe_layers(config) * config.num_experts * 9
+ if len(wanted) != expected:
+ raise ValueError(f"{spec.desc}: found {len(wanted)} expert tensors, expected {expected}")
+
+ def _serial():
+ by_shard: dict[str, list[str]] = collections.defaultdict(list)
+ for name, shard in weight_map.items():
+ if name in wanted:
+ by_shard[shard].append(name)
+ for shard in tqdm(sorted(by_shard), desc=f"Loading {spec.desc}", disable=not primary):
path = os.path.join(folder, shard)
+ drop(path)
with safetensors.safe_open(path, framework="pt", device="cpu") as f:
- for name, match, bank_layer_id in weight_shards[shard]:
- layer = int(match.group("layer"))
- expert = int(match.group("expert"))
- proj = match.group("proj")
- role = spec.proj_to_role[proj]
- kind = _canon_kind(spec, match.group("kind"))
+ for name in by_shard[shard]:
tensor = f.get_tensor(name)
- if kind == "weight":
- if role == "gate":
- gate_up_packed[bank_layer_id][expert, :I] = tensor
- elif role == "up":
- gate_up_packed[bank_layer_id][expert, I:] = tensor
- elif role == "down":
- down_packed[bank_layer_id][expert] = tensor
- else:
- raise ValueError(f"{spec.desc}: unknown projection role {role!r}")
- else:
- global_scale = globals_map[(layer, expert, proj)]
- if role == "gate":
- gate_up_scale[bank_layer_id][expert, :I] = tensor
- gate_up_global[bank_layer_id][expert, :I] = global_scale
- elif role == "up":
- gate_up_scale[bank_layer_id][expert, I:] = tensor
- gate_up_global[bank_layer_id][expert, I:] = global_scale
- elif role == "down":
- down_scale[bank_layer_id][expert] = tensor
- down_global[bank_layer_id][expert] = global_scale
- else:
- raise ValueError(f"{spec.desc}: unknown projection role {role!r}")
- tracker.note(bank_layer_id)
- placed += 1
- drop_page_cache(path)
- return placed
-
- if layer_sink is not None:
- placed = _load(layer_sink)
- else:
- with PinPipeline() as pins:
- placed = _load(pins)
-
- expected = num_layers * E * 6
- assert placed == expected, f"{spec.desc}: loaded {placed} expert tensors, expected {expected}"
- return {
- "gate_up_packed": gate_up_packed,
- "gate_up_scale": gate_up_scale,
- "gate_up_global": gate_up_global,
- "down_packed": down_packed,
- "down_scale": down_scale,
- "down_global": down_global,
- }
-
-
-def load_nvfp4_expert_source_banks_parallel(
- model_path: str,
- config,
- spec: Nvfp4ExpertSourceSpec,
- *,
- drop_page_cache: DropPageCache,
- primary: bool,
- workers: int = 8,
- chunk: int = 8 << 20,
- layer_sink=None,
-) -> dict[str, list[torch.Tensor]]:
- """parallel counterpart of :func:`load_nvfp4_expert_source_banks`, byte-for-byte same
- placement. bulk weight/weight_scale read via chunked multi-threaded O_DIRECT reader
- (iter_expert_tensors_parallel); tiny globals (``weight_scale_2``) stay serial (negligible
- bytes). ``layer_sink``: see :func:`load_nvfp4_expert_source_banks`."""
- from freetoken.models.weight import iter_expert_tensors_parallel
-
- folder = download_hf_weight(model_path)
- with open(os.path.join(folder, "model.safetensors.index.json"), encoding="utf-8") as f:
- weight_map = json.load(f)["weight_map"]
-
- E = config.num_experts
- H = config.hidden_size
- I = config.moe_intermediate_size
- num_layers = _num_moe_layers(config)
-
- weight_info: dict[str, tuple[re.Match[str], int]] = {} # name -> (match, bank_layer)
- global_names_by_shard: dict[str, list[str]] = collections.defaultdict(list)
- for name, shard in weight_map.items():
- match = spec.key_pattern.match(name)
- if match is None:
- continue
- bank_layer = _bank_layer(spec, int(match.group("layer")), config)
- if bank_layer is None:
- continue
- kind = _canon_kind(spec, match.group("kind"))
- if kind == "weight_scale_2":
- global_names_by_shard[shard].append(name)
- elif kind in {"weight", "weight_scale"}:
- weight_info[name] = (match, bank_layer)
- else:
- raise ValueError(f"{spec.desc}: unknown NVFP4 expert tensor kind {kind!r}")
-
- # Pass 1: tiny per-tensor global scales (serial; data is scalar-per-expert).
- globals_map: dict[tuple[int, int, str], torch.Tensor] = {}
- for shard in sorted(global_names_by_shard):
- path = os.path.join(folder, shard)
- drop_page_cache(path)
- with safetensors.safe_open(path, framework="pt", device="cpu") as f:
- for name in global_names_by_shard[shard]:
- m = spec.key_pattern.match(name)
- globals_map[(int(m.group("layer")), int(m.group("expert")), m.group("proj"))] = (
- _ingest_global(spec, f.get_tensor(name))
- )
- drop_page_cache(path)
-
- _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill
- gate_up_packed = [b.tensor for b in _hb["gate_up_packed"]]
- gate_up_scale = [b.tensor for b in _hb["gate_up_scale"]]
- gate_up_global = [b.tensor for b in _hb["gate_up_global"]]
- down_packed = [b.tensor for b in _hb["down_packed"]]
- down_scale = [b.tensor for b in _hb["down_scale"]]
- down_global = [b.tensor for b in _hb["down_global"]]
-
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline
+ if wanted[name][2].endswith("_global"):
+ tensor = _ingest_global(spec, tensor)
+ yield name, tensor
+ drop(path)
- # Pass 2: bulk weight/weight_scale via the common parallel reader; place by name.
- def _load(sink) -> int:
- tracker = LayerCompletionTracker(E * 6, _hb, sink)
- placed = 0
- for name, tensor in iter_expert_tensors_parallel(
- folder, lambda n: n in weight_info, workers=workers, chunk=chunk
- ):
- match, bank_layer_id = weight_info[name]
- layer = int(match.group("layer"))
- expert = int(match.group("expert"))
- proj = match.group("proj")
- role = spec.proj_to_role[proj]
- kind = _canon_kind(spec, match.group("kind"))
- if kind == "weight":
- if role == "gate":
- gate_up_packed[bank_layer_id][expert, :I] = tensor
- elif role == "up":
- gate_up_packed[bank_layer_id][expert, I:] = tensor
- else:
- down_packed[bank_layer_id][expert] = tensor
- else:
- g = globals_map[(layer, expert, proj)]
- if role == "gate":
- gate_up_scale[bank_layer_id][expert, :I] = tensor
- gate_up_global[bank_layer_id][expert, :I] = g
- elif role == "up":
- gate_up_scale[bank_layer_id][expert, I:] = tensor
- gate_up_global[bank_layer_id][expert, I:] = g
- else:
- down_scale[bank_layer_id][expert] = tensor
- down_global[bank_layer_id][expert] = g
- tracker.note(bank_layer_id)
- placed += 1
- return placed
+ def _parallel():
+ from freetoken.models.weight import iter_expert_tensors_parallel
- if layer_sink is not None:
- placed = _load(layer_sink)
- else:
- with PinPipeline() as pins:
- placed = _load(pins)
+ for name, tensor in iter_expert_tensors_parallel(folder, lambda n: n in wanted, workers=workers, chunk=chunk):
+ if wanted[name][2].endswith("_global"):
+ tensor = _ingest_global(spec, tensor)
+ yield name, tensor
- expected = num_layers * E * 6
- assert placed == expected, f"{spec.desc}: loaded {placed} expert tensors, expected {expected}"
- return {
- "gate_up_packed": gate_up_packed,
- "gate_up_scale": gate_up_scale,
- "gate_up_global": gate_up_global,
- "down_packed": down_packed,
- "down_scale": down_scale,
- "down_global": down_global,
- }
+ return per_expert_pieces(_parallel() if parallel else _serial(), wanted.get, tensors_per_expert=9)
-__all__ = [
- "Nvfp4ExpertSourceSpec",
- "load_nvfp4_expert_source_banks",
- "load_nvfp4_expert_source_banks_parallel",
-]
+__all__ = ["Nvfp4ExpertSourceSpec", "iter_nvfp4_expert_pieces"]
diff --git a/python/freetoken/models/quant_linear.py b/python/freetoken/models/quant_linear.py
deleted file mode 100644
index 6ba67a9e..00000000
--- a/python/freetoken/models/quant_linear.py
+++ /dev/null
@@ -1,78 +0,0 @@
-"""Quant-aware dense-linear factories, shared by the models that serve quantized
-dense projections (qwen3_5_moe, muse_glimmer).
-
-Maps the model's quant config (``expert_quant`` for the dense MLP / shared-expert path,
-``attn_quant`` for attention + GatedDeltaNet projections) to the right ``BaseOP`` linear:
-block-FP8, per-tensor-FP8 and NVFP4 implementations live under ``freetoken.kernel.triton``;
-the bf16 fallback is the framework's TP-aware ``freetoken.layers``. Only the *dispatch*
-(config -> layer class) lives here.
-"""
-
-from __future__ import annotations
-
-
-def make_col_merged_quant(expert_quant: str, attn_quant: str, in_f: int,
- output_sizes: list[int], has_bias: bool = False):
- """Column-merged linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""
- if expert_quant == "fp8_block":
- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged
-
- return Fp8BlockColMerged(in_f, output_sizes, has_bias)
- if attn_quant == "fp8_pertensor":
- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
-
- return Fp8PerTensorColMerged(in_f, output_sizes, has_bias)
- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention (q/k/v fused)
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged
-
- return Nvfp4DenseColMerged(in_f, output_sizes, has_bias)
- from freetoken.layers import LinearColParallelMerged
-
- return LinearColParallelMerged(in_f, output_sizes, has_bias=has_bias)
-
-
-def make_replicated_quant(expert_quant: str, attn_quant: str, in_f: int, out_f: int,
- has_bias: bool = False):
- """Replicated linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""
- if expert_quant == "fp8_block":
- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockLinear
-
- return Fp8BlockLinear(in_f, out_f, has_bias)
- if attn_quant == "fp8_pertensor":
- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear
-
- return Fp8PerTensorLinear(in_f, out_f, has_bias)
- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention o_proj / GDN out_proj
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseLinear
-
- return Nvfp4DenseLinear(in_f, out_f, has_bias)
- from freetoken.layers import LinearReplicated
-
- return LinearReplicated(in_f, out_f, has_bias=has_bias)
-
-
-def make_replicated(config, in_f: int, out_f: int, has_bias: bool = False):
- """Config-driven replicated linear: ``Fp8BlockLinear`` under block-fp8, ``Fp8PerTensorLinear``
- under per-tensor-fp8 attention, ``Nvfp4DenseLinear`` under nvfp4, else ``LinearReplicated``."""
- return make_replicated_quant(
- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),
- in_f, out_f, has_bias,
- )
-
-
-def make_col_merged(config, in_f: int, output_sizes: list[int], has_bias: bool = False):
- """Config-driven column-merged linear: ``Fp8BlockColMerged`` under block-fp8,
- ``Fp8PerTensorColMerged`` under per-tensor-fp8 attention, ``Nvfp4DenseColMerged`` under
- nvfp4, else ``LinearColParallelMerged``."""
- return make_col_merged_quant(
- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),
- in_f, output_sizes, has_bias,
- )
-
-
-__all__ = [
- "make_col_merged_quant",
- "make_replicated_quant",
- "make_replicated",
- "make_col_merged",
-]
diff --git a/python/freetoken/models/qwen2/attention.py b/python/freetoken/models/qwen2/attention.py
index 5ac6c2c0..60bb2624 100644
--- a/python/freetoken/models/qwen2/attention.py
+++ b/python/freetoken/models/qwen2/attention.py
@@ -22,6 +22,7 @@ class Qwen2Attention(BaseOP):
*,
has_attn_bias: bool = False,
has_qk_norm: bool = False,
+ prefix: str = "",
):
head_dim = config.head_dim
self.layer_id = layer_id
@@ -37,6 +38,8 @@ class Qwen2Attention(BaseOP):
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=has_attn_bias,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
)
if has_qk_norm:
self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
@@ -59,6 +62,8 @@ class Qwen2Attention(BaseOP):
head_dim * config.num_qo_heads,
config.hidden_size,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.o_proj",
)
@nvtx_annotate("MHA")
diff --git a/python/freetoken/models/qwen2/model.py b/python/freetoken/models/qwen2/model.py
index 762529da..ed56fde2 100644
--- a/python/freetoken/models/qwen2/model.py
+++ b/python/freetoken/models/qwen2/model.py
@@ -16,9 +16,9 @@ if TYPE_CHECKING:
class Qwen2DecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = Qwen2Attn(config, layer_id, has_qk_norm=False, has_attn_bias=True)
- self.mlp = Qwen2MLP(config)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = Qwen2Attn(config, layer_id, has_qk_norm=False, has_attn_bias=True, prefix=f"{prefix}.self_attn")
+ self.mlp = Qwen2MLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
@@ -42,13 +42,16 @@ class Qwen2DecoderLayer(BaseOP):
class Qwen2Model(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [Qwen2DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ Qwen2DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(
size=config.hidden_size,
@@ -71,6 +74,8 @@ class Qwen2ForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/qwen3/attention.py b/python/freetoken/models/qwen3/attention.py
index 74b6891e..039dc50d 100644
--- a/python/freetoken/models/qwen3/attention.py
+++ b/python/freetoken/models/qwen3/attention.py
@@ -22,6 +22,7 @@ class Qwen3Attention(BaseOP):
*,
has_attn_bias: bool = False,
has_qk_norm: bool = False,
+ prefix: str = "",
):
head_dim = config.head_dim
self.layer_id = layer_id
@@ -37,6 +38,8 @@ class Qwen3Attention(BaseOP):
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=has_attn_bias,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
)
if has_qk_norm:
self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
@@ -59,6 +62,8 @@ class Qwen3Attention(BaseOP):
head_dim * config.num_qo_heads,
config.hidden_size,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.o_proj",
)
@nvtx_annotate("MHA")
diff --git a/python/freetoken/models/qwen3/model.py b/python/freetoken/models/qwen3/model.py
index d8becaa0..dc35930d 100644
--- a/python/freetoken/models/qwen3/model.py
+++ b/python/freetoken/models/qwen3/model.py
@@ -16,9 +16,9 @@ if TYPE_CHECKING:
class Qwen3DecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True)
- self.mlp = Qwen3MLP(config)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True, prefix=f"{prefix}.self_attn")
+ self.mlp = Qwen3MLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
@@ -42,13 +42,16 @@ class Qwen3DecoderLayer(BaseOP):
class Qwen3Model(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [Qwen3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ Qwen3DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(
size=config.hidden_size,
@@ -71,6 +74,8 @@ class Qwen3ForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.py
index 98936e9f..8620ce9c 100644
--- a/python/freetoken/models/qwen3_5_moe/__init__.py
+++ b/python/freetoken/models/qwen3_5_moe/__init__.py
@@ -1,19 +1,12 @@
from .config import parse_config
from .model import Qwen3_5MoEForCausalLM
-from .weight import (
- iter_weights,
- iter_weights_parallel,
- load_nvfp4_expert_sources,
- load_nvfp4_expert_sources_parallel,
- setup_offload_expert_banks,
-)
+from .weight import iter_expert_pieces, iter_weights, iter_weights_parallel, nvfp4_expert_spec
__all__ = [
"Qwen3_5MoEForCausalLM",
"parse_config",
"iter_weights",
"iter_weights_parallel",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
- "setup_offload_expert_banks",
+ "iter_expert_pieces",
+ "nvfp4_expert_spec",
]
diff --git a/python/freetoken/models/qwen3_5_moe/attention.py b/python/freetoken/models/qwen3_5_moe/attention.py
index 2421264e..12a91bb3 100644
--- a/python/freetoken/models/qwen3_5_moe/attention.py
+++ b/python/freetoken/models/qwen3_5_moe/attention.py
@@ -4,12 +4,10 @@ from typing import TYPE_CHECKING
import torch
from freetoken.core import get_global_ctx
-from freetoken.layers import BaseOP, GemmaRMSNorm
+from freetoken.layers import BaseOP, GemmaRMSNorm, LinearColParallelMerged, LinearReplicated
from freetoken.layers.rotary import get_rope
from freetoken.utils import nvtx_annotate
-from .quant_linear import make_col_merged, make_replicated
-
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
@@ -27,7 +25,7 @@ class Qwen3_5Attention(BaseOP):
column/row-parallel for tensor parallelism later.
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
head_dim = config.head_dim
self.layer_id = layer_id
self.num_q = config.num_qo_heads
@@ -39,10 +37,10 @@ class Qwen3_5Attention(BaseOP):
# Fused q/k/v projection (one GEMM instead of three); q half is 2x for the
# output gate. Split sizes: [num_q*head_dim*2, num_kv*head_dim, num_kv*head_dim].
self._qkv_split = [self.num_q * head_dim * 2, self.kv_attn_dim, self.kv_attn_dim]
- # Block-fp8 (Fp8BlockColMerged) when the checkpoint is quantized, else bf16
- # LinearColParallelMerged. q/k/v out dims are all /128, so the merged fp8 weight +
- # weight_scale_inv concatenate cleanly along the output dim.
- self.qkv_proj = make_col_merged(config, config.hidden_size, self._qkv_split, has_bias=False)
+ self.qkv_proj = LinearColParallelMerged(
+ config.hidden_size, self._qkv_split, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.qkv_proj",
+ )
# Qwen3.5 uses Gemma-style (1+weight) RMSNorm; the weight loader bakes the +1
# into the stored weight (GemmaRMSNorm scales by the raw weight).
self.q_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps)
@@ -58,7 +56,10 @@ class Qwen3_5Attention(BaseOP):
else None
),
)
- self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False)
+ self.o_proj = LinearReplicated(
+ self.qo_attn_dim, config.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.o_proj",
+ )
def _project(self, x: torch.Tensor):
"""Returns (q, k, v, gate): q [N, num_q, head_dim] post qk-norm+rope,
diff --git a/python/freetoken/models/qwen3_5_moe/config.py b/python/freetoken/models/qwen3_5_moe/config.py
index 2ac4b607..dbcabc74 100644
--- a/python/freetoken/models/qwen3_5_moe/config.py
+++ b/python/freetoken/models/qwen3_5_moe/config.py
@@ -44,7 +44,7 @@ def _expert_quant(hf_config: Any) -> str:
offload cache). The nvidia/modelopt checkpoints are either plain NVFP4 (``quant_algo``
``NVFP4``) or ``MIXED_PRECISION`` (per-layer ``quantized_layers`` map); in the mixed
case the routed experts carry their own ``W4A16_NVFP4``/``FP8`` algo. Dense quantized
- weights (attention/shared-expert/lm_head) are handled separately by dequant-at-load."""
+ weights (attention / shared expert / lm_head) are served through their own schemes."""
get = _quant_accessor(hf_config)
if get is None:
return "none"
@@ -109,7 +109,7 @@ def _attn_quant(hf_config: Any) -> str:
``MIXED_PRECISION`` checkpoints tag ``self_attn.{q,k,v,o}_proj`` and
``linear_attn.{in_proj_qkv,in_proj_z,out_proj}`` with ``quant_algo`` ``FP8`` (fp8-e4m3
weight + a scalar ``weight_scale``; W8A16). Returns ``"fp8_pertensor"`` when present,
- else ``"none"`` (NVFP4 dense weights -- shared_expert/lm_head -- stay dequant-at-load)."""
+ else ``"none"`` (NVFP4 dense weights are covered by ``dense_quant`` / ``lm_head_quant``)."""
get = _quant_accessor(hf_config)
if get is None:
return "none"
diff --git a/python/freetoken/models/qwen3_5_moe/gdn.py b/python/freetoken/models/qwen3_5_moe/gdn.py
index 2e732005..d5fe2aa8 100644
--- a/python/freetoken/models/qwen3_5_moe/gdn.py
+++ b/python/freetoken/models/qwen3_5_moe/gdn.py
@@ -4,13 +4,10 @@ import torch
import torch.nn.functional as F
from freetoken.core import get_global_ctx
from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen
-from freetoken.layers import BaseOP, LinearColParallelMerged
-
-from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged
-from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
+from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicated
+from freetoken.layers.quantization import QuantConfig
from .gdn_kernels import gdn_decode_fla, gdn_prefill_chunk_fla
-from .quant_linear import make_replicated_quant
class _DepthwiseConv1d(BaseOP):
@@ -20,26 +17,6 @@ class _DepthwiseConv1d(BaseOP):
self.weight = torch.empty(conv_dim, 1, kernel)
-class _GatedRMSNorm(BaseOP):
- """RMSNorm of x followed by a silu(z) gate (HF Qwen3_5MoeRMSNormGated).
-
- Uses the fused fla ``rms_norm_gated`` triton kernel (norm(x) * silu(z) in one
- kernel) instead of the unfused pow/mean/rsqrt/mul/silu chain, matching sglang's
- ``RMSNormGated`` -- collapses ~8 elementwise kernels per GDN layer into one."""
-
- def __init__(self, dim: int, eps: float):
- self.weight = torch.empty(dim)
- self.eps = eps
-
- def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
- from freetoken.kernel.fla import rms_norm_gated
-
- return rms_norm_gated(
- x=x, weight=self.weight, bias=None, z=z, eps=self.eps,
- is_rms_norm=True, norm_before_gate=True, activation="silu",
- )
-
-
class Qwen3_5GatedDeltaNet(BaseOP):
"""GatedDeltaNet op using the vendored flash-linear-attention triton kernels
(``freetoken.kernel.fla``) for the recurrence and a per-request
@@ -52,8 +29,8 @@ class Qwen3_5GatedDeltaNet(BaseOP):
def __init__(
self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,
- conv_kernel_size, rms_norm_eps, layer_id, expert_quant: str = "none",
- attn_quant: str = "none",
+ conv_kernel_size, rms_norm_eps, layer_id, *, quant_config: QuantConfig | None = None,
+ prefix: str = "",
):
self.layer_id = layer_id
# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as
@@ -71,25 +48,27 @@ class Qwen3_5GatedDeltaNet(BaseOP):
self.value_dim = num_v_heads * head_v_dim
self.conv_dim = 2 * self.key_dim + self.value_dim
self.conv_kernel_size = conv_kernel_size
- # qkv|z carry a weight scale (block-fp8 weight_scale_inv, or per-tensor FP8
- # weight_scale); b|a stay bf16. Both quant modes therefore split the four-way
- # fusion into an fp8 qkvz GEMM + a bf16 ba GEMM (matches sglang/vLLM).
- self._block_fp8 = expert_quant == "fp8_block"
- self._pertensor_fp8 = attn_quant == "fp8_pertensor"
- self._fp8 = self._block_fp8 or self._pertensor_fp8
+ # quantized checkpoints quantize qkv|z but not b|a, so the fusion splits into a qkvz GEMM and a ba GEMM with their own schemes (matches sglang / vLLM)
+ self._split_in_proj = (
+ quant_config is not None and quant_config.scheme_for(f"{prefix}.in_proj_qkvz") is not None
+ )
self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads]
- if self._fp8:
- ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged
- self.in_proj_qkvz = ColMerged(
- hidden_size, [self.conv_dim, self.value_dim], has_bias=False
+ if self._split_in_proj:
+ self.in_proj_qkvz = LinearColParallelMerged(
+ hidden_size, [self.conv_dim, self.value_dim], has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.in_proj_qkvz",
)
self.in_proj_ba = LinearColParallelMerged(
- hidden_size, [num_v_heads, num_v_heads], has_bias=False
+ hidden_size, [num_v_heads, num_v_heads], has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.in_proj_ba",
)
else:
# Fused input projection (one GEMM instead of four): qkv | z | b | a.
- self.in_proj = LinearColParallelMerged(hidden_size, self._in_proj_split, has_bias=False)
+ self.in_proj = LinearColParallelMerged(
+ hidden_size, self._in_proj_split, has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.in_proj",
+ )
self.conv1d = _DepthwiseConv1d(self.conv_dim, conv_kernel_size)
# Recurrence-gating params kept in fp32 (exp/softplus is precision-sensitive,
# and the fla kernel reads them as fp32) -- matches HF/sglang, and avoids a
@@ -97,12 +76,10 @@ class Qwen3_5GatedDeltaNet(BaseOP):
# *.A_log / *.dt_bias from the model-dtype downcast.
self.dt_bias = torch.empty(num_v_heads, dtype=torch.float32)
self.A_log = torch.empty(num_v_heads, dtype=torch.float32)
- self.norm = _GatedRMSNorm(head_v_dim, eps=rms_norm_eps)
- # out_proj follows the checkpoint quant: block-fp8 / per-tensor-fp8 / compressed-tensors
- # NVFP4 (W4A16) / bf16. in_proj_* stay bf16 in every mode (above), so a compressed-tensors
- # NVFP4 checkpoint (attn_quant=="nvfp4") only makes out_proj native FP4.
- self.out_proj = make_replicated_quant(
- expert_quant, attn_quant, self.value_dim, hidden_size, has_bias=False
+ self.norm = GatedRMSNorm(head_v_dim, eps=rms_norm_eps)
+ self.out_proj = LinearReplicated(
+ self.value_dim, hidden_size, has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.out_proj",
)
def _gate_params(self, a: torch.Tensor, b: torch.Tensor):
@@ -161,7 +138,7 @@ class Qwen3_5GatedDeltaNet(BaseOP):
fla = build_fla_metadata(batch, hidden_states.device)
batch.fla_metadata = fla
- if self._fp8:
+ if self._split_in_proj:
qkvz = self.in_proj_qkvz.forward(hidden_states)
conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1)
ba = self.in_proj_ba.forward(hidden_states)
diff --git a/python/freetoken/models/qwen3_5_moe/model.py b/python/freetoken/models/qwen3_5_moe/model.py
index eba7fd24..d34793ad 100644
--- a/python/freetoken/models/qwen3_5_moe/model.py
+++ b/python/freetoken/models/qwen3_5_moe/model.py
@@ -27,7 +27,7 @@ class Qwen3_5DecoderLayer(BaseOP):
where the mixer is a GatedDeltaNet (linear layers) or gated attention (full layers).
All norms are Gemma-style (1+weight)."""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
self._layer_id = layer_id
self._is_linear = config.is_linear_layer(layer_id)
if self._is_linear:
@@ -42,14 +42,18 @@ class Qwen3_5DecoderLayer(BaseOP):
conv_kernel_size=g.conv_kernel_dim,
rms_norm_eps=config.rms_norm_eps,
layer_id=layer_id,
- expert_quant=config.expert_quant,
- attn_quant=config.attn_quant,
+ quant_config=config.quant,
+ prefix=f"{prefix}.linear_attn",
)
else:
- self.self_attn = Qwen3_5Attention(config, layer_id)
+ self.self_attn = Qwen3_5Attention(config, layer_id, prefix=f"{prefix}.self_attn")
# Dense variants (num_experts==0, e.g. Qwen3.6-27B) use a plain SwiGLU MLP instead of
# the routed MoE block; both expose ``forward(hidden)->hidden`` and the same key prefix.
- self.mlp = Qwen3_5MoE(config, layer_id) if config.moe_enabled else Qwen3_5DenseMLP(config)
+ self.mlp = (
+ Qwen3_5MoE(config, layer_id, prefix=f"{prefix}.mlp")
+ if config.moe_enabled
+ else Qwen3_5DenseMLP(config, prefix=f"{prefix}.mlp")
+ )
self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -69,13 +73,16 @@ class Qwen3_5DecoderLayer(BaseOP):
class Qwen3_5Model(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [Qwen3_5DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ Qwen3_5DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -91,22 +98,14 @@ class Qwen3_5Model(BaseOP):
class Qwen3_5MoEForCausalLM(BaseLLMModel):
def __init__(self, config: ModelConfig):
self.model = Qwen3_5Model(config)
- if getattr(config, "lm_head_quant", "none") == "nvfp4":
- # checkpoint stores the (untied) lm_head as NVFP4: keep it native (W4A16) -- the
- # bf16 dequant of this ~1 GB matrix was the single largest decode kernel.
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead
-
- assert not config.tie_word_embeddings, "NVFP4 lm_head assumes untied embeddings"
- self.lm_head = Nvfp4LMHead(
- num_embeddings=config.vocab_size, embedding_dim=config.hidden_size
- )
- else:
- self.lm_head = ParallelLMHead(
- num_embeddings=config.vocab_size,
- embedding_dim=config.hidden_size,
- tie_word_embeddings=config.tie_word_embeddings,
- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
- )
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
+ )
super().__init__()
def forward(self) -> torch.Tensor:
diff --git a/python/freetoken/models/qwen3_5_moe/moe.py b/python/freetoken/models/qwen3_5_moe/moe.py
index b5ab1cf3..766059e5 100644
--- a/python/freetoken/models/qwen3_5_moe/moe.py
+++ b/python/freetoken/models/qwen3_5_moe/moe.py
@@ -12,8 +12,6 @@ from freetoken.layers import (
silu_and_mul,
)
-from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged, Fp8BlockLinear
-
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
@@ -21,25 +19,17 @@ if TYPE_CHECKING:
class _SharedExpert(BaseOP):
"""Always-present shared SwiGLU expert of width ``shared_expert_intermediate_size``."""
- def __init__(self, config: ModelConfig, hidden_size: int, intermediate_size: int):
- if getattr(config, "expert_quant", "none") == "fp8_block":
- self.gate_up_proj = Fp8BlockColMerged(
- hidden_size, [intermediate_size, intermediate_size], has_bias=False
- )
- self.down_proj = Fp8BlockLinear(intermediate_size, hidden_size, has_bias=False)
- elif getattr(config, "dense_quant", "none") == "nvfp4":
- # NVFP4 checkpoint: keep the shared expert's NVFP4 weights native (W4A16).
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear
-
- self.gate_up_proj = Nvfp4DenseColMerged(
- hidden_size, [intermediate_size, intermediate_size], has_bias=False
- )
- self.down_proj = Nvfp4DenseLinear(intermediate_size, hidden_size, has_bias=False)
- else:
- self.gate_up_proj = LinearColParallelMerged(
- hidden_size, [intermediate_size, intermediate_size], has_bias=False
- )
- self.down_proj = LinearRowParallel(intermediate_size, hidden_size, has_bias=False)
+ def __init__(
+ self, config: ModelConfig, hidden_size: int, intermediate_size: int, *, prefix: str = ""
+ ):
+ self.gate_up_proj = LinearColParallelMerged(
+ hidden_size, [intermediate_size, intermediate_size], has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.gate_up_proj",
+ )
+ self.down_proj = LinearRowParallel(
+ intermediate_size, hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.down_proj",
+ )
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.down_proj.forward(silu_and_mul(self.gate_up_proj.forward(x)))
@@ -47,13 +37,12 @@ class _SharedExpert(BaseOP):
class Qwen3_5DenseMLP(_SharedExpert):
"""Dense (non-MoE) SwiGLU MLP for dense Qwen3.x checkpoints (e.g. 27B): ``gate_up_proj``
- (fused gate|up) + ``down_proj`` at full ``intermediate_size``. Same structure (and quant
- dispatch) as the shared expert -- NVFP4 (W4A16) when ``dense_quant=="nvfp4"``, else bf16 --
- so it reuses ``_SharedExpert`` directly and keeps the state-dict keys flat
+ (fused gate|up) + ``down_proj`` at full ``intermediate_size``. Same structure as the shared
+ expert, so it reuses ``_SharedExpert`` directly and keeps the state-dict keys flat
(``...layers.N.mlp.{gate_up_proj,down_proj}``)."""
- def __init__(self, config: ModelConfig):
- super().__init__(config, config.hidden_size, config.intermediate_size)
+ def __init__(self, config: ModelConfig, *, prefix: str = ""):
+ super().__init__(config, config.hidden_size, config.intermediate_size, prefix=prefix)
class Qwen3_5MoE(BaseOP):
@@ -64,19 +53,19 @@ class Qwen3_5MoE(BaseOP):
Router softmaxes over all experts, takes top-k, and renormalizes (HF semantics).
"""
- def __init__(self, config: ModelConfig, layer_id: int | None = None):
- weight_format = (
- "fp8_block" if getattr(config, "expert_quant", "none") == "fp8_block" else "bf16"
- )
+ def __init__(self, config: ModelConfig, layer_id: int | None = None, *, prefix: str = ""):
self.experts = make_moe_layer(
config,
layer_id=layer_id,
renormalize=config.norm_topk_prob,
- weight_format=weight_format,
+ quant_config=config.quant,
+ prefix=f"{prefix}.experts",
)
+ # routers stay bf16 whatever the checkpoint quantizes
self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
self.shared_expert = _SharedExpert(
- config, config.hidden_size, config.shared_expert_intermediate_size
+ config, config.hidden_size, config.shared_expert_intermediate_size,
+ prefix=f"{prefix}.shared_expert",
)
self.shared_expert_gate = LinearReplicated(config.hidden_size, 1, has_bias=False)
diff --git a/python/freetoken/models/qwen3_5_moe/quant_linear.py b/python/freetoken/models/qwen3_5_moe/quant_linear.py
deleted file mode 100644
index cd0443b6..00000000
--- a/python/freetoken/models/qwen3_5_moe/quant_linear.py
+++ /dev/null
@@ -1,19 +0,0 @@
-"""Re-export of the shared quant-aware dense-linear factories (models/quant_linear.py).
-
-The dispatch used to live here; muse_glimmer needing the same NVFP4/bf16 pair promoted
-it to a model-agnostic home. Kept as a module so existing imports stay valid.
-"""
-
-from freetoken.models.quant_linear import (
- make_col_merged,
- make_col_merged_quant,
- make_replicated,
- make_replicated_quant,
-)
-
-__all__ = [
- "make_col_merged_quant",
- "make_replicated_quant",
- "make_replicated",
- "make_col_merged",
-]
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index d07f18cd..45adb48b 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -7,6 +7,7 @@ from typing import Iterator
import safetensors
import torch
+from freetoken.layers.quantization import QuantKind
from freetoken.distributed import get_tp_info
from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4
from freetoken.models.loader import (
@@ -18,10 +19,7 @@ from freetoken.models.loader import (
iter_weight_files,
nvfp4_parts_ct,
)
-from freetoken.models.nvfp4_banks import (
- Nvfp4ExpertSourceSpec,
- load_nvfp4_expert_source_banks,
-)
+from freetoken.models.nvfp4_banks import Nvfp4ExpertSourceSpec
from freetoken.utils import cached_load_hf_config, download_hf_weight
from tqdm import tqdm
@@ -191,7 +189,7 @@ def iter_weights(
if config.expert_quant == "fp8_block":
# Dense (attn/GDN/shared-expert) weights are always block-fp8; routed experts are
# yielded here only for the resident path (include_moe_experts=True). Under offload
- # they are excluded and loaded by setup_offload_expert_banks instead.
+ # they are excluded and loaded from expert pieces instead.
yield from _iter_weights_fp8(
model_path, device,
include_non_moe=include_non_moe, include_moe_experts=include_moe_experts,
@@ -230,7 +228,7 @@ def iter_weights(
with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
keyset = set(f.keys())
for raw_name in f.keys():
- # Per-expert NVFP4 tensors go to the offload cache (load_nvfp4_expert_sources),
+ # Per-expert NVFP4 tensors go to the offload cache (expert pieces),
# not the dense pass. bf16-base stacked experts (experts.gate_up_proj) have no
# ``.mlp.experts.<int>.`` so they are unaffected and still hit _PACKED_EXPERT.
if _NVFP4_EXPERT_RE.search(raw_name):
@@ -670,7 +668,7 @@ def iter_weights_parallel(
Qwen3.5 stores experts pre-fused/pre-stacked per layer (already ``[E, ...]``), so no
merge/stack -- just rename and yield; bank builder places by name as the serial path."""
assert include_moe_experts and not include_non_moe, (
- "qwen3_5_moe parallel reader is experts-only (used by load_moe_expert_sources)"
+ "qwen3_5_moe parallel reader is experts-only (used by the expert piece reader)"
)
from freetoken.models.weight import iter_expert_tensors_parallel
@@ -760,7 +758,7 @@ def _iter_weights_fp8(
q/k/v -> qkv_proj, GDN in_proj_qkv|z -> in_proj_qkvz (fp8) and in_proj_b|a -> in_proj_ba
(bf16), shared_expert gate|up -> gate_up_proj; Gemma (1+w) norms get +1.
- Routed experts: skipped under offload (loaded by setup_offload_expert_banks). Under the
+ Routed experts: skipped under offload (loaded from expert pieces). Under the
resident (non-offload) path ``include_moe_experts`` is True -> per-layer stacked fp8
experts for the Fp8ResidentMoE buffers are yielded too."""
if get_tp_info().size > 1:
@@ -787,20 +785,40 @@ def _iter_weights_fp8(
assert not fuse_buf, f"Incomplete fp8 fusions: {sorted(k for k, _ in fuse_buf)}"
if include_moe_experts:
- # Resident (non-offload) experts: build the stacked fp8 banks once via the shared
- # parallel reader (pageable host -- the engine copies the per-layer slices to GPU),
- # then yield per-layer views into the Fp8ResidentMoE buffers.
+ # resident experts: stack the per-expert pieces into the per-layer tensors the resident MoE method declares (pageable host; the engine copies to GPU)
config = parse_config(cached_load_hf_config(model_path))
if not config.is_moe:
return # dense checkpoint: no routed experts to build as resident banks
- L, E, H, I, dense = _moe_dims(config)
- banks = _build_fp8_expert_banks(model_path, config, dummy=False, pin=False)
- for li in range(L):
+ yield from _resident_fp8_experts(model_path, config)
+
+
+def _resident_fp8_experts(model_path, config):
+ from freetoken.kernel.triton.fp8_block_linear import FP8
+
+ B = 128
+ L, E, H, I, dense = _moe_dims(config)
+ shapes = {
+ "gate_up_proj": ((E, 2 * I, H), FP8),
+ "gate_up_scale_inv": ((E, 2 * I // B, H // B), torch.bfloat16),
+ "down_proj": ((E, H, I), FP8),
+ "down_scale_inv": ((E, H // B, I // B), torch.bfloat16),
+ }
+ layers: dict[int, dict[str, torch.Tensor]] = {}
+ placed = [0] * L
+ for li, e0, e1, piece in iter_expert_pieces(model_path, config, "fp8_block", parallel=None):
+ stack = layers.setdefault(li, {n: torch.empty(shape, dtype=dt) for n, (shape, dt) in shapes.items()})
+ stack["gate_up_proj"][e0:e1, :I] = piece["gate"]
+ stack["gate_up_proj"][e0:e1, I:] = piece["up"]
+ stack["gate_up_scale_inv"][e0:e1, : I // B] = piece["gate_scale"]
+ stack["gate_up_scale_inv"][e0:e1, I // B :] = piece["up_scale"]
+ stack["down_proj"][e0:e1] = piece["down"]
+ stack["down_scale_inv"][e0:e1] = piece["down_scale"]
+ placed[li] += e1 - e0
+ if placed[li] == E:
pre = f"model.layers.{dense + li}.mlp.experts"
- yield f"{pre}.gate_up_proj", banks["gate_up"][li]
- yield f"{pre}.gate_up_scale_inv", banks["gate_up_scale"][li]
- yield f"{pre}.down_proj", banks["down"][li]
- yield f"{pre}.down_scale_inv", banks["down_scale"][li]
+ for name, tensor in layers.pop(li).items():
+ yield f"{pre}.{name}", tensor
+ assert not layers, f"incomplete resident fp8 experts for layers {sorted(layers)}"
class _ShardReader:
@@ -831,49 +849,6 @@ class _ShardReader:
self._handles.clear()
-def setup_offload_expert_banks(
- model_path: str, model_config, *, device: torch.device, dtype: torch.dtype,
- dummy: bool = False, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20,
- decode_target: str = "gpu", layer_sink=None,
-):
- """Build the routed-expert offload banks. The qwen3_5_moe module always exports this hook,
- so it intercepts *every* qwen3_5_moe offload load -- defer non-block-fp8 checkpoints (plain
- BF16, or modelopt NVFP4) to the generic provider for that ``expert_quant``.
-
- block-fp8 default ("fp8"): keep experts block-fp8 -- ``gate_up``/``down`` fp8 banks + their
- bf16 ``weight_scale_inv`` banks (half the host/cache bytes; routed rows are dequantized on
- demand in ``_expert_gemm``). ``FREETOKEN_FP8_EXPERTS=bf16`` instead dequantizes every expert
- to bf16 at load (reuses the bf16 offload path; ~2x the memory). Both modes build per-layer
- :class:`HostBank` banks (pin-after-fill), so the converter's ``layer_sink`` streams each
- completed layer straight through; ``layer_sink`` also reaches the deferred (nvfp4/none)
- providers for non-block-fp8 checkpoints.
-
- ``decode_target`` is forwarded so the cpu backend gets CPU-readable (native, non-
- GPU-tiled) bank layouts -- e.g. native ``nvfp4`` rows rather than marlin/b12x."""
- eq = getattr(model_config, "expert_quant", "none")
- if eq != "fp8_block":
- from freetoken.moe.expert_banks import _PROVIDERS # nvfp4 -> _nvfp4_banks, none -> _bf16_banks
-
- return _PROVIDERS[eq](model_path, model_config, device, dtype, dummy,
- parallel=parallel, workers=workers, chunk=chunk,
- decode_target=decode_target, layer_sink=layer_sink)
- if get_tp_info().size > 1:
- raise NotImplementedError("qwen3_5_moe fp8 expert banks support TP=1 only")
- from freetoken.moe.expert_banks import ExpertBanks
-
- mode = os.environ.get("FREETOKEN_FP8_EXPERTS", "fp8").strip().lower()
- if mode not in ("fp8", "bf16"):
- raise ValueError(f"FREETOKEN_FP8_EXPERTS must be 'fp8' or 'bf16', got {mode!r}")
- sink = None if dummy else layer_sink
- if mode == "bf16":
- return _setup_bf16_dequant_banks(model_path, model_config, device, dummy, layer_sink=sink)
- banks = _build_fp8_expert_banks(
- model_path, model_config, dummy=dummy, parallel=parallel, workers=workers, chunk=chunk,
- pin=True, layer_sink=sink,
- )
- return ExpertBanks("fp8_block", banks, streamed=sink is not None)
-
-
def _moe_dims(model_config):
L = model_config.num_moe_layers
return (
@@ -889,214 +864,59 @@ def _expert_reader(model_path, device):
return _ShardReader(folder, weight_map, device)
-def _build_fp8_expert_banks(
- model_path, config, *, dummy: bool, parallel: bool | None = None,
- workers: int = 8, chunk: int = 8 << 20, pin: bool = True, layer_sink=None,
-) -> dict[str, list[torch.Tensor]]:
- """The single block-fp8 routed-expert reader, shared by offload (``pin=True`` -> per-layer
- :class:`HostBank` banks, pin-after-fill / streamable) and resident (``pin=False`` -> plain
- pageable tensors, never pinned or streamed). Reads via the common chunked multi-threaded
- O_DIRECT reader (drops page cache, no per-tensor serial overhead) when the experts are
- scattered per-tensor; else a serial shard fallback. Stacks gate|up -> gate_up into one
- ``[E, ...]`` tensor per layer per bank.
-
- Each expert contributes 6 ``place()`` writes per layer ({gate,up,down} x {weight,
- weight_scale_inv}), so a layer completes after ``E * 6`` writes. ``layer_sink=None``
- (serving) pins each layer's 4 banks as it completes via an internally-owned
- :class:`PinPipeline`; ``layer_sink`` given (converter) fires the completion tracker into it
- instead (nothing pinned; released banks -- caller owns that tradeoff). ``pin=False`` and the
- CUDA-less host stay on the plain materialize path (no pin, no stream)."""
- from freetoken.kernel.triton.fp8_block_linear import FP8
+def iter_expert_pieces(model_path, config, kind: QuantKind, *, parallel: bool | None = False, workers: int = 8, chunk: int = 8 << 20):
+ """Block-fp8 routed experts, one piece per expert: ``{gate, up, down}`` fp8 codes and their
+ ``_scale`` (bf16 block ``weight_scale_inv``) companions. Other expert kinds use the generic readers."""
+ if kind is not QuantKind.FP8_BLOCK:
+ return None
+ if get_tp_info().size > 1:
+ raise NotImplementedError("qwen3_5_moe fp8 expert banks support TP=1 only")
from freetoken.models.weight import experts_scattered, iter_expert_tensors_parallel
+ from freetoken.moe.expert_pieces import per_expert_pieces
- B = 128
L, E, H, I, dense = _moe_dims(config)
+ suffix = {"weight": "", "weight_scale_inv": "_scale"}
- # 16B-align the per-expert scale rows (Qwen3.8: down_scale is 20x5 bf16 = 200 B) so the
- # fused multi-bank copy engages; the GEMMs read scales through explicit strides, so the
- # padding is inert. Unconditional: one layout per format, shared with the byte formulas.
- from freetoken.moe.offload_cache import fp8_block_scale_pad as _pad_cols
-
- specs = {
- "gate_up": ((E, 2 * I, H), FP8),
- "gate_up_scale": ((E, 2 * I // B, _pad_cols(2 * I // B, H // B)), torch.bfloat16),
- "down": ((E, H, I), FP8),
- "down_scale": ((E, H // B, _pad_cols(H // B, I // B)), torch.bfloat16),
- }
- hb = None
- if pin:
- from freetoken.moe.host_banks import alloc_layer_banks
-
- hb = alloc_layer_banks(specs, L) # lazy anon mmaps (unpinned)
- banks = {name: [b.tensor for b in hb[name]] for name in specs}
- else: # resident dequant source: plain pageable tensors (never pinned / streamed)
- banks = {name: [torch.empty(shape, dtype=dt) for _ in range(L)] for name, (shape, dt) in specs.items()}
- gate_up, gate_up_scale, down, down_scale = (
- banks["gate_up"], banks["gate_up_scale"], banks["down"], banks["down_scale"]
- )
- if dummy:
- for li in range(L):
- gate_up[li].view(torch.uint8).random_(0, 16) # small fp8 codes (avoid NaN/inf)
- down[li].view(torch.uint8).random_(0, 16)
- gate_up_scale[li].fill_(1.0)
- down_scale[li].fill_(1.0)
- if hb is not None and torch.cuda.is_available():
- from freetoken.moe.host_banks import pin_banks
-
- pin_banks(hb) # pin-after-fill (match the other dummies)
- return banks
-
- def place(raw_name: str, t: torch.Tensor) -> int | None:
+ def locate(raw_name: str):
m = _FP8_EXPERT_RE.match(raw_name)
if m is None:
return None
- li, e = int(m["layer"]) - dense, int(m["expert"])
- proj, kind = m["proj"], m["kind"]
- if kind == "weight":
- (gate_up[li][e, :I] if proj == "gate" else
- gate_up[li][e, I:] if proj == "up" else down[li][e]).copy_(t)
- else: # weight_scale_inv
- (gate_up_scale[li][e, : I // B, : H // B] if proj == "gate" else
- gate_up_scale[li][e, I // B :, : H // B] if proj == "up" else
- down_scale[li][e, :, : I // B]).copy_(t)
- return li
+ li = int(m["layer"]) - dense
+ if not 0 <= li < L:
+ raise ValueError(f"unexpected routed-expert layer in {raw_name}")
+ return li, int(m["expert"]), m["proj"] + suffix[m["kind"]]
if parallel is None:
parallel = experts_scattered(model_path)
+ if parallel:
+ tensors = iter_expert_tensors_parallel(
+ model_path, lambda n: _FP8_EXPERT_RE.match(n) is not None, workers=workers, chunk=chunk
+ )
+ return per_expert_pieces(tensors, locate, tensors_per_expert=6)
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline
-
- def _load(sink) -> None:
- # {gate,up,down} x {weight, weight_scale_inv} per expert -> E*6 writes/layer.
- tracker = LayerCompletionTracker(E * 6, hb, sink) if sink is not None else None
- if parallel:
- for raw_name, t in iter_expert_tensors_parallel(
- model_path, lambda n: _FP8_EXPERT_RE.match(n) is not None, workers=workers, chunk=chunk
- ):
- li = place(raw_name, t)
- if tracker is not None and li is not None:
- tracker.note(li)
- else:
- reader = _expert_reader(model_path, torch.device("cpu"))
- primary = get_tp_info().is_primary()
- try:
- for li in tqdm(range(L), desc="Loading fp8 experts (serial)", disable=not primary):
- layer = dense + li
- for e in range(E):
- p = f"model.language_model.layers.{layer}.mlp.experts.{e}"
- for proj in ("gate", "up", "down"):
- for kind in ("weight", "weight_scale_inv"):
- key = f"{p}.{proj}_proj.{kind}"
- place(key, reader.get(key))
- if tracker is not None:
- tracker.note(li)
- finally:
- reader.close()
-
- if not pin:
- assert layer_sink is None, "pin=False (resident source) cannot stream to a layer_sink"
- _load(None)
- elif layer_sink is not None:
- _load(layer_sink)
- elif torch.cuda.is_available():
- with PinPipeline() as pins:
- _load(pins)
- else:
- _load(None) # CUDA-less: mmap banks stay pageable, never pinned
- return banks
-
-
-def _setup_bf16_dequant_banks(model_path, model_config, device, dummy: bool, *, layer_sink=None):
- from freetoken.models.weight import dummy_moe_expert_sources
- from freetoken.moe.expert_banks import ExpertBanks
-
- if dummy:
- gate_up, down = dummy_moe_expert_sources(model_config, dtype=torch.bfloat16)
- return ExpertBanks("bf16", {"gate_up": gate_up, "down": down})
-
- from freetoken.kernel.triton.fp8_block_linear import dequant_block_fp8
- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks
-
- L, E, H, I, dense = _moe_dims(model_config)
- specs = {"gate_up": ((E, 2 * I, H), torch.bfloat16), "down": ((E, H, I), torch.bfloat16)}
- hb = alloc_layer_banks(specs, L) # lazy anon mmaps (unpinned)
- banks = {name: [b.tensor for b in hb[name]] for name in specs}
- gate_up, down = banks["gate_up"], banks["down"]
- reader = _expert_reader(model_path, device)
- primary = get_tp_info().is_primary()
-
- def _deq(prefix: str) -> torch.Tensor:
- return dequant_block_fp8(reader.get(f"{prefix}.weight"), reader.get(f"{prefix}.weight_scale_inv"))
-
- def _load(sink) -> None:
- # Whole-layer gate_up + down copies -> 2 writes/layer.
- tracker = LayerCompletionTracker(2, hb, sink) if sink is not None else None
+ def _serial():
+ reader = _expert_reader(model_path, torch.device("cpu"))
try:
- for li in tqdm(range(L), desc="Loading fp8 experts (dequant->bf16)", disable=not primary):
- layer = dense + li
- gu_rows = torch.empty(E, 2 * I, H, dtype=torch.bfloat16, device=device)
- dn_rows = torch.empty(E, H, I, dtype=torch.bfloat16, device=device)
+ for li in tqdm(range(L), desc="Loading fp8 experts (serial)", disable=not get_tp_info().is_primary()):
for e in range(E):
- p = f"model.language_model.layers.{layer}.mlp.experts.{e}"
- gu_rows[e, :I] = _deq(f"{p}.gate_proj")
- gu_rows[e, I:] = _deq(f"{p}.up_proj")
- dn_rows[e] = _deq(f"{p}.down_proj")
- gate_up[li].copy_(gu_rows)
- if tracker is not None:
- tracker.note(li)
- down[li].copy_(dn_rows)
- if tracker is not None:
- tracker.note(li)
+ base = f"model.language_model.layers.{dense + li}.mlp.experts.{e}"
+ for proj in ("gate", "up", "down"):
+ for kind, suf in suffix.items():
+ name = f"{base}.{proj}_proj.{kind}"
+ yield name, reader.get(name)
finally:
reader.close()
- if layer_sink is not None:
- _load(layer_sink)
- elif torch.cuda.is_available():
- with PinPipeline() as pins:
- _load(pins)
- else:
- _load(None) # CUDA-less: mmap banks stay pageable, never pinned
- return ExpertBanks("bf16", banks, streamed=layer_sink is not None)
-
-
-def load_nvfp4_expert_sources(
- model_path: str, config, *, layer_sink=None
-) -> dict[str, torch.Tensor]:
- """Build the CPU NVFP4 expert source banks for the offload cache (gate/up fused on the
- output-row axis, down separate; weight_scale_2 carried as the per-row global scale)."""
- return load_nvfp4_expert_source_banks(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- layer_sink=layer_sink,
- )
+ return per_expert_pieces(_serial(), locate, tensors_per_expert=6)
-def load_nvfp4_expert_sources_parallel(
- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
-):
- """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""
- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
-
- return load_nvfp4_expert_source_banks_parallel(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- workers=workers,
- chunk=chunk,
- layer_sink=layer_sink,
- )
+def nvfp4_expert_spec(model_path: str, config):
+ return _NVFP4_SOURCE_SPEC
__all__ = [
"iter_weights",
"iter_weights_parallel",
- "setup_offload_expert_banks",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
+ "iter_expert_pieces",
+ "nvfp4_expert_spec",
]
diff --git a/python/freetoken/models/qwen3_moe/attention.py b/python/freetoken/models/qwen3_moe/attention.py
index d1fef63c..8a737b27 100644
--- a/python/freetoken/models/qwen3_moe/attention.py
+++ b/python/freetoken/models/qwen3_moe/attention.py
@@ -22,6 +22,7 @@ class Qwen3MoeAttention(BaseOP):
*,
has_attn_bias: bool = False,
has_qk_norm: bool = False,
+ prefix: str = "",
):
head_dim = config.head_dim
self.layer_id = layer_id
@@ -37,6 +38,8 @@ class Qwen3MoeAttention(BaseOP):
num_qo_heads=config.num_qo_heads,
num_kv_heads=config.num_kv_heads,
has_bias=has_attn_bias,
+ quant_config=config.quant,
+ prefix=f"{prefix}.qkv_proj",
)
if has_qk_norm:
self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
@@ -59,6 +62,8 @@ class Qwen3MoeAttention(BaseOP):
head_dim * config.num_qo_heads,
config.hidden_size,
has_bias=False,
+ quant_config=config.quant,
+ prefix=f"{prefix}.o_proj",
)
@nvtx_annotate("MHA")
diff --git a/python/freetoken/models/qwen3_moe/model.py b/python/freetoken/models/qwen3_moe/model.py
index e37437d2..4918092e 100644
--- a/python/freetoken/models/qwen3_moe/model.py
+++ b/python/freetoken/models/qwen3_moe/model.py
@@ -17,9 +17,9 @@ if TYPE_CHECKING:
class Qwen3DecoderLayer(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int):
- self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True)
- self.mlp = Qwen3MLP(config, layer_id)
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.self_attn = Qwen3Attn(config, layer_id, has_qk_norm=True, prefix=f"{prefix}.self_attn")
+ self.mlp = Qwen3MLP(config, layer_id, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNormFused(
size=config.hidden_size,
eps=config.rms_norm_eps,
@@ -43,13 +43,16 @@ class Qwen3DecoderLayer(BaseOP):
class Qwen3Model(BaseOP):
- def __init__(self, config: ModelConfig):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [Qwen3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ Qwen3DecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
self.norm = RMSNormFused(
size=config.hidden_size,
@@ -72,6 +75,8 @@ class Qwen3MoeForCausalLM(BaseLLMModel):
embedding_dim=config.hidden_size,
tie_word_embeddings=config.tie_word_embeddings,
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
)
super().__init__()
diff --git a/python/freetoken/models/qwen3_moe/moe.py b/python/freetoken/models/qwen3_moe/moe.py
index 973cfd95..c5a73305 100644
--- a/python/freetoken/models/qwen3_moe/moe.py
+++ b/python/freetoken/models/qwen3_moe/moe.py
@@ -11,8 +11,10 @@ if TYPE_CHECKING:
class Qwen3MoeMLP(BaseOP):
- def __init__(self, config: ModelConfig, layer_id: int | None = None):
- self.experts = make_moe_layer(config, layer_id=layer_id)
+ def __init__(self, config: ModelConfig, layer_id: int | None = None, *, prefix: str = ""):
+ self.experts = make_moe_layer(
+ config, layer_id=layer_id, quant_config=config.quant, prefix=f"{prefix}.experts"
+ )
self.gate = LinearReplicated(
config.hidden_size,
config.num_experts,
diff --git a/python/freetoken/models/qwen3_moe/weight.py b/python/freetoken/models/qwen3_moe/weight.py
index a147640c..a8b480d0 100644
--- a/python/freetoken/models/qwen3_moe/weight.py
+++ b/python/freetoken/models/qwen3_moe/weight.py
@@ -89,7 +89,7 @@ def iter_weights_parallel(
"""experts-only iter_weights: raw experts read via the common chunked multi-threaded
O_DIRECT reader, then same merge+stack pipeline."""
assert include_moe_experts and not include_non_moe, (
- "qwen3_moe parallel reader is experts-only (used by load_moe_expert_sources)"
+ "qwen3_moe parallel reader is experts-only (used by the expert piece reader)"
)
from freetoken.models.weight import iter_expert_tensors_parallel
diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.py
index 04c29d77..1a74d4ae 100644
--- a/python/freetoken/models/qwen4_exp/__init__.py
+++ b/python/freetoken/models/qwen4_exp/__init__.py
@@ -12,24 +12,20 @@ Contracts shared across modules (do not rename):
from .config import parse_config
from .model import Qwen4ExpForCausalLM
from .weight import (
+ nvfp4_expert_spec,
iter_weights,
- load_nvfp4_expert_sources,
- load_nvfp4_expert_sources_parallel,
load_ple_table,
)
# Official FP8 checkpoints share qwen3_5_moe's block-fp8 expert layout (same
-# model.language_model.layers.* keys), so reuse its bank hook; for every other
-# expert_quant it defers to the per-quant providers, which resolve this module's
-# load_nvfp4_expert_sources via the model spec.
-from freetoken.models.qwen3_5_moe.weight import setup_offload_expert_banks
+# model.language_model.layers.* keys), so reuse its expert reader.
+from freetoken.models.qwen3_5_moe.weight import iter_expert_pieces
__all__ = [
+ "nvfp4_expert_spec",
"Qwen4ExpForCausalLM",
"iter_weights",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
"load_ple_table",
"parse_config",
- "setup_offload_expert_banks",
+ "iter_expert_pieces",
]
diff --git a/python/freetoken/models/qwen4_exp/attention.py b/python/freetoken/models/qwen4_exp/attention.py
index d6ab2867..66211c7c 100644
--- a/python/freetoken/models/qwen4_exp/attention.py
+++ b/python/freetoken/models/qwen4_exp/attention.py
@@ -76,7 +76,7 @@ class QSAAttentionBackend(Protocol):
class Qwen4ExpIndexer(BaseOP):
"""QSA indexer weights (checkpoint prefix ``self_attn.indexer``); the scoring lives in the backend."""
- def __init__(self, config: ModelConfig, layer_id: int) -> None:
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = "") -> None:
args = config.qwen4_args
self.layer_id = layer_id
self.num_heads = args.index_n_heads
@@ -84,7 +84,10 @@ class Qwen4ExpIndexer(BaseOP):
self.head_dim = args.index_head_dim
self.eps = config.rms_norm_eps
self._split = [self.num_heads * self.head_dim, self.num_kv_heads * self.head_dim]
- self.index_qk_proj = LinearReplicated(args.hidden_size, sum(self._split), has_bias=False)
+ self.index_qk_proj = LinearReplicated(
+ args.hidden_size, sum(self._split), has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.index_qk_proj",
+ )
self.q_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)
self.k_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)
@@ -113,7 +116,7 @@ class Qwen4ExpAttention(BaseOP):
(both zero-centered, loaded RAW), ``indexer.*``.
"""
- def __init__(self, config: ModelConfig, layer_id: int) -> None:
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = "") -> None:
self.layer_id = layer_id
self.num_q = config.num_qo_heads
self.num_kv = config.num_kv_heads
@@ -122,9 +125,13 @@ class Qwen4ExpAttention(BaseOP):
self.kv_attn_dim = self.num_kv * self.head_dim
self._qkv_split = [self.qo_attn_dim * 2, self.kv_attn_dim, self.kv_attn_dim]
self.qkv_proj = LinearColParallelMerged(
- config.hidden_size, self._qkv_split, has_bias=False
+ config.hidden_size, self._qkv_split, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.qkv_proj",
+ )
+ self.o_proj = LinearReplicated(
+ self.qo_attn_dim, config.hidden_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.o_proj",
)
- self.o_proj = LinearReplicated(self.qo_attn_dim, config.hidden_size, has_bias=False)
self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)
rotary = config.rotary_config
@@ -135,7 +142,7 @@ class Qwen4ExpAttention(BaseOP):
base=rotary.base,
rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,
)
- self.indexer = Qwen4ExpIndexer(config, layer_id)
+ self.indexer = Qwen4ExpIndexer(config, layer_id, prefix=f"{prefix}.indexer")
@nvtx_annotate("QSA")
def forward(self, x: torch.Tensor, batch: Batch) -> torch.Tensor:
diff --git a/python/freetoken/models/qwen4_exp/gdn.py b/python/freetoken/models/qwen4_exp/gdn.py
index 69838153..ed49dd42 100644
--- a/python/freetoken/models/qwen4_exp/gdn.py
+++ b/python/freetoken/models/qwen4_exp/gdn.py
@@ -4,15 +4,9 @@ import torch
import torch.nn.functional as F
from freetoken.core import get_global_ctx
from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen
-from freetoken.layers import BaseOP, LinearColParallelMerged
-
-from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged
-from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
+from freetoken.layers import BaseOP, GatedRMSNorm, LinearColParallelMerged, LinearReplicated
+from freetoken.layers.quantization import QuantConfig
from freetoken.models.qwen3_5_moe.gdn_kernels import gdn_decode_fla, gdn_prefill_chunk_fla
-from freetoken.models.quant_linear import make_replicated_quant
-
-
-_GATE_ACTIVATIONS = ("silu", "swish", "sigmoid")
class _DepthwiseConv1d(BaseOP):
@@ -22,30 +16,6 @@ class _DepthwiseConv1d(BaseOP):
self.weight = torch.empty(conv_dim, 1, kernel)
-class _GatedRMSNorm(BaseOP):
- """RMSNorm of x followed by an ``activation(z)`` gate (HF Qwen4ExpTextRMSNormGated).
-
- Uses the fused fla ``rms_norm_gated`` triton kernel (norm(x) * act(z) in one
- kernel) instead of the unfused pow/mean/rsqrt/mul/act chain, matching sglang's
- ``RMSNormGated`` -- collapses ~8 elementwise kernels per GDN layer into one.
- Qwen3.8-Flash-Next gates with sigmoid where Qwen3.5 gates with silu."""
-
- def __init__(self, dim: int, eps: float, activation: str):
- # rms_norm_gated drops the gate entirely (no error) for a name it does not know.
- assert activation in _GATE_ACTIVATIONS, f"unsupported GDN output gate {activation!r}"
- self.weight = torch.empty(dim)
- self.eps = eps
- self.activation = activation
-
- def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
- from freetoken.kernel.fla import rms_norm_gated
-
- return rms_norm_gated(
- x=x, weight=self.weight, bias=None, z=z, eps=self.eps,
- is_rms_norm=True, norm_before_gate=True, activation=self.activation,
- )
-
-
class Qwen4ExpGatedDeltaNet(BaseOP):
"""GatedDeltaNet op using the vendored flash-linear-attention triton kernels
(``freetoken.kernel.fla``) for the recurrence and a per-request
@@ -62,7 +32,7 @@ class Qwen4ExpGatedDeltaNet(BaseOP):
def __init__(
self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,
conv_kernel_size, rms_norm_eps, layer_id, output_gate: str = "sigmoid",
- expert_quant: str = "none", attn_quant: str = "none",
+ *, quant_config: QuantConfig | None = None, prefix: str = "",
):
self.layer_id = layer_id
# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as
@@ -80,25 +50,27 @@ class Qwen4ExpGatedDeltaNet(BaseOP):
self.value_dim = num_v_heads * head_v_dim
self.conv_dim = 2 * self.key_dim + self.value_dim
self.conv_kernel_size = conv_kernel_size
- # qkv|z carry a weight scale (block-fp8 weight_scale_inv, or per-tensor FP8
- # weight_scale); b|a stay bf16. Both quant modes therefore split the four-way
- # fusion into an fp8 qkvz GEMM + a bf16 ba GEMM (matches sglang/vLLM).
- self._block_fp8 = expert_quant == "fp8_block"
- self._pertensor_fp8 = attn_quant == "fp8_pertensor"
- self._fp8 = self._block_fp8 or self._pertensor_fp8
+ # quantized checkpoints quantize qkv|z but not b|a, so the fusion splits into a qkvz GEMM and a ba GEMM with their own schemes (matches sglang / vLLM)
+ self._split_in_proj = (
+ quant_config is not None and quant_config.scheme_for(f"{prefix}.in_proj_qkvz") is not None
+ )
self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads]
- if self._fp8:
- ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged
- self.in_proj_qkvz = ColMerged(
- hidden_size, [self.conv_dim, self.value_dim], has_bias=False
+ if self._split_in_proj:
+ self.in_proj_qkvz = LinearColParallelMerged(
+ hidden_size, [self.conv_dim, self.value_dim], has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.in_proj_qkvz",
)
self.in_proj_ba = LinearColParallelMerged(
- hidden_size, [num_v_heads, num_v_heads], has_bias=False
+ hidden_size, [num_v_heads, num_v_heads], has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.in_proj_ba",
)
else:
# Fused input projection (one GEMM instead of four): qkv | z | b | a.
- self.in_proj = LinearColParallelMerged(hidden_size, self._in_proj_split, has_bias=False)
+ self.in_proj = LinearColParallelMerged(
+ hidden_size, self._in_proj_split, has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.in_proj",
+ )
self.conv1d = _DepthwiseConv1d(self.conv_dim, conv_kernel_size)
# Recurrence-gating params kept in fp32 (exp/softplus is precision-sensitive,
# and the fla kernel reads them as fp32) -- matches HF/sglang, and avoids a
@@ -106,12 +78,10 @@ class Qwen4ExpGatedDeltaNet(BaseOP):
# *.A_log / *.dt_bias from the model-dtype downcast.
self.dt_bias = torch.empty(num_v_heads, dtype=torch.float32)
self.A_log = torch.empty(num_v_heads, dtype=torch.float32)
- self.norm = _GatedRMSNorm(head_v_dim, eps=rms_norm_eps, activation=output_gate)
- # out_proj follows the checkpoint quant: block-fp8 / per-tensor-fp8 / compressed-tensors
- # NVFP4 (W4A16) / bf16. in_proj_* stay bf16 in every mode (above), so a compressed-tensors
- # NVFP4 checkpoint (attn_quant=="nvfp4") only makes out_proj native FP4.
- self.out_proj = make_replicated_quant(
- expert_quant, attn_quant, self.value_dim, hidden_size, has_bias=False
+ self.norm = GatedRMSNorm(head_v_dim, eps=rms_norm_eps, activation=output_gate)
+ self.out_proj = LinearReplicated(
+ self.value_dim, hidden_size, has_bias=False,
+ quant_config=quant_config, prefix=f"{prefix}.out_proj",
)
def _gate_params(self, a: torch.Tensor, b: torch.Tensor):
@@ -170,7 +140,7 @@ class Qwen4ExpGatedDeltaNet(BaseOP):
fla = build_fla_metadata(batch, hidden_states.device)
batch.fla_metadata = fla
- if self._fp8:
+ if self._split_in_proj:
qkvz = self.in_proj_qkvz.forward(hidden_states)
conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1)
ba = self.in_proj_ba.forward(hidden_states)
diff --git a/python/freetoken/models/qwen4_exp/hc.py b/python/freetoken/models/qwen4_exp/hc.py
index 6fed76c0..f13808dc 100644
--- a/python/freetoken/models/qwen4_exp/hc.py
+++ b/python/freetoken/models/qwen4_exp/hc.py
@@ -85,7 +85,7 @@ class GatedResidual(BaseOP):
Launch budget on CUDA: ``mix`` is 3 kernels around 2 GEMMs, ``combine`` is 1.
"""
- def __init__(self, config: ModelConfig, use_combine: bool = True) -> None:
+ def __init__(self, config: ModelConfig, use_combine: bool = True, *, prefix: str = "") -> None:
args = config.qwen4_args
self.hc_count = args.hc_count
self.hidden_size = args.hidden_size
@@ -97,12 +97,19 @@ class GatedResidual(BaseOP):
# 16-row alignment for the merged skinny GEMM (vLLM hyperconnection.py:98)
self.pad_size = (-(self.lowrank + self.hc_count)) % 16
self.input_mix_weight_down_block_inject = LinearReplicated(
- width, self.lowrank + self.hc_count + self.pad_size, has_bias=False
+ width, self.lowrank + self.hc_count + self.pad_size, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_down_block_inject",
)
else:
self.pad_size = 0
- self.input_mix_weight_down = LinearReplicated(width, self.lowrank, has_bias=False)
- self.input_mix_weight_up = LinearReplicated(self.lowrank, width, has_bias=False)
+ self.input_mix_weight_down = LinearReplicated(
+ width, self.lowrank, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_down",
+ )
+ self.input_mix_weight_up = LinearReplicated(
+ self.lowrank, width, has_bias=False,
+ quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_up",
+ )
def _down(self, rn: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:
"""Run the down GEMM and split off the raw inject logits; the pad columns are dropped."""
diff --git a/python/freetoken/models/qwen4_exp/model.py b/python/freetoken/models/qwen4_exp/model.py
index 19c31e2f..a4b63362 100644
--- a/python/freetoken/models/qwen4_exp/model.py
+++ b/python/freetoken/models/qwen4_exp/model.py
@@ -33,7 +33,7 @@ if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
-def build_linear_mixer(config: ModelConfig, layer_id: int) -> BaseOP:
+def build_linear_mixer(config: ModelConfig, layer_id: int, prefix: str) -> BaseOP:
"""GDN mixer of a linear_attention layer (Qwen3.5's GDN with a configurable output gate)."""
from .gdn import Qwen4ExpGatedDeltaNet
@@ -48,28 +48,26 @@ def build_linear_mixer(config: ModelConfig, layer_id: int) -> BaseOP:
rms_norm_eps=config.rms_norm_eps,
layer_id=layer_id,
output_gate=g.output_gate,
- # Qwen3.8's block-fp8 checkpoint keeps the GDN projections bf16 (only the routed
- # experts are quantized), so do not let expert_quant flip them to Fp8Block.
- expert_quant="none" if config.expert_quant == "fp8_block" else config.expert_quant,
- attn_quant=config.attn_quant,
+ quant_config=config.quant,
+ prefix=prefix,
)
class Qwen4ExpDecoderLayer(BaseOP):
"""One decoder layer over the hyper-connection streams (see the module docstring for the flow)."""
- def __init__(self, config: ModelConfig, layer_id: int) -> None:
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = "") -> None:
self._layer_id = layer_id
self._is_linear = config.is_linear_layer(layer_id)
if self._is_linear:
- self.linear_attn = build_linear_mixer(config, layer_id)
+ self.linear_attn = build_linear_mixer(config, layer_id, f"{prefix}.linear_attn")
else:
- self.self_attn = Qwen4ExpAttention(config, layer_id)
- self.mlp = Qwen4ExpMoE(config, layer_id)
- self.attn_hyper_connection = GatedResidual(config)
- self.mlp_hyper_connection = GatedResidual(config)
+ self.self_attn = Qwen4ExpAttention(config, layer_id, prefix=f"{prefix}.self_attn")
+ self.mlp = Qwen4ExpMoE(config, layer_id, prefix=f"{prefix}.mlp")
+ self.attn_hyper_connection = GatedResidual(config, prefix=f"{prefix}.attn_hyper_connection")
+ self.mlp_hyper_connection = GatedResidual(config, prefix=f"{prefix}.mlp_hyper_connection")
self.ple = (
- PLELayer(config, layer_id) if layer_id in config.qwen4_args.ple_layer_ids else None
+ PLELayer(config, layer_id, prefix=f"{prefix}.ple") if layer_id in config.qwen4_args.ple_layer_ids else None
)
@nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
@@ -87,16 +85,19 @@ class Qwen4ExpDecoderLayer(BaseOP):
class Qwen4ExpModel(BaseOP):
- def __init__(self, config: ModelConfig) -> None:
+ def __init__(self, config: ModelConfig, *, prefix: str = "model") -> None:
self.hc_count = config.qwen4_args.hc_count
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
)
self.layers = OPList(
- [Qwen4ExpDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ [
+ Qwen4ExpDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
+ for layer_id in range(config.num_layers)
+ ]
)
- self.hyper_connection_mixer = GatedResidual(config, use_combine=False)
+ self.hyper_connection_mixer = GatedResidual(config, use_combine=False, prefix=f"{prefix}.hyper_connection_mixer")
# plain tuple (not an OP child), so it never shows up in the state dict
self._ple = tuple(layer.ple for layer in self.layers.op_list if layer.ple is not None)
@@ -127,20 +128,14 @@ class Qwen4ExpForCausalLM(BaseLLMModel):
def __init__(self, config: ModelConfig) -> None:
self._config = config
self.model = Qwen4ExpModel(config)
- if getattr(config, "lm_head_quant", "none") == "nvfp4":
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead
-
- assert not config.tie_word_embeddings, "NVFP4 lm_head assumes untied embeddings"
- self.lm_head = Nvfp4LMHead(
- num_embeddings=config.vocab_size, embedding_dim=config.hidden_size
- )
- else:
- self.lm_head = ParallelLMHead(
- num_embeddings=config.vocab_size,
- embedding_dim=config.hidden_size,
- tie_word_embeddings=config.tie_word_embeddings,
- tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
- )
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant,
+ prefix="lm_head",
+ )
super().__init__()
def load_host_tables(self, engine_config) -> int:
diff --git a/python/freetoken/models/qwen4_exp/moe.py b/python/freetoken/models/qwen4_exp/moe.py
index 9ef65c0a..3687f280 100644
--- a/python/freetoken/models/qwen4_exp/moe.py
+++ b/python/freetoken/models/qwen4_exp/moe.py
@@ -1,11 +1,9 @@
from __future__ import annotations
-from dataclasses import replace
from typing import TYPE_CHECKING
import torch
from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid
-from freetoken.layers.moe import make_moe_layer
from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE
if TYPE_CHECKING:
@@ -18,21 +16,6 @@ class Qwen4ExpMoE(Qwen3_5MoE):
Same weights, same state dict. The gate reduction stays ahead of the routed experts, which may write into ``hidden_states`` in place.
"""
- def __init__(self, config: ModelConfig, layer_id: int | None = None) -> None:
- if getattr(config, "expert_quant", "none") != "fp8_block":
- super().__init__(config, layer_id=layer_id)
- return
- # Qwen3.8's block-fp8 checkpoint quantizes only the routed experts; the shared
- # expert stays bf16, so hide expert_quant from _SharedExpert's fp8 branch and
- # rebuild the routed experts with the fp8_block bank layout.
- super().__init__(replace(config, expert_quant="none"), layer_id=layer_id)
- self.experts = make_moe_layer(
- config,
- layer_id=layer_id,
- renormalize=config.norm_topk_prob,
- weight_format="fp8_block",
- )
-
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
diff --git a/python/freetoken/models/qwen4_exp/ple.py b/python/freetoken/models/qwen4_exp/ple.py
index 5100229a..13157cb8 100644
--- a/python/freetoken/models/qwen4_exp/ple.py
+++ b/python/freetoken/models/qwen4_exp/ple.py
@@ -542,7 +542,7 @@ class PLELayer(BaseOP):
"""
def __init__(
- self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None
+ self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None, *, prefix: str = ""
) -> None:
args = config.qwen4_args
self.args = args
@@ -554,8 +554,12 @@ class PLELayer(BaseOP):
self.state_len = args.ple_conv_state_len
width = args.ple_state_width
self.ple_embedding = NGramEmbedding(args, table)
- self.key_proj = LinearReplicated(args.ple_embed_dim, width, has_bias=False)
- self.value_proj = LinearReplicated(args.ple_embed_dim, args.hidden_size, has_bias=False)
+ self.key_proj = LinearReplicated(
+ args.ple_embed_dim, width, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.key_proj"
+ )
+ self.value_proj = LinearReplicated(
+ args.ple_embed_dim, args.hidden_size, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.value_proj"
+ )
self.norm_key = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
self.norm_query = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
self.norm_conv = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.py
index f8d2a749..da4bea48 100644
--- a/python/freetoken/models/qwen4_exp/weight.py
+++ b/python/freetoken/models/qwen4_exp/weight.py
@@ -1,10 +1,10 @@
-"""Qwen3.8-Flash-Next (RadixArk NVFP4) checkpoint reader.
+"""Qwen3.8-Flash-Next checkpoint reader (the NVFP4 and the official block-fp8 releases).
Three separate paths, because the checkpoint's three weight classes live in different places:
* :func:`iter_weights` -- every dense (non-expert) tensor, with the ``model.language_model.`` prefix stripped and fused where the model expects one buffer. See ``_FUSIONS``.
* :func:`load_ple_table` -- the 47.7 GiB FP8 n-gram table, 128 checkpoint shards concatenated into one pinned :class:`HostBank`.
-* :func:`load_nvfp4_expert_sources` -- the routed NVFP4 experts, into the offload cache's source banks.
+* :func:`nvfp4_expert_spec` -- how the routed NVFP4 experts are named, for the offload cache's expert reader.
Dropped: ``mtp.*`` (speculative head, including its stacked ``mtp.layers.0.mlp.experts.*``) and ``model.visual.*`` (served text-only).
"""
@@ -24,7 +24,6 @@ from freetoken.distributed import get_tp_info
from freetoken.models.loader import drop_page_cache, iter_weight_files
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
- load_nvfp4_expert_source_banks,
)
from freetoken.moe.host_banks import HostBank, read_range_into
from freetoken.utils import download_hf_weight
@@ -147,15 +146,16 @@ def iter_weights(
"""Yield the dense (non-expert) weights, prefix-stripped and fused to the model's buffers.
Keys keep the checkpoint's module names below the stripped prefix, so the emitted set is the
- model's state dict minus the routed experts. Nothing here is quantized: the modelopt
- ``ignore`` list covers everything except those experts, so attention, GDN, HC, PLE, the shared
- expert and lm_head are all plain bf16 (the n-gram hash constants stay int64). Fusions:
+ model's state dict minus the routed experts. Nothing here is quantized: every release's skip
+ list (modelopt ``ignore``, fp8 ``modules_to_not_convert``) covers everything except those experts,
+ so attention, GDN, HC, PLE, the shared expert and lm_head are all plain bf16 (the n-gram hash
+ constants stay int64). Fusions:
attention q|k|v -> ``qkv_proj``, GDN ``in_proj_{qkv,z,b,a}`` -> ``in_proj``, shared-expert
gate|up -> ``gate_up_proj``, and each per-layer HC's ``input_mix_weight_down`` |
``block_inject_weight`` -> a zero-padded ``input_mix_weight_down_block_inject``.
``include_moe_experts`` is accepted for the loader contract but never yields anything: the
- routed experts are NVFP4 and always come from :func:`load_nvfp4_expert_sources`.
+ routed experts are NVFP4 and always come from the offload cache's expert reader.
"""
if get_tp_info().size > 1:
raise NotImplementedError("qwen4_exp weight loading supports TP=1 only")
@@ -289,40 +289,13 @@ def load_ple_table(model_path: str, qwen4_args, *, pin: bool = True,
# ======================================================================================
-def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None) -> dict:
- """Build the CPU NVFP4 expert source banks for the offload cache (gate/up fused on the output-row axis, down separate; weight_scale_2 carried as the per-row global scale)."""
- return load_nvfp4_expert_source_banks(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- layer_sink=layer_sink,
- )
-
-
-def load_nvfp4_expert_sources_parallel(
- model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
-) -> dict:
- """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""
- from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
-
- return load_nvfp4_expert_source_banks_parallel(
- model_path,
- config,
- _NVFP4_SOURCE_SPEC,
- drop_page_cache=drop_page_cache,
- primary=get_tp_info().is_primary(),
- workers=workers,
- chunk=chunk,
- layer_sink=layer_sink,
- )
+def nvfp4_expert_spec(model_path: str, config):
+ return _NVFP4_SOURCE_SPEC
__all__ = [
+ "nvfp4_expert_spec",
"PleTable",
"iter_weights",
- "load_nvfp4_expert_sources",
- "load_nvfp4_expert_sources_parallel",
"load_ple_table",
]
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index b8c17322..3778d3c3 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -14,28 +14,75 @@ class ModelSpec:
model_cls: str
parse_config: str = "parse_config"
iter_weights: str = "iter_weights"
+ # attribute-path root -> checkpoint root, for quantization_config lookups
+ checkpoint_roots: tuple[tuple[str, str], ...] = ()
+ # inner attribute path -> checkpoint path (feed_forward.shared_mlp -> mlp)
+ checkpoint_segments: tuple[tuple[str, str], ...] = ()
+ # fused leaf -> source leaves (qkv_proj -> q_proj, k_proj, v_proj)
+ packed_modules_mapping: tuple[tuple[str, tuple[str, ...]], ...] = ()
+ # checkpoint-name globs the family serves in bf16 although the quantization_config covers them
+ unquantized_modules: tuple[str, ...] = ()
+# Multimodal wrappers store the text tower under model.language_model.
+_LANGUAGE_MODEL_ROOT = (("model", "model.language_model"),)
+# Fused projections split back into their HF leaves so the quantization_config lookup sees the stored names.
+_DENSE_PACKED = (
+ ("qkv_proj", ("q_proj", "k_proj", "v_proj")),
+ ("gate_up_proj", ("gate_proj", "up_proj")),
+)
+# per-expert checkpoints: probe the first expert's projections for the experts container
+_EXPERTS_PACKED = (("experts", ("experts.0.gate_proj", "experts.0.up_proj", "experts.0.down_proj")),)
+_EXPERTS_W123_PACKED = (("experts", ("experts.0.w1", "experts.0.w2", "experts.0.w3")),)
+_QWEN3_5_PACKED = _DENSE_PACKED + (
+ ("in_proj_qkvz", ("in_proj_qkv", "in_proj_z")),
+ ("in_proj_ba", ("in_proj_b", "in_proj_a")),
+ ("in_proj", ("in_proj_qkv", "in_proj_z", "in_proj_b", "in_proj_a")),
+) + _EXPERTS_PACKED
+# Qwen3.8's per-layer hyper-connections fuse the down projection with the block-inject rows.
+_QWEN4_EXP_PACKED = _QWEN3_5_PACKED + (
+ ("input_mix_weight_down_block_inject", ("input_mix_weight_down", "block_inject_weight")),
+)
+# GLM-5.3-Flash's KDA fuses its six input projections.
+_GLM5_NEXT_PACKED = _EXPERTS_PACKED + (
+ ("in_proj", ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")),
+)
+# Gemma 4 keeps HF's flat layer children (mlp / experts / router) under one feed_forward block.
+_GEMMA4_SEGMENTS = (
+ ("feed_forward.shared_mlp", "mlp"),
+ ("feed_forward.experts", "experts"),
+ ("feed_forward.router", "router"),
+)
+_GEMMA4_PACKED = _DENSE_PACKED + _EXPERTS_PACKED
+_MINIMAX_M3_PACKED = _DENSE_PACKED + (
+ ("index_qk_proj", ("index_q_proj", "index_k_proj")),
+) + _EXPERTS_W123_PACKED
+
_MODEL_REGISTRY: dict[str, ModelSpec] = {
"LlamaForCausalLM": ModelSpec(
"freetoken.models.llama",
"LlamaForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED,
),
"Qwen2ForCausalLM": ModelSpec(
"freetoken.models.qwen2",
"Qwen2ForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED,
),
"Qwen3ForCausalLM": ModelSpec(
"freetoken.models.qwen3",
"Qwen3ForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED,
),
"Qwen3MoeForCausalLM": ModelSpec(
"freetoken.models.qwen3_moe",
"Qwen3MoeForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,
),
"MiniMaxM2ForCausalLM": ModelSpec(
"freetoken.models.minimax_m2",
"MiniMaxM2ForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_W123_PACKED,
),
# MiniMax-M3 (model_type minimax_m3_vl): multimodal wrapper config (text tower in
# text_config, weights under language_model.); served text-only. GQA + block-sparse
@@ -44,19 +91,29 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"MiniMaxM3SparseForConditionalGeneration": ModelSpec(
"freetoken.models.minimax_m3",
"MiniMaxM3ForCausalLM",
+ checkpoint_roots=(("model", "language_model.model"), ("lm_head", "language_model.lm_head")),
+ packed_modules_mapping=_MINIMAX_M3_PACKED,
),
# Text-only sibling (the text_config's own architectures entry).
"MiniMaxM3SparseForCausalLM": ModelSpec(
"freetoken.models.minimax_m3",
"MiniMaxM3ForCausalLM",
+ packed_modules_mapping=_MINIMAX_M3_PACKED,
),
"DeepseekV4ForCausalLM": ModelSpec(
"freetoken.models.deepseek_v4",
"DeepseekV4ForCausalLM",
+ # the checkpoint has no ``model.`` root
+ checkpoint_roots=(("model.layers", "layers"), ("model.head", "head")),
+ packed_modules_mapping=_EXPERTS_W123_PACKED,
+ # the head, the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert
+ unquantized_modules=("head", "*.compressor.wkv", "*.compressor.wgate", "*.indexer.weights_proj"),
),
"Qwen3_5MoeForConditionalGeneration": ModelSpec(
"freetoken.models.qwen3_5_moe",
"Qwen3_5MoEForCausalLM",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=_QWEN3_5_PACKED,
),
# Qwen3.8-Flash-Next (model_type qwen4_exp): multimodal wrapper config (text tower in
# text_config, weights under model.language_model.); served text-only. 36 GDN + 12 QSA
@@ -65,6 +122,8 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"Qwen4ExpForConditionalGeneration": ModelSpec(
"freetoken.models.qwen4_exp",
"Qwen4ExpForCausalLM",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=_QWEN4_EXP_PACKED,
),
# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the
# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the
@@ -72,6 +131,8 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"Qwen3_5ForConditionalGeneration": ModelSpec(
"freetoken.models.qwen3_5_moe",
"Qwen3_5MoEForCausalLM",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=_QWEN3_5_PACKED,
),
# Muse-Glimmer-30B (model_type muse_glimmer): multimodal wrapper config (text tower in
# text_config, weights under model.language_model.); served text-only. Dense gated GQA
@@ -81,10 +142,16 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"MuseGlimmerForConditionalGeneration": ModelSpec(
"freetoken.models.muse_glimmer",
"MuseGlimmerForCausalLM",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=(
+ ("qkvg_proj", ("q_proj", "k_proj", "v_proj", "gate_proj")),
+ ("gate_up_proj", ("gate_proj", "up_proj")),
+ ),
),
"MistralForCausalLM": ModelSpec(
"freetoken.models.mistral",
"MistralForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED,
),
"Mistral3ForConditionalGeneration": ModelSpec(
"freetoken.models.mistral",
@@ -93,20 +160,30 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"Gemma4ForConditionalGeneration": ModelSpec(
"freetoken.models.gemma4",
"Gemma4ForCausalLM",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ checkpoint_segments=_GEMMA4_SEGMENTS,
+ packed_modules_mapping=_GEMMA4_PACKED,
),
"Gemma4ForCausalLM": ModelSpec(
"freetoken.models.gemma4",
"Gemma4ForCausalLM",
+ checkpoint_segments=_GEMMA4_SEGMENTS,
+ packed_modules_mapping=_GEMMA4_PACKED,
),
# Dense text tower of the gemma-4-12B "Unified"/omni model (model_type gemma4_unified_text).
# Same decoder as gemma4; the dense feed-forward is selected via config.is_moe.
"Gemma4UnifiedForConditionalGeneration": ModelSpec(
"freetoken.models.gemma4",
"Gemma4ForCausalLM",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ checkpoint_segments=_GEMMA4_SEGMENTS,
+ packed_modules_mapping=_GEMMA4_PACKED,
),
"Gemma4UnifiedForCausalLM": ModelSpec(
"freetoken.models.gemma4",
"Gemma4ForCausalLM",
+ checkpoint_segments=_GEMMA4_SEGMENTS,
+ packed_modules_mapping=_GEMMA4_PACKED,
),
# GGUF (native Q4_0/Q6_K) gemma4: same model classes, GGUF config + weight loaders.
"Gemma4GGUFForCausalLM": ModelSpec(
@@ -118,10 +195,12 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"GptOssForCausalLM": ModelSpec(
"freetoken.models.gpt_oss",
"GptOssForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED,
),
"Glm4MoeForCausalLM": ModelSpec(
"freetoken.models.glm4_moe",
"Glm4MoeForCausalLM",
+ packed_modules_mapping=_EXPERTS_PACKED,
),
# GLM-5.2 (model_type glm_moe_dsa): DeepSeek-V3.2-class MLA + DSA sparse attention
# with GLM-4-style sigmoid/noaux_tc MoE routing; NVFP4 routed experts served from
@@ -129,6 +208,7 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"GlmMoeDsaForCausalLM": ModelSpec(
"freetoken.models.glm_moe_dsa",
"GlmMoeDsaForCausalLM",
+ packed_modules_mapping=_EXPERTS_PACKED,
),
# GLM-5.3-Flash (model_type glm5_next): hybrid KDA linear attention (34/45 layers)
# + NoPE-MLA/DSA with a kpool-compressed indexer (11/45), mHC x4 residual streams,
@@ -137,11 +217,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"Glm5NextForConditionalGeneration": ModelSpec(
"freetoken.models.glm5_next",
"Glm5NextForCausalLM",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=_GLM5_NEXT_PACKED,
),
# Text-only sibling (the text_config's own architectures entry).
"Glm5NextForCausalLM": ModelSpec(
"freetoken.models.glm5_next",
"Glm5NextForCausalLM",
+ packed_modules_mapping=_GLM5_NEXT_PACKED,
),
}
diff --git a/python/freetoken/models/weight.py b/python/freetoken/models/weight.py
index 6a34f3b9..03b11a1d 100644
--- a/python/freetoken/models/weight.py
+++ b/python/freetoken/models/weight.py
@@ -22,10 +22,7 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Iterator, Tuple
import torch
-from freetoken.distributed import get_tp_info
-from freetoken.kernel.pinned import alloc_pinned_tensor, copy_to_pinned_tensor
-from freetoken.models.loader import stream_moe_expert_sources
-from freetoken.utils import cached_load_hf_config, div_even
+from freetoken.utils import cached_load_hf_config
from .register import _load_attr, get_model_spec
@@ -256,74 +253,6 @@ def load_weight(
)
-def load_moe_expert_sources(
- model_path: str,
- *,
- dtype: torch.dtype,
- dummy: bool = False,
- parallel: bool = False,
- workers: int = 8,
- chunk: int = 8 << 20,
- layer_sink=None,
-) -> Tuple[list[torch.Tensor], list[torch.Tensor]]:
- config, spec = _spec_for_model_path(model_path)
- if not config.is_moe:
- raise ValueError(
- f"{config.architectures[0]} does not provide MoE expert source loading"
- )
- if dummy:
- builder = _model_override(spec, "dummy_moe_expert_sources") or dummy_moe_expert_sources
- return builder(config, dtype=dtype)
- if parallel: # parallel: experts read via the common chunked multi-threaded O_DIRECT reader
- iter_weights = _model_override(spec, "iter_weights_parallel")
- if iter_weights is None: # model has no parallel reader -> let the caller fall back to serial
- raise NotImplementedError(
- f"{spec.module} provides no iter_weights_parallel")
- src = iter_weights(model_path, torch.device("cpu"), include_moe_experts=True,
- include_non_moe=False, workers=workers, chunk=chunk)
- else:
- iter_weights = _load_attr(spec.module, spec.iter_weights)
- src = iter_weights(model_path, torch.device("cpu"), include_moe_experts=True,
- include_non_moe=False)
- return stream_moe_expert_sources(
- src,
- config,
- dtype=dtype,
- layer_sink=layer_sink,
- )
-
-
-def load_nvfp4_moe_expert_sources(
- model_path: str,
- model_config,
- *,
- dummy: bool = False,
- parallel: bool = False,
- workers: int = 8,
- chunk: int = 8 << 20,
- layer_sink=None,
-) -> dict:
- """Load (or fabricate, with ``dummy=True``) packed NVFP4 expert source banks.
- ``parallel=True`` uses the model's ``load_nvfp4_expert_sources_parallel`` (common
- chunked multi-threaded O_DIRECT reader). ``layer_sink``: see
- ``models.nvfp4_banks.load_nvfp4_expert_source_banks``; forwarded to the per-model
- loader, which forwards it on."""
- _config, spec = _spec_for_model_path(model_path)
- if dummy:
- builder = (
- _model_override(spec, "dummy_nvfp4_expert_sources") or dummy_nvfp4_expert_sources
- )
- return builder(model_config)
- if parallel:
- loader = _model_override(spec, "load_nvfp4_expert_sources_parallel")
- if loader is None: # no parallel reader -> let the caller fall back to serial
- raise NotImplementedError(
- f"{spec.module} provides no load_nvfp4_expert_sources_parallel")
- return loader(model_path, model_config, workers=workers, chunk=chunk, layer_sink=layer_sink)
- loader = _load_attr(spec.module, "load_nvfp4_expert_sources")
- return loader(model_path, model_config, layer_sink=layer_sink)
-
-
def load_q4_0_moe_expert_sources(
model_path: str,
model_config,
@@ -342,72 +271,8 @@ def load_q4_0_moe_expert_sources(
return loader(model_path, model_config, layer_sink=layer_sink)
-def _num_moe_layers(config) -> int:
- value = getattr(config, "num_moe_layers", None)
- if value is not None:
- return int(value)
- return int(config.num_layers) - int(getattr(config, "first_k_dense_replace", 0))
-
-
-def dummy_moe_expert_sources(
- config, *, dtype: torch.dtype
-) -> Tuple[list[torch.Tensor], list[torch.Tensor]]:
- """Random BF16 expert banks shaped like ``stream_moe_expert_sources`` output:
- one independently allocated ``[num_experts, ...]`` tensor per layer."""
- num_layers = _num_moe_layers(config)
- intermediate_size = div_even(config.moe_intermediate_size, get_tp_info().size)
- gate_up = [
- torch.randn(config.num_experts, 2 * intermediate_size, config.hidden_size, dtype=dtype)
- for _ in range(num_layers)
- ]
- down = [
- torch.randn(config.num_experts, config.hidden_size, intermediate_size, dtype=dtype)
- for _ in range(num_layers)
- ]
- if torch.cuda.is_available():
- gate_up = [copy_to_pinned_tensor(t) for t in gate_up]
- down = [copy_to_pinned_tensor(t) for t in down]
- return gate_up, down
-
-
-def dummy_nvfp4_expert_sources(config) -> dict[str, list[torch.Tensor]]:
- """Random NVFP4 (ModelOpt-layout) expert banks shaped like the real loader's.
-
- Same pinned allocation and per-layer bank shapes as ``load_nvfp4_expert_sources``,
- so the repack/offload path downstream is exercised unchanged. Packed codes are
- random nibbles; block scales are 1.0 and globals small because random e4m3 bytes
- reach 448 (and include NaN encodings), which would blow up the dummy activations.
- """
- num_layers = _num_moe_layers(config)
- E = config.num_experts
- H, I = config.hidden_size, config.moe_intermediate_size
- fp8 = torch.float8_e4m3fn
-
- def bank(*shape: int, dtype: torch.dtype) -> list[torch.Tensor]:
- return [alloc_pinned_tensor(*shape, dtype=dtype) for _ in range(num_layers)]
-
- sources = {
- "gate_up_packed": bank(E, 2 * I, H // 2, dtype=torch.uint8),
- "gate_up_scale": bank(E, 2 * I, H // 16, dtype=fp8),
- "gate_up_global": bank(E, 2 * I, dtype=torch.float16),
- "down_packed": bank(E, H, I // 2, dtype=torch.uint8),
- "down_scale": bank(E, H, I // 16, dtype=fp8),
- "down_global": bank(E, H, dtype=torch.float16),
- }
- for t in sources["gate_up_packed"] + sources["down_packed"]:
- t.random_(0, 256)
- for t in sources["gate_up_scale"] + sources["down_scale"]:
- t.fill_(1.0)
- for t in sources["gate_up_global"] + sources["down_global"]:
- t.fill_(0.01)
- return sources
-
-
__all__ = [
"load_weight",
- "load_moe_expert_sources",
- "load_nvfp4_moe_expert_sources",
- "dummy_moe_expert_sources",
- "dummy_nvfp4_expert_sources",
+ "load_q4_0_moe_expert_sources",
"iter_expert_tensors_parallel",
]
diff --git a/python/freetoken/moe/__init__.py b/python/freetoken/moe/__init__.py
index 5fc41911..8794abce 100644
--- a/python/freetoken/moe/__init__.py
+++ b/python/freetoken/moe/__init__.py
@@ -1,74 +1,17 @@
from __future__ import annotations
-from typing import Protocol
+# Where routed experts live. The offload family (offload / cpu / hybrid) serves experts from
+# pinned host banks through an ``OffloadMoeCache`` -- the GPU only holds the two-layer prefill
+# double buffer -- and differs only in how decode gets the experts: ``offload`` streams the
+# missing experts over PCIe into a GPU slot cache, ``cpu`` computes them on the CPU from the
+# host banks, ``hybrid`` fetches at most K missing experts per layer and computes the rest on
+# the CPU, overlapped. ``fused`` keeps every expert resident on the GPU.
+MOE_STRATEGIES = ("fused", "offload", "cpu", "hybrid")
+OFFLOAD_MOE_STRATEGIES = frozenset({"offload", "cpu", "hybrid"})
-from freetoken.utils import Registry, init_logger
-from .base import BaseMoeBackend
+def is_offload_moe_strategy(strategy: str) -> bool:
+ return strategy in OFFLOAD_MOE_STRATEGIES
-logger = init_logger(__name__)
-
-class MoeBackendCreator(Protocol):
- def __call__(self) -> BaseMoeBackend: ...
-
-
-SUPPORTED_MOE_BACKENDS = Registry[MoeBackendCreator]("MoE Backend")
-
-# Backends that serve experts from CPU (pinned) host banks through an
-# ``OffloadMoeCache`` -- the GPU only holds the two-layer prefill double buffer.
-# They differ only in how *decode* gets the experts: ``offload`` streams the
-# missing experts over PCIe into a GPU slot cache and runs the GEMM on the GPU;
-# ``cpu`` ships the activations to the CPU, computes the experts there (high RAM
-# bandwidth), and ships the results back; ``hybrid`` keeps a full GPU slot cache
-# AND a CPU executor -- it fetches at most K missing experts/layer over PCIe
-# (computed on the GPU with the cache hits) and computes the remaining misses on
-# the CPU, overlapped, then merges (capped PCIe + CPU overflow). All build their
-# cache the same way, so model layer construction and the engine wiring key off
-# this set rather than a bare ``== "offload"`` check.
-OFFLOAD_MOE_BACKENDS = frozenset({"offload", "cpu", "hybrid"})
-
-
-def is_offload_moe_backend(backend: str) -> bool:
- return backend in OFFLOAD_MOE_BACKENDS
-
-
-@SUPPORTED_MOE_BACKENDS.register("fused")
-def create_fused_moe_backend():
- from .fused import FusedMoe
-
- return FusedMoe()
-
-
-@SUPPORTED_MOE_BACKENDS.register("offload")
-def create_offload_moe_backend():
- from .offload import OffloadMoeBackend
-
- return OffloadMoeBackend()
-
-
-@SUPPORTED_MOE_BACKENDS.register("cpu")
-def create_cpu_moe_backend():
- from .cpu_offload import CpuOffloadMoeBackend
-
- return CpuOffloadMoeBackend()
-
-
-@SUPPORTED_MOE_BACKENDS.register("hybrid")
-def create_hybrid_moe_backend():
- from .cpu_offload import HybridMoeBackend
-
- return HybridMoeBackend()
-
-
-def create_moe_backend(backend: str) -> BaseMoeBackend:
- return SUPPORTED_MOE_BACKENDS[backend]()
-
-
-__all__ = [
- "BaseMoeBackend",
- "create_moe_backend",
- "SUPPORTED_MOE_BACKENDS",
- "OFFLOAD_MOE_BACKENDS",
- "is_offload_moe_backend",
-]
+__all__ = ["MOE_STRATEGIES", "OFFLOAD_MOE_STRATEGIES", "is_offload_moe_strategy"]
diff --git a/python/freetoken/moe/base.py b/python/freetoken/moe/base.py
deleted file mode 100644
index f5a2ce7a..00000000
--- a/python/freetoken/moe/base.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from abc import ABC, abstractmethod
-
-import torch
-
-
-class BaseMoeBackend(ABC):
- @abstractmethod
- def forward(
- self,
- hidden_states: torch.Tensor,
- w1: torch.Tensor,
- w2: torch.Tensor,
- gating_output: torch.Tensor,
- topk: int,
- renormalize: bool,
- activation: str,
- apply_router_weight_on_input: bool,
- ) -> torch.Tensor: ...
diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py
index d9cb43c1..e2e48d14 100644
--- a/python/freetoken/moe/cpu_executor.py
+++ b/python/freetoken/moe/cpu_executor.py
@@ -1,4 +1,4 @@
-"""Python wrapper around the ``_cpu_moe`` C++ executor (--moe-backend cpu).
+"""Python wrapper around the ``_cpu_moe`` C++ executor (--moe-strategy cpu).
Owns the persistent CPU worker pool, the per-batch-size pinned IO buffers, and
the per-(layer, batch-size) host-func task descriptors. ``decode`` issues the
@@ -157,15 +157,17 @@ class CpuMoeExecutor:
device: torch.device,
swiglu_alpha: float = 1.702,
swiglu_limit: float | None = None,
+ fmt: str | None = None,
) -> None:
from freetoken.kernel import _cpu_moe
+ from freetoken.moe.legacy_format import canonical_role
- fmt = cache.quant_format
+ fmt = fmt or cache.quant_format
if fmt not in _WFMT_IDS:
raise NotImplementedError(
- f"--moe-backend cpu/hybrid computes experts on the CPU and supports "
+ f"--moe-strategy cpu/hybrid computes experts on the CPU and supports "
f"{sorted(_WFMT_IDS)} formats, but this checkpoint's experts are "
- f"{fmt!r}; use --moe-backend offload (GPU-side dequant) instead."
+ f"{fmt!r}; use --moe-strategy offload (GPU-side dequant) instead."
)
if activation not in _ACT_IDS:
raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")
@@ -193,7 +195,9 @@ class CpuMoeExecutor:
# The per-layer tensors and their pointer tables must outlive the executor
# (C++ holds raw addresses into both).
self._banks: list[torch.Tensor] = []
- ptrs, (self.H, self.I) = self._resolve_banks(cache.bank_sources, fmt)
+ ptrs, (self.H, self.I) = self._resolve_banks(
+ {canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()}, fmt
+ )
# Decide the flag handshake up front (env + device + a functional stream-memop
# probe): its coordinator needs a core of its own, which the auto thread sizing
@@ -375,8 +379,8 @@ class CpuMoeExecutor:
return self._resolve_dsfp4_banks(banks)
# nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.
- gup, gus, gug = banks["gate_up_packed"], banks["gate_up_scale"], banks["gate_up_global"]
- dnp, dns, dng = banks["down_packed"], banks["down_scale"], banks["down_global"]
+ gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]
+ dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]
assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)
assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"
assert gug[0].dtype == torch.float16 and dng[0].dtype == torch.float16, (gug[0].dtype, dng[0].dtype)
@@ -432,8 +436,8 @@ class CpuMoeExecutor:
(N innermost) + per-output-row biases. The C++ kernel streams K and
accumulates a contiguous N-block, so the GPU-tiled layout is read in place
(no repack, no extra host memory). Block scales are e8m0 (1 byte / 32 K)."""
- gub, gus, gob = banks["gate_up_blocks"], banks["gate_up_scales"], banks["gate_up_bias"]
- dnb, dns, dob = banks["down_blocks"], banks["down_scales"], banks["down_bias"]
+ gub, gus, gob = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_bias"]
+ dnb, dns, dob = banks["down"], banks["down_scale"], banks["down_bias"]
assert gub[0].dtype == torch.uint8 and dnb[0].dtype == torch.uint8, (gub[0].dtype, dnb[0].dtype)
assert gus[0].dtype == torch.uint8 and dns[0].dtype == torch.uint8, (gus[0].dtype, dns[0].dtype)
assert gob[0].dtype == torch.bfloat16 and dob[0].dtype == torch.bfloat16, (gob[0].dtype, dob[0].dtype)
@@ -463,8 +467,8 @@ class CpuMoeExecutor:
scales, no global, no bias. Layout matches nvfp4 (K contiguous per output row),
so the C++ GEMV reads it in place. The kernel additionally FP8-round-trips the
activations (block 128) to match DSV4's W4A8 reference, hence the %128 dims."""
- gup, gus = banks["gate_up_packed"], banks["gate_up_scale"]
- dnp, dns = banks["down_packed"], banks["down_scale"]
+ gup, gus = banks["gate_up"], banks["gate_up_scale"]
+ dnp, dns = banks["down"], banks["down_scale"]
assert gup[0].dtype == torch.uint8 and dnp[0].dtype == torch.uint8, (gup[0].dtype, dnp[0].dtype)
assert gus[0].element_size() == 1 and dns[0].element_size() == 1, "block scales must be 1 byte"
I = int(gup[0].shape[1] // 2)
diff --git a/python/freetoken/moe/cpu_offload.py b/python/freetoken/moe/cpu_offload.py
deleted file mode 100644
index 8d2738ad..00000000
--- a/python/freetoken/moe/cpu_offload.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from __future__ import annotations
-
-import torch
-
-from .base import BaseMoeBackend
-
-
-class CpuOffloadMoeBackend(BaseMoeBackend):
- """Marker backend for CPU-compute expert offload.
-
- Like :class:`OffloadMoeBackend`, the real work lives in ``OffloadMoELayer``:
- prefill streams whole expert layers into the GPU double buffer and runs the
- GEMM on the GPU (identical to ``offload``), while decode ships the
- activations to the CPU, computes the experts there (reading the pinned host
- banks at full RAM bandwidth), and ships the results back. The whole decode
- path is CUDA-graph capturable via ``cudaLaunchHostFunc`` submit/sync nodes.
- """
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- w1: torch.Tensor,
- w2: torch.Tensor,
- gating_output: torch.Tensor,
- topk: int,
- renormalize: bool,
- activation: str,
- apply_router_weight_on_input: bool,
- ) -> torch.Tensor:
- raise RuntimeError("CPU-offload MoE is handled by OffloadMoELayer")
-
-
-class HybridMoeBackend(BaseMoeBackend):
- """Marker backend for hybrid GPU-cache + CPU-overflow expert decode.
-
- Prefill is identical to ``offload`` (stream whole layers into the GPU double
- buffer, GEMM on the GPU). Decode keeps a full GPU slot cache but caps PCIe
- fetches per (layer, step) -- at ``hybrid_max_fetch`` experts, or at the
- bandwidth-matched ``hybrid_fetch_fraction`` of that step's misses (default:
- the benched pcie_bw/cpu_bw ratio, so the fetch and the CPU compute overlap
- perfectly): the cache hits plus the freshly-fetched experts run on the GPU,
- the remaining misses run on the CPU (reading the pinned host banks), and the
- two partial results are summed. The real work lives in the model's MoE layer
- (``OffloadMoELayer`` / DSV4 ``MoE``).
- """
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- w1: torch.Tensor,
- w2: torch.Tensor,
- gating_output: torch.Tensor,
- topk: int,
- renormalize: bool,
- activation: str,
- apply_router_weight_on_input: bool,
- ) -> torch.Tensor:
- raise RuntimeError("Hybrid MoE is handled by the model's MoE layer")
diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py
index 8b6116ba..e3a768b6 100644
--- a/python/freetoken/moe/expert_banks.py
+++ b/python/freetoken/moe/expert_banks.py
@@ -1,29 +1,24 @@
-"""Per-quant-format expert bank providers for the offload MoE cache.
-
-A provider owns everything between "checkpoint on disk" and "banks ready for
-``OffloadMoeCache.set_bank_sources``" for one ``expert_quant`` value: loading the
-pinned host banks, picking a kernel backend, repacking into its layout, and
-extracting the GPU-resident alphas if the format folds its global scales. The
-engine stays quant-agnostic: it calls :func:`load_expert_banks` and wires the
-returned bundle into the cache.
-
-A format is fully described by three per-format tables: ``_BANK_SCHEMAS``
-(offload_cache, bank layout), ``_PROVIDERS`` (here, loading/repack), and
-``OffloadMoELayer._expert_gemm`` (kernel dispatch). Adding a format touches those
-three places and nothing else.
+"""Expert banks for the offload MoE cache: load, pack and pin the routed experts.
+
+The expert kernel (``QuantMethod.kernel``) owns the bank layout and the pack step; the
+checkpoint side delivers pieces (``moe.expert_pieces``) and this module fills the pinned host
+banks from them (``build_expert_banks``). The GGUF q4_0 experts still
+use their own providers until they get a method.
"""
from __future__ import annotations
import glob
+import math
import os
-import threading
from dataclasses import dataclass, field
import torch
+from freetoken.layers.quantization import QuantKind
from freetoken.utils import init_logger
+from .host_banks import alloc_layer_banks
from .offload_cache import _BANK_BYTES_PER_EXPERT, _BANK_SCHEMAS
logger = init_logger(__name__)
@@ -49,187 +44,112 @@ class ExpertBanks:
# streamed straight to its sink instead of staying materialized here) -- set by
# convert.py's per-format streaming gate; ``sources`` may hold released tensors.
streamed: bool = False
+ # the expert (kind, kernel) the banks were packed for; None for the legacy providers
+ kind: QuantKind | None = None
+ kernel: str | None = None
+ layout: dict | None = None
+
+
+def _dummy_fill(role: str, tensor: torch.Tensor) -> None:
+ """Random but finite bank contents for --use-dummy-weight."""
+ if role.endswith("_scale"):
+ if tensor.dtype is torch.uint8:
+ tensor.fill_(127) # e8m0 exponent code for 1.0
+ else:
+ tensor.fill_(1.0)
+ elif role.endswith("_global"):
+ tensor.fill_(0.01)
+ elif tensor.dtype in (torch.uint8, torch.int32):
+ tensor.view(torch.uint8).random_(0, 256)
+ elif tensor.dtype is torch.float8_e4m3fn:
+ tensor.view(torch.uint8).random_(0, 16) # small codes, no NaN / inf
+ else:
+ tensor.normal_()
-_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader
-
+def build_expert_banks(
+ method,
+ num_layers: int,
+ pieces,
+ *,
+ device: torch.device,
+ layer_sink=None,
+ dummy: bool = False,
+) -> ExpertBanks:
+ """Fill host banks in the kernel's layout from a stream of expert pieces.
-def _v4_unsupported(quant):
- raise NotImplementedError(
- f"parallel expert reader not implemented for quant {quant!r} yet; "
- "add a load_*_expert_sources_parallel using freetoken.models.weight."
- "iter_expert_tensors_parallel (only ds_fp4 implemented so far)"
- )
+ ``pieces`` yields ``(layer_id, e0, e1, {role: tensor[e1 - e0, ...]})`` in any order;
+ each batch is packed in place into rows ``e0:e1`` of that layer's banks. A layer is
+ complete once its ``num_experts`` rows have arrived: with ``layer_sink=None`` its banks
+ are pinned in the background, otherwise the sink receives them (converter). ``dummy``
+ skips the pieces and fills the banks with finite random contents.
+ """
+ from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, pin_banks
+ from freetoken.moe.legacy_format import legacy_format_for
+
+ kernel = method.kernel
+ layout = method.layout()
+ E = method.cfg.num_experts
+ specs = {role: ((E, *spec.shape), spec.dtype) for role, spec in layout.items() if not spec.resident}
+ hb = alloc_layer_banks(specs, num_layers)
+ banks = {role: [b.tensor for b in hb[role]] for role in specs}
+ alphas = {
+ role: torch.empty(num_layers * E, dtype=spec.dtype, device=device)
+ for role, spec in layout.items() if spec.resident
+ }
+ if dummy:
+ for role, per_layer in banks.items():
+ for tensor in per_layer:
+ _dummy_fill(role, tensor)
+ for alpha in alphas.values():
+ alpha.fill_(1.0)
+ if torch.cuda.is_available():
+ pin_banks(hb)
+ return ExpertBanks(
+ legacy_format_for(method.kind, kernel.name), banks,
+ gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),
+ kind=method.kind, kernel=kernel.name, layout=layout,
+ )
-def _bf16_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:
- from freetoken.models.weight import load_moe_expert_sources
+ def _fill(sink) -> None:
+ tracker = LayerCompletionTracker(E, hb, sink) if sink is not None else None
+ # a reader that skips a layer or mislabels a piece must fail here, not serve uninitialized rows
+ written = torch.zeros(num_layers, E, dtype=torch.int32)
+ for layer_id, e0, e1, piece in pieces:
+ if not (0 <= layer_id < num_layers and 0 <= e0 < e1 <= E):
+ raise ValueError(f"expert piece out of range: layer {layer_id}, experts {e0}:{e1} of {num_layers} x {E}")
+ # refuse before writing: a duplicate row would also complete the layer early and hand the sink a half-filled bank
+ if written[layer_id, e0:e1].any():
+ raise ValueError(f"expert rows written more than once: layer {layer_id}, experts {e0}:{e1}")
+ written[layer_id, e0:e1] = 1
+ out = {role: banks[role][layer_id][e0:e1] for role in specs}
+ got = method.pack(piece, out)
+ for role, values in got.items():
+ alphas[role][layer_id * E + e0 : layer_id * E + e1] = values.to(alphas[role].dtype)
+ if tracker is not None:
+ for _ in range(e1 - e0):
+ tracker.note(layer_id)
+ missing = (written == 0).nonzero().tolist()
+ if missing:
+ raise ValueError(f"expert banks were not filled: {len(missing)} (layer, expert) rows missing (first {missing[:4]})")
+
+ if layer_sink is not None:
+ _fill(layer_sink)
+ elif torch.cuda.is_available():
+ with PinPipeline() as pins:
+ _fill(pins)
+ else:
+ _fill(None)
- # bf16 banks are written as-loaded -> streamable (dummy fabricates in one shot, so a
- # sink given alongside dummy=True would never fire; keep it materialize-only there).
- sink = None if dummy else layer_sink
- gate_up_source, down_source = load_moe_expert_sources(
- model_path, dtype=dtype, dummy=dummy, parallel=parallel, workers=workers, chunk=chunk,
- layer_sink=sink,
- )
return ExpertBanks(
- "bf16", {"gate_up": gate_up_source, "down": down_source}, streamed=sink is not None
- )
-
-
-class _RepackedBank:
- """A repacked per-layer bank forwarded to the converter's outer sink: the post-repack
- tensor (a reinterpreted view of the native source's storage) plus a ``release`` that
- frees that native source. Exposes just the ``.tensor``/``.nbytes``/``.release()`` the
- :class:`~freetoken.checkpoint.convert._ConvertSink` reads."""
-
- __slots__ = ("tensor", "nbytes", "_src")
-
- def __init__(self, tensor: torch.Tensor, src) -> None:
- self.tensor = tensor
- self.nbytes = tensor.numel() * tensor.element_size()
- self._src = src # the native HostBank whose storage `tensor` reinterprets
-
- def release(self) -> None:
- self._src.release()
-
-
-class _Nvfp4RepackSink:
- """Streaming layer sink (converter only) for the nvfp4 marlin/b12x backends: repack
- each completed layer's 6 native banks into the backend's 4-bank layout in place, then
- forward the renamed banks to the outer FTW sink and stash the per-layer alphas.
-
- Runs from the loader's reader threads and calls CUDA ops (the repack), so the whole
- per-layer body -- ``set_device`` + repack + forward + stash -- is serialized under one
- lock (conversion is disk/compute-bound, not concurrency-bound). The two native
- ``*_global`` banks fold into the alphas, so they are released here (the outer sink only
- sees, and releases, the 4 weight banks)."""
-
- def __init__(self, repack_layer, config, device: torch.device, outer) -> None:
- from freetoken.moe.nvfp4_backends import _POST_NVFP4_BANKS
-
- self._repack_layer = repack_layer
- self._config = config
- self._device = device
- self._outer = outer
- self._post_names = _POST_NVFP4_BANKS
- self._lock = threading.Lock()
- self._gate_up_alpha: dict[int, torch.Tensor] = {}
- self._down_alpha: dict[int, torch.Tensor] = {}
- # post-repack per-layer views, kept only to reassemble ExpertBanks.sources (they
- # alias storage the outer sink released after writing -- released-tensor caveat).
- self._post: dict[str, dict[int, torch.Tensor]] = {n: {} for n in self._post_names}
-
- def __call__(self, layer_id: int, banks: dict) -> None:
- from freetoken.moe.nvfp4_backends import _NATIVE_NVFP4_BANKS
-
- with self._lock:
- if self._device.type == "cuda":
- torch.cuda.set_device(self._device)
- layer_tensors = {name: banks[name].tensor for name in _NATIVE_NVFP4_BANKS}
- post, gate_up_alpha, down_alpha = self._repack_layer(
- layer_tensors, self._config, self._device
- )
- self._gate_up_alpha[layer_id] = gate_up_alpha
- self._down_alpha[layer_id] = down_alpha
- forwarded = {}
- for name in self._post_names:
- self._post[name][layer_id] = post[name]
- # post[name] reinterprets banks[name]'s storage in place; release it via that bank.
- forwarded[name] = _RepackedBank(post[name], banks[name])
- # the two *_global banks were consumed into the alphas; they are not forwarded,
- # so release them here instead of leaving them resident.
- banks["gate_up_global"].release()
- banks["down_global"].release()
- self._outer(layer_id, forwarded)
-
- def assemble(self, num_layers: int):
- """After the load: flat layer-major ``[L*E]`` alphas + the 4-name post-repack
- per-layer source lists. Asserts every layer streamed through."""
- for by_layer, what in (
- (self._gate_up_alpha, "gate_up_alpha"),
- (self._down_alpha, "down_alpha"),
- ):
- missing = [l for l in range(num_layers) if l not in by_layer]
- assert not missing, f"nvfp4 repack sink never saw layers {missing} ({what})"
- gate_up_alpha = torch.cat([self._gate_up_alpha[l] for l in range(num_layers)])
- down_alpha = torch.cat([self._down_alpha[l] for l in range(num_layers)])
- sources = {
- name: [self._post[name][l] for l in range(num_layers)] for name in self._post_names
- }
- return sources, gate_up_alpha, down_alpha
-
-
-def _nvfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:
- from freetoken.models.weight import load_nvfp4_moe_expert_sources
- from freetoken.moe.nvfp4_backends import (
- b12x_repack_layer,
- b12x_repack_sources_inplace,
- marlin_repack_layer,
- marlin_repack_sources_inplace,
- select_nvfp4_backend,
+ legacy_format_for(method.kind, kernel.name), banks,
+ gate_up_alpha=alphas.get("gate_up_alpha"), down_alpha=alphas.get("down_alpha"),
+ streamed=layer_sink is not None, kind=method.kind, kernel=kernel.name, layout=layout,
)
- # Backend pick is a pure hardware/config decision, independent of the loaded data, so
- # it's resolved up front. The native "nvfp4" layout (decode_target=="cpu", or the
- # "triton" backend) is written as-loaded straight through the sink. marlin/b12x repack
- # per expert: when converting (layer_sink given), a wrapper sink repacks each layer as
- # it completes and forwards the renamed banks to the outer sink; serving (no sink)
- # repacks the whole materialized bank set in place after load.
- native = decode_target == "cpu"
- backend = None
- if not native:
- backend = select_nvfp4_backend(device, getattr(model_config, "moe_intermediate_size", None),
- getattr(model_config, "nvfp4_backend", "auto"),
- activation=getattr(model_config, "hidden_act", "silu"))
- native = backend == "triton"
-
- repack_sink = None
- if not native and not dummy and layer_sink is not None:
- repack_layer = {"marlin": marlin_repack_layer, "b12x": b12x_repack_layer}[backend]
- repack_sink = _Nvfp4RepackSink(repack_layer, model_config, device, layer_sink)
-
- if native:
- sink = None if dummy else layer_sink
- else:
- sink = repack_sink # None for serving/dummy marlin/b12x; the repack wrapper when converting
- # parallel only parallelizes the source read; the backend repack below is reused unchanged.
- sources = load_nvfp4_moe_expert_sources(
- model_path, model_config, dummy=dummy, parallel=parallel, workers=workers, chunk=chunk,
- layer_sink=sink,
- )
- # CPU-compute decode (cpu/hybrid) reads the native ModelOpt rows directly (its
- # dequant-in-GEMV kernel), so keep the native "nvfp4" layout and skip the GPU-tiled
- # marlin/b12x repacks (which only the GPU W4A16 kernels can read).
- if decode_target == "cpu":
- return ExpertBanks("nvfp4", {name: sources[name] for name in _BANK_SCHEMAS["nvfp4"]},
- streamed=sink is not None)
- # Pick the expert-GEMM backend by compute capability (and MoE width: auto keeps
- # small-I MoE on the Triton M=1 GEMV, which beats b12x's tensor cores at single-stream
- # decode) and repack the banks (in place; the tiled blocks are byte-identical per
- # expert) into that backend's layout. One layout per process: prefill and decode read it.
- logger.info(f"NVFP4 expert backend: {backend}")
- if backend == "triton":
- return ExpertBanks("nvfp4", {name: sources[name] for name in _BANK_SCHEMAS["nvfp4"]},
- streamed=sink is not None)
- quant_format = f"nvfp4_{backend}"
- if repack_sink is not None:
- # Streamed conversion: each layer was already repacked + written by the wrapper.
- num_layers = len(sources["gate_up_packed"])
- post_sources, gate_up_alpha, down_alpha = repack_sink.assemble(num_layers)
- return ExpertBanks(
- quant_format, post_sources,
- gate_up_alpha=gate_up_alpha, down_alpha=down_alpha, streamed=True,
- )
- repack = {"marlin": marlin_repack_sources_inplace, "b12x": b12x_repack_sources_inplace}
- sources = repack[backend](sources, model_config, device)
- return ExpertBanks(
- quant_format,
- {name: sources[name] for name in _BANK_SCHEMAS[quant_format]},
- gate_up_alpha=sources["gate_up_alpha"],
- down_alpha=sources["down_alpha"],
- )
+_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader
def _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:
@@ -252,91 +172,18 @@ def _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False,
)
-def _dsfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:
- args = model_config.dsv4_args
- assert args is not None, "ds_fp4 expert banks require dsv4_args on the model config"
- # DeepSeek-FP4: packed e2m1 + e8m0 per-32 block scales, no global scale -> 4 banks,
- # no alphas. DeepSeek-V4's own grouped GEMV kernels read them via bank_views().
- # Written as-loaded -> streamable (dummy fabricates in one shot; never streamed).
- sink = None if dummy else layer_sink
- if dummy:
- from freetoken.models.deepseek_v4.weight import dummy_dsfp4_expert_sources
-
- banks = dummy_dsfp4_expert_sources(args)
- elif parallel: # parallel: common chunked multi-threaded O_DIRECT reader
- from freetoken.models.deepseek_v4.weight import load_dsfp4_expert_sources_parallel
-
- banks = load_dsfp4_expert_sources_parallel(
- model_path, args, workers=workers, chunk=chunk, layer_sink=sink
- )
- else:
- from freetoken.models.deepseek_v4.weight import load_dsfp4_expert_sources
-
- banks = load_dsfp4_expert_sources(model_path, args, layer_sink=sink)
- return ExpertBanks(
- "ds_fp4", {name: banks[name] for name in _BANK_SCHEMAS["ds_fp4"]}, streamed=sink is not None
- )
-
-
-def _model_setup_override(model_config):
- architectures = getattr(model_config, "architectures", None)
- if not architectures:
- return None
-
- from freetoken.models.register import _load_attr, get_model_spec
-
- try:
- spec = get_model_spec(architectures[0])
- except ValueError:
- return None
- try:
- return _load_attr(spec.module, "setup_offload_expert_banks")
- except AttributeError:
- return None
-
-
-# ModelConfig.expert_quant -> provider
+# expert formats that still load through their own provider (GGUF)
_PROVIDERS = {
- "none": _bf16_banks,
- "nvfp4": _nvfp4_banks,
- "ds_fp4": _dsfp4_banks,
"q4_0": _q4_0_banks,
}
-def _build_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk, decode_target="gpu", layer_sink=None) -> ExpertBanks:
- """Dispatch to the model's setup-override or the per-quant provider. ``parallel=True``
- is the parallel read; a provider that hasn't implemented it raises NotImplementedError (the
- caller falls back to serial). ``decode_target`` lets the cpu backend force CPU-readable
- (native, non-GPU-tiled) bank layouts. ``layer_sink`` (converter only) is forwarded to
- setups/providers that declare the parameter; the rest ignore it and stay on the
- materialize-and-write path (``ExpertBanks.streamed`` reports which happened)."""
- setup = _model_setup_override(model_config)
- if setup is not None:
- import inspect
-
- params = inspect.signature(setup).parameters
- supports_parallel = "parallel" in params
- if parallel and not supports_parallel:
- arch = getattr(model_config, "architectures", ["?"])[0]
- raise NotImplementedError(
- f"parallel reader not implemented for {arch} (model owns expert setup "
- "via setup_offload_expert_banks; add a parallel path there)"
- )
- kw = dict(device=device, dtype=dtype, dummy=dummy)
- if supports_parallel:
- kw.update(parallel=parallel, workers=workers, chunk=chunk)
- if "decode_target" in params:
- kw["decode_target"] = decode_target
- if "layer_sink" in params and layer_sink is not None:
- kw["layer_sink"] = layer_sink
- return setup(model_path, model_config, **kw)
-
+def _legacy_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk, decode_target="gpu", layer_sink=None) -> ExpertBanks:
expert_quant = model_config.expert_quant
if expert_quant not in _PROVIDERS:
raise ValueError(
- f"no expert-bank provider for expert_quant={expert_quant!r} "
- f"(known: {sorted(_PROVIDERS)})"
+ f"{expert_quant!r} experts load through their MoE quant method; "
+ f"only {sorted(_PROVIDERS)} still have a format provider"
)
return _PROVIDERS[expert_quant](
model_path, model_config, device, dtype, dummy,
@@ -345,6 +192,18 @@ def _build_expert_banks(model_path, model_config, device, dtype, dummy, parallel
)
+def _method_expert_banks(model_path, model_config, method, device, dummy, parallel, workers, chunk, layer_sink=None) -> ExpertBanks:
+ from freetoken.moe.expert_pieces import iter_expert_pieces
+
+ num_layers = model_config.num_moe_layers
+ if dummy:
+ return build_expert_banks(method, num_layers, None, device=device, dummy=True)
+ pieces = iter_expert_pieces(
+ model_path, model_config, method.kind, parallel=parallel, workers=workers, chunk=chunk
+ )
+ return build_expert_banks(method, num_layers, pieces, device=device, layer_sink=layer_sink)
+
+
def _host_ram_fits_parallel(model_path: str) -> bool:
"""Best-effort: can free host RAM hold the expert banks plus the parallel reader's one
extra (non-reclaimable) whole-shard buffer? Unknown (non-local path / no /proc) -> True,
@@ -386,11 +245,19 @@ def ftw_bank_bytes(model_path: str) -> int | None:
return sum(t["nbytes"] for t in tensors if t.get("kind") == "experts_bank")
-def bank_bytes_estimate(model_config) -> int | None:
- """Estimated total expert-bank bytes of a raw checkpoint, from the model config alone.
+def bank_bytes_estimate(model_config, method=None) -> int | None:
+ """Estimated total expert-bank bytes of a raw checkpoint before loading it.
- Sizes the pin-budget decisions where FTW metadata is not available; ``None`` for unknown formats or missing dims (callers then skip the pre-load sizing).
- nvfp4 uses the native-row formula, a slight over-estimate for the repacked backends."""
+ With a bound expert ``method`` the kernel's layout gives the exact host bytes; otherwise the
+ format-tag table sizes the GGUF format. ``None`` for unknown formats or missing dims
+ (callers then skip the pre-load sizing)."""
+ layers = getattr(model_config, "num_moe_layers", None)
+ if method is not None and layers:
+ per_expert = sum(
+ math.prod(spec.shape) * torch.empty((), dtype=spec.dtype).element_size()
+ for spec in method.layout().values() if not spec.resident
+ )
+ return layers * method.cfg.num_experts * per_expert
expert_quant = getattr(model_config, "expert_quant", "none")
fmt = expert_quant if expert_quant != "none" else (
getattr(model_config, "moe_weight_format", None) or "bf16"
@@ -409,6 +276,7 @@ def load_expert_banks(
model_path: str,
model_config,
*,
+ method=None,
device: torch.device,
dtype: torch.dtype,
dummy: bool = False,
@@ -436,6 +304,10 @@ def load_expert_banks(
provider only engages it (and reports ``ExpertBanks.streamed=True``) for its own
streamable formats, so callers must check ``streamed`` rather than assume it fired.
+ ``method`` (the bound expert quant method of the model's offload layers) selects
+ the generic path: the family's pieces packed by the method's kernel. Without it only the
+ GGUF q4_0 format loads, through its own provider.
+
``layer_residency``: per-layer ``HostResidency`` labels applied at settle time -- explicitly on the FTW fast path, ambiently (``requested_residency``) in the slow-path providers.
Applied labels are echoed on ``ExpertBanks.layer_residency``; a loader that settles some other way leaves it ``None`` (CPU-layer decode still works on pinned banks, it just saves no pin quota).
"""
@@ -480,16 +352,19 @@ def load_expert_banks(
# allocation) falls back to serial.
from freetoken.moe.host_banks import requested_residency
+ def _build(par: bool) -> ExpertBanks:
+ if method is not None:
+ return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink)
+ return _legacy_expert_banks(model_path, model_config, device, dtype, dummy, par, workers, chunk, decode_target, layer_sink)
+
with requested_residency(layer_residency) as residency_plan:
try:
- banks = _build_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk,
- decode_target, layer_sink)
+ banks = _build(parallel)
except NotImplementedError as exc:
if not parallel:
raise
logger.warning_rank0(f"parallel reader unavailable ({exc}); falling back to serial build")
- banks = _build_expert_banks(model_path, model_config, device, dtype, dummy, False, workers, chunk,
- decode_target, layer_sink)
+ banks = _build(False)
return _echo_residency(banks, layer_residency, residency_plan)
diff --git a/python/freetoken/moe/expert_pieces.py b/python/freetoken/moe/expert_pieces.py
new file mode 100644
index 00000000..79f655a9
--- /dev/null
+++ b/python/freetoken/moe/expert_pieces.py
@@ -0,0 +1,157 @@
+"""Expert pieces: the checkpoint-side stream ``build_expert_banks`` packs into host banks.
+
+A piece is ``(bank_layer, e0, e1, {role: tensor[e1 - e0, ...]})``: rows ``e0:e1`` of one MoE
+layer's experts, in the checkpoint's own form (per-projection ``gate`` / ``up`` / ``down`` with
+their ``_scale`` / ``_global`` companions, or an already fused ``gate_up``). The expert kernel's
+``pack`` turns pieces into its bank layout, so readers never know the layout.
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Iterable, Iterator
+
+import torch
+from freetoken.layers.quantization import QuantKind
+from freetoken.models.register import _load_attr, get_model_spec
+
+Piece = tuple[int, int, int, dict[str, torch.Tensor]]
+
+
+def num_moe_layers(config) -> int:
+ value = getattr(config, "num_moe_layers", None)
+ if value is not None:
+ return int(value)
+ return int(config.num_layers) - int(getattr(config, "first_k_dense_replace", 0) or 0)
+
+
+def bank_layer_of(config, layer: int) -> int | None:
+ """MoE-layer index of checkpoint layer ``layer``; None for the leading dense layers."""
+ bank_layer = layer - int(getattr(config, "first_k_dense_replace", 0) or 0)
+ return bank_layer if 0 <= bank_layer < num_moe_layers(config) else None
+
+
+def _model_hook(spec, name: str):
+ try:
+ return _load_attr(spec.module, name)
+ except AttributeError:
+ return None
+
+
+def iter_expert_pieces(
+ model_path: str, config, kind: QuantKind, *, parallel: bool = False, workers: int = 8, chunk: int = 8 << 20
+) -> Iterator[Piece]:
+ """The pieces of ``model_path``'s routed experts, stored as ``kind``.
+
+ A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, parallel, workers,
+ chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16
+ experts come from the family's stacked ``iter_weights`` and NVFP4 experts from its
+ ``nvfp4_expert_spec``. The reader is resolved here, before any bank is allocated, so a
+ missing parallel reader raises ``NotImplementedError`` while a serial fallback is still cheap.
+ """
+ spec = get_model_spec(config.architectures[0])
+ hook = _model_hook(spec, "iter_expert_pieces")
+ if hook is not None:
+ pieces = hook(model_path, config, kind, parallel=parallel, workers=workers, chunk=chunk)
+ if pieces is not None:
+ return pieces
+ if kind is QuantKind.NONE:
+ return _bf16_pieces(model_path, config, spec, parallel=parallel, workers=workers, chunk=chunk)
+ if kind is QuantKind.NVFP4:
+ spec_hook = _model_hook(spec, "nvfp4_expert_spec")
+ if spec_hook is None:
+ raise NotImplementedError(f"{spec.module} provides no nvfp4_expert_spec")
+ from freetoken.models.nvfp4_banks import iter_nvfp4_expert_pieces
+
+ return iter_nvfp4_expert_pieces(
+ model_path, config, spec_hook(model_path, config), parallel=parallel, workers=workers, chunk=chunk
+ )
+ raise NotImplementedError(f"{spec.module} provides no expert reader for {kind!r} experts")
+
+
+def packed_expert_source_info(key: str) -> tuple[int, str] | None:
+ """``model.layers.N....experts.{gate_up_proj,down_proj}`` -> (N, leaf); None otherwise."""
+ parts = key.split(".")
+ if len(parts) < 5 or parts[0] != "model" or parts[1] != "layers":
+ return None
+ if parts[-2] != "experts" or parts[-1] not in {"gate_up_proj", "down_proj"}:
+ return None
+ try:
+ return int(parts[2]), parts[-1]
+ except ValueError:
+ return None
+
+
+def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config) -> Iterator[Piece]:
+ """Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer."""
+ pending: dict[int, dict[str, torch.Tensor]] = {}
+ num_experts = config.num_experts
+ for name, tensor in tensors:
+ info = packed_expert_source_info(name)
+ if info is None:
+ raise ValueError(f"Unexpected expert weight key: {name}")
+ layer, packed_name = info
+ bank_layer = bank_layer_of(config, layer)
+ if bank_layer is None:
+ raise ValueError(f"Unexpected MoE expert layer {layer}; expected a routed-expert layer")
+ if tensor.size(0) != num_experts:
+ raise ValueError(f"Unexpected {packed_name} expert count {tensor.size(0)}; expected {num_experts}")
+ piece = pending.setdefault(bank_layer, {})
+ piece["gate_up" if packed_name == "gate_up_proj" else "down"] = tensor
+ if len(piece) == 2:
+ del pending[bank_layer]
+ yield bank_layer, 0, num_experts, piece
+ if pending:
+ raise ValueError(f"Missing MoE expert source layers: {sorted(pending)}")
+
+
+def _bf16_pieces(model_path: str, config, spec, *, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:
+ device = torch.device("cpu")
+ if parallel:
+ iter_weights = _model_hook(spec, "iter_weights_parallel")
+ if iter_weights is None:
+ raise NotImplementedError(f"{spec.module} provides no iter_weights_parallel")
+ tensors = iter_weights(
+ model_path, device, include_moe_experts=True, include_non_moe=False, workers=workers, chunk=chunk
+ )
+ else:
+ iter_weights = _load_attr(spec.module, spec.iter_weights)
+ tensors = iter_weights(model_path, device, include_moe_experts=True, include_non_moe=False)
+ return stacked_expert_pieces(tensors, config)
+
+
+def per_expert_pieces(
+ tensors: Iterable[tuple[str, torch.Tensor]],
+ locate: Callable[[str], tuple[int, int, str] | None],
+ *,
+ tensors_per_expert: int,
+) -> Iterator[Piece]:
+ """Group per-expert tensors into one piece per expert, in whatever order they arrive.
+
+ ``locate(name)`` -> ``(bank_layer, expert, role)`` or None to skip; a role tensor is one
+ expert's ``[...]`` row and becomes the piece's ``[1, ...]`` entry. Globals may be scalars.
+ """
+ pending: dict[tuple[int, int], dict[str, torch.Tensor]] = {}
+ for name, tensor in tensors:
+ where = locate(name)
+ if where is None:
+ continue
+ bank_layer, expert, role = where
+ piece = pending.setdefault((bank_layer, expert), {})
+ piece[role] = tensor.reshape(1, -1) if role.endswith("_global") else tensor.unsqueeze(0)
+ if len(piece) == tensors_per_expert:
+ del pending[(bank_layer, expert)]
+ yield bank_layer, expert, expert + 1, piece
+ if pending:
+ missing = sorted(pending)[:8]
+ raise ValueError(f"Incomplete expert tensors in checkpoint (layer, expert): {missing}")
+
+
+__all__ = [
+ "Piece",
+ "bank_layer_of",
+ "iter_expert_pieces",
+ "num_moe_layers",
+ "packed_expert_source_info",
+ "per_expert_pieces",
+ "stacked_expert_pieces",
+]
diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.py
index ecc45eae..a90aa4c7 100644
--- a/python/freetoken/moe/fused.py
+++ b/python/freetoken/moe/fused.py
@@ -5,7 +5,6 @@ from pathlib import Path
from typing import Dict, Tuple
import torch
-from freetoken.moe import BaseMoeBackend
from freetoken.utils import div_ceil
@@ -198,13 +197,15 @@ def fused_experts_impl(
topk_ids: torch.Tensor,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
+ act_alpha: float = 1.0,
+ act_limit: float = float("inf"),
) -> torch.Tensor:
"""Returns ``hidden_states`` itself, overwritten with the routed output. A caller that
still needs the input afterwards (a shared expert, a residual) must read it BEFORE this
call or pass a copy. ``fused_experts_decode_impl`` allocates instead, so the contract is
not shared; the resident bf16 path routes decode through here too."""
from freetoken.kernel import fused_moe_kernel_triton, moe_sum_reduce_triton
- from freetoken.layers import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul
+ from freetoken.layers import gated_act_and_mul
padded_size = 0
assert hidden_states.shape[1] == w1.shape[2] - padded_size, "Hidden size mismatch"
@@ -273,8 +274,7 @@ def fused_experts_impl(
config,
compute_type=compute_type,
)
- FN_MAP = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
- FN_MAP[activation](intermediate_cache1.view(-1, N), intermediate_cache2)
+ gated_act_and_mul(activation, intermediate_cache1.view(-1, N), intermediate_cache2, alpha=act_alpha, limit=act_limit)
fused_moe_kernel_triton(
intermediate_cache2,
w2,
@@ -305,9 +305,11 @@ def fused_experts_decode_impl(
topk_ids: torch.Tensor,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
+ act_alpha: float = 1.0,
+ act_limit: float = float("inf"),
) -> torch.Tensor:
from freetoken.kernel import fused_moe_decode_kernel_triton, moe_sum_reduce_triton
- from freetoken.layers import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul
+ from freetoken.layers import gated_act_and_mul
assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
assert w1.shape[0] == w2.shape[0], "Expert cache size mismatch"
@@ -323,9 +325,6 @@ def fused_experts_decode_impl(
assert w1.dtype == hidden_states.dtype and w2.dtype == hidden_states.dtype
assert topk_weights.dtype == torch.float32
assert topk_ids.dtype == torch.int32
- if activation not in {"silu", "gelu", "gelu_tanh"}:
- raise ValueError(f"Unsupported activation: {activation}")
-
M, _ = hidden_states.shape
top_k = topk_ids.shape[1]
gate_up_dim = w1.shape[1]
@@ -358,8 +357,7 @@ def fused_experts_decode_impl(
device=hidden_states.device,
dtype=hidden_states.dtype,
)
- FN_MAP = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
- FN_MAP[activation](intermediate_cache1.view(-1, gate_up_dim), intermediate_cache2)
+ gated_act_and_mul(activation, intermediate_cache1.view(-1, gate_up_dim), intermediate_cache2, alpha=act_alpha, limit=act_limit)
intermediate_cache3 = torch.empty(
(M, top_k, w2.shape[1]),
@@ -381,32 +379,3 @@ def fused_experts_decode_impl(
out_hidden_states = torch.empty_like(hidden_states)
moe_sum_reduce_triton(intermediate_cache3, out_hidden_states)
return out_hidden_states
-
-
-class FusedMoe(BaseMoeBackend):
- def forward(
- self,
- hidden_states: torch.Tensor,
- w1: torch.Tensor,
- w2: torch.Tensor,
- gating_output: torch.Tensor,
- topk: int,
- renormalize: bool,
- activation: str = "silu",
- apply_router_weight_on_input: bool = False,
- ) -> torch.Tensor:
- topk_weights, topk_ids = fused_topk(
- hidden_states=hidden_states,
- gating_output=gating_output,
- topk=topk,
- renormalize=renormalize,
- )
- return fused_experts_impl(
- hidden_states,
- w1,
- w2,
- topk_weights,
- topk_ids,
- activation,
- apply_router_weight_on_input=apply_router_weight_on_input,
- )
diff --git a/python/freetoken/moe/fused_fp8_block.py b/python/freetoken/moe/fused_fp8_block.py
index bd3dbdc1..4c7a58d4 100644
--- a/python/freetoken/moe/fused_fp8_block.py
+++ b/python/freetoken/moe/fused_fp8_block.py
@@ -16,7 +16,7 @@ from __future__ import annotations
def fused_experts_fp8_block(
hidden_states, gate_up, gate_up_scale, down, down_scale,
topk_weights, topk_ids, num_experts, activation="silu",
- apply_router_weight_on_input=False,
+ apply_router_weight_on_input=False, act_alpha=1.0, act_limit=float("inf"),
):
"""Prefill: W8A8 fused grouped GEMM over the materialized-layer banks
(``[num_experts, ...]``, position == expert id)."""
@@ -25,13 +25,14 @@ def fused_experts_fp8_block(
return fused_experts_fp8_blockscale(
hidden_states, gate_up, gate_up_scale, down, down_scale,
- topk_weights, topk_ids, num_experts, activation,
+ topk_weights, topk_ids, num_experts, activation, act_alpha, act_limit,
)
def fused_experts_decode_fp8_block(
hidden_states, gate_up, gate_up_scale, down, down_scale,
topk_weights, topk_ids, activation="silu", apply_router_weight_on_input=False,
+ act_alpha=1.0, act_limit=float("inf"),
):
"""Decode: W8A16 fused inline-dequant grouped GEMV -- reads the routed experts' fp8 rows
directly (``topk_ids`` index the banks) and dequantizes in the K-loop. CUDA-graph safe."""
@@ -40,7 +41,7 @@ def fused_experts_decode_fp8_block(
return fused_experts_decode_fp8_blockscale(
hidden_states, gate_up, gate_up_scale, down, down_scale,
- topk_weights, topk_ids, activation,
+ topk_weights, topk_ids, activation, act_alpha, act_limit,
)
diff --git a/python/freetoken/moe/fused_mxfp4.py b/python/freetoken/moe/fused_mxfp4.py
index 82088aa8..ac92190b 100644
--- a/python/freetoken/moe/fused_mxfp4.py
+++ b/python/freetoken/moe/fused_mxfp4.py
@@ -9,7 +9,7 @@ import torch
from freetoken.moe.fused import moe_align_block_size, try_get_optimal_moe_config
-# Token-count threshold used by GptOssMxfp4TritonMoELayer.forward to dispatch: batches at
+# Token-count threshold used by the resident gpt-oss mxfp4 kernel to dispatch: batches at
# or below this bound take the gather decode path (no sort), larger batches take the
# grouped/sorted prefill kernel. GPT-OSS decode targets small batches (max running
# requests), matching the gather path.
diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.py
index f736b80c..7988fa9e 100644
--- a/python/freetoken/moe/fused_nvfp4.py
+++ b/python/freetoken/moe/fused_nvfp4.py
@@ -21,38 +21,9 @@ from freetoken.kernel.triton.nvfp4_fused_moe import (
_e2m1_lut,
_prefill_nvfp4_moe_kernel,
)
-from freetoken.layers import (
- gelu_and_mul,
- gelu_tanh_and_mul,
- silu_and_mul,
- swiglu_clamp_and_mul,
- swigluoai_and_mul,
-)
+from freetoken.layers import gated_act_and_mul
from freetoken.moe.fused import moe_align_block_size
-_ACT = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
-
-
-def _run_act(
- activation: str,
- gate_up: torch.Tensor,
- out: torch.Tensor,
- act_alpha: float,
- act_limit: float,
-) -> None:
- """gemm1 -> gemm2 activation dispatch. ``swigluoai`` (MiniMax-M3, clamped
- gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) and
- ``swiglu_clamp`` (GLM-5.3, same clamp without the +1 up bias) carry the
- per-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kinds
- ignore them."""
- if activation == "swigluoai":
- swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)
- return
- if activation == "swiglu_clamp":
- swiglu_clamp_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)
- return
- _ACT[activation](gate_up, out)
-
# Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune,
# which benchmarks at run time). Tuned offline against the NVFP4 decode kernels.
# These drive the original LUT-gather decode (_decode_gemm), kept only for A/B.
@@ -196,7 +167,7 @@ def _fused_experts_decode_nvfp4(
ic1, topk_weights, topk_ids, apply_router_weight_on_input, False,
)
ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)
- _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)
+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)
ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)
gemm_fn(
ic2, down_packed, down_scale, down_global,
@@ -346,7 +317,7 @@ def fused_experts_nvfp4(
apply_router_weight_on_input, cfg,
)
ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)
- _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)
+ gated_act_and_mul(activation, ic1.view(-1, two_i), ic2, alpha=act_alpha, limit=act_limit)
ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)
_prefill_gemm(
ic2, down_packed, down_scale, down_global, ic3,
diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py
index 436f52d2..91e26487 100644
--- a/python/freetoken/moe/host_banks.py
+++ b/python/freetoken/moe/host_banks.py
@@ -36,6 +36,10 @@ logger = init_logger(__name__)
_BLK = 4096 # O_DIRECT alignment (page size)
+class PinFailed(RuntimeError):
+ """cudaHostRegister refused a bank: the host is out of pinnable RAM or over its pin quota."""
+
+
class HostResidency(str, Enum):
"""Residency class of a host bank layer.
@@ -136,9 +140,7 @@ class HostBank:
try:
host_register(self.addr, len(self._buf))
except RuntimeError as exc:
- raise RuntimeError(
- f"cudaHostRegister failed for {len(self._buf) / 2**30:.1f} GiB"
- ) from exc
+ raise PinFailed(f"cudaHostRegister failed for {len(self._buf) / 2**30:.1f} GiB") from exc
self._pinned = True
def release(self) -> None:
diff --git a/python/freetoken/moe/legacy_format.py b/python/freetoken/moe/legacy_format.py
new file mode 100644
index 00000000..973a3dba
--- /dev/null
+++ b/python/freetoken/moe/legacy_format.py
@@ -0,0 +1,46 @@
+"""Bank and format names on disk (FTW), in the CPU executor and in the GGUF provider; the kernels use the canonical roles."""
+
+from __future__ import annotations
+
+from freetoken.layers.quantization import QuantKind
+
+CANONICAL_ROLE = {
+ "gate_up_packed": "gate_up",
+ "gate_up_blocks": "gate_up",
+ "gate_up_scales": "gate_up_scale",
+ "down_packed": "down",
+ "down_blocks": "down",
+ "down_scales": "down_scale",
+}
+
+
+def canonical_role(bank_name: str) -> str:
+ return CANONICAL_ROLE.get(bank_name, bank_name)
+
+
+def legacy_bank_names(quant_format: str) -> dict[str, str]:
+ """Canonical role -> the bank name FTW files store for ``quant_format``."""
+ from freetoken.moe.offload_cache import _BANK_SCHEMAS
+
+ return {canonical_role(name): name for name in _BANK_SCHEMAS[quant_format]}
+
+
+# (kind, kernel) <-> the quant_format tag
+LEGACY_FORMAT = {
+ (QuantKind.NONE, "fused"): "bf16",
+ (QuantKind.FP8_BLOCK, "triton"): "fp8_block",
+ (QuantKind.NVFP4, "triton"): "nvfp4",
+ (QuantKind.NVFP4, "marlin"): "nvfp4_marlin",
+ (QuantKind.NVFP4, "b12x"): "nvfp4_b12x",
+ (QuantKind.MXFP4, "triton_gptoss"): "mxfp4_triton",
+ (QuantKind.MXFP4, "triton"): "ds_fp4",
+}
+_KIND_KERNEL = {fmt: kk for kk, fmt in LEGACY_FORMAT.items()}
+
+
+def legacy_format_for(kind: QuantKind, kernel: str) -> str:
+ return LEGACY_FORMAT[(kind, kernel)]
+
+
+def kind_kernel_for(legacy_format: str) -> tuple[QuantKind, str]:
+ return _KIND_KERNEL[legacy_format]
diff --git a/python/freetoken/moe/nvfp4_backends.py b/python/freetoken/moe/nvfp4_backends.py
deleted file mode 100644
index e8a3c6f6..00000000
--- a/python/freetoken/moe/nvfp4_backends.py
+++ /dev/null
@@ -1,790 +0,0 @@
-"""NVFP4 expert-GEMM backends borrowed from vLLM (Marlin) and flashinfer (b12x).
-
-FreeToken stores MoE experts as ModelOpt NVFP4 (packed e2m1 codes + fp8-e4m3 per-16
-block scales + per-tensor global scale) in pinned host banks and gathers the routed
-experts into a GPU slot cache. This module picks the fastest fused-MoE kernel for
-that format by compute capability and owns the matching bank layout:
-
-========== ========================== =============================================
-backend compute capability kernel / weight layout
-========== ========================== =============================================
-marlin sm_80 .. sm_99 vLLM ``fused_marlin_moe`` (W4A16
- dequant-in-kernel); Marlin-tiled weights
-b12x sm_120+ and CUDA>=13 ``flashinfer`` SM12x CuTe-DSL MoE (W4A16);
- b12x-packed weights
-triton anything (fallback) FreeToken's own Triton kernels; the native
- row-major ModelOpt layout (used on sm_120 +
- CUDA 12.x, where b12x cannot run)
-========== ========================== =============================================
-
-The Marlin kernels come from vLLM (not sgl-kernel) deliberately: vLLM ships the
-NVFP4 instantiations in its AOT wheel together with the matching host-side layout
-transforms, so the pair stays consistent by construction. sgl-kernel's AOT wheel
-excludes the NVFP4 MoE Marlin kernels to cut wheel size (its python transforms pair
-with sglang's *JIT* kernel tree, whose scale encoding has since diverged).
-
-Hard rules:
-
-* A backend owns ``{load-time pack, forward}`` as a unit. Prefill and decode consume
- the *same* bank layout -- there is never a second copy of the experts.
-* Every pack keeps the expert dimension outermost with per-expert blocks contiguous
- and byte-identical in size to the native layout, so the banks are repacked *in
- place* and the offload cache's slot gather (``copy_missing``) works unchanged.
-* The per-tensor global scales never enter the offload banks: they are tiny
- ``[L*E]`` vectors kept resident on the GPU and gathered per forward call
- (device-side, CUDA-graph safe).
-
-The small layout-transform helpers are ported from sglang
-(``srt/layers/quantization/marlin_utils{,_fp4}.py``, Apache-2.0); the kernels
-themselves are imported, not vendored.
-"""
-
-from __future__ import annotations
-
-import os
-
-import torch
-
-from freetoken.utils import init_logger
-
-logger = init_logger(__name__)
-
-# The 6 native ModelOpt NVFP4 source bank names (== _BANK_SCHEMAS["nvfp4"]) the repack
-# reads, and the 4 post-repack bank names both marlin and b12x emit (== the "nvfp4_marlin"
-# / "nvfp4_b12x" schemas); the two *_global banks fold into per-expert alphas, not banks.
-_NATIVE_NVFP4_BANKS = (
- "gate_up_packed", "gate_up_scale", "gate_up_global",
- "down_packed", "down_scale", "down_global",
-)
-_POST_NVFP4_BANKS = ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale")
-
-# ---------------------------------------------------------------------------
-# Backend selection
-# ---------------------------------------------------------------------------
-
-
-def _donor_symbols_ok(backend: str) -> bool:
- """Probe the exact donor symbols the pack/forward paths below use.
-
- ``find_spec`` only proves the top-level package exists; an incompatible donor
- version would otherwise crash mid-init, after minutes of bank loading. Probing
- at selection time degrades to triton before any bank is touched.
- """
- try:
- if backend == "marlin":
- from vllm import _custom_ops # noqa: F401
- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( # noqa: F401
- fused_marlin_moe,
- )
- from vllm.model_executor.layers.quantization.utils.marlin_utils import ( # noqa: F401
- marlin_permute_scales,
- )
- from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401
- nvfp4_marlin_process_global_scale,
- nvfp4_marlin_process_scales,
- )
- from vllm.scalar_type import scalar_types # noqa: F401
- else:
- assert backend == "b12x"
- # Probe exactly what the pack/forward use: the row-major->swizzled scale
- # transform, the W4A16 prepare, and the prepared-weights launch (which also
- # transitively imports the CuTe-DSL kernel + cutlass).
- from flashinfer import nvfp4_block_scale_interleave # noqa: F401
- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import ( # noqa: F401
- _launch_sm120_w4a16_moe,
- )
- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import ( # noqa: F401
- W4A16PackedWeights,
- _make_workspace,
- prepare_w4a16_packed_weights,
- )
- except Exception as exc:
- logger.warning(
- f"NVFP4 {backend} backend is installed but unusable ({exc!r}); "
- "falling back to the Triton inline-dequant backend"
- )
- return False
- return True
-
-
-def _flashinfer_cuda_major() -> int | None:
- """flashinfer's own toolkit-CUDA major; None if it can't be determined. Only a
- fallback proxy when the driver version is unavailable -- the b12x kernel is JIT
- PTX-compiled through the driver, so the toolkit major is *not* what gates it."""
- try:
- from flashinfer.jit.cpp_ext import get_cuda_version
-
- return int(get_cuda_version().major)
- except Exception:
- return None
-
-
-def _b12x_unusable_reason(cc: tuple[int, int]) -> str | None:
- """None if the flashinfer b12x decode/prefill kernel can actually run on this device,
- else a human-readable reason. The b12x *pack* works anywhere, but the SM12x CuTe-DSL
- kernel itself requires sm_120+ and a CUDA>=13 *driver* (it JIT-compiles PTX through the
- driver), so selection must check the runtime here -- not just that flashinfer imports --
- or the model loads in the b12x layout and then crashes on the first decode."""
- import importlib.util
-
- if cc < (12, 0):
- return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"
- if importlib.util.find_spec("flashinfer") is None:
- return "flashinfer is not installed"
- from freetoken.kernel.backend import driver_cuda_version
-
- drv = driver_cuda_version()
- if drv is not None:
- if drv < 13000:
- return (
- "b12x fused MoE requires a CUDA>=13 driver "
- f"(driver supports CUDA {drv // 1000}.{(drv % 1000) // 10})"
- )
- else:
- # Driver version undetermined: fall back to flashinfer's toolkit major as a
- # conservative proxy rather than risk loading the b12x layout and crashing.
- major = _flashinfer_cuda_major()
- if major is not None and major < 13:
- return (
- "b12x fused MoE requires CUDA>=13 (driver undetermined; "
- f"flashinfer toolkit is CUDA {major}.x)"
- )
- if not _donor_symbols_ok("b12x"):
- return "flashinfer b12x donor symbols are unusable"
- return None
-
-
-def _b12x_min_intermediate() -> int:
- """Smallest ``moe_intermediate_size`` for which ``auto`` prefers b12x over Triton.
-
- One bank layout is chosen per process (b12x's prepared/tiled weights and Triton's
- native row-major layout are not byte-compatible, so they can't coexist without a 2x
- expert-memory copy), so this is a *load-time* pick by shape. The default (1024) is the
- M=1 single-stream decode crossover measured on RTX 5090 (CUDA-graph %HBM, top_k=8):
-
- I=512 (Qwen3.x-35B-A3B): triton 43% vs b12x 27% -> triton
- I=768 (Qwen3-30B-A3B): triton 41% vs b12x 41% -> triton (tie; b12x wins batched)
- I=1536 (MiniMax-M2): triton 65% vs b12x 82% -> b12x
-
- b12x always wins *batched* decode (M>=~4) thanks to tensor cores, so a throughput
- deployment of a small-I model should force it with ``--nvfp4-backend flashinfer``.
- Override the threshold with ``FREETOKEN_NVFP4_B12X_MIN_I``."""
- raw = os.environ.get("FREETOKEN_NVFP4_B12X_MIN_I")
- if raw is None:
- return 1024
- try:
- return int(raw)
- except ValueError as exc:
- raise ValueError(
- f"bad FREETOKEN_NVFP4_B12X_MIN_I={raw!r}; expected an integer"
- ) from exc
-
-
-def select_nvfp4_backend(
- device: torch.device,
- intermediate_size: int | None = None,
- requested: str = "auto",
- activation: str = "silu",
-) -> str:
- """Pick the NVFP4 expert-GEMM backend for ``device`` (and, in ``auto``, ``intermediate_size``).
-
- ``requested`` is the ``--nvfp4-backend`` choice; a forced backend fails loudly (never
- silently degrades) when it cannot run on this hardware/toolkit:
-
- * ``auto`` (default) -- capability pick: ``marlin`` on sm_80-99 (+ vLLM), flashinfer
- ``b12x`` on sm_120+ (CUDA>=13 driver) when the MoE is wide enough that its tensor cores
- beat Triton at single-stream M=1 decode (``intermediate_size >= _b12x_min_intermediate()``),
- else the portable Triton inline-dequant GEMV.
- * ``marlin`` / ``flashinfer`` / ``triton`` -- force that backend (and its bank layout),
- raising if it cannot run rather than degrading.
-
- ``activation`` is the model's routed-expert activation: the borrowed marlin/b12x
- kernels hard-code silu (their fused epilogue), so any other activation (MiniMax-M3's
- ``swigluoai``) resolves ``auto`` to the Triton kernels -- which dispatch the
- activation as a separate elementwise op -- and rejects a forced marlin/flashinfer.
-
- Returns the internal backend name (``marlin`` / ``b12x`` / ``triton``); ``flashinfer``
- maps to ``b12x``.
- """
- import importlib.util
-
- requested = (requested or "auto").strip().lower()
- if requested not in ("auto", "marlin", "flashinfer", "triton"):
- raise ValueError(
- f"bad --nvfp4-backend={requested!r}; expected auto, marlin, flashinfer or triton"
- )
- if requested == "triton":
- return "triton"
- if activation != "silu":
- if requested != "auto":
- raise RuntimeError(
- f"--nvfp4-backend={requested} only supports silu routed experts (its fused "
- f"epilogue); this model's experts use {activation!r} -- use "
- "--nvfp4-backend triton (or auto)."
- )
- logger.info(
- f"NVFP4 auto backend: routed experts use {activation!r}, which only the "
- "Triton kernels support; using the Triton inline-dequant kernels"
- )
- return "triton"
- if requested == "marlin":
- if device.type != "cuda":
- raise RuntimeError("--nvfp4-backend=marlin requires a CUDA device")
- if not _donor_symbols_ok("marlin"):
- raise RuntimeError(
- "--nvfp4-backend=marlin is unusable: vLLM Marlin donor symbols are unavailable"
- )
- return "marlin"
- if requested == "flashinfer":
- cc = torch.cuda.get_device_capability(device) if device.type == "cuda" else (0, 0)
- reason = _b12x_unusable_reason(cc)
- if reason is not None:
- raise RuntimeError(f"--nvfp4-backend=flashinfer is unavailable: {reason}")
- return "b12x"
-
- # auto: capability pick, but only land on marlin/b12x when they can really run.
- if device.type != "cuda":
- return "triton"
- cc = torch.cuda.get_device_capability(device)
- if (
- (8, 0) <= cc < (10, 0)
- and importlib.util.find_spec("vllm") is not None
- and _donor_symbols_ok("marlin")
- ):
- return "marlin"
- if cc >= (12, 0):
- reason = _b12x_unusable_reason(cc)
- if reason is None:
- thr = _b12x_min_intermediate()
- if intermediate_size is None or intermediate_size >= thr:
- return "b12x"
- logger.info(
- f"NVFP4 auto backend: b12x is runnable but moe_intermediate_size="
- f"{intermediate_size} < {thr}; using the Triton decode kernels "
- "(faster at single-stream M=1 decode for small-I MoE; set "
- "--nvfp4-backend flashinfer to force b12x for batched throughput)"
- )
- return "triton"
- logger.info(
- f"NVFP4 auto backend: flashinfer b12x unavailable ({reason}); "
- "using the Triton inline-dequant decode kernels"
- )
- return "triton" # sm_100/103, b12x unusable, or the donor package is missing
-
-
-# ---------------------------------------------------------------------------
-# Marlin pack (kernels and layout transforms borrowed from vLLM; mirrors its
-# prepare_moe_fp4_layer_for_marlin so the host prep always matches the AOT op)
-# ---------------------------------------------------------------------------
-
-
-def _marlin_pack_proj(
- packed: torch.Tensor,
- scale: torch.Tensor,
- row_global: torch.Tensor,
- *,
- size_k: int,
- size_n: int,
-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
- """Pack one expert projection (on GPU) into the Marlin layout.
-
- ``packed`` ``[N, K//2]`` uint8 e2m1 pairs, ``scale`` ``[N, K//16]`` e4m3,
- ``row_global`` ``[N]`` fp16 per-row global (rows of a merged gate_up may carry
- w1's and w3's different globals). Returns ``(qweight [K//16, 2N] int32,
- scales [K//16, N] e4m3-coded, global scalar bf16)``.
-
- Marlin takes one global scalar per projection, so when the row globals differ
- the ratio is folded into the fp16 block scales before they are re-encoded
- (exact when all rows share one global, <=1 scale ulp otherwise).
- """
- from vllm import _custom_ops as ops
- from vllm.model_executor.layers.quantization.utils.marlin_utils import (
- marlin_permute_scales,
- )
- from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
- nvfp4_marlin_process_global_scale,
- nvfp4_marlin_process_scales,
- )
-
- assert size_n % 64 == 0, f"Marlin requires N % 64 == 0, got {size_n}"
- qweight = ops.gptq_marlin_repack(
- b_q_weight=packed.view(torch.int32).T.contiguous(),
- perm=torch.empty(0, dtype=torch.int, device=packed.device),
- size_k=size_k,
- size_n=size_n,
- num_bits=4,
- )
-
- g = row_global.to(torch.float32)
- g_max = g.max()
- s = scale.to(torch.bfloat16)
- if not torch.all(g == g_max):
- s = (s.float() * (g / g_max).unsqueeze(1)).to(torch.bfloat16)
- s = marlin_permute_scales(s.T, size_k=size_k, size_n=size_n, group_size=16)
- s = nvfp4_marlin_process_scales(s)
- if isinstance(s, tuple): # newer vLLM returns (scales, scale_factor)
- s, factor = s
- g_max = g_max / factor
- g_out = nvfp4_marlin_process_global_scale(g_max.to(torch.bfloat16).reshape(1))
- return qweight, s, g_out
-
-
-def marlin_repack_layer(
- layer_banks: dict[str, torch.Tensor],
- config,
- device: torch.device,
- *,
- chunk: int = 32,
-) -> tuple[dict[str, torch.Tensor], torch.Tensor, torch.Tensor]:
- """Repack ONE layer's 6 native NVFP4 banks into the Marlin layout, in place.
-
- ``layer_banks`` are that layer's native ``[E, ...]`` tensors keyed by the 6 native
- NVFP4 bank names (``gate_up_packed/scale/global``, ``down_packed/scale/global``).
- The Marlin blocks are byte-identical in size, so the weight/scale storage is
- reinterpreted rather than reallocated; the two ``*_global`` banks are consumed into
- per-expert alphas and are not part of the output layout.
-
- Returns ``({post-repack bank name -> reinterpreted tensor}, gate_up_alpha, down_alpha)``
- where the dict is keyed by the 4 ``nvfp4_marlin`` bank names and the two alphas are
- ``[E]`` bf16 on ``device``. Staging ``.to(device, non_blocking=True)`` from the native
- source works whether or not it is pinned (pageable -> synchronous copy)."""
- H = config.hidden_size
- I = config.moe_intermediate_size
- gu_packed_l = layer_banks["gate_up_packed"]
- gu_scale_l = layer_banks["gate_up_scale"]
- gu_global_l = layer_banks["gate_up_global"]
- dn_packed_l = layer_banks["down_packed"]
- dn_scale_l = layer_banks["down_scale"]
- dn_global_l = layer_banks["down_global"]
- E = gu_packed_l.size(0)
-
- gate_up_q = gu_packed_l.view(torch.int32).view(E, H // 16, 4 * I)
- gate_up_s = gu_scale_l.view(E, H // 16, 2 * I)
- down_q = dn_packed_l.view(torch.int32).view(E, I // 16, 2 * H)
- down_s = dn_scale_l.view(E, I // 16, H)
- gate_up_alpha = torch.empty(E, dtype=torch.bfloat16, device=device)
- down_alpha = torch.empty(E, dtype=torch.bfloat16, device=device)
-
- for start in range(0, E, chunk):
- end = min(start + chunk, E)
- # Stage the native rows on GPU before their (in-place) storage is overwritten.
- gu_p = gu_packed_l[start:end].to(device, non_blocking=True)
- gu_s = gu_scale_l[start:end].to(device, non_blocking=True)
- gu_g = gu_global_l[start:end].to(device, non_blocking=True)
- dn_p = dn_packed_l[start:end].to(device, non_blocking=True)
- dn_s = dn_scale_l[start:end].to(device, non_blocking=True)
- dn_g = dn_global_l[start:end].to(device, non_blocking=True)
- for i in range(end - start):
- qw, sc, al = _marlin_pack_proj(gu_p[i], gu_s[i], gu_g[i], size_k=H, size_n=2 * I)
- gate_up_q[start + i].copy_(qw, non_blocking=True)
- gate_up_s[start + i].copy_(sc.view(torch.float8_e4m3fn), non_blocking=True)
- gate_up_alpha[start + i] = al[0]
- qw, sc, al = _marlin_pack_proj(dn_p[i], dn_s[i], dn_g[i], size_k=I, size_n=H)
- down_q[start + i].copy_(qw, non_blocking=True)
- down_s[start + i].copy_(sc.view(torch.float8_e4m3fn), non_blocking=True)
- down_alpha[start + i] = al[0]
- torch.cuda.synchronize(device)
-
- return (
- {
- "gate_up_packed": gate_up_q,
- "gate_up_scale": gate_up_s,
- "down_packed": down_q,
- "down_scale": down_s,
- },
- gate_up_alpha,
- down_alpha,
- )
-
-
-def marlin_repack_sources_inplace(
- sources: dict[str, list[torch.Tensor]],
- config,
- device: torch.device,
- *,
- chunk: int = 32,
-) -> dict[str, list[torch.Tensor] | torch.Tensor]:
- """Convert the native NVFP4 banks from ``load_nvfp4_expert_sources`` to the
- Marlin layout, in place per layer (thin per-layer loop over
- :func:`marlin_repack_layer`; the Marlin blocks are byte-identical in size, so each
- layer's bank tensor is reinterpreted rather than reallocated).
-
- Returns the per-layer bank lists for ``OffloadMoeCache.set_bank_sources`` plus
- the processed per-expert global-scale vectors (``gate_up_alpha`` / ``down_alpha``,
- flat ``[L*E]`` bf16 on ``device``, layer-major) that the forward path gathers
- per slot.
- """
- num_layers = len(sources["gate_up_packed"])
-
- gate_up_q_layers: list[torch.Tensor] = []
- gate_up_s_layers: list[torch.Tensor] = []
- down_q_layers: list[torch.Tensor] = []
- down_s_layers: list[torch.Tensor] = []
- gate_up_alpha_layers: list[torch.Tensor] = []
- down_alpha_layers: list[torch.Tensor] = []
-
- for layer_id in range(num_layers):
- layer_banks = {name: sources[name][layer_id] for name in _NATIVE_NVFP4_BANKS}
- post, gate_up_alpha, down_alpha = marlin_repack_layer(
- layer_banks, config, device, chunk=chunk
- )
- gate_up_q_layers.append(post["gate_up_packed"])
- gate_up_s_layers.append(post["gate_up_scale"])
- down_q_layers.append(post["down_packed"])
- down_s_layers.append(post["down_scale"])
- gate_up_alpha_layers.append(gate_up_alpha)
- down_alpha_layers.append(down_alpha)
-
- return {
- "gate_up_packed": gate_up_q_layers,
- "gate_up_scale": gate_up_s_layers,
- "down_packed": down_q_layers,
- "down_scale": down_s_layers,
- "gate_up_alpha": torch.cat(gate_up_alpha_layers),
- "down_alpha": torch.cat(down_alpha_layers),
- }
-
-
-# ---------------------------------------------------------------------------
-# Marlin forward (vLLM's fused_marlin_moe, borrowed wholesale)
-# ---------------------------------------------------------------------------
-
-
-def marlin_fused_experts(
- hidden_states: torch.Tensor,
- gate_up_q: torch.Tensor,
- gate_up_s: torch.Tensor,
- gate_up_alpha: torch.Tensor,
- down_q: torch.Tensor,
- down_s: torch.Tensor,
- down_alpha: torch.Tensor,
- topk_weights: torch.Tensor,
- topk_ids: torch.Tensor,
- activation: str,
- apply_router_weight_on_input: bool,
-) -> torch.Tensor:
- """Marlin W4A16 fused MoE (two grouped GEMMs + activation + reduce).
-
- Two calling regimes share this entry: decode passes the full ``[S]`` slot cache
- with ``topk_ids`` rewritten to slot ids; full-layer prefill passes banks whose
- position == expert id (the materialized ``[:E]`` slot view or the overlap double
- buffer views), so the raw routing ids arrive unmapped. The ``*_alpha`` vectors
- are the matching per-row global scales in both regimes.
- vLLM's implementation is device-side only (no host syncs), so the decode call is
- CUDA-graph capturable.
- """
- from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe
- from vllm.scalar_type import scalar_types
-
- assert activation == "silu", "Marlin NVFP4 backend supports gated silu only"
- return fused_marlin_moe(
- hidden_states,
- gate_up_q,
- down_q,
- None, # bias1
- None, # bias2
- gate_up_s,
- down_s,
- gating_output=None,
- topk_weights=topk_weights,
- topk_ids=topk_ids,
- quant_type_id=scalar_types.float4_e2m1f.id,
- apply_router_weight_on_input=apply_router_weight_on_input,
- global_num_experts=gate_up_q.size(0),
- activation=activation,
- global_scale1=gate_up_alpha,
- global_scale2=down_alpha,
- )
-
-
-# ---------------------------------------------------------------------------
-# b12x (flashinfer SM12x CuTe-DSL, W4A16) -- the pack (prepare_w4a16_packed_weights)
-# runs on any CUDA build, but the fused-MoE kernel needs sm_120/121 AND a CUDA>=13
-# *driver* (it JIT-compiles PTX at runtime). select_nvfp4_backend gates on that, so
-# this path is only reached on hardware that can actually run it.
-#
-# Layout contract (verified against flashinfer 0.6.12 prepare_w4a16_packed_weights):
-# * prepare() takes the block scales already *swizzled* (it calls unswizzle_expert_
-# scales internally), so the native row-major fp8 scales are run through
-# nvfp4_block_scale_interleave first.
-# * prepare() reorders the w13 rows with reorder_w13_to_gate_up == cat([second, first]),
-# i.e. it expects the merged proj as [up, gate] and emits [gate, up]. FreeToken stores
-# [gate, up], so the halves are swapped before prepare() to come out [gate, up] again
-# (the kernel computes silu(first) * second == silu(gate) * up).
-# * modelopt globals are passed straight through (no 1/g inversion).
-# The fused forward reconstructs the W4A16PackedWeights from the banks and uses the
-# prepared-weights launch (_launch_sm120_w4a16_moe) directly, so weights are prepared
-# exactly once at load time (b12x_fused_moe would re-prepare + ptr-cache every call).
-# ---------------------------------------------------------------------------
-
-
-def _b12x_swizzle_block_scales(scale: torch.Tensor) -> torch.Tensor:
- """Row-major per-16 fp8 block scales ``[n, N, K//16]`` -> flashinfer's expert-leading
- *swizzled* storage, which is what :func:`prepare_w4a16_packed_weights` expects on input
- (it unswizzles internally). The 128x4 interleave is per-expert; ``N`` is a multiple of
- 128 and ``K//16`` a multiple of 4 for the supported models, so the shape is preserved."""
- from flashinfer import nvfp4_block_scale_interleave
-
- pieces = [
- nvfp4_block_scale_interleave(scale[e].view(torch.uint8).contiguous())
- for e in range(scale.shape[0])
- ]
- return torch.stack(pieces, dim=0).view(torch.float8_e4m3fn)
-
-
-def b12x_repack_layer(
- layer_banks: dict[str, torch.Tensor],
- config,
- device: torch.device,
- *,
- chunk: int = 32,
-) -> tuple[dict[str, torch.Tensor], torch.Tensor, torch.Tensor]:
- """Repack ONE layer's 6 native NVFP4 banks into flashinfer's SM12x W4A16 layout, in place.
-
- Mirrors :func:`marlin_repack_layer`; the b12x packed blocks are also byte-identical
- per expert (4-bit weights + e4m3-coded scales), so the weight/scale storage is
- reinterpreted and the two ``*_global`` banks fold into ``[E]`` fp32 alphas. Uses
- flashinfer's own prepare helpers so the layout always matches the installed kernel.
-
- Returns ``({post-repack bank name -> reinterpreted tensor}, gate_up_alpha, down_alpha)``
- keyed by the 4 ``nvfp4_b12x`` bank names. Staging from the native source works whether
- or not it is pinned (pageable ``.to(device)`` is a synchronous copy)."""
- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (
- prepare_w4a16_packed_weights,
- )
-
- I = config.moe_intermediate_size
- gu_packed_l = layer_banks["gate_up_packed"]
- gu_scale_l = layer_banks["gate_up_scale"]
- gu_global_l = layer_banks["gate_up_global"]
- dn_packed_l = layer_banks["down_packed"]
- dn_scale_l = layer_banks["down_scale"]
- dn_global_l = layer_banks["down_global"]
- E = gu_packed_l.size(0)
-
- gate_up_alpha = torch.empty(E, dtype=torch.float32, device=device)
- down_alpha = torch.empty(E, dtype=torch.float32, device=device)
- gate_up_q = gate_up_s = down_q = down_s = None
-
- for start in range(0, E, chunk):
- end = min(start + chunk, E)
- n = end - start
- gu_g = gu_global_l[start:end].to(device)
- # The merged gate_up rows carry w1/w3 globals; b12x also takes one alpha per
- # expert, so fold any ratio into the block scales like the Marlin pack does.
- g = gu_g.float()
- g_max = g.max(dim=1, keepdim=True).values
- gu_s_native = gu_scale_l[start:end].to(device).to(torch.float16)
- ratio = g / g_max
- if not torch.all(ratio == 1.0):
- gu_s_native = (gu_s_native.float() * ratio.unsqueeze(-1)).to(torch.float16)
- # The down bank stores one w2 global broadcast per output row; b12x takes a
- # single alpha per expert, so a row-varying global cannot be represented (the
- # marlin pack folds the ratio into the block scales instead -- do not let such
- # a checkpoint be silently truncated to row 0 here).
- dn_g = dn_global_l[start:end].to(device)
- if not torch.all(dn_g == dn_g[:, :1]):
- raise ValueError("b12x pack requires a row-constant per-expert down_proj global scale")
- # prepare() emits [gate, up] from a [up, gate] input (reorder_w13_to_gate_up),
- # so swap FreeToken's native [gate, up] halves here; swap the (ratio-folded)
- # block scales identically to keep weight/scale rows aligned.
- gu_p = gu_packed_l[start:end].to(device)
- gu_p = torch.cat([gu_p[:, I:], gu_p[:, :I]], dim=1).contiguous()
- gu_s_native = torch.cat(
- [gu_s_native[:, I:], gu_s_native[:, :I]], dim=1
- ).contiguous()
- prepared = prepare_w4a16_packed_weights(
- gu_p,
- _b12x_swizzle_block_scales(gu_s_native.to(torch.float8_e4m3fn)),
- g_max.squeeze(1),
- dn_packed_l[start:end].to(device),
- _b12x_swizzle_block_scales(dn_scale_l[start:end].to(device)),
- dn_g[:, 0].float(),
- activation="silu",
- params_dtype=torch.bfloat16,
- source_format="modelopt",
- )
- if gate_up_q is None:
- def _as_bank(bank: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
- assert t[0].nbytes * E == bank.nbytes, (
- f"b12x block size mismatch: {t[0].nbytes} * {E} != {bank.nbytes}"
- )
- return bank.view(t.dtype).view(E, *t.shape[1:])
-
- gate_up_q = _as_bank(gu_packed_l, prepared.w13)
- gate_up_s = _as_bank(gu_scale_l, prepared.w13_scale)
- down_q = _as_bank(dn_packed_l, prepared.w2)
- down_s = _as_bank(dn_scale_l, prepared.w2_scale)
- gate_up_q[start:end].copy_(prepared.w13[:n])
- gate_up_s[start:end].copy_(prepared.w13_scale[:n])
- down_q[start:end].copy_(prepared.w2[:n])
- down_s[start:end].copy_(prepared.w2_scale[:n])
- gate_up_alpha[start:end] = prepared.w13_global_scale[:n].float()
- down_alpha[start:end] = prepared.w2_global_scale[:n].float()
-
- return (
- {
- "gate_up_packed": gate_up_q,
- "gate_up_scale": gate_up_s,
- "down_packed": down_q,
- "down_scale": down_s,
- },
- gate_up_alpha,
- down_alpha,
- )
-
-
-def b12x_repack_sources_inplace(
- sources: dict[str, list[torch.Tensor]],
- config,
- device: torch.device,
- *,
- chunk: int = 32,
-) -> dict[str, list[torch.Tensor] | torch.Tensor]:
- """Convert the native NVFP4 banks to flashinfer's SM12x W4A16 layout in place, per
- layer (thin per-layer loop over :func:`b12x_repack_layer`).
-
- Mirrors :func:`marlin_repack_sources_inplace`; the b12x packed blocks are also
- byte-identical per expert (4-bit weights + e4m3-coded scales), so each layer's
- bank tensor is reinterpreted.
- """
- num_layers = len(sources["gate_up_packed"])
-
- gate_up_q_layers: list[torch.Tensor] = []
- gate_up_s_layers: list[torch.Tensor] = []
- down_q_layers: list[torch.Tensor] = []
- down_s_layers: list[torch.Tensor] = []
- gate_up_alpha_layers: list[torch.Tensor] = []
- down_alpha_layers: list[torch.Tensor] = []
-
- for layer_id in range(num_layers):
- layer_banks = {name: sources[name][layer_id] for name in _NATIVE_NVFP4_BANKS}
- post, gate_up_alpha, down_alpha = b12x_repack_layer(
- layer_banks, config, device, chunk=chunk
- )
- gate_up_q_layers.append(post["gate_up_packed"])
- gate_up_s_layers.append(post["gate_up_scale"])
- down_q_layers.append(post["down_packed"])
- down_s_layers.append(post["down_scale"])
- gate_up_alpha_layers.append(gate_up_alpha)
- down_alpha_layers.append(down_alpha)
-
- return {
- "gate_up_packed": gate_up_q_layers,
- "gate_up_scale": gate_up_s_layers,
- "down_packed": down_q_layers,
- "down_scale": down_s_layers,
- "gate_up_alpha": torch.cat(gate_up_alpha_layers),
- "down_alpha": torch.cat(down_alpha_layers),
- }
-
-
-# Per-device scratch (sized sms*4+2) that run_w4a16_moe reads off the prepared object;
-# tiny and shape-independent, so one per device is reused across all layers/decodes.
-_B12X_WORKSPACE: dict[torch.device, torch.Tensor] = {}
-
-
-def _b12x_small_workspace(device: torch.device) -> torch.Tensor:
- ws = _B12X_WORKSPACE.get(device)
- if ws is None:
- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (
- _make_workspace,
- )
-
- ws = _make_workspace(device, max_blocks_per_sm=4)
- _B12X_WORKSPACE[device] = ws
- return ws
-
-
-def b12x_fused_experts(
- hidden_states: torch.Tensor,
- gate_up_q: torch.Tensor,
- gate_up_s: torch.Tensor,
- gate_up_alpha: torch.Tensor,
- down_q: torch.Tensor,
- down_s: torch.Tensor,
- down_alpha: torch.Tensor,
- topk_weights: torch.Tensor,
- topk_ids: torch.Tensor,
- activation: str,
- apply_router_weight_on_input: bool,
-) -> torch.Tensor:
- """SM12x W4A16 fused MoE over cache slots (same calling convention as
- :func:`marlin_fused_experts`). Requires sm_120/121 + a CUDA>=13 driver; reached only
- when :func:`select_nvfp4_backend` has confirmed the runtime supports it.
-
- The banks already hold flashinfer's prepared (tiled) layout from
- :func:`b12x_repack_sources_inplace`, so this wraps them back into a
- ``W4A16PackedWeights`` and calls the prepared-weights launch directly -- the public
- ``b12x_fused_moe`` would re-prepare the raw modelopt weights (and ptr-cache them) on
- every call, which both costs a prepare per step and breaks for the offload cache,
- whose slot contents move between calls.
-
- CUDA-graph note: the launch resolves a (shape-keyed, module-cached) scratch workspace
- on first use and flashinfer raises if that happens *during* capture, so the decode
- path must be warmed once eagerly before graph capture (FreeToken already does)."""
- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import (
- _launch_sm120_w4a16_moe,
- )
- from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (
- W4A16PackedWeights,
- )
-
- assert activation == "silu", "b12x backend supports gated silu only"
- assert not apply_router_weight_on_input
- num_experts = gate_up_q.size(0)
- hidden_size = hidden_states.size(-1)
- # down bank is the prepared w2 == [E, K_tiles, ...] with K_tiles == intermediate//16.
- intermediate_size = down_q.size(1) * 16
- prepared = W4A16PackedWeights(
- w13=gate_up_q,
- w13_scale=gate_up_s,
- w13_global_scale=gate_up_alpha,
- w2=down_q,
- w2_scale=down_s,
- w2_global_scale=down_alpha,
- workspace=_b12x_small_workspace(hidden_states.device),
- hidden_size=hidden_size,
- intermediate_size=intermediate_size,
- num_experts=num_experts,
- is_gated=True,
- params_dtype=hidden_states.dtype,
- source_format="modelopt",
- )
- out = torch.empty(
- hidden_states.size(0), hidden_size, dtype=hidden_states.dtype, device=hidden_states.device
- )
- _launch_sm120_w4a16_moe(
- a=hidden_states,
- topk_ids=topk_ids,
- topk_weights=topk_weights,
- w1_weight=gate_up_q,
- w1_weight_sf=gate_up_s,
- w1_alpha=gate_up_alpha,
- w2_weight=down_q,
- w2_weight_sf=down_s,
- w2_alpha=down_alpha,
- num_experts=num_experts,
- top_k=topk_ids.size(1),
- num_local_experts=num_experts,
- scatter_output=out,
- activation="silu",
- source_format="modelopt",
- _prepared_weights=prepared,
- )
- return out
-
-
-__all__ = [
- "select_nvfp4_backend",
- "marlin_repack_layer",
- "marlin_repack_sources_inplace",
- "marlin_fused_experts",
- "b12x_repack_layer",
- "b12x_repack_sources_inplace",
- "b12x_fused_experts",
-]
diff --git a/python/freetoken/moe/offload.py b/python/freetoken/moe/offload.py
deleted file mode 100644
index 0e688a7d..00000000
--- a/python/freetoken/moe/offload.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from __future__ import annotations
-
-import torch
-
-from .base import BaseMoeBackend
-
-
-class OffloadMoeBackend(BaseMoeBackend):
- def forward(
- self,
- hidden_states: torch.Tensor,
- w1: torch.Tensor,
- w2: torch.Tensor,
- gating_output: torch.Tensor,
- topk: int,
- renormalize: bool,
- activation: str,
- apply_router_weight_on_input: bool,
- ) -> torch.Tensor:
- raise RuntimeError("Offload MoE is handled by OffloadMoELayer")
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index e1f20dd2..7abb9c80 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -141,18 +141,22 @@ class OffloadMoeCache:
# pcie_bw / cpu_bw ratio so the PCIe fetch and the CPU overflow GEMV take equal
# time (perfect overlap): fetched : cpu = pcie : cpu - pcie.
hybrid_fetch_fraction: float = 0.0
+ # bank layout from the expert kernel (a BankSpec per role); when given it replaces the _BANK_SCHEMAS lookup and the slot cap comes from max_slots
+ layout: dict | None = None
+ max_slots: int | None = None
def __post_init__(self) -> None:
policy_ids = {"lru": 0}
assert self.cache_policy in policy_ids
assert self.decode_target in ("gpu", "cpu", "hybrid"), self.decode_target
- assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"
+ if self.layout is None:
+ assert self.quant_format in _BANK_SCHEMAS, f"unknown quant_format {self.quant_format!r}"
# Attached by the engine for decode_target == "cpu" (CpuMoeExecutor); None
# for the GPU decode path.
self.cpu_executor = None
# MoE layer ids whose decode runs on the CPU executor; the rest use the GPU
# offload/PCIe path. Set by the engine after construction (empty = all-GPU,
- # all layers = the plain --moe-backend cpu case).
+ # all layers = the plain --moe-strategy cpu case).
self.cpu_layer_ids: frozenset = frozenset()
# num_experts floor + nvfp4_marlin slot cap, shared with the runtime-rebuild path.
self.validate_rebuild(self.cache_size)
@@ -199,7 +203,10 @@ class OffloadMoeCache:
# carry independent host attributes -- see layer_residency) and their GPU
# slot caches, keyed by the format's bank schema (attached by
# set_bank_sources). The GPU slot cache stays one unified pool per bank.
- self.bank_schema = _BANK_SCHEMAS[self.quant_format]
+ if self.layout is not None:
+ self.bank_schema = tuple(role for role, spec in self.layout.items() if not spec.resident)
+ else:
+ self.bank_schema = _BANK_SCHEMAS[self.quant_format]
self.bank_sources: dict[str, list[torch.Tensor]] = {}
self.bank_caches: dict[str, torch.Tensor] = {}
# per-layer host residency: the GPU movement paths require "pinned"; LOCKED/PAGEABLE layers decode on the CPU executor and prefill via copy_missing's pageable branch
@@ -294,18 +301,22 @@ class OffloadMoeCache:
per layer (independent allocations, so each layer can carry its own host
attributes); each slot cache mirrors the bank's row shape and dtype as one
unified GPU pool. The row layouts are produced by the weight loaders /
- repackers (see ``_BANK_SCHEMAS`` and :mod:`freetoken.moe.nvfp4_backends`)
+ repackers (see ``_BANK_SCHEMAS`` and :mod:`freetoken.layers.quantization.moe.nvfp4`)
-- the cache machinery is layout-agnostic and just moves rows.
``layer_residency`` labels each layer with a ``HostResidency`` value (default: all pinned).
Non-pinned (LOCKED/PAGEABLE) layers have no device address: they must already be routed to the CPU executor (``cpu_layer_ids``, set BEFORE this call), the copy plan skips their rows, and their only movement is ``copy_missing``'s whole-layer pageable prefill branch -- which is why prefill overlap is incompatible with them.
"""
+ from freetoken.moe.legacy_format import canonical_role
from freetoken.moe.host_banks import HostResidency
- assert set(sources) == set(self.bank_schema), (
- f"banks {sorted(sources)} do not match the {self.quant_format!r} "
- f"schema {self.bank_schema}"
- )
+ # loaders and FTW files may still name the banks the old way (gate_up_packed, ...)
+ by_role = {canonical_role(name): per_layer for name, per_layer in sources.items()}
+ if set(by_role) != {canonical_role(n) for n in self.bank_schema}:
+ raise AssertionError(
+ f"banks {sorted(sources)} do not match the {self.quant_format!r} schema {self.bank_schema}"
+ )
+ sources = {name: by_role[canonical_role(name)] for name in self.bank_schema}
residency = layer_residency or [HostResidency.PINNED.value] * self.num_layers
assert len(residency) == self.num_layers, (len(residency), self.num_layers)
unpinned = frozenset(
@@ -329,6 +340,13 @@ class OffloadMoeCache:
per_layer = sources[name]
assert len(per_layer) == self.num_layers, (name, len(per_layer))
head = per_layer[0]
+ if self.layout is not None:
+ spec = self.layout[name]
+ if tuple(head.shape[1:]) != tuple(spec.shape) or head.dtype != spec.dtype:
+ raise ValueError(
+ f"bank {name!r} rows are {tuple(head.shape[1:])} {head.dtype} but the expert kernel's layout "
+ f"wants {tuple(spec.shape)} {spec.dtype}; the banks were packed for another kernel"
+ )
for layer_id, source in enumerate(per_layer):
assert source.is_contiguous(), f"bank {name!r} layer {layer_id} must be contiguous"
assert source.size(0) == self.num_experts, (name, layer_id, source.shape)
@@ -432,11 +450,16 @@ class OffloadMoeCache:
"""
if cache_size < self.num_experts:
raise ValueError(f"cache_size {cache_size} < num_experts {self.num_experts}")
- if self.quant_format == "nvfp4_marlin" and cache_size > MARLIN_MAX_CACHE_SIZE:
+ if self.max_slots is not None and cache_size > self.max_slots:
+ raise ValueError(
+ f"moe_cache_size={cache_size} exceeds the expert kernel's slot limit of {self.max_slots}; "
+ f"pass --moe-cache-size {self.max_slots} or less, or let the default kernel serve the experts"
+ )
+ if self.layout is None and self.quant_format == "nvfp4_marlin" and cache_size > MARLIN_MAX_CACHE_SIZE:
raise ValueError(
f"moe_cache_size={cache_size} exceeds the marlin backend's slot limit of "
f"{MARLIN_MAX_CACHE_SIZE} (vLLM moe_align_block_size caps padded experts at "
- "1024); reduce moe_cache_size or force --nvfp4-backend triton"
+ "1024); reduce moe_cache_size or force --quant-backend moe.nvfp4=triton"
)
def rebuild(self, cache_size: int) -> None:
@@ -1033,14 +1056,6 @@ class OffloadMoeCache:
def iter_offload_moe_layers(model) -> Iterator:
from freetoken.layers import BaseOP, OffloadMoELayer
- # A model whose MoE blocks are bespoke nn.Modules (not OffloadMoELayer) declares its
- # offload layers explicitly via this hook (e.g. DeepSeek-V4-Flash); attach_offload_moe_cache
- # then sets .offload_cache on each yielded layer just like the OffloadMoELayer walk.
- hook = getattr(model, "_iter_offload_moe_layers", None)
- if hook is not None:
- yield from hook()
- return
-
if isinstance(model, OffloadMoELayer):
yield model
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index 5b4db587..32e8d126 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -10,6 +10,27 @@ from freetoken.distributed import DistributedInfo
from freetoken.scheduler import SchedulerConfig
from freetoken.utils import init_logger
+logger = init_logger(__name__)
+
+
+class _DeprecatedAlias(argparse.Action):
+ """An old flag: warns at parse time, converts the value if asked, stores it."""
+
+ def __init__(self, *args, new_flag: str, convert=None, **kwargs):
+ self.new_flag, self.convert = new_flag, convert
+ super().__init__(*args, **kwargs)
+
+ def __call__(self, parser, namespace, values, option_string=None):
+ logger.warning("%s is deprecated; use %s", option_string, self.new_flag)
+ setattr(namespace, self.dest, self.convert(values) if self.convert else values)
+
+
+def _nvfp4_entry(value: str) -> str:
+ """The --quant-backend entry an old --nvfp4-backend value stands for; auto stands for none."""
+ if value == "auto":
+ return ""
+ return "moe.nvfp4=" + {"flashinfer": "b12x"}.get(value, value)
+
@dataclass(frozen=True)
class ServerArgs(SchedulerConfig):
@@ -93,7 +114,16 @@ def parse_args(
"""
from freetoken.attention import validate_attn_backend
from freetoken.kvcache import SUPPORTED_CACHE_MANAGER
- from freetoken.moe import SUPPORTED_MOE_BACKENDS
+ from freetoken.moe import MOE_STRATEGIES
+
+ def _parse_quant_backend(value: str) -> str:
+ from freetoken.layers.quantization import QuantBackend
+
+ try:
+ QuantBackend.parse(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError(str(exc)) from None
+ return value
def _parse_moe_cache_rate(value: str) -> float:
try:
@@ -467,16 +497,37 @@ def parse_args(
)
parser.add_argument(
- "--moe-backend",
- default=ServerArgs.moe_backend,
- choices=["auto"] + SUPPORTED_MOE_BACKENDS.supported_names(),
+ "--moe-strategy",
+ default=ServerArgs.moe_strategy,
+ choices=["auto", *MOE_STRATEGIES],
help=(
- "The MoE backend to use. 'auto' resolves a MoE model to the offload family "
+ "How the routed experts are served. 'auto' resolves a MoE model to the offload family "
"(offload, or hybrid when a `ft bench bw` profile recommends it); resident "
"'fused' experts must be requested explicitly."
),
)
+ parser.add_argument(
+ "--moe-backend",
+ dest="moe_strategy",
+ action=_DeprecatedAlias,
+ new_flag="--moe-strategy",
+ default=argparse.SUPPRESS,
+ choices=["auto", *MOE_STRATEGIES],
+ help="[Deprecated] Use --moe-strategy.",
+ )
+
+ parser.add_argument(
+ "--quant-backend",
+ default=None,
+ type=_parse_quant_backend,
+ help=(
+ "Kernel per quantized layer type: comma-separated layer[.kind]=name entries, e.g. "
+ "'linear=marlin,moe=b12x' or 'moe.nvfp4=triton'. A layer-level entry applies to every "
+ "kind whose kernel table lists the name; unlisted tables stay automatic."
+ ),
+ )
+
parser.add_argument(
"--ple-backend",
default=ServerArgs.ple_backend,
@@ -489,13 +540,12 @@ def parse_args(
parser.add_argument(
"--nvfp4-backend",
- default=ServerArgs.nvfp4_backend,
+ action=_DeprecatedAlias,
+ new_flag="--quant-backend moe.nvfp4=<marlin|b12x|triton>",
+ convert=_nvfp4_entry,
+ default=argparse.SUPPRESS,
choices=["auto", "marlin", "flashinfer", "triton"],
- help=(
- "NVFP4 routed-expert GEMM backend (default: triton, the portable inline-dequant "
- "kernel). auto picks by GPU (marlin on sm80-99 + vLLM; flashinfer b12x on sm120+ "
- "& CUDA>=13; else triton). Force one to override; it fails loudly if it cannot run."
- ),
+ help="[Deprecated] Use --quant-backend moe.nvfp4=<marlin|b12x|triton> ('flashinfer' is b12x).",
)
parser.add_argument(
@@ -552,7 +602,7 @@ def parse_args(
type=int,
default=ServerArgs.moe_cpu_threads,
help=(
- "Number of CPU worker threads for --moe-backend cpu decode experts. "
+ "Number of CPU worker threads for --moe-strategy cpu decode experts. "
"0 = auto (physical cores)."
),
)
@@ -562,13 +612,15 @@ def parse_args(
type=str,
default=ServerArgs.moe_cpu_layers,
help=(
- "With --moe-backend offload/hybrid: which MoE layers compute on the "
+ "With --moe-strategy offload/hybrid: which MoE layers compute on the "
"CPU executor instead of the GPU offload/PCIe path (where CUDA pinning "
"is quota-capped, e.g. WSL, their banks are OS-locked instead of pinned). Explicit id list ('3,7,11'), a count ('8' = 8 "
- "layers evenly strided), or a fraction ('0.5'). Unset = automatic where "
- "CUDA pinning is quota-capped, e.g. WSL (locks just enough head+tail "
- "layers when the banks exceed the pin budget, none otherwise); '0' "
- "forces all layers on GPU."
+ "layers evenly strided), a fraction ('0.5'), or 'auto'. 'auto' is for Windows/WSL "
+ "only, where CUDA pinned memory is capped: it locks just enough head+tail layers "
+ "for the banks over the pin budget. Any value, 'auto' included, commits to CPU "
+ "decode before the model is built, so the expert format must have a CPU executor "
+ "path (bf16, nvfp4, mxfp4); do not pass it on Linux. Unset = every layer on the "
+ "GPU; a boot whose banks exceed a known pin budget stops and asks for this flag."
),
)
@@ -577,7 +629,7 @@ def parse_args(
type=int,
default=ServerArgs.moe_hybrid_max_fetch,
help=(
- "For --moe-backend hybrid: max experts fetched over PCIe per (layer, decode "
+ "For --moe-strategy hybrid: max experts fetched over PCIe per (layer, decode "
"step); the rest of that step's misses are computed on the CPU, overlapped. "
"-1 (default) = auto: fetch the benched pcie/cpu bandwidth fraction of each "
"step's misses (perfect overlap; needs an `ft bench bw` profile, else 1). "
@@ -661,6 +713,14 @@ def parse_args(
kwargs["max_running_req"] = 1
kwargs["silent_output"] = True
+ # the old flag stands in for one --quant-backend entry; next to the real flag it is a usage error
+ entry = kwargs.pop("nvfp4_backend", None)
+ if entry is not None:
+ if kwargs["quant_backend"] is not None:
+ parser.error("--nvfp4-backend cannot be combined with --quant-backend; write --quant-backend moe.nvfp4=... instead")
+ if entry:
+ kwargs["quant_backend"] = entry
+
if kwargs["model_path"].startswith("~"):
kwargs["model_path"] = os.path.expanduser(kwargs["model_path"])
@@ -681,14 +741,14 @@ def parse_args(
# sizing flag at all, default to --moe-cache-auto so a bare `ft serve <FTW MoE>` works
# out of the box (the scheduler resolves the size from free VRAM). Explicit
# size/rate/auto is preserved.
- from freetoken.moe import is_offload_moe_backend
+ from freetoken.moe import is_offload_moe_strategy
_no_cache_flag = (
kwargs["moe_cache_size"] == 0
and not kwargs["moe_cache_auto"]
and (kwargs["moe_cache_rate"] is None or kwargs["moe_cache_rate"] == 0)
)
- if is_offload_moe_backend(kwargs["moe_backend"]) and _no_cache_flag:
+ if is_offload_moe_strategy(kwargs["moe_strategy"]) and _no_cache_flag:
kwargs["moe_cache_auto"] = True
if kwargs["model_source"] == "modelscope":
@@ -726,6 +786,5 @@ def parse_args(
del kwargs["tensor_parallel_size"]
result = ServerArgs(**kwargs)
- logger = init_logger(__name__)
logger.info(f"Parsed arguments:\n{result}")
return result, run_shell
diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.py
index dbc38679..a28146e6 100644
--- a/python/freetoken/utils/hf.py
+++ b/python/freetoken/utils/hf.py
@@ -6,6 +6,7 @@ from typing import Any
from typing import FrozenSet
from huggingface_hub import hf_hub_download, snapshot_download
+from huggingface_hub.utils import EntryNotFoundError
from tqdm.asyncio import tqdm
from transformers import (
AutoConfig,
@@ -170,6 +171,17 @@ class RawConfigShim:
return json.loads(json.dumps(self._data)) # deep copy, callers may mutate
+def optional_hf_file(model_path: str, filename: str) -> str | None:
+ """Local path of ``filename`` in a checkpoint dir or Hub repo; None when the checkpoint has no such file."""
+ if os.path.isdir(model_path):
+ path = os.path.join(model_path, filename)
+ return path if os.path.isfile(path) else None
+ try:
+ return hf_hub_download(repo_id=model_path, filename=filename)
+ except EntryNotFoundError:
+ return None
+
+
def _raw_config_json(model_path: str) -> dict:
if os.path.isdir(model_path):
path = os.path.join(model_path, "config.json")
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 00000000..aa33bef8
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,10 @@
+import pytest
+
+
[email protected](autouse=True)
+def _default_quant_backend():
+ """Tests that install kernel requests must not leak them into the next test."""
+ yield
+ from freetoken.layers.quantization import QuantBackend, set_quant_backend
+
+ set_quant_backend(QuantBackend())
diff --git a/tests/e2e/test_aime.py b/tests/e2e/test_aime.py
index f2803539..3dde4cad 100644
--- a/tests/e2e/test_aime.py
+++ b/tests/e2e/test_aime.py
@@ -154,6 +154,16 @@ def recommended_sampling(model_dir: Path) -> tuple[SamplingParams, int]:
return sp, n_samples
+def format_chat(tokenizer, messages: list[dict]) -> str:
+ """The thinking-mode prompt: the checkpoint's chat template, or DeepSeek-V4's encoder module."""
+ from freetoken.tokenizer.tokenize import _apply_dsv4_chat_encoder, _load_dsv4_encoder_if_needed
+
+ encoder = _load_dsv4_encoder_if_needed(tokenizer)
+ if encoder is not None:
+ return _apply_dsv4_chat_encoder(encoder, messages, None, {"enable_thinking": True})
+ return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=True)
+
+
def build_llm(model_path: Path) -> LLM:
"""Resident by default; set FREETOKEN_TEST_MOE_CACHE_SIZE>0 to run an offload model
(fp8 Qwen3.5-MoE, GLM-4.7, MiniMax, ... -- routed experts that do not fit in HBM)."""
@@ -167,12 +177,15 @@ def build_llm(model_path: Path) -> LLM:
max_running_req=1,
max_extend_tokens=8192,
)
+ if quant_backend := os.environ.get("FREETOKEN_TEST_QUANT_BACKEND"):
+ kwargs["quant_backend"] = quant_backend # --quant-backend syntax, e.g. moe.nvfp4=b12x
cache_size = int(os.environ.get("FREETOKEN_TEST_MOE_CACHE_SIZE", "0"))
cache_auto = os.environ.get("FREETOKEN_TEST_MOE_CACHE_AUTO") == "1"
if cache_size > 0 or cache_auto:
kwargs.update(
- moe_backend=os.environ.get("FREETOKEN_TEST_MOE_BACKEND", "offload"),
+ moe_strategy=os.environ.get("FREETOKEN_TEST_MOE_STRATEGY", "offload"),
moe_cpu_threads=int(os.environ.get("FREETOKEN_TEST_MOE_CPU_THREADS", "0")),
+ moe_cpu_layers=os.environ.get("FREETOKEN_TEST_MOE_CPU_LAYERS"),
moe_cache_auto=cache_auto,
moe_cache_size=cache_size,
kv_reserve_tokens=int(os.environ.get("FREETOKEN_TEST_KV_RESERVE", "8192")),
@@ -251,13 +264,7 @@ def test_aime():
for idx in selected:
row = rows[idx]
prompt, expected = row["prompt"], str(row["answer"])
- messages = [{"role": "user", "content": prompt}]
- formatted_prompt = tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=True,
- enable_thinking=True,
- )
+ formatted_prompt = format_chat(tokenizer, [{"role": "user", "content": prompt}])
# Decode at the model's recommended params and check pass@N: the expected answer must
# appear in at least one of N samples (single greedy/sampled runs are brittle -- see
# recommended_sampling docstring).
diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py
index 9ac2a4f4..a4e47aa0 100644
--- a/tests/engine/test_cache_budget.py
+++ b/tests/engine/test_cache_budget.py
@@ -100,24 +100,24 @@ def test_expert_bytes_per_slot_sums_row_bytes_over_banks():
assert expert_bytes_per_slot(sources) == 512 + 256
-def test_resolve_auto_applies_ratio_once_and_marlin_cap():
+def test_resolve_auto_applies_ratio_once():
# baseline 1000, weights 100, ratio 0.9 -> budget = 900 - 100 - 0(fixed) = 800
size, pages, overlap = resolve_moe_cache_auto(
baseline_free=1000, weights_bytes=100, memory_ratio=0.9,
cache_per_page=10, fixed_cache_size=0, per_expert_bytes=50,
num_experts=4, total_experts=8, prefill_overlap=True,
- kv_reserve_tokens=0, page_size=1, quant_format="bf16",
+ kv_reserve_tokens=0, page_size=1,
)
# budget 800: experts cap at 8 -> 400 bytes; KV = 400//10 = 40 pages
assert size == 8 and pages == 40 and overlap is True
-def test_resolve_auto_marlin_caps_slots():
+def test_resolve_auto_caps_slots_at_the_kernel_limit():
size, _, _ = resolve_moe_cache_auto(
baseline_free=10_000_000, weights_bytes=0, memory_ratio=1.0,
cache_per_page=10, fixed_cache_size=0, per_expert_bytes=100,
num_experts=128, total_experts=4000, prefill_overlap=False,
- kv_reserve_tokens=0, page_size=1, quant_format="nvfp4_marlin",
+ kv_reserve_tokens=0, page_size=1, max_slots=992,
)
assert size == 992
@@ -136,7 +136,7 @@ def _dsv4_adjust_cfg(**over):
moe_cache_auto = True
moe_cache_size = 0
moe_cache_rate = None
- moe_backend = "offload"
+ moe_strategy = "offload"
max_running_req = 1
cuda_graph_max_bs = 1
cuda_graph_bs = [1]
@@ -145,7 +145,6 @@ def _dsv4_adjust_cfg(**over):
page_size = 1
attention_backend = "dsv4_sparse"
moe_cpu_layers = None
- nvfp4_backend = "auto"
num_page_override = None
num_token_override = None
@@ -166,7 +165,7 @@ def test_adjust_config_allows_auto_for_dsv4():
cfg = _dsv4_adjust_cfg()
_adjust_config(cfg) # must not raise
- assert cfg.moe_backend == "offload"
+ assert cfg.moe_strategy == "offload"
assert cfg.moe_cache_auto is True # resolved later at engine init, not here
assert cfg.page_size == 128 # DSV4's KV page is the P-token window page
@@ -212,14 +211,13 @@ def test_adjust_config_resolves_num_tokens_generic():
moe_cache_auto = False
moe_cache_size = 0
moe_cache_rate = None
- moe_backend = "auto"
+ moe_strategy = "auto"
max_running_req = 4
cuda_graph_max_bs = 2
cuda_graph_bs = [1, 2]
max_seq_len = 1024
page_size = 1
attention_backend = "fi"
- nvfp4_backend = "auto"
num_page_override = None
num_token_override = 5000
@@ -303,7 +301,6 @@ def test_engine_resolve_auto_moe_cache_size_maps_kwargs():
size = 1
class StubBanks:
- quant_format = "bf16"
# 2 layers (num_moe_layers above) x 4 experts each -- per-layer host bank contract.
sources = {
"gate_up": [torch.zeros(4, 32, 8, dtype=torch.float16)] * 2, # row = 32*8*2 = 512
@@ -329,10 +326,17 @@ def test_engine_resolve_auto_moe_cache_size_maps_kwargs():
cache_per_page=cache_per_page, fixed_cache_size=fixed,
per_expert_bytes=expert_bytes_per_slot(StubBanks.sources),
num_experts=4, total_experts=8, prefill_overlap=True,
- kv_reserve_tokens=0, page_size=16, quant_format="bf16",
+ kv_reserve_tokens=0, page_size=16,
)
assert (size, pages, overlap) == expected
+ class StubMethod:
+ def slot_limit(self):
+ return 5
+
+ size, _, _ = engine._resolve_auto_moe_cache_size(StubConfig(), StubBanks(), StubMethod())
+ assert size == 5
+
# ---------------------------------------------------------------------------
# offload-cache sizing guard + auto-resolution (_require_offload_cache_size / _adjust_config),
@@ -342,7 +346,7 @@ def test_engine_resolve_auto_moe_cache_size_maps_kwargs():
def _offload_engine_config(**overrides):
"""A frozen EngineConfig for a quantized-experts MoE checkpoint in the bare-invocation state
- (moe_backend="auto") unless overridden — the shared fixture for the _adjust_config tests."""
+ (moe_strategy="auto") unless overridden — the shared fixture for the _adjust_config tests."""
from freetoken.distributed import DistributedInfo
from freetoken.engine.config import EngineConfig
@@ -364,7 +368,7 @@ def _offload_engine_config(**overrides):
num_moe_layers=10,
num_experts=8,
expert_quant="nvfp4", # quantized experts -> must resolve to an offload backend
- moe_backend="auto",
+ moe_strategy="auto",
),
)
return config
@@ -389,7 +393,7 @@ def test_adjust_config_defaults_moe_cache_auto_for_auto_resolved_offload_backend
"""Bare `ft serve <FTW MoE checkpoint>`: no --moe-backend, no --moe-cache-* flags at all.
args.py's parse-time default only fires when the backend is *already*
- offload-family at parse time -- but a bare invocation leaves moe_backend="auto" at parse
+ offload-family at parse time -- but a bare invocation leaves moe_strategy="auto" at parse
time, and the "auto" -> offload/cpu/hybrid resolution only happens later, in _adjust_config,
once the model_config (and its expert_quant) is known. This proves the engine-level
resolution: a quantized-experts model auto-resolving to an offload-family backend also gets
@@ -397,7 +401,7 @@ def test_adjust_config_defaults_moe_cache_auto_for_auto_resolved_offload_backend
reached with moe_cache_size still 0.
"""
from freetoken.engine.engine import _adjust_config
- from freetoken.moe import is_offload_moe_backend
+ from freetoken.moe import is_offload_moe_strategy
config = _offload_engine_config()
_adjust_config(config)
@@ -405,7 +409,7 @@ def test_adjust_config_defaults_moe_cache_auto_for_auto_resolved_offload_backend
# Which member of the family gets picked is not this test's claim, and is not ours to
# decide: a bare "auto" consults ~/.cache/freetoken/benchbw.json, so a box that has run
# `ft bench bw` resolves nvfp4 experts to hybrid instead. Assert the family, not the member.
- assert is_offload_moe_backend(config.moe_backend)
+ assert is_offload_moe_strategy(config.moe_strategy)
assert config.moe_cache_auto is True
assert config.moe_cache_size == 0 # still unresolved -- the scheduler sizes it from VRAM
@@ -439,14 +443,13 @@ def _generic_rotary_cfg(max_position, override):
moe_cache_auto = False
moe_cache_size = 0
moe_cache_rate = None
- moe_backend = "auto"
+ moe_strategy = "auto"
max_running_req = 4
cuda_graph_max_bs = 2
cuda_graph_bs = [1, 2]
max_seq_len = 1024
page_size = 1
attention_backend = "triton"
- nvfp4_backend = "auto"
num_page_override = None
num_token_override = None
max_seq_len_override = None
diff --git a/tests/engine/test_moe_cpu_layers.py b/tests/engine/test_moe_cpu_layers.py
index dee36a11..feb19300 100644
--- a/tests/engine/test_moe_cpu_layers.py
+++ b/tests/engine/test_moe_cpu_layers.py
@@ -46,7 +46,7 @@ def test_out_of_range_raises(spec):
def _cfg(backend, spec=None):
- return SimpleNamespace(moe_backend=backend, moe_cpu_layers=spec)
+ return SimpleNamespace(moe_strategy=backend, moe_cpu_layers=spec)
def test_resolve_backend_dispatch():
diff --git a/tests/kernels/test_fp8_blockscale_moe.py b/tests/kernels/test_fp8_blockscale_moe.py
new file mode 100644
index 00000000..d776104f
--- /dev/null
+++ b/tests/kernels/test_fp8_blockscale_moe.py
@@ -0,0 +1,74 @@
+"""The block-fp8 MoE kernels against a torch reference, for each gate-up epilogue they accept."""
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+E, H, I, TOPK, M = 4, 256, 256, 2, 8
+BLOCK = 128
+
+
+def _quant_block(w):
+ n, k = w.shape
+ blocks = w.float().view(n // BLOCK, BLOCK, k // BLOCK, BLOCK)
+ scale = blocks.abs().amax(dim=(1, 3), keepdim=True).clamp(min=1e-12) / 448.0
+ q = (blocks / scale).to(torch.float8_e4m3fn).view(n, k)
+ return q, scale.squeeze(1).squeeze(-1).to(torch.bfloat16)
+
+
+def _dequant(q, s):
+ n, k = q.shape
+ return (q.float().view(n // BLOCK, BLOCK, k // BLOCK, BLOCK) * s.float()[:, None, :, None]).view(n, k)
+
+
+def _experts():
+ torch.manual_seed(0)
+ gu = [_quant_block(torch.randn(2 * I, H, device="cuda") / H**0.5) for _ in range(E)]
+ dn = [_quant_block(torch.randn(H, I, device="cuda") / I**0.5) for _ in range(E)]
+ stack = lambda xs, i: torch.stack([x[i] for x in xs]).contiguous()
+ return stack(gu, 0), stack(gu, 1), stack(dn, 0), stack(dn, 1)
+
+
+def _routing():
+ ids = torch.stack([torch.randperm(E, device="cuda")[:TOPK] for _ in range(M)]).to(torch.int32)
+ w = torch.softmax(torch.randn(M, TOPK, device="cuda"), dim=-1)
+ return w, ids
+
+
+def _reference(x, gate_up, gate_up_scale, down, down_scale, w, ids, activation, alpha, limit):
+ from freetoken.layers import gated_act_and_mul
+
+ out = torch.zeros(M, H, device="cuda", dtype=torch.float32)
+ for e in range(E):
+ gu = _dequant(gate_up[e], gate_up_scale[e])
+ dn = _dequant(down[e], down_scale[e])
+ for t, k in zip(*torch.nonzero(ids == e, as_tuple=True)):
+ h = (x[t].float() @ gu.T).to(x.dtype).view(1, -1)
+ a = torch.empty(1, I, device="cuda", dtype=x.dtype)
+ gated_act_and_mul(activation, h, a, alpha=alpha, limit=limit)
+ out[t] += w[t, k] * (a[0].float() @ dn.T)
+ return out.to(x.dtype)
+
+
[email protected]("activation, alpha, limit", [("silu", 1.0, float("inf")), ("swiglu_clamp", 1.0, 0.5), ("gelu_tanh", 1.0, float("inf"))])
+def test_fp8_block_moe_epilogues_match_the_reference(activation, alpha, limit):
+ from freetoken.kernel.triton.fp8_blockscale_moe import fused_experts_decode_fp8_blockscale, fused_experts_fp8_blockscale
+
+ gate_up, gate_up_scale, down, down_scale = _experts()
+ w, ids = _routing()
+ x = torch.randn(M, H, device="cuda", dtype=torch.bfloat16)
+ ref = _reference(x, gate_up, gate_up_scale, down, down_scale, w, ids, activation, alpha, limit).float()
+ plain = _reference(x, gate_up, gate_up_scale, down, down_scale, w, ids, "silu", 1.0, float("inf")).float()
+ if activation != "silu":
+ assert not torch.allclose(ref, plain, rtol=1e-2, atol=1e-3), "the epilogue under test must change the result"
+
+ decode = fused_experts_decode_fp8_blockscale(x, gate_up, gate_up_scale, down, down_scale, w, ids, activation, alpha, limit).float()
+ # W8A16 decode: only bf16 accumulation-order differences remain
+ assert torch.nn.functional.cosine_similarity(decode.flatten(), ref.flatten(), dim=0) > 0.999
+ assert (decode - ref).abs().max() <= 2e-2 * ref.abs().max() + 1e-3
+
+ prefill = fused_experts_fp8_blockscale(x, gate_up, gate_up_scale, down, down_scale, w, ids, E, activation, alpha, limit).float()
+ # W8A8 prefill quantizes the activations per 128-group, so the tolerance is the fp8 activation error
+ assert torch.nn.functional.cosine_similarity(prefill.flatten(), ref.flatten(), dim=0) > 0.99
+ assert (prefill - ref).abs().max() <= 8e-2 * ref.abs().max() + 1e-3
diff --git a/tests/kernels/test_fp8_pertensor_linear.py b/tests/kernels/test_fp8_pertensor_linear.py
index 441da408..b03ed374 100644
--- a/tests/kernels/test_fp8_pertensor_linear.py
+++ b/tests/kernels/test_fp8_pertensor_linear.py
@@ -105,32 +105,6 @@ def test_batch_size_does_not_change_the_numeric_scheme():
assert torch.equal(alone, batched[:1])
-def test_layer_load_marks_uniform_scale_and_optional_input_scale():
- from freetoken.kernel.triton.fp8_pertensor_linear import (
- Fp8PerTensorColMerged,
- Fp8PerTensorLinear,
- )
-
- K = 512
- w8, scale = _quant_parts([256, 64, 64], K)
- merged = Fp8PerTensorColMerged(K, [256, 64, 64])
- merged.load_state_dict({
- "weight": w8, "weight_scale": scale, "input_scale": torch.tensor(0.01, device=DEV),
- })
- assert merged._uniform_scale is False
- assert merged.input_scale is not None
-
- single = Fp8PerTensorLinear(K, 384)
- flat = scale[:1].expand(384).contiguous()
- single.load_state_dict({"weight": w8, "weight_scale": flat}) # no input_scale
- assert single._uniform_scale is True
- assert single.input_scale is None
-
- # a reload must not trip over the input_scale it kept from the first load
- single.load_state_dict({"weight": w8, "weight_scale": flat})
- assert single.input_scale is None
-
-
@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+")
@pytest.mark.parametrize("M", [1, 4, 64, 300])
def test_per_part_path_matches_rowwise(M: int, monkeypatch):
@@ -160,16 +134,19 @@ def test_fused_layer_forward_on_a_side_stream_completes():
stall fails the test instead of hanging the session."""
script = textwrap.dedent("""
import os, time, torch
- from freetoken.kernel.triton.fp8_pertensor_linear import FP8, Fp8PerTensorColMerged
+ from freetoken.kernel.triton.fp8_pertensor_linear import (
+ FP8, fp8_pertensor_linear, rowwise_scaled_mm_ok, weight_scale_segments,
+ )
torch.manual_seed(0)
K, parts = 2048, [8192, 512, 512] # a prefill-sized fused qkv
w8 = (torch.randn(sum(parts), K, device="cuda") * 8).clamp(-448, 448).to(FP8)
scale = torch.cat([torch.full((p,), 0.01 * (i + 1), device="cuda")
for i, p in enumerate(parts)])
- layer = Fp8PerTensorColMerged(K, parts)
- layer.load_state_dict({"weight": w8, "weight_scale": scale,
- "input_scale": torch.tensor(0.02, device="cuda")})
+ input_scale = torch.tensor(0.02, device="cuda")
+ # same load-time decisions the torch fp8_tensor kernel makes in finalize
+ segments = weight_scale_segments(scale)
+ rowwise_scaled_mm_ok()
x = torch.randn(2010, K, device="cuda", dtype=torch.bfloat16) # #182 shape
torch.cuda.synchronize()
@@ -177,7 +154,7 @@ def test_fused_layer_forward_on_a_side_stream_completes():
events = []
with torch.cuda.stream(stream):
for _ in range(128):
- layer.forward(x)
+ fp8_pertensor_linear(x, w8, scale, None, input_scale, False, scale_segments=segments)
ev = torch.cuda.Event()
ev.record(stream)
events.append(ev)
diff --git a/tests/kernels/test_mxfp8_linear.py b/tests/kernels/test_mxfp8_linear.py
index 1f1557d4..f762cb38 100644
--- a/tests/kernels/test_mxfp8_linear.py
+++ b/tests/kernels/test_mxfp8_linear.py
@@ -42,18 +42,6 @@ def test_mxfp8_linear_matches_dequant_reference(M: int, N: int, K: int):
assert rel.item() < 2e-2, rel.item()
-def test_mxfp8_module_shapes_and_forward():
- from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear
-
- lin = Mxfp8Linear(6144, 512)
- assert lin.weight.shape == (512, 6144) and lin.weight.dtype == torch.float8_e4m3fn
- assert lin.weight_scale_inv.shape == (512, 192) and lin.weight_scale_inv.dtype == torch.uint8
- w8, codes = _make_mxfp8(512, 6144, seed=42)
- lin.weight, lin.weight_scale_inv = w8, codes
- x = torch.randn(3, 6144, device=DEV, dtype=torch.bfloat16)
- assert lin.forward(x).shape == (3, 512)
-
-
def test_gemma_plus_one_norm_matches_flashinfer_semantics():
"""Triton fallback vs the (1+w) definition; per-head 3D strided in-place."""
from freetoken.kernel.triton.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm
diff --git a/tests/models/qwen4_exp/test_skeleton.py b/tests/models/qwen4_exp/test_skeleton.py
index 00f7c43f..fa75d6ee 100644
--- a/tests/models/qwen4_exp/test_skeleton.py
+++ b/tests/models/qwen4_exp/test_skeleton.py
@@ -367,7 +367,7 @@ def test_qsa_layer_matches_hf_dense():
class _StubLinearMixer(BaseOP):
"""Stands in for the GDN layer; same [T, hidden] -> [T, hidden] shape."""
- def __init__(self, config, layer_id):
+ def __init__(self, config, layer_id, prefix=""):
self.out_proj = LinearReplicated(config.hidden_size, config.hidden_size, has_bias=False)
def forward(self, x):
@@ -379,7 +379,6 @@ def test_shared_expert_gate_fusion_matches_eager():
"""Qwen4ExpMoE only swaps qwen3_5's gemv+sigmoid+mul+add gate chain for two triton kernels."""
from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE
from freetoken.models.qwen4_exp.moe import Qwen4ExpMoE
- from freetoken.moe.fused import FusedMoe
from freetoken.utils.torch_utils import torch_dtype
config = _config()
@@ -387,7 +386,7 @@ def test_shared_expert_gate_fusion_matches_eager():
with torch.device(device), torch_dtype(dtype):
moe = Qwen4ExpMoE(config, 0)
_fill(moe, torch.Generator(device=device).manual_seed(21), scale=0.2)
- _fresh_ctx(moe_backend=FusedMoe())
+ _fresh_ctx(_batch=SimpleNamespace(is_prefill=True))
x = torch.randn(6, config.hidden_size, device=device, dtype=dtype) * 0.5
fused = moe.forward(x.clone())
@@ -433,7 +432,6 @@ def test_decoder_stack_prefill_and_decode(monkeypatch):
from freetoken.models.qwen4_exp import model as model_module
from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference
from freetoken.models.qwen4_exp.ple import GpuResidentTable
- from freetoken.moe.fused import FusedMoe
from freetoken.utils.torch_utils import torch_dtype
torch.manual_seed(8)
@@ -462,7 +460,6 @@ def test_decoder_stack_prefill_and_decode(monkeypatch):
prompts = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12], [9, 10, 11, 12, 13]]
ctx = _fresh_ctx(
attn_backend=TorchDenseQSAReference(config, num_slots, max_len, device, dtype),
- moe_backend=FusedMoe(),
linear_state_pool=pool,
)
reqs = [
diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.py
index 747d22f2..82f9a01d 100644
--- a/tests/models/qwen4_exp/test_weight_ckpt.py
+++ b/tests/models/qwen4_exp/test_weight_ckpt.py
@@ -19,7 +19,7 @@ import torch
from freetoken.distributed import set_tp_info, try_get_tp_info
from freetoken.kernel.aot_models import expert_bank_row_bytes
-from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks
+from freetoken.models.nvfp4_banks import iter_nvfp4_expert_pieces
from freetoken.models.qwen4_exp.config import parse_config
from freetoken.models.qwen4_exp.weight import (
_NVFP4_SOURCE_SPEC,
@@ -27,6 +27,7 @@ from freetoken.models.qwen4_exp.weight import (
iter_weights,
load_ple_table,
)
+from freetoken.moe.expert_banks import build_expert_banks
from freetoken.moe.host_banks import HostResidency
from freetoken.utils import cached_load_hf_config
@@ -287,9 +288,23 @@ def layer0_expert_banks():
)
config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,
num_moe_layers=1)
- return load_nvfp4_expert_source_banks(
+ layer = _triton_layer()
+ pieces = iter_nvfp4_expert_pieces(
MODEL_PATH, config, spec, drop_page_cache=lambda path: None, primary=False
)
+ return build_expert_banks(layer.quant_method, 1, pieces, device=torch.device("cuda")).sources
+
+
+def _triton_layer():
+ """An offload NVFP4 layer bound to the Triton kernel, whose banks are the native rows."""
+ from freetoken.layers.moe import OffloadMoELayer
+ from freetoken.layers.quantization import QuantBackend, QuantConfig, set_quant_backend
+
+ if try_get_tp_info() is None:
+ set_tp_info(0, 1)
+ set_quant_backend(QuantBackend.parse("moe.nvfp4=triton"))
+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})
+ return OffloadMoELayer(0, E, 10, H, I, quant_config=quant, prefix="model.layers.0.mlp.experts")
@pytest.mark.slow
@@ -297,11 +312,11 @@ def test_sampled_experts_match_the_checkpoint(layer0_expert_banks, reader):
banks = layer0_expert_banks
for expert in random.Random(1).sample(range(E), 8):
base = f"{LM}.layers.0.mlp.experts.{expert}"
- assert torch.equal(banks["gate_up_packed"][0][expert, :I],
+ assert torch.equal(banks["gate_up"][0][expert, :I],
reader.get(f"{base}.gate_proj.weight"))
- assert torch.equal(banks["gate_up_packed"][0][expert, I:],
+ assert torch.equal(banks["gate_up"][0][expert, I:],
reader.get(f"{base}.up_proj.weight"))
- assert torch.equal(banks["down_packed"][0][expert],
+ assert torch.equal(banks["down"][0][expert],
reader.get(f"{base}.down_proj.weight"))
for proj, bank, rows in (("gate_proj", "gate_up_scale", slice(0, I)),
("up_proj", "gate_up_scale", slice(I, 2 * I)),
@@ -320,18 +335,9 @@ def test_expert_bank_bytes_match_the_aot_row_table(layer0_expert_banks):
@pytest.mark.skipif(not torch.cuda.is_available(), reason="dummy banks are pinned")
-def test_dummy_expert_sources_have_the_real_bank_shapes(layer0_expert_banks):
- from freetoken.models.weight import _model_override, dummy_nvfp4_expert_sources
- from freetoken.models.register import get_model_spec
-
- spec = get_model_spec("Qwen4ExpForConditionalGeneration")
- # No dummy_* override, so --use-dummy-weight goes through the generic builders.
- for hook in ("dummy_nvfp4_expert_sources", "dummy_moe_expert_sources", "dummy_q4_0_expert_sources"):
- assert _model_override(spec, hook) is None
-
- config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,
- num_moe_layers=1)
- dummy = dummy_nvfp4_expert_sources(config)
+def test_dummy_expert_banks_have_the_real_bank_shapes(layer0_expert_banks):
+ layer = _triton_layer()
+ dummy = build_expert_banks(layer.quant_method, 1, None, device=torch.device("cuda"), dummy=True).sources
assert set(dummy) == set(layer0_expert_banks)
for name, banks in dummy.items():
assert banks[0].shape == layer0_expert_banks[name][0].shape
diff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.py
index 8f0bc518..54060e16 100644
--- a/tests/models/test_glm5_next_config.py
+++ b/tests/models/test_glm5_next_config.py
@@ -202,8 +202,7 @@ def test_moe_and_scalars():
assert cfg.attn_sm_scale == pytest.approx(256**-0.5)
assert cfg.num_moe_layers == _NUM_LAYERS - 3
assert cfg.is_moe
- # Checkpoint-faithful default; the FREETOKEN_GLM5_*_FP8 env flags opt into
- # the W8A16 fp8 load.
+ # checkpoint-faithful: the resident projections follow the quantization_config, never a switch
assert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3
# Text-only serving: the vision tower is never built.
assert cfg.vision_config is None
diff --git a/tests/models/test_glm5_next_kda_op.py b/tests/models/test_glm5_next_kda_op.py
index 11adbf11..588e5e00 100644
--- a/tests/models/test_glm5_next_kda_op.py
+++ b/tests/models/test_glm5_next_kda_op.py
@@ -53,7 +53,7 @@ def _make_args():
def _make_op(seed=0):
from freetoken.models.glm5_next.kda import Glm5NextKDA
- cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none")
+ cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none", quant=None)
op = Glm5NextKDA(cfg, layer_id=0)
torch.manual_seed(seed)
dev, dt = "cuda", torch.bfloat16
diff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.py
index 9cd8c7fb..1062261e 100644
--- a/tests/models/test_minimax_m3.py
+++ b/tests/models/test_minimax_m3.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import pytest
import torch
+from freetoken.layers.quantization import MoEConfig
from freetoken.attention.base import AttnType
from freetoken.models.minimax_m3.config import parse_config
@@ -148,7 +149,7 @@ def test_parse_config_full_model(monkeypatch):
assert cfg.hidden_act_alpha == pytest.approx(1.702)
assert cfg.swiglu_limit == pytest.approx(7.0)
assert cfg.expert_quant == "nvfp4"
- # Default env: MXFP8 dense native, bf16 lm_head.
+ # quantized release: MXFP8 dense native, bf16 lm_head.
assert cfg.attn_quant == "mxfp8" and cfg.dense_quant == "mxfp8"
assert cfg.lm_head_quant == "none"
@@ -212,13 +213,6 @@ def test_sparse_ablation_env(monkeypatch):
assert resolve_pool_class(cfg) is MHAKVCache
-def test_mxfp8_ablation_env(monkeypatch):
- monkeypatch.setenv("FREETOKEN_M3_ATTN_MXFP8", "0")
- monkeypatch.setenv("FREETOKEN_M3_MLP_MXFP8", "0")
- cfg = parse_config(_hf_config())
- assert cfg.attn_quant == "none" and cfg.dense_quant == "none"
-
-
def test_registry_resolves_both_architectures():
from freetoken.models.register import get_model_spec
@@ -238,13 +232,19 @@ def test_auto_backend_resolution():
assert attention_backend_info("m3_sparse").page_sizes == (128,)
-def test_nvfp4_backend_restricted_to_triton_for_swigluoai():
- from freetoken.moe.nvfp4_backends import select_nvfp4_backend
+def test_nvfp4_experts_restricted_to_triton_for_swigluoai(monkeypatch):
+ """The borrowed marlin / b12x kernels hard-code silu; swigluoai experts land on the Triton kernels."""
+ from freetoken.kernel import backend
+ from freetoken.layers.quantization import KernelSelectionError, ModelOptConfig, select_kernel
+ from freetoken.layers.quantization.moe import Nvfp4MoEMethod
- dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- assert select_nvfp4_backend(dev, 3072, "auto", activation="swigluoai") == "triton"
- with pytest.raises(RuntimeError):
- select_nvfp4_backend(dev, 3072, "marlin", activation="swigluoai")
+ monkeypatch.setattr(backend, "device_capability", lambda: (12, 0))
+ monkeypatch.setattr(backend, "is_vllm_installed", lambda: True)
+ monkeypatch.setattr(backend, "is_flashinfer_installed", lambda: True)
+ cfg = MoEConfig(num_experts=256, hidden=6144, intermediate=3072, top_k=4, scheme=ModelOptConfig.SCHEMES["NVFP4"], strategy="offload", activation="swigluoai", alpha=1.702, limit=7.0)
+ assert select_kernel(Nvfp4MoEMethod.candidates, "auto", cfg).name == "triton"
+ with pytest.raises(KernelSelectionError):
+ select_kernel(Nvfp4MoEMethod.candidates, "marlin", cfg)
def test_expert_source_spec_layer_to_bank():
diff --git a/tests/models/test_models_loader.py b/tests/models/test_models_loader.py
index 7254739c..b0ac2b13 100644
--- a/tests/models/test_models_loader.py
+++ b/tests/models/test_models_loader.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import re
from types import SimpleNamespace
+import pytest
import torch
@@ -181,8 +182,8 @@ def test_stack_expert_tensors_after_all_experts_arrive():
assert packed[0][1][1].tolist() == [[11.0, 11.0]]
-def test_stream_moe_expert_sources_writes_layers_into_final_banks():
- from freetoken.models.loader import stream_moe_expert_sources
+def test_stacked_expert_pieces_pair_each_layer_in_arrival_order():
+ from freetoken.moe.expert_pieces import stacked_expert_pieces
config = SimpleNamespace(num_layers=2, num_experts=2)
tensors = [
@@ -192,16 +193,11 @@ def test_stream_moe_expert_sources_writes_layers_into_final_banks():
("model.layers.0.mlp.experts.down_proj", torch.full((2, 4, 3), 10.0)),
]
- gate_up_source, down_source = stream_moe_expert_sources(
- tensors,
- config,
- dtype=torch.bfloat16,
- )
+ pieces = list(stacked_expert_pieces(tensors, config))
- assert len(gate_up_source) == 2 and all(t.shape == (2, 3, 4) for t in gate_up_source)
- assert len(down_source) == 2 and all(t.shape == (2, 4, 3) for t in down_source)
- assert all(t.dtype == torch.bfloat16 for t in gate_up_source + down_source)
- torch.testing.assert_close(gate_up_source[0], torch.full_like(gate_up_source[0], 2.0))
- torch.testing.assert_close(gate_up_source[1], torch.full_like(gate_up_source[1], 3.0))
- torch.testing.assert_close(down_source[0], torch.full_like(down_source[0], 10.0))
- torch.testing.assert_close(down_source[1], torch.full_like(down_source[1], 11.0))
+ assert [(layer, e0, e1) for layer, e0, e1, _ in pieces] == [(1, 0, 2), (0, 0, 2)]
+ assert torch.equal(pieces[0][3]["gate_up"], torch.full((2, 3, 4), 3.0))
+ assert torch.equal(pieces[0][3]["down"], torch.full((2, 4, 3), 11.0))
+ assert torch.equal(pieces[1][3]["gate_up"], torch.full((2, 3, 4), 2.0))
+ with pytest.raises(ValueError, match="Missing MoE expert source layers"):
+ list(stacked_expert_pieces(tensors[:3], config))
diff --git a/tests/models/test_muse_glimmer.py b/tests/models/test_muse_glimmer.py
index cbd579e4..e9b51ce6 100644
--- a/tests/models/test_muse_glimmer.py
+++ b/tests/models/test_muse_glimmer.py
@@ -59,6 +59,7 @@ def _hf_config(num_layers: int = 52, quantized: bool = False) -> _Cfg:
data["quantization_config"] = {
"quant_method": "compressed-tensors",
"format": "nvfp4-pack-quantized",
+ "ignore": ["lm_head"],
"config_groups": {
"group_0": {
"format": "nvfp4-pack-quantized",
@@ -447,16 +448,22 @@ def test_model_state_dict_matches_loader_keys():
assert layers[3].self_attn.attn_spec.sliding_window is None
assert layers[0].self_attn.attn_spec.sm_scale == pytest.approx(3.87 * 128**-0.5)
- # NVFP4 build swaps every text Linear for the W4A16 kernels.
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear
+ # NVFP4 build: every text Linear the config targets gets the W4A16 method.
+ from freetoken.layers.quantization import NameMap, QuantConfig
+ from freetoken.layers.quantization.linear import Nvfp4LinearMethod
+ from freetoken.layers.quantization.linear import UnquantizedLinearMethod
+ from freetoken.models.register import get_model_spec
- qcfg = parse_config(_hf_config(num_layers=4, quantized=True))
+ hf_q = _hf_config(num_layers=4, quantized=True)
+ qcfg = parse_config(hf_q)
+ spec = get_model_spec("MuseGlimmerForConditionalGeneration")
+ object.__setattr__(qcfg, "quant", QuantConfig.from_hf(
+ hf_q, name_map=NameMap(roots=spec.checkpoint_roots, packed=spec.packed_modules_mapping)
+ ))
qmodel = MuseGlimmerForCausalLM(qcfg)
attn = qmodel.model.layers.op_list[0].self_attn
mlp = qmodel.model.layers.op_list[0].mlp
- assert isinstance(attn.qkvg_proj, Nvfp4DenseColMerged)
- assert isinstance(attn.o_proj, Nvfp4DenseLinear)
- assert isinstance(mlp.gate_up_proj, Nvfp4DenseColMerged)
- assert isinstance(mlp.down_proj, Nvfp4DenseLinear)
- assert type(qmodel.lm_head).__name__ == "ParallelLMHead" # lm_head stays bf16
+ for proj in (attn.qkvg_proj, attn.o_proj, mlp.gate_up_proj, mlp.down_proj):
+ assert type(proj.quant_method) is Nvfp4LinearMethod
+ assert type(qmodel.lm_head.quant_method) is UnquantizedLinearMethod # lm_head stays bf16
del model, qmodel, torch
diff --git a/tests/models/test_quant_config.py b/tests/models/test_quant_config.py
new file mode 100644
index 00000000..b0043cde
--- /dev/null
+++ b/tests/models/test_quant_config.py
@@ -0,0 +1,523 @@
+"""What each checkpoint's quant config resolves to: the dialect, the method behind every layer, and agreement with the stored tensors.
+
+Rows need the checkpoint's config.json locally (/mnt/nvme/models or the HF cache); absent ones skip.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import struct
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, Callable
+
+import pytest
+import torch
+
+from freetoken.distributed.info import set_tp_info, try_get_tp_info
+from freetoken.engine.config import EngineConfig, checkpoint_quant_config
+from freetoken.layers import set_rope_device
+from freetoken.layers.quantization import (
+ CompressedTensorsConfig,
+ Fp8BlockConfig,
+ KernelSelectionError,
+ ModelOptConfig,
+ Mxfp4Config,
+ NoQuantConfig,
+ QuantConfig,
+ QuantKind,
+)
+from freetoken.layers.quantization.linear import (
+ Fp8BlockLinearMethod,
+ Fp8TensorLinearMethod,
+ Mxfp8LinearMethod,
+ Nvfp4LinearMethod,
+ UnquantizedLinearMethod,
+)
+from freetoken.layers.quantization.moe import Fp8BlockMoEMethod, Mxfp4MoEMethod, Nvfp4MoEMethod, UnquantizedMoEMethod
+from freetoken.models import create_model
+from freetoken.models.gpt_oss.moe import GptOssMoELayer, GptOssOffloadMoELayer
+from freetoken.models.register import get_model_spec
+from freetoken.utils.torch_utils import torch_dtype
+
+MODELS = "/mnt/nvme/models"
+HF_CACHE = os.path.expanduser("~/.cache/huggingface/hub")
+
+BF16, FP8B, FP8T, MXFP8, NVFP4 = UnquantizedLinearMethod, Fp8BlockLinearMethod, Fp8TensorLinearMethod, Mxfp8LinearMethod, Nvfp4LinearMethod
+
+
+def model_dir(name: str) -> str | None:
+ path = os.path.join(MODELS, name.split("/")[-1])
+ if os.path.isfile(os.path.join(path, "config.json")): # a download in progress has the dir but no config yet
+ return path
+ snaps = glob.glob(os.path.join(HF_CACHE, "models--" + name.replace("/", "--"), "snapshots", "*", "config.json"))
+ return os.path.dirname(snaps[0]) if snaps else None
+
+
+def op_at(model, path: str):
+ op = model
+ for part in path.split("."):
+ op = op.op_list[int(part)] if part.isdigit() else getattr(op, part)
+ return op
+
+
+def build_meta_model(path: str, strategy: str):
+ if try_get_tp_info() is None:
+ set_tp_info(0, 1)
+ set_rope_device(torch.device("cpu"))
+ config = EngineConfig(model_path=path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy=strategy)
+ from freetoken.engine.engine import _decode_target
+
+ object.__setattr__(config.model_config, "moe_strategy", strategy)
+ object.__setattr__(config.model_config, "decode_target", _decode_target(config))
+ with torch.device("meta"), torch_dtype(torch.bfloat16):
+ return create_model(config.model_config), config.model_config
+
+
+def _layer_indices(model) -> dict[str, int | None]:
+ """First layer of each kind, for the {moe} {dense} {lin} {full} {dsa} {kda} {ple} placeholders in probe paths."""
+ layers = model.model.layers.op_list
+
+ def first(pred):
+ return next((i for i, layer in enumerate(layers) if pred(layer)), None)
+
+ def has_experts(layer):
+ if getattr(layer, "is_moe_layer", None) is not None:
+ return layer.is_moe_layer
+ block = next((getattr(layer, n) for n in ("mlp", "feed_forward", "block_sparse_moe", "ffn") if hasattr(layer, n)), None)
+ return block is not None and hasattr(block, "experts")
+
+ def is_linear_attn(layer):
+ return getattr(layer, "_is_linear", False) or hasattr(layer, "linear_attn")
+
+ attn = lambda layer: getattr(layer, "self_attn", None)
+ ple = getattr(getattr(model, "_config", None), "qwen4_args", None)
+ return {
+ "moe": first(has_experts),
+ "dense": first(lambda l: not has_experts(l)),
+ "lin": first(is_linear_attn),
+ "full": first(lambda l: not is_linear_attn(l)),
+ "dsa": first(lambda l: getattr(attn(l), "indexer", None) is not None),
+ "kda": first(lambda l: hasattr(attn(l), "in_proj")),
+ "ple": ple.ple_layer_ids[0] if ple is not None else None,
+ }
+
+
+@dataclass(frozen=True)
+class Case:
+ name: str
+ dialect: type
+ methods: dict[str, Any] # probe path -> method class, (class, kernel name), or None for a layer without a method
+ strategy: str = "offload"
+ raises: type | None = None
+ skip_if: Callable[[], bool] | None = None
+ env: dict[str, str] = field(default_factory=dict)
+ check: Callable[[Any, dict], None] | None = None # family extras beyond the method table
+
+
+def _installed(module: str) -> bool:
+ try:
+ __import__(module)
+ return True
+ except ImportError:
+ return False
+
+
+def _qwen35_fp8(model, idx):
+ assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj
+
+
+def _qwen35_nvfp4(model, idx):
+ assert model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj
+ assert op_at(model, f"model.layers.{idx['lin']}.linear_attn.in_proj_qkvz").quant_method.scheme.has("input_scale")
+ assert model.lm_head.weight.dtype is torch.uint8
+
+
+def _no_split(model, idx):
+ assert not model.model.layers.op_list[idx["lin"]].linear_attn._split_in_proj
+
+
+def _dsv4(model, idx):
+ block = op_at(model, "model.layers.2")
+ for proj in (block.attn.wq_a, block.ffn.shared_experts.w1):
+ assert proj.weight.dtype == torch.float8_e4m3fn and proj.weight_scale_inv.dtype == torch.float8_e8m0fnu
+ assert proj.weight_scale_inv.shape == (proj.out_features // 128, proj.in_features // 128) and proj.bias is None
+ assert set(block.attn.compressor.wkv.state_dict()) == {"weight"}
+ experts = block.ffn.experts
+ assert experts.limit == 10.0 and experts.quant_method.cpu_format == "ds_fp4" and not experts.has_bias
+ H, I = 4096, 2048
+ assert {r: (s.shape, s.dtype) for r, s in experts.quant_method.layout().items()} == {
+ "gate_up": ((2 * I, H // 2), torch.uint8), "gate_up_scale": ((2 * I, H // 32), torch.float8_e8m0fnu),
+ "down": ((H, I // 2), torch.uint8), "down_scale": ((H, I // 32), torch.float8_e8m0fnu),
+ }
+ keys = set(model.state_dict())
+ assert "model.layers.2.attn.wq_a.weight_scale_inv" in keys and "model.layers.2.attn.wq_a.scale" not in keys
+ assert "model.layers.2.attn.compressor.wkv.weight_scale_inv" not in keys
+ assert "model.head.weight" in keys and not any(".ffn.experts." in k for k in keys)
+
+
+def _gpt_oss(cls):
+ def check(model, idx):
+ experts = op_at(model, "model.layers.0.mlp.experts")
+ assert type(experts) is cls
+ assert experts.interleaved and experts.alpha == pytest.approx(1.702) and experts.limit == 7.0
+ if cls is GptOssMoELayer:
+ assert experts.gate_up_proj_blocks.dtype is torch.uint8 and experts.gate_up_proj_blocks.dim() == 4
+
+ return check
+
+
+def _m3_sparse(model, idx):
+ layers = model.model.layers.op_list
+ sparse = next((i for i, l in enumerate(layers) if getattr(l.self_attn, "is_sparse", False)), None)
+ if sparse is not None:
+ assert type(op_at(model, f"model.layers.{sparse}.self_attn.index_qk_proj").quant_method) is MXFP8
+
+
+def _gemma_tied_head(model, idx):
+ assert model.lm_head.quant_method is None
+
+
+CASES = [
+ Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {
+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
+ "model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,
+ "model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": FP8B,
+ "model.layers.{lin}.mlp.shared_expert.down_proj": FP8B, "model.layers.{lin}.mlp.experts": Fp8BlockMoEMethod,
+ "model.layers.{lin}.mlp.gate": BF16, "model.layers.{lin}.mlp.shared_expert_gate": BF16, "lm_head": BF16,
+ }, check=_qwen35_fp8),
+ Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {
+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
+ "model.layers.{full}.self_attn.qkv_proj": FP8T, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": NVFP4,
+ "model.layers.{lin}.mlp.shared_expert.down_proj": NVFP4, "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod,
+ "lm_head": NVFP4,
+ }, check=_qwen35_nvfp4),
+ # resident NVFP4 experts have no kernel without vLLM's Marlin
+ Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {}, strategy="fused", raises=KernelSelectionError,
+ skip_if=lambda: not _installed("flashinfer") or _installed("vllm")),
+ Case("Qwen3.8-27B-NVFP4", ModelOptConfig, {
+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{full}.self_attn.o_proj": FP8T,
+ "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,
+ }, strategy="auto"),
+ Case("Qwen3.6-27B", NoQuantConfig, {
+ "model.layers.{lin}.linear_attn.in_proj": BF16, "model.layers.{lin}.mlp.gate_up_proj": BF16, "lm_head": BF16,
+ }, strategy="auto", check=_no_split),
+ Case("Qwen3.6-35B-A3B", NoQuantConfig, {
+ "model.layers.{lin}.mlp.experts": UnquantizedMoEMethod, "model.layers.{lin}.mlp.shared_expert.down_proj": BF16,
+ }),
+ Case("nvidia/Qwen3.6-27B-NVFP4", ModelOptConfig, {
+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8T, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
+ "model.layers.{lin}.linear_attn.out_proj": FP8T, "model.layers.{full}.self_attn.qkv_proj": FP8T,
+ "model.layers.{lin}.mlp.gate_up_proj": NVFP4, "model.layers.{lin}.mlp.down_proj": NVFP4, "lm_head": NVFP4,
+ }, strategy="auto"),
+ *[Case(name, Fp8BlockConfig, {
+ "model.layers.{lin}.linear_attn.in_proj_qkvz": FP8B, "model.layers.{lin}.linear_attn.in_proj_ba": BF16,
+ "model.layers.{lin}.linear_attn.out_proj": FP8B, "model.layers.{full}.self_attn.qkv_proj": FP8B,
+ "model.layers.{full}.self_attn.o_proj": FP8B, "model.layers.{lin}.mlp.gate_up_proj": FP8B,
+ "model.layers.{lin}.mlp.down_proj": FP8B, "lm_head": BF16,
+ }, strategy="auto") for name in ("Qwen/Qwen3.8-27B-FP8", "Qwen/Qwen3.6-27B-FP8")],
+ Case("RedHatAI/Muse-Glimmer-30B-NVFP4", CompressedTensorsConfig, {
+ "model.layers.0.self_attn.qkvg_proj": NVFP4, "model.layers.0.self_attn.o_proj": NVFP4,
+ "model.layers.0.mlp.gate_up_proj": NVFP4, "model.layers.0.mlp.down_proj": NVFP4, "lm_head": BF16,
+ }, strategy="auto"),
+ Case("nvidia/Gemma-4-31B-IT-NVFP4", ModelOptConfig, {
+ "model.layers.0.feed_forward.shared_mlp.gate_up_proj": NVFP4, "model.layers.0.feed_forward.shared_mlp.down_proj": NVFP4,
+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
+ }, strategy="auto", check=_gemma_tied_head),
+ Case("google/gemma-4-12B-it", NoQuantConfig, {
+ "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16, "model.layers.0.self_attn.qkv_proj": BF16,
+ }, strategy="auto", check=_gemma_tied_head),
+ Case("Qwen/Qwen3-4B", NoQuantConfig, {
+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
+ "model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,
+ }, strategy="auto"),
+ *[Case(name, NoQuantConfig, {
+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
+ "model.layers.0.mlp.gate_up_proj": BF16, "model.layers.0.mlp.down_proj": BF16,
+ }, strategy="auto") for name in ("Qwen/Qwen2.5-1.5B-Instruct", "unsloth/Llama-3.2-1B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3")],
+ *[Case("Qwen/Qwen3-30B-A3B", NoQuantConfig, {
+ "model.layers.0.mlp.experts": UnquantizedMoEMethod, "model.layers.0.mlp.gate": BF16,
+ "model.layers.0.self_attn.qkv_proj": BF16, "lm_head": BF16,
+ }, strategy=strategy) for strategy in ("fused", "offload")],
+ Case("RadixArk/Qwen3.8-Flash-Next-NVFP4", ModelOptConfig, {
+ "model.layers.{lin}.mlp.experts": Nvfp4MoEMethod, "model.layers.{lin}.linear_attn.in_proj": BF16,
+ "model.layers.{lin}.linear_attn.out_proj": BF16, "model.layers.{full}.self_attn.qkv_proj": BF16,
+ "model.layers.{full}.self_attn.o_proj": BF16, "model.layers.{lin}.mlp.shared_expert.gate_up_proj": BF16,
+ "model.layers.{lin}.mlp.shared_expert.down_proj": BF16, "model.layers.{lin}.mlp.gate": BF16, "lm_head": BF16,
+ # the indexer, hyper-connection and PLE projections ask the config too
+ "model.layers.{full}.self_attn.indexer.index_qk_proj": BF16,
+ "model.layers.{lin}.attn_hyper_connection.input_mix_weight_down_block_inject": BF16,
+ "model.layers.{lin}.mlp_hyper_connection.input_mix_weight_up": BF16,
+ "model.hyper_connection_mixer.input_mix_weight_down": BF16,
+ "model.layers.{ple}.ple.key_proj": BF16, "model.layers.{ple}.ple.value_proj": BF16,
+ }, check=_no_split),
+ Case("deepseek-ai/DeepSeek-V4-Flash-0731", Fp8BlockConfig, {
+ **{f"model.layers.2.{p}": (FP8B, "dsv4") for p in (
+ "attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "attn.indexer.wq_b",
+ "ffn.shared_experts.w1", "ffn.shared_experts.w2", "ffn.shared_experts.w3")},
+ # the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert
+ **{f"model.layers.2.{p}": (BF16, "torch") for p in (
+ "attn.compressor.wkv", "attn.compressor.wgate", "attn.indexer.weights_proj", "attn.indexer.compressor.wkv")},
+ "model.head": (BF16, "torch"), "model.layers.2.ffn.experts": (Mxfp4MoEMethod, "triton"),
+ }, check=_dsv4),
+ Case("nvidia/MiniMax-M2.5-NVFP4", ModelOptConfig, {
+ "model.layers.3.block_sparse_moe.experts": Nvfp4MoEMethod, "model.layers.3.self_attn.qkv_proj": BF16,
+ "model.layers.3.self_attn.o_proj": BF16, "model.layers.3.block_sparse_moe.gate": BF16, "lm_head": BF16,
+ }),
+ Case("nvidia/MiniMax-M3-NVFP4", ModelOptConfig, {
+ "model.layers.{moe}.block_sparse_moe.experts": Nvfp4MoEMethod,
+ "model.layers.{moe}.block_sparse_moe.shared_experts.gate_up_proj": MXFP8,
+ "model.layers.{moe}.block_sparse_moe.shared_experts.down_proj": MXFP8, "model.layers.{dense}.mlp.gate_up_proj": MXFP8,
+ "model.layers.{moe}.self_attn.qkv_proj": MXFP8, "model.layers.{moe}.self_attn.o_proj": MXFP8, "lm_head": BF16,
+ }, env={"FREETOKEN_M3_MAX_LAYERS": "8"}, check=_m3_sparse),
+ Case("nvidia/Gemma-4-26B-A4B-NVFP4", ModelOptConfig, {
+ "model.layers.0.feed_forward.experts": Nvfp4MoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,
+ "model.layers.0.feed_forward.shared_mlp.down_proj": BF16, "model.layers.0.feed_forward.router.proj": BF16,
+ "model.layers.0.self_attn.qkv_proj": BF16, "model.layers.0.self_attn.o_proj": BF16,
+ }, check=_gemma_tied_head),
+ Case("google/gemma-4-26B-A4B-it", NoQuantConfig, {
+ "model.layers.0.feed_forward.experts": UnquantizedMoEMethod, "model.layers.0.feed_forward.shared_mlp.gate_up_proj": BF16,
+ }),
+ Case("RedHatAI/GLM-5.3-Flash-NVFP4", CompressedTensorsConfig, {
+ "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": BF16,
+ "model.layers.{moe}.mlp.shared_experts.down_proj": BF16, "model.layers.{dense}.mlp.gate_proj": BF16,
+ "model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,
+ # the indexer, the MLA kv_b_proj and the KDA projections ask the config too
+ "model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,
+ "model.layers.{kda}.self_attn.in_proj": BF16, "model.layers.{kda}.self_attn.f_b_proj": BF16,
+ "model.layers.{kda}.self_attn.g_b_proj": BF16,
+ }),
+ Case("zai-org/GLM-5.3-Flash", Fp8BlockConfig, {
+ "model.layers.{moe}.mlp.experts": (Fp8BlockMoEMethod, "triton"), "model.layers.{moe}.mlp.shared_experts.gate_proj": FP8B,
+ "model.layers.{moe}.mlp.shared_experts.down_proj": FP8B, "model.layers.{dense}.mlp.gate_proj": FP8B,
+ "model.layers.{dsa}.self_attn.q_b_proj": FP8B, "model.layers.{dsa}.self_attn.kv_b_proj": BF16,
+ "model.layers.{dsa}.self_attn.o_proj": FP8B, "model.layers.{kda}.self_attn.in_proj": BF16,
+ "model.layers.{kda}.self_attn.o_proj": BF16, "lm_head": BF16,
+ }),
+ Case("nvidia/GLM-4.7-NVFP4", ModelOptConfig, {
+ "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.shared_experts.gate_proj": NVFP4,
+ "model.layers.{moe}.mlp.shared_experts.down_proj": NVFP4, "model.layers.{moe}.mlp.gate": BF16, "lm_head": BF16,
+ }),
+ Case("nvidia/GLM-5.3-NVFP4", ModelOptConfig, {
+ "model.layers.{moe}.mlp.experts": Nvfp4MoEMethod, "model.layers.{moe}.mlp.gate": BF16,
+ "model.layers.{moe}.mlp.shared_experts.gate_proj": BF16, "model.layers.{moe}.self_attn.o_proj": BF16, "lm_head": BF16,
+ "model.layers.{dsa}.self_attn.indexer.wq_b": BF16, "model.layers.{moe}.self_attn.kv_b_proj": BF16,
+ }),
+ *[Case("openai/gpt-oss-20b", Mxfp4Config, {
+ "model.layers.0.mlp.experts": Mxfp4MoEMethod, "model.layers.0.self_attn.qkv_proj": BF16,
+ "model.layers.0.self_attn.o_proj": BF16, "model.layers.0.mlp.router": BF16, "lm_head": BF16,
+ }, strategy=strategy, check=_gpt_oss(cls)) for strategy, cls in (
+ ("fused", GptOssMoELayer), ("offload", GptOssOffloadMoELayer), ("cpu", GptOssOffloadMoELayer), ("hybrid", GptOssOffloadMoELayer))],
+ # hybrid decodes on the CPU executor too, so the expert kernel must have a CPU format
+ Case("Qwen3.6-35B-A3B-NVFP4", ModelOptConfig, {"model.layers.{lin}.mlp.experts": (Nvfp4MoEMethod, "triton")}, strategy="hybrid"),
+ Case("Qwen3.6-35B-A3B", NoQuantConfig, {"model.layers.{lin}.mlp.experts": (UnquantizedMoEMethod, "fused")}, strategy="hybrid"),
+ Case("Qwen3.6-35B-A3B-FP8", Fp8BlockConfig, {}, strategy="hybrid", raises=KernelSelectionError),
+]
+
+
[email protected]("case", CASES, ids=[f"{c.name.split('/')[-1]}:{c.strategy}" for c in CASES])
+def test_probed_layers_get_the_method_their_config_says(case: Case, monkeypatch):
+ path = model_dir(case.name)
+ if path is None:
+ pytest.skip(f"{case.name} not present locally")
+ if case.skip_if is not None and case.skip_if():
+ pytest.skip("kernel availability differs on this host")
+ for key, value in case.env.items():
+ monkeypatch.setenv(key, value)
+ if case.raises is not None:
+ with pytest.raises(case.raises):
+ build_meta_model(path, case.strategy)
+ return
+ model, model_config = build_meta_model(path, case.strategy)
+ assert type(model_config.quant) is case.dialect
+ idx = _layer_indices(model)
+ for template, expected in case.methods.items():
+ probe = template.format(**{k: v for k, v in idx.items() if v is not None})
+ op = op_at(model, probe)
+ if getattr(op, "prefix", ""): # routers are built without one
+ assert op.prefix == probe
+ if expected is None:
+ assert op.quant_method is None, probe
+ continue
+ cls, kernel = expected if isinstance(expected, tuple) else (expected, None)
+ assert type(op.quant_method) is cls, f"{probe}: {type(op.quant_method).__name__}"
+ if kernel is not None:
+ assert op.quant_method.kernel.name == kernel, probe
+ if case.check is not None:
+ case.check(model, idx)
+
+
+# --------------------------------------------------------------------------- config without local weights
+
+
+def test_hub_ids_fetch_the_modelopt_sidecar(tmp_path, monkeypatch):
+ """An old ModelOpt export keeps its quantization config only in hf_quant_config.json."""
+ from huggingface_hub.utils import EntryNotFoundError
+
+ sidecar = tmp_path / "hf_quant_config.json"
+ sidecar.write_text(json.dumps({"producer": {"name": "modelopt", "version": "0.29.0"}, "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head"]}}))
+
+ def fake_download(repo_id, filename, **_):
+ assert repo_id == "org/old-modelopt-fp8"
+ if filename == "hf_quant_config.json":
+ return str(sidecar)
+ raise EntryNotFoundError(f"no {filename}")
+
+ monkeypatch.setattr("freetoken.utils.hf.hf_hub_download", fake_download)
+ quant = checkpoint_quant_config("org/old-modelopt-fp8", SimpleNamespace(architectures=["Qwen3MoeForCausalLM"]), get_model_spec("Qwen3MoeForCausalLM"))
+ assert type(quant) is ModelOptConfig
+ assert quant.scheme_for("model.layers.0.self_attn.q_proj").kind is QuantKind.FP8_TENSOR
+ assert quant.scheme_for("lm_head") is None
+
+
+def test_unsupported_dialects_fail_closed(tmp_path):
+ hf_config = SimpleNamespace(architectures=["Qwen3MoeForCausalLM"], quantization_config={"quant_method": "gptq", "bits": 4})
+ with pytest.raises(NotImplementedError, match="gptq"):
+ checkpoint_quant_config(str(tmp_path), hf_config, get_model_spec("Qwen3MoeForCausalLM"))
+
+
+# --------------------------------------------------------------------------- config against the stored tensors
+
+# leaf names FreeToken builds as Linear / MoE layers; routers, norms, convs never ask for a scheme
+_LINEAR_LEAVES = {
+ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3",
+ "in_proj_qkv", "in_proj_z", "in_proj_a", "in_proj_b", "in_proj_ba", "out_proj", "qkv_proj", "gate_up_proj",
+ "lm_head", "q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "wq_a", "wq_b", "wkv_a", "wkv_b", "wo",
+ "wkv", "wo_a", "wo_b", "wgate", "weights_proj", "experts",
+ "b_proj", "f_a_proj", "f_b_proj", "g_a_proj", "g_b_proj", "index_qk_proj", "key_proj", "value_proj",
+ "input_mix_weight_down", "input_mix_weight_up", "block_inject_weight",
+}
+_META_SUFFIXES = {"bias", "k_scale", "v_scale"}
+_QUANT_SUFFIXES = {
+ "weight_scale", "weight_scale_2", "weight_scale_inv", "weight_packed", "weight_global_scale",
+ "input_scale", "input_scale_2", "input_global_scale", "scale",
+ "gate_up_proj_blocks", "gate_up_proj_scales", "gate_up_proj_bias", "down_proj_blocks", "down_proj_scales", "down_proj_bias",
+}
+# unsupported dialects: from_hf must refuse them, nothing else is checked
+_UNSUPPORTED = {"quark", "mxfp8", "fp_quant"}
+# tables the model code reads itself; RadixArk lists the fp8 PLE table in ``ignore`` yet ships it quantized
+_MODEL_READS = ("ple.ple_embedding",)
+# non-Linear leaves whose parameters happen to be called scale
+_NOT_LINEAR = {"router", "gate", "shared_expert_gate"}
+
+
+def _weight_map(p: Path) -> dict[str, str] | None:
+ """Tensor name -> shard file, from the index or from a single model.safetensors."""
+ if (p / "model.safetensors.index.json").exists():
+ return json.load(open(p / "model.safetensors.index.json"))["weight_map"]
+ if (p / "model.safetensors").exists():
+ return {name: "model.safetensors" for name in _safetensors_header(p / "model.safetensors") if name != "__metadata__"}
+ return None
+
+
+def _candidate_dirs() -> list[Path]:
+ roots = glob.glob(os.path.join(HF_CACHE, "models--*/snapshots/*/")) + glob.glob(os.path.join(MODELS, "*/"))
+ return [Path(d) for d in roots if (Path(d) / "config.json").exists() and _weight_map(Path(d)) is not None]
+
+
+def _label(p: Path) -> str:
+ m = re.search(r"models--([^/]+)/snapshots", str(p))
+ name = m.group(1).replace("--", "/") if m else f"nvme:{p.name}"
+ # an HF cache entry may hold the config and index without the shards; the tensor check then only has the names
+ shards = set(_weight_map(p).values())
+ return name if all((p / s).exists() for s in shards) else f"{name}[index-only]"
+
+
+def _safetensors_header(path: Path) -> dict:
+ with open(path, "rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ return json.loads(f.read(n))
+
+
+def _tensor_info(p: Path, weight_map: dict[str, str]) -> dict[str, dict]:
+ info: dict[str, dict] = {}
+ for shard in sorted(set(weight_map.values())):
+ if not (p / shard).exists():
+ return {}
+ for name, meta in _safetensors_header(p / shard).items():
+ if name != "__metadata__":
+ info[name] = meta
+ return info
+
+
+def _expected_kinds(suffixes: set[str], tensors: dict[str, dict], module: str) -> set[QuantKind]:
+ s = suffixes - _META_SUFFIXES
+ if not s & {"weight", "weight_packed", "gate_up_proj_blocks"} or not s & _QUANT_SUFFIXES:
+ return {QuantKind.NONE}
+ if "weight_packed" in s or "weight_scale_2" in s:
+ return {QuantKind.NVFP4}
+ if "gate_up_proj_blocks" in s:
+ return {QuantKind.MXFP4}
+
+ def dtype(suffix):
+ return (tensors.get(f"{module}.{suffix}") or {}).get("dtype")
+
+ def shape(suffix):
+ return (tensors.get(f"{module}.{suffix}") or {}).get("shape")
+
+ if "weight_scale_inv" in s:
+ if tensors:
+ return {QuantKind.MXFP8} if dtype("weight_scale_inv") == "U8" else {QuantKind.FP8_BLOCK}
+ return {QuantKind.FP8_BLOCK, QuantKind.MXFP8}
+ if "scale" in s:
+ if tensors:
+ return {QuantKind.FP8_BLOCK} if dtype("weight") == "F8_E4M3" else {QuantKind.MXFP4}
+ return {QuantKind.FP8_BLOCK, QuantKind.MXFP4}
+ if "weight_scale" in s:
+ if tensors:
+ if dtype("weight") == "U8":
+ return {QuantKind.MXFP4}
+ sh = shape("weight_scale") or []
+ return {QuantKind.FP8_BLOCK} if len(sh) == 2 and sh[0] > 1 and sh[1] > 1 else {QuantKind.FP8_TENSOR}
+ return {QuantKind.FP8_TENSOR, QuantKind.FP8_BLOCK, QuantKind.MXFP4}
+ return {QuantKind.NONE}
+
+
+def _probes(weight_map: dict[str, str]):
+ mods: dict[str, set[str]] = defaultdict(set)
+ for name in weight_map:
+ module, _, suffix = name.rpartition(".")
+ mods[module].add(suffix)
+ for module, suffixes in mods.items():
+ leaf = module.rpartition(".")[2]
+ if leaf in _NOT_LINEAR or any(m in module for m in _MODEL_READS) or suffixes <= _META_SUFFIXES:
+ continue
+ if leaf in _LINEAR_LEAVES or suffixes & _QUANT_SUFFIXES:
+ yield module, suffixes
+
+
+DIRS = _candidate_dirs()
+
+
[email protected](not DIRS, reason="no local checkpoint with config.json + safetensors")
[email protected]("ckpt", DIRS, ids=_label)
+def test_scheme_for_agrees_with_the_stored_tensors(ckpt: Path):
+ cfg = json.load(open(ckpt / "config.json"))
+ quant = cfg.get("quantization_config") or (cfg.get("text_config") or {}).get("quantization_config") or {}
+ if str(quant.get("quant_method") or "").lower() in _UNSUPPORTED:
+ with pytest.raises(NotImplementedError):
+ QuantConfig.from_hf(cfg)
+ return
+ hfq = json.load(open(ckpt / "hf_quant_config.json")) if (ckpt / "hf_quant_config.json").exists() else None
+ try:
+ spec = get_model_spec((cfg.get("architectures") or [""])[0])
+ except Exception:
+ spec = None
+ qc = QuantConfig.from_hf(cfg, hf_quant_config=hfq, unquantized=spec.unquantized_modules if spec else ())
+ weight_map = _weight_map(ckpt)
+ tensors = _tensor_info(ckpt, weight_map)
+ mismatches, checked = [], 0
+ for module, suffixes in _probes(weight_map):
+ expected = _expected_kinds(suffixes, tensors, module)
+ scheme = qc.scheme_for(module)
+ kind = scheme.kind if scheme else QuantKind.NONE
+ checked += 1
+ if kind not in expected:
+ mismatches.append(f"{module}: config says {kind}, tensors {sorted(suffixes)} say {sorted(k.value for k in expected)}")
+ assert checked > 0
+ assert not mismatches, f"{len(mismatches)}/{checked} modules disagree:\n " + "\n ".join(mismatches[:40])
diff --git a/tests/moe/test_gpt_oss.py b/tests/moe/test_gpt_oss.py
index eadbd4c1..1dbf1f16 100644
--- a/tests/moe/test_gpt_oss.py
+++ b/tests/moe/test_gpt_oss.py
@@ -61,17 +61,28 @@ def tp1(monkeypatch):
return tp_info
+def _offload_layer(config, layer_id, cache):
+ """gpt-oss offload experts as GptOssMLP builds them, kernel bound the way the engine does."""
+ from freetoken.layers.quantization import QuantConfig
+ from freetoken.models.gpt_oss.moe import GptOssOffloadMoELayer
+
+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "mxfp4", "modules_to_not_convert": []}})
+ layer = GptOssOffloadMoELayer(
+ layer_id, config.num_experts, config.num_experts_per_tok, config.hidden_size,
+ config.moe_intermediate_size, renormalize=True, activation="gpt_oss_swiglu",
+ alpha=config.hidden_act_alpha, limit=config.swiglu_limit, interleaved=True, has_bias=True,
+ quant_config=quant, prefix=f"model.layers.{layer_id}.mlp.experts",
+ )
+ layer.offload_cache = cache
+ return layer
+
+
def _make_offload_cache(config, device, *, cache_size=None, prefill_overlap=False):
- from freetoken.moe.expert_banks import load_expert_banks
+ from freetoken.moe.expert_banks import build_expert_banks
from freetoken.moe.offload_cache import OffloadMoeCache
- banks = load_expert_banks(
- None,
- config,
- device=device,
- dtype=torch.bfloat16,
- dummy=True,
- )
+ probe = _offload_layer(config, 0, None)
+ banks = build_expert_banks(probe.quant_method, config.num_layers, None, device=device, dummy=True)
cache = OffloadMoeCache(
num_layers=config.num_layers,
num_experts=config.num_experts,
@@ -80,6 +91,7 @@ def _make_offload_cache(config, device, *, cache_size=None, prefill_overlap=Fals
cache_policy="lru",
prefill_overlap=prefill_overlap,
quant_format=banks.quant_format,
+ layout=banks.layout,
)
cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency)
cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
@@ -188,7 +200,6 @@ def test_run_mxfp4_prefill_experts_t_matches_reference(M):
@CUDA
def test_offload_decode_bit_identical_under_eviction(tp1):
"""Offload decode matches direct split-K across a real evict+reload cycle."""
- from freetoken.models.gpt_oss.moe import GptOssMxfp4OffloadMoELayer
from freetoken.moe.fused_mxfp4 import (
run_mxfp4_splitk_decode_experts as _run_mxfp4_splitk_decode_experts,
)
@@ -199,10 +210,8 @@ def test_offload_decode_bit_identical_under_eviction(tp1):
cache = _make_offload_cache(c, dev, cache_size=E) # 4 slots, 8 total experts -> eviction
- layer1 = GptOssMxfp4OffloadMoELayer(c, layer_id=1)
- layer1.offload_cache = cache
- layer0 = GptOssMxfp4OffloadMoELayer(c, layer_id=0)
- layer0.offload_cache = cache
+ layer1 = _offload_layer(c, 1, cache)
+ layer0 = _offload_layer(c, 0, cache)
torch.manual_seed(42)
hidden = 0.1 * torch.randn(1, H, device=dev, dtype=torch.bfloat16)
@@ -213,8 +222,8 @@ def test_offload_decode_bit_identical_under_eviction(tp1):
tid_ref = torch.tensor([expert_ids], dtype=torch.int32, device=dev)
return _run_mxfp4_splitk_decode_experts(
hidden, tw, tid_ref,
- g("gate_up_blocks"), g("gate_up_scales"), g("gate_up_bias"),
- g("down_blocks"), g("down_scales"), g("down_bias"),
+ g("gate_up"), g("gate_up_scale"), g("gate_up_bias"),
+ g("down"), g("down_scale"), g("down_bias"),
top_k=tk, hidden_act_alpha=c.hidden_act_alpha, swiglu_limit=c.swiglu_limit,
)
@@ -241,7 +250,7 @@ def test_offload_decode_bit_identical_under_eviction(tp1):
@pytest.mark.parametrize("M", [16])
def test_offload_prefill_overlap_matches_reference(M, tp1):
"""Prefill overlap must not crash and must match the direct _t kernel."""
- from freetoken.models.gpt_oss.moe import GptOssMxfp4OffloadMoELayer
+ from freetoken.kernel import gpt_oss_fused_routing
from freetoken.moe.fused_mxfp4 import (
run_mxfp4_prefill_experts_t as _run_mxfp4_prefill_experts_t,
)
@@ -253,21 +262,20 @@ def test_offload_prefill_overlap_matches_reference(M, tp1):
cache = _make_offload_cache(c, dev, prefill_overlap=True)
# layer_id=0 is mandatory: _wait_prefill_overlap gates begin_prefill() on layer_id==0,
# so only layer 0 exercises the previously-crashing code path.
- layer = GptOssMxfp4OffloadMoELayer(c, layer_id=0)
- layer.offload_cache = cache
+ layer = _offload_layer(c, 0, cache)
torch.manual_seed(2)
hidden = 0.1 * torch.randn(M, H, device=dev, dtype=torch.bfloat16)
logits = torch.randn(M, E, device=dev, dtype=torch.bfloat16)
- tw, tid = layer._topk(logits.contiguous())
+ tw, tid = gpt_oss_fused_routing(logits.contiguous(), tk)
out_overlap = layer._prefill_routed(hidden, tw, tid.clone()) # must not raise
g = lambda k: cache.bank_sources[k][0].to(dev) # layer-0 source rows
out_ref = _run_mxfp4_prefill_experts_t(
hidden, tw, tid,
- g("gate_up_blocks"), g("gate_up_scales"), g("gate_up_bias"),
- g("down_blocks"), g("down_scales"), g("down_bias"),
+ g("gate_up"), g("gate_up_scale"), g("gate_up_bias"),
+ g("down"), g("down_scale"), g("down_bias"),
top_k=tk, hidden_act_alpha=c.hidden_act_alpha, swiglu_limit=c.swiglu_limit,
)
max_diff = (out_overlap - out_ref).abs().max().item()
diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.py
index 7005a1b4..0546fbcd 100644
--- a/tests/moe/test_nvfp4_backends.py
+++ b/tests/moe/test_nvfp4_backends.py
@@ -9,15 +9,13 @@ Coverage by hardware:
- Triton (any CUDA GPU): prefill + the production fast decode GEMV, plus a fast-vs-
baseline-kernel equality guard. This is the path used on sm_120 + CUDA 12.x.
- Marlin (sm_80..sm_99, e.g. H100): prefill + decode + overlap.
- - b12x (sm_120 + CUDA>=13): pure-torch pack everywhere; the fused decode forward is
- gated and skipped where the kernel cannot run.
-The ``--nvfp4-backend`` selection + CUDA-13 gate is checked without a GPU.
+ - b12x (sm_120 + CUDA>=13): pack + fused decode, skipped where the kernel cannot run.
+Banks are built the way the loader builds them: the bound kernel object packs native pieces.
"""
from __future__ import annotations
import importlib.util
-import types
import pytest
import torch
@@ -117,33 +115,63 @@ def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids, activation="silu
return out
-def _marlin_cache(device, *, cache_size=S, prefill_overlap=False):
- from freetoken.moe.nvfp4_backends import marlin_repack_sources_inplace
- from freetoken.moe.offload_cache import OffloadMoeCache
+def _bound_layer(kernel: str):
+ """An NVFP4 offload layer with ``kernel`` bound, as the engine binds it."""
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+ from freetoken.layers.moe import OffloadMoELayer
+ from freetoken.layers.quantization import QuantBackend, QuantConfig, set_quant_backend
- sources = _make_native_sources(device)
- ref_sources = {k: [t.clone() for t in v] for k, v in sources.items()} # repack is in place
- cfg = types.SimpleNamespace(hidden_size=H, moe_intermediate_size=I)
- packed = marlin_repack_sources_inplace(sources, cfg, device, chunk=5)
+ if try_get_tp_info() is None:
+ set_tp_info(0, 1)
+ set_quant_backend(QuantBackend.parse(f"moe.nvfp4={kernel}"))
+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})
+ return OffloadMoELayer(0, E, TOPK, H, I, quant_config=quant, prefix="model.layers.0.mlp.experts")
+
+def _pieces(sources):
+ for layer_id in range(L):
+ yield layer_id, 0, E, {
+ "gate_up": sources["gate_up_packed"][layer_id],
+ "gate_up_scale": sources["gate_up_scale"][layer_id],
+ "gate_up_global": sources["gate_up_global"][layer_id],
+ "down": sources["down_packed"][layer_id],
+ "down_scale": sources["down_scale"][layer_id],
+ "down_global": sources["down_global"][layer_id],
+ }
+
+
+def _packed_cache(device, kernel: str, *, seed=0, cache_size=S, prefill_overlap=False):
+ """Native sources packed by ``kernel`` into an offload cache; the sources stay the dequant reference."""
+ from freetoken.moe.expert_banks import build_expert_banks
+ from freetoken.moe.offload_cache import OffloadMoeCache
+
+ sources = _make_native_sources(device, seed=seed)
+ layer = _bound_layer(kernel)
+ banks = build_expert_banks(layer.quant_method, L, _pieces(sources), device=device)
cache = OffloadMoeCache(
num_layers=L,
num_experts=E,
cache_size=cache_size,
device=device,
- quant_format="nvfp4_marlin",
+ quant_format=banks.quant_format,
prefill_overlap=prefill_overlap,
+ layout=banks.layout,
+ max_slots=layer.quant_method.slot_limit(),
)
- cache.set_bank_sources({name: packed[name] for name in cache.bank_schema})
- cache.set_alphas(packed["gate_up_alpha"], packed["down_alpha"])
+ cache.set_bank_sources(banks.sources)
+ cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
cache.reset()
- return cache, ref_sources
+ return cache, sources
+
+
+def _marlin_cache(device, *, cache_size=S, prefill_overlap=False):
+ return _packed_cache(device, "marlin", cache_size=cache_size, prefill_overlap=prefill_overlap)
@cuda
@marlin
def test_marlin_prefill_matches_dequant_reference():
- from freetoken.moe.nvfp4_backends import marlin_fused_experts
+ from freetoken.layers.quantization.moe.nvfp4 import marlin_fused_experts
device = torch.device("cuda")
cache, ref_sources = _marlin_cache(device)
@@ -172,7 +200,7 @@ def test_marlin_prefill_matches_dequant_reference():
def test_marlin_decode_matches_dequant_reference_after_prefill_stomp():
"""Decode through the slot cache, including the request-B-after-request-A pattern
that B1 guarded against: a layer-1 full-layer prefill between two layer-0 decodes."""
- from freetoken.moe.nvfp4_backends import marlin_fused_experts
+ from freetoken.layers.quantization.moe.nvfp4 import marlin_fused_experts
device = torch.device("cuda")
cache, ref_sources = _marlin_cache(device)
@@ -211,7 +239,7 @@ def test_marlin_overlap_prefill_matches_dequant_reference():
(cache_size == 2E, so every slot is buffer-backed): if the prefetch failed to
invalidate them, the post-prefill decode would "hit" stale mappings and read other
experts' bytes; we assert it misses both experts instead."""
- from freetoken.moe.nvfp4_backends import marlin_fused_experts
+ from freetoken.layers.quantization.moe.nvfp4 import marlin_fused_experts
device = torch.device("cuda")
cache, ref_sources = _marlin_cache(device, cache_size=2 * E, prefill_overlap=True)
@@ -411,49 +439,18 @@ def test_triton_decode_marlin_matches_baseline_kernel():
torch.testing.assert_close(marlin.float(), base.float(), rtol=2e-3, atol=2e-3)
-def test_nvfp4_backend_selection():
- """--nvfp4-backend selection + the flashinfer/marlin device gates -- runs without a GPU
- via the CPU branch (forced backends need a usable device, so they error loudly there)."""
- from freetoken.moe.nvfp4_backends import select_nvfp4_backend
-
- cpu = torch.device("cpu")
- assert select_nvfp4_backend(cpu, None, "triton") == "triton"
- assert select_nvfp4_backend(cpu, None, "auto") == "triton" # auto on CPU
- with pytest.raises(RuntimeError):
- select_nvfp4_backend(cpu, None, "flashinfer") # b12x needs a CUDA device
- with pytest.raises(RuntimeError):
- select_nvfp4_backend(cpu, None, "marlin") # marlin needs a CUDA device
- with pytest.raises(ValueError):
- select_nvfp4_backend(cpu, None, "bogus")
-
-
@cuda
def test_b12x_decode_matches_dequant_reference():
"""sm_120 + CUDA>=13 only: the flashinfer b12x W4A16 fused MoE over the slot cache
vs the dequant reference (skipped on hardware/toolkits where b12x cannot run)."""
- from freetoken.moe.nvfp4_backends import (
- _b12x_unusable_reason,
- b12x_fused_experts,
- b12x_repack_sources_inplace,
- )
- from freetoken.moe.offload_cache import OffloadMoeCache
+ from freetoken.layers.quantization.moe.nvfp4 import _b12x_unusable_reason, b12x_fused_experts
device = torch.device("cuda")
reason = _b12x_unusable_reason(torch.cuda.get_device_capability(device))
if reason is not None:
pytest.skip(f"b12x not runnable here: {reason}")
- sources = _make_native_sources(device, seed=8)
- ref_sources = {k: [t.clone() for t in v] for k, v in sources.items()} # repack is in place
- cfg = types.SimpleNamespace(hidden_size=H, moe_intermediate_size=I)
- packed = b12x_repack_sources_inplace(sources, cfg, device, chunk=6)
-
- cache = OffloadMoeCache(
- num_layers=L, num_experts=E, cache_size=S, device=device, quant_format="nvfp4_b12x"
- )
- cache.set_bank_sources({name: packed[name] for name in cache.bank_schema})
- cache.set_alphas(packed["gate_up_alpha"], packed["down_alpha"])
- cache.reset()
+ cache, ref_sources = _packed_cache(device, "b12x", seed=8)
torch.manual_seed(2)
hidden = torch.randn(1, H, dtype=torch.bfloat16, device=device) / 4
@@ -472,27 +469,23 @@ def test_b12x_decode_matches_dequant_reference():
@cuda
-def test_dummy_nvfp4_sources_match_loader_contract():
- """--use-dummy-weight banks must match the real loader's shapes/dtypes/pinning so the
- engine repack/offload path is exercised unchanged. The marlin repack + offload gather
- tail (which needs vllm) lives in test_dummy_nvfp4_sources_marlin_repack."""
- from freetoken.models.weight import dummy_nvfp4_expert_sources
-
- cfg = types.SimpleNamespace(
- num_layers=L, num_experts=E, hidden_size=H, moe_intermediate_size=I
- )
- sources = dummy_nvfp4_expert_sources(cfg)
+def test_dummy_nvfp4_banks_match_the_triton_layout():
+ """--use-dummy-weight banks follow the bound kernel's layout (shapes, dtypes, pinning)."""
+ from freetoken.moe.expert_banks import build_expert_banks
+
+ layer = _bound_layer("triton")
+ banks = build_expert_banks(layer.quant_method, L, None, device=torch.device("cuda"), dummy=True)
expected = {
- "gate_up_packed": ((E, 2 * I, H // 2), torch.uint8),
+ "gate_up": ((E, 2 * I, H // 2), torch.uint8),
"gate_up_scale": ((E, 2 * I, H // 16), torch.float8_e4m3fn),
"gate_up_global": ((E, 2 * I), torch.float16),
- "down_packed": ((E, H, I // 2), torch.uint8),
+ "down": ((E, H, I // 2), torch.uint8),
"down_scale": ((E, H, I // 16), torch.float8_e4m3fn),
"down_global": ((E, H), torch.float16),
}
- assert sources.keys() == expected.keys()
+ assert banks.sources.keys() == expected.keys()
for name, (shape, dtype) in expected.items():
- layers = sources[name]
+ layers = banks.sources[name]
assert len(layers) == L, (name, len(layers))
for t in layers:
assert t.shape == shape and t.dtype == dtype and t.is_pinned(), name
@@ -500,54 +493,42 @@ def test_dummy_nvfp4_sources_match_loader_contract():
@cuda
@marlin
-def test_dummy_nvfp4_sources_marlin_repack():
- """The --use-dummy-weight banks drop into the same marlin repack + offload path as the
- real loader's (in-place repack). The gather kernel reads the banks zero-copy from the
- GPU, which requires the allocator's memory to be device-mapped, not merely page-locked."""
- from freetoken.models.weight import dummy_nvfp4_expert_sources
- from freetoken.moe.nvfp4_backends import marlin_repack_sources_inplace
+def test_dummy_nvfp4_banks_marlin_pack_gathers_zero_copy():
+ """Dummy banks packed by marlin drop into the offload gather path like the real loader's."""
+ from freetoken.moe.expert_banks import build_expert_banks
from freetoken.moe.offload_cache import OffloadMoeCache
- cfg = types.SimpleNamespace(
- num_layers=L, num_experts=E, hidden_size=H, moe_intermediate_size=I
- )
- sources = dummy_nvfp4_expert_sources(cfg)
-
device = torch.device("cuda")
- packed = marlin_repack_sources_inplace(sources, cfg, device, chunk=5)
- assert torch.isfinite(packed["gate_up_alpha"].float()).all()
- assert torch.isfinite(packed["down_alpha"].float()).all()
+ layer = _bound_layer("marlin")
+ banks = build_expert_banks(layer.quant_method, L, None, device=device, dummy=True)
+ assert torch.isfinite(banks.gate_up_alpha.float()).all()
+ assert torch.isfinite(banks.down_alpha.float()).all()
cache = OffloadMoeCache(
- num_layers=L, num_experts=E, cache_size=S, device=device, quant_format="nvfp4_marlin"
+ num_layers=L, num_experts=E, cache_size=S, device=device, quant_format=banks.quant_format,
+ layout=banks.layout, max_slots=layer.quant_method.kernel.max_slots,
)
- cache.set_bank_sources({name: packed[name] for name in cache.bank_schema})
- cache.set_alphas(packed["gate_up_alpha"], packed["down_alpha"])
+ cache.set_bank_sources(banks.sources)
+ cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
cache.reset()
cache.materialize_layer(0)
cache.copy_missing()
torch.cuda.synchronize()
- assert torch.equal(cache.bank_caches["gate_up_packed"][:E].cpu(), packed["gate_up_packed"][0])
+ assert torch.equal(cache.bank_caches["gate_up"][:E].cpu(), banks.sources["gate_up"][0])
@cuda
@pytest.mark.slow
-def test_b12x_pack_is_byte_compatible_with_native_banks():
- """The b12x kernel needs sm_120, but its pack is pure torch: verify the prepared
- blocks drop into the native banks byte-for-byte (the in-place repack contract)."""
- from freetoken.moe.nvfp4_backends import b12x_repack_sources_inplace
+def test_b12x_pack_keeps_per_layer_banks_and_flat_alphas():
+ """The b12x pack is pure torch: its banks stay per-layer lists with flat [L*E] alphas."""
+ from freetoken.layers.quantization.moe.nvfp4 import _b12x_unusable_reason
device = torch.device("cuda")
- sources = _make_native_sources(device, seed=3)
- cfg = types.SimpleNamespace(hidden_size=H, moe_intermediate_size=I)
- try:
- packed = b12x_repack_sources_inplace(sources, cfg, device, chunk=6)
- except Exception as exc: # pragma: no cover - depends on flashinfer internals
- pytest.skip(f"flashinfer w4a16 prepare unavailable off-target: {exc}")
+ reason = _b12x_unusable_reason(torch.cuda.get_device_capability(device))
+ if reason is not None:
+ pytest.skip(f"b12x not runnable here: {reason}")
+ cache, _ = _packed_cache(device, "b12x", seed=3)
total = L * E
- # packed banks stay per-layer lists; alphas are the one flat [L*E] exception (see
- # cache_budget.expert_bytes_per_slot).
- assert len(packed["gate_up_packed"]) == L
- assert sum(t.shape[0] for t in packed["gate_up_packed"]) == total
- assert packed["gate_up_alpha"].shape == (total,)
- assert packed["down_packed"][0].dtype == torch.int32
+ assert len(cache.bank_sources["gate_up"]) == L
+ assert sum(t.shape[0] for t in cache.bank_sources["gate_up"]) == total
+ assert cache.gate_up_alpha.shape == (total,)
diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.py
index 422ca867..b29ce531 100644
--- a/tests/moe/test_offload.py
+++ b/tests/moe/test_offload.py
@@ -4,6 +4,7 @@ import pytest
import torch
from freetoken.distributed import set_tp_info, try_get_tp_info
+from freetoken.layers.quantization import QuantKind
def _init_tp():
@@ -11,18 +12,22 @@ def _init_tp():
set_tp_info(rank=0, size=1)
-def _make_layer_and_cache():
+def _bf16_offload_layer(layer_id: int, num_experts: int, top_k: int, hidden_size: int, intermediate_size: int):
+ """A bf16 offload layer on the fused kernel."""
from freetoken.layers.moe import OffloadMoELayer
+ from freetoken.layers.quantization import NoQuantConfig
+
+ return OffloadMoELayer(
+ layer_id, num_experts, top_k, hidden_size, intermediate_size,
+ quant_config=NoQuantConfig(), prefix=f"model.layers.{layer_id}.mlp.experts",
+ )
+
+
+def _make_layer_and_cache():
from freetoken.moe.offload_cache import OffloadMoeCache
_init_tp()
- layer = OffloadMoELayer(
- layer_id=0,
- num_experts=4,
- top_k=2,
- hidden_size=8,
- intermediate_size=16,
- )
+ layer = _bf16_offload_layer(0, 4, 2, 8, 16)
cache = OffloadMoeCache(
num_layers=1,
num_experts=4,
@@ -34,34 +39,41 @@ def _make_layer_and_cache():
return layer, cache
-def test_dummy_expert_sources_use_moe_layer_count(monkeypatch):
- from types import SimpleNamespace
+def test_dummy_expert_banks_follow_the_kernel_layout(monkeypatch):
- import freetoken.models.weight as weight
+ from freetoken.kernel import backend
+ from freetoken.layers.quantization import QuantBackend, QuantConfig, set_quant_backend
+ from freetoken.moe.expert_banks import build_expert_banks
_init_tp()
- config = SimpleNamespace(
- num_layers=5,
- num_moe_layers=3,
- num_experts=4,
- hidden_size=6,
- moe_intermediate_size=8,
- )
-
- gate_up, down = weight.dummy_moe_expert_sources(config, dtype=torch.float16)
-
- assert len(gate_up) == 3 and all(t.shape == (4, 16, 6) for t in gate_up)
- assert len(down) == 3 and all(t.shape == (4, 6, 8) for t in down)
-
- monkeypatch.setattr(
- weight,
- "alloc_pinned_tensor",
- lambda *shape, dtype: torch.empty(*shape, dtype=dtype),
- )
- banks = weight.dummy_nvfp4_expert_sources(config)
-
- assert {len(layers) for layers in banks.values()} == {3}
- assert {t.shape[0] for layers in banks.values() for t in layers} == {4}
+ L, E, H, I = 3, 4, 64, 32
+ monkeypatch.setattr(backend, "device_capability", lambda: (0, 0))
+ monkeypatch.setattr(backend, "is_vllm_installed", lambda: False)
+ monkeypatch.setattr(backend, "is_flashinfer_installed", lambda: False)
+
+ def _bound(quant):
+ layer = _bf16_offload_layer(0, E, 2, H, I) if quant is None else None
+ if layer is None:
+ from freetoken.layers.moe import OffloadMoELayer
+
+ layer = OffloadMoELayer(0, E, 2, H, I, quant_config=quant, prefix="model.layers.0.mlp.experts")
+ return layer
+
+ bf16 = _bound(None)
+ banks = build_expert_banks(bf16.quant_method, L, None, device=torch.device("cpu"), dummy=True)
+ assert banks.kind is QuantKind.NONE and set(banks.sources) == {"gate_up", "down"}
+ assert len(banks.sources["gate_up"]) == L and all(t.shape == (E, 2 * I, H) for t in banks.sources["gate_up"])
+ assert all(t.shape == (E, H, I) for t in banks.sources["down"])
+
+ set_quant_backend(QuantBackend.parse("moe.nvfp4=triton"))
+ quant = QuantConfig.from_hf({"quantization_config": {"quant_method": "modelopt", "quant_algo": "NVFP4", "ignore": ["lm_head"]}})
+ nvfp4 = _bound(quant)
+ banks = build_expert_banks(nvfp4.quant_method, L, None, device=torch.device("cpu"), dummy=True)
+ assert banks.kind is QuantKind.NVFP4 and banks.kernel == "triton"
+ assert {len(layers) for layers in banks.sources.values()} == {L}
+ assert {t.shape[0] for layers in banks.sources.values() for t in layers} == {E}
+ assert torch.all(banks.sources["gate_up_scale"][0].float() == 1.0)
+ assert torch.all(banks.sources["gate_up_global"][0].float() > 0)
def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypatch):
@@ -87,6 +99,8 @@ def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypa
got_topk_ids,
activation,
apply_router_weight_on_input,
+ act_alpha=1.0,
+ act_limit=float("inf"),
):
calls["w1"] = w1
calls["w2"] = w2
@@ -94,7 +108,7 @@ def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypa
calls["topk_ids"] = got_topk_ids.clone()
return hidden_states
- monkeypatch.setattr("freetoken.layers.moe.fused_experts_impl", fake_fused)
+ monkeypatch.setattr("freetoken.moe.fused.fused_experts_impl", fake_fused)
out = layer.prefill_forward(hidden_states, router_logits)
@@ -112,22 +126,12 @@ def test_offload_moe_layer_prefill_forward_uses_single_layer_cache_view(monkeypa
def test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(monkeypatch):
- from freetoken.layers.moe import OffloadMoELayer
from freetoken.moe.offload_cache import OffloadMoeCache
_init_tp()
num_layers = 3
num_experts = 4
- layers = [
- OffloadMoELayer(
- layer_id=layer_id,
- num_experts=num_experts,
- top_k=2,
- hidden_size=8,
- intermediate_size=16,
- )
- for layer_id in range(num_layers)
- ]
+ layers = [_bf16_offload_layer(layer_id, num_experts, 2, 8, 16) for layer_id in range(num_layers)]
cache = OffloadMoeCache(
num_layers=num_layers,
num_experts=num_experts,
@@ -172,6 +176,8 @@ def test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(mo
got_topk_ids,
activation,
apply_router_weight_on_input,
+ act_alpha=1.0,
+ act_limit=float("inf"),
):
layer_id = len(fused_calls)
fused_calls.append(
@@ -186,7 +192,7 @@ def test_offload_moe_layer_prefill_overlap_prefetches_layers_into_two_buffers(mo
)
return hidden_states + layer_id
- monkeypatch.setattr("freetoken.layers.moe.fused_experts_impl", fake_fused)
+ monkeypatch.setattr("freetoken.moe.fused.fused_experts_impl", fake_fused)
out = hidden_states
for layer in layers:
@@ -353,6 +359,8 @@ def test_offload_moe_layer_decode_forward_uses_remapped_slot_ids(monkeypatch):
got_topk_ids,
activation,
apply_router_weight_on_input,
+ act_alpha=1.0,
+ act_limit=float("inf"),
):
calls["w1"] = w1
calls["w2"] = w2
@@ -360,7 +368,7 @@ def test_offload_moe_layer_decode_forward_uses_remapped_slot_ids(monkeypatch):
calls["topk_ids"] = got_topk_ids.clone()
return hidden_states
- monkeypatch.setattr("freetoken.layers.moe.fused_experts_decode_impl", fake_fused_decode)
+ monkeypatch.setattr("freetoken.moe.fused.fused_experts_decode_impl", fake_fused_decode)
out = layer.decode_forward(hidden_states, router_logits)
@@ -429,17 +437,17 @@ def test_adjust_config_converts_moe_cache_rate_to_cache_size():
num_moe_layers=10,
num_experts=8,
expert_quant="none",
- moe_backend="auto",
+ moe_strategy="auto",
),
)
_adjust_config(config)
- from freetoken.moe import is_offload_moe_backend
+ from freetoken.moe import is_offload_moe_strategy
assert config.moe_cache_size == 24
# Family, not member: a box with a benchbw profile resolves bf16 experts to hybrid.
- assert is_offload_moe_backend(config.moe_backend)
+ assert is_offload_moe_strategy(config.moe_strategy)
def test_graph_capture_reuses_warm_offload_cache_before_capture(monkeypatch):