Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(quant)!: keep nvfp4 global scales in fp32 (#739)

Xiaoze Fan0db5023b60ec
diff --git a/docs/ftw-hotfix.md b/docs/ftw-hotfix.md
index 299d522d..a88c0066 100644
--- a/docs/ftw-hotfix.md
+++ b/docs/ftw-hotfix.md
@@ -32,6 +32,7 @@ byte range, never the whole checkpoint; `--revision` pins the Hub revision to re
| `KeyError: 'model.embed.weight'` | deepseek-ai/DeepSeek-V4-Flash-0731 | rename the index entries | none |
| `RuntimeError: Unexpected keys ... .weight_scale` | nvidia/GLM-5.2-NVFP4 | dequantize the old runtime-fp8 weights back to bf16 | none |
| `PLE shard indices are not contiguous 0..N-1: []` | Qwen3.8-Flash-Next (FTWs converted before #420) | write the PLE table as `ple-table-*.safetensors` | 47.7 GiB |
+| `ValueError: ... stores NVFP4 global scales as fp16` | NVFP4 FTWs converted by older builds | none: the fp16 rounding cannot be undone, so the script refuses; reconvert with `ft checkpoint` | - |
| `ValueError: ... holds no vision encoder tensors` | image-input families (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next, Gemma-4, GLM-5.3-Flash, Muse-Glimmer, MiniMax-M3) converted before the family served images | add the vision encoder: the tower's checkpoint tensors are fetched and the family's own weight reader emits them as the converter would | 1 to 4 GiB |
The script decides by itself: it builds the current model from the FTW's `config.json`, compares
diff --git a/docs/models.md b/docs/models.md
index fd124ea3..9c35ece2 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -86,6 +86,8 @@ unset on PCIe GPUs: wider grids add no bandwidth there.
see [ftw-hotfix.md](ftw-hotfix.md) for the affected checkpoints and the repair tool.
- Muse-Glimmer keeps the attention gate as its own linear next to `qkv_proj`; an FTW converted
when the gate was fused into `qkvg_proj` no longer loads, reconvert it with `ft checkpoint`.
+- NVFP4 global scales are kept in fp32. An NVFP4 FTW that stores them as fp16 (converted by an
+ older build) is refused at load; reconvert it with `ft checkpoint`.
- An FTW converted before its family served images holds no vision encoder: `ft serve`
refuses it unless started with `--text-model-only` (or `--mm-disable vision`); reconvert it
with `ft checkpoint`, or add the encoder in place with [scripts/ftw_hotfix.py](ftw-hotfix.md).
diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.py
index ae97a98d..ca9c33dc 100644
--- a/python/freetoken/checkpoint/ftw.py
+++ b/python/freetoken/checkpoint/ftw.py
@@ -102,6 +102,11 @@ def _elsize(dt: torch.dtype) -> int:
return torch.empty((), dtype=dt).element_size()
+def _is_nvfp4_global(name: str) -> bool:
+ m = _LAYER_ENTRY_RE.match(name)
+ return (m["base"] if m else name).endswith(("gate_up_global", "down_global", ".weight_global"))
+
+
def is_ftw_checkpoint(path: str) -> bool:
"""True if ``path`` is a directory holding a FreeToken Weight (FTW) index."""
return os.path.isfile(os.path.join(path, INDEX_NAME))
@@ -221,6 +226,12 @@ class FTWReader:
self.dir = path
self.shards = sorted(self.index["shards"], key=lambda s: s["global_off"])
self.tensors = {t["name"]: t for t in self.index["tensors"]}
+ fp16_globals = [n for n, t in self.tensors.items() if t["dtype"] == "float16" and _is_nvfp4_global(n)]
+ if fp16_globals:
+ raise ValueError(
+ f"{path!r} stores NVFP4 global scales as fp16 (e.g. {fp16_globals[0]!r}) but FreeToken reads them as fp32; "
+ "reconvert it from the source checkpoint with `ft checkpoint`"
+ )
self._fds: dict[str, int] = {}
self._maps: dict[str, tuple[mmap.mmap, memoryview]] = {}
# O_DIRECT (DMA straight from disk, bypassing the page cache) is the fast path but a
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 1c367aa4..fd67d565 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -92,7 +92,7 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
"down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2,
}
if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"):
- # models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp16
+ # models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp32
# per-row globals; marlin/b12x repacks are byte-identical with the globals
# folded into GPU-resident alphas (layers/quantization/moe/nvfp4.py), so no global banks.
banks = {
@@ -102,8 +102,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
"down_scale": H * (I // 16),
}
if fmt == "nvfp4":
- banks["gate_up_global"] = 2 * I * 2
- banks["down_global"] = H * 2
+ banks["gate_up_global"] = 2 * I * 4
+ banks["down_global"] = H * 4
return banks
if fmt == "mxfp4_triton":
# gpt-oss mxfp4 expert banks: transposed split-K blocks/scales + bf16 bias
diff --git a/python/freetoken/kernel/csrc/cpu_moe/common.h b/python/freetoken/kernel/csrc/cpu_moe/common.h
index adf7e145..0edee015 100644
--- a/python/freetoken/kernel/csrc/cpu_moe/common.h
+++ b/python/freetoken/kernel/csrc/cpu_moe/common.h
@@ -92,33 +92,6 @@ constexpr int PF_AHEAD = 512;
const float kE2M1[16] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f,
-0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f};
-inline float fp16_to_f32(uint16_t h) {
- const uint32_t sign = (uint32_t)(h & 0x8000u) << 16;
- uint32_t exp = (h >> 10) & 0x1Fu;
- uint32_t man = h & 0x3FFu;
- uint32_t f;
- if (exp == 0) {
- if (man == 0) {
- f = sign;
- } else {
- exp = 127 - 15 + 1;
- while ((man & 0x400u) == 0) {
- man <<= 1;
- --exp;
- }
- man &= 0x3FFu;
- f = sign | (exp << 23) | (man << 13);
- }
- } else if (exp == 0x1Fu) {
- f = sign | 0x7F800000u | (man << 13);
- } else {
- f = sign | ((exp + (127 - 15)) << 23) | (man << 13);
- }
- float out;
- std::memcpy(&out, &f, sizeof(out));
- return out;
-}
-
// e4m3 (OCP "fn": finite, max-normal 448, exp bias 7). Decoded into a 256-entry LUT.
inline float e4m3_decode(uint8_t v) {
const float sign = (v & 0x80u) ? -1.0f : 1.0f;
diff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp
index 429955fa..3c5a56d5 100644
--- a/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp
+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp
@@ -7,7 +7,7 @@ namespace {
// --------------------------- NVFP4 (W4A16) dequant ---------------------------
// Weights: e2m1 4-bit codes (2/byte, low nibble first), per-16 block scale in
-// fp8-e4m3, per-output-row global scale in fp16. Dequant matches the GPU kernels
+// fp8-e4m3, per-output-row global scale in fp32. Dequant matches the GPU kernels
// (freetoken/kernel/triton/nvfp4_dequant.py): w = E2M1[code] * e4m3(scale) * global.
// Activations stay bf16 (W4A16); the GEMV dequantizes weights inside the K-loop.
@@ -405,7 +405,7 @@ class Nvfp4Format final : public MoeFormat {
const int H = p.H;
const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));
const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));
- const uint16_t* global = static_cast<const uint16_t*>(tbl_at(gu_global_, layer));
+ const float* global = static_cast<const float*>(tbl_at(gu_global_, layer));
const float* xe = xe_.data() + (size_t)tok * (H / 2);
const float* xo = xo_.data() + (size_t)tok * (H / 2);
const int8_t* xi8 = nvi8dot_ != nullptr ? xi8_.data() + (size_t)tok * H : nullptr;
@@ -414,9 +414,9 @@ class Nvfp4Format final : public MoeFormat {
const size_t r = (size_t)e * (2 * p.I) + row;
if (nvi8dot_ != nullptr)
return nvi8dot_(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16),
- fp16_to_f32(global[r]), xi8, H, e4m3_lut_, xas);
+ global[r], xi8, H, e4m3_lut_, xas);
return nvdot_(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16),
- fp16_to_f32(global[r]), xe, xo, H, e2m1_lut_, e4m3_lut_);
+ global[r], xe, xo, H, e2m1_lut_, e4m3_lut_);
});
}
@@ -424,15 +424,15 @@ class Nvfp4Format final : public MoeFormat {
const int I = p.I;
const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(down_, layer));
const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));
- const uint16_t* global = static_cast<const uint16_t*>(tbl_at(dn_global_, layer));
+ const float* global = static_cast<const float*>(tbl_at(dn_global_, layer));
for (int h = h0; h < h1; ++h) {
for (int j = 0; j < n; ++j) {
const size_t r = (size_t)routes[j].r;
const size_t row = (size_t)routes[j].e * p.H + h;
out[j * kRowBlock + h - h0] = nvi8dot_ != nullptr
- ? nvi8dot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), fp16_to_f32(global[row]),
+ ? nvi8dot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), global[row],
gi8_.data() + r * I, I, e4m3_lut_, gas_.data() + r * (I / 16))
- : nvdot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), fp16_to_f32(global[row]),
+ : nvdot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), global[row],
ge_.data() + r * (I / 2), go_.data() + r * (I / 2), I, e2m1_lut_, e4m3_lut_);
}
}
@@ -441,7 +441,7 @@ class Nvfp4Format final : public MoeFormat {
private:
nvdot_fn nvdot_;
nvi8dot_fn nvi8dot_; // AVX-VNNI W4A8 nvfp4 dot (nullptr -> use fp32 nvdot)
- // ModelOpt rows: packed e2m1 [E,2I,H/2], e4m3 scales per 16 along K [E,2I,H/16], fp16 row globals [E,2I]; down likewise with K = I.
+ // ModelOpt rows: packed e2m1 [E,2I,H/2], e4m3 scales per 16 along K [E,2I,H/16], fp32 row globals [E,2I]; down likewise with K = I.
const uint64_t* gate_up_ = nullptr;
const uint64_t* gu_scale_ = nullptr;
const uint64_t* gu_global_ = nullptr;
diff --git a/python/freetoken/kernel/triton/nvfp4_dequant.py b/python/freetoken/kernel/triton/nvfp4_dequant.py
index a949e7fa..214e6ab7 100644
--- a/python/freetoken/kernel/triton/nvfp4_dequant.py
+++ b/python/freetoken/kernel/triton/nvfp4_dequant.py
@@ -26,7 +26,7 @@ def _e2m1_lut(device_index: int) -> torch.Tensor:
def _dequant_nvfp4_kernel(
packed_ptr, # [S, OUT, IN // 2] uint8
scale_ptr, # [S, OUT, IN // 16] fp8-e4m3 per-block scale (checkpoint native)
- global_ptr, # [S, OUT] fp16 per-output-row global scale (weight_scale_2)
+ global_ptr, # [S, OUT] fp32 per-output-row global scale (weight_scale_2)
slots_ptr, # [N] int32 -> cache slot for each output expert
out_ptr, # [N, OUT, IN] compute dtype
lut_ptr, # [16] float32 E2M1 values
diff --git a/python/freetoken/kernel/triton/nvfp4_fused_moe.py b/python/freetoken/kernel/triton/nvfp4_fused_moe.py
index 61e867b1..08f2f777 100644
--- a/python/freetoken/kernel/triton/nvfp4_fused_moe.py
+++ b/python/freetoken/kernel/triton/nvfp4_fused_moe.py
@@ -1,14 +1,14 @@
"""Inline-dequant NVFP4 fused-MoE Triton kernels.
These kernels read the NVFP4 expert cache (packed FP4 codes + fp8 block scales +
-fp16 per-row global scale) *directly* and dequantize inside the GEMM K-loop, so the
+fp32 per-row global scale) *directly* and dequantize inside the GEMM K-loop, so the
grouped MoE GEMM never materializes a BF16 copy of the experts (no separate dequant
pass, no HBM round-trip of the 4x larger BF16 weights).
Layout, for an expert weight ``W[N, K]``:
- ``packed[slot, n, k//2]`` uint8: two FP4 codes per byte, low nibble = even k.
- ``scale[slot, n, k//16]`` fp8-e4m3: per 16-wide block scale.
- - ``global[slot, n]`` fp16: per-output-row scale (``weight_scale_2``).
+ - ``global[slot, n]`` fp32: per-output-row scale (``weight_scale_2``).
- ``W[n, k] = E2M1[code] * scale[n, k//16] * global[n]``.
The global scale is constant along K, so it is applied once after the K-loop.
@@ -52,7 +52,7 @@ def _decode_nvfp4_moe_kernel(
a_ptr, # [M, K] activations (compute dtype)
packed_ptr, # [S, N, K // 2] uint8
scale_ptr, # [S, N, K // 16] fp8-e4m3
- global_ptr, # [S, N] fp16
+ global_ptr, # [S, N] fp32
c_ptr, # [M, TOP_K, N] output (compute dtype)
topk_weights_ptr, # [M, TOP_K] fp32
topk_ids_ptr, # [M, TOP_K] int32 -> cache slot
@@ -163,7 +163,7 @@ def _decode_nvfp4_marlin_kernel(
a_ptr, # [M, K] activations (compute dtype)
packed_ptr, # [S, N, K // 8] int32 (8 fp4 codes per word, nibble j -> k=8*w+j)
scale_ptr, # [S, N, K // 16] fp8-e4m3
- global_ptr, # [S, N] fp16
+ global_ptr, # [S, N] fp32
c_ptr, # [M, TOP_K, N] output (compute dtype)
topk_weights_ptr, # [M, TOP_K] fp32
topk_ids_ptr, # [M, TOP_K] int32 -> cache slot
@@ -357,7 +357,7 @@ def _prefill_nvfp4_moe_kernel(
a_ptr, # [M, K] activations
packed_ptr, # [S, N, K // 2] uint8
scale_ptr, # [S, N, K // 16] fp8-e4m3 (uint8 view before sm_89)
- global_ptr, # [S, N] fp16
+ global_ptr, # [S, N] fp32
c_ptr, # [num_valid_tokens, N] output (flat over M*top_k)
topk_weights_ptr,
sorted_token_ids_ptr,
diff --git a/python/freetoken/kernel/triton/nvfp4_linear.py b/python/freetoken/kernel/triton/nvfp4_linear.py
index 53abdedc..38f148f0 100644
--- a/python/freetoken/kernel/triton/nvfp4_linear.py
+++ b/python/freetoken/kernel/triton/nvfp4_linear.py
@@ -158,7 +158,7 @@ def _nvfp4_gemv_kernel(
a_ptr, # [K] activation (compute dtype), contiguous
packed_ptr, # [N, K // 8] int32 (8 fp4 codes per word, nibble j -> k = 8*w + j)
scale_ptr, # [N, K // 16] fp8-e4m3 per-16 block scale
- gscale_ptr, # [N] fp16 per-output-row global scale (weight_scale_2)
+ gscale_ptr, # [N] fp32 per-output-row global scale (weight_scale_2)
out_ptr, # [N] out dtype
N, K,
stride_pn, stride_pkw,
@@ -226,7 +226,7 @@ def _nvfp4_gemv_splitk_kernel(
a_ptr, # [K] activation (compute dtype), contiguous
packed_ptr, # [N, K // 8] int32
scale_ptr, # [N, K // 16] fp8-e4m3
- gscale_ptr, # [N] fp16 per-output-row global scale
+ gscale_ptr, # [N] fp32 per-output-row global scale
part_ptr, # [SPLIT_K, N] fp32 partial sums (pre-global-scale)
counter_ptr, # [cdiv(N, BLOCK_N)] int32 arrival counters, zeroed (self-resetting)
out_ptr, # [N] out dtype
@@ -311,7 +311,7 @@ def _gemv(a: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tensor,
gscale: torch.Tensor, out_dtype: torch.dtype, transposed: bool) -> torch.Tensor:
"""M==1 W4A16 GEMV. ``a`` [K] compute-dtype; ``packed_i32`` logical [N, K//8] int32
(row-major or a K-major transposed view); ``scale`` logical [N, K//16] fp8; ``gscale``
- [N] fp16. Picks split-K so small ``N`` (shared expert) still fills the SMs; large ``N``
+ [N] fp32. Picks split-K so small ``N`` (shared expert) still fills the SMs; large ``N``
(lm_head) stays single-pass (split_k=1)."""
out_tl = _TL_DTYPE[out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16]
N = packed_i32.shape[0]
@@ -389,7 +389,7 @@ def _nvfp4_gemm_kernel(
a_ptr, # [M, K] activations
packed_ptr, # logical [N, K // 8] int32 (row- or K-major via strides)
scale_ptr, # logical [N, K // 16] fp8-e4m3
- gscale_ptr, # [N] fp16
+ gscale_ptr, # [N] fp32
c_ptr, # [M, N] output (written when SPLIT_K == 1)
part_ptr, # [SPLIT_K, M, N] fp32 partials (written when SPLIT_K > 1)
M, N, K, K_WORDS, tiles_per,
@@ -612,7 +612,7 @@ def _gemm_inkernel(a: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tenso
def _nvfp4_dequant_rows_kernel(
packed_ptr, # [N, K // 8] int32
scale_ptr, # [N, K // 16] fp8-e4m3
- gscale_ptr, # [N] fp16
+ gscale_ptr, # [N] fp32
out_ptr, # [N, K] compute dtype
K_WORDS,
stride_pn, stride_pkw,
@@ -646,7 +646,7 @@ def _nvfp4_dequant_rows_kernel(
def _nvfp4_dequant_cols_kernel(
packed_ptr, # K-major: element (n, kw) at kw * stride_pkw + n (stride_pn == 1)
scale_ptr,
- gscale_ptr, # [N] fp16
+ gscale_ptr, # [N] fp32
out_ptr, # [n_rows, K] compute dtype, row-major
N, K_WORDS,
stride_pn, stride_pkw,
@@ -781,7 +781,7 @@ def nvfp4_dense_linear(
weight_global: torch.Tensor, bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""``y = x @ dequant(weight)^T`` on the checkpoint-native row-major layout.
- ``weight`` [N, K//2] uint8, ``weight_scale`` [N, K//16] fp8-e4m3, ``weight_global`` [N] fp16."""
+ ``weight`` [N, K//2] uint8, ``weight_scale`` [N, K//16] fp8-e4m3, ``weight_global`` [N] fp32."""
return _linear_impl(
x, weight.view(torch.int32), weight_scale, weight_global, bias, x.dtype, transposed=False
)
diff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.py
index 6a131653..1002a70b 100644
--- a/python/freetoken/layers/quantization/linear/nvfp4.py
+++ b/python/freetoken/layers/quantization/linear/nvfp4.py
@@ -1,4 +1,4 @@
-"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp16 global per output row."""
+"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp32 global per output row."""
from __future__ import annotations
@@ -256,7 +256,7 @@ class MarlinNvfp4LinearKernel(LinearKernel):
dev = layer.weight.device
layer.weight = torch.empty(0, dtype=torch.uint8, device=dev)
layer.weight_scale = torch.empty(0, dtype=FP8, device=dev)
- layer.weight_global = torch.empty(0, dtype=torch.float16, device=dev)
+ layer.weight_global = torch.empty(0, dtype=torch.float32, device=dev)
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import apply_fp4_marlin_linear
@@ -301,6 +301,6 @@ class Nvfp4LinearMethod(LinearMethod):
raise ValueError(f"nvfp4 needs in_features divisible by {GROUP}, got {g.in_features}")
layer.weight = torch.empty(g.out_features, g.in_features // 2, dtype=torch.uint8)
layer.weight_scale = torch.empty(g.out_features, g.in_features // GROUP, dtype=FP8)
- layer.weight_global = torch.empty(g.out_features, dtype=torch.float16)
+ layer.weight_global = torch.empty(g.out_features, dtype=torch.float32)
# no W4A16 kernel reads it; declared so a W4A4 checkpoint loads complete and a W4A4 kernel finds it in place
layer.input_scale = torch.empty((), dtype=torch.float32) if self.scheme.has("input_scale") else None
diff --git a/python/freetoken/layers/quantization/moe/base.py b/python/freetoken/layers/quantization/moe/base.py
index 5ae89c97..b17a9135 100644
--- a/python/freetoken/layers/quantization/moe/base.py
+++ b/python/freetoken/layers/quantization/moe/base.py
@@ -96,8 +96,8 @@ def fused_piece(pieces: dict[str, torch.Tensor], role: str) -> torch.Tensor:
def global_rows(piece: torch.Tensor, rows: int) -> torch.Tensor:
- """Per-expert global scale as one fp16 value per output row (a scalar per expert is broadcast)."""
- flat = piece.reshape(piece.shape[0], -1).to(torch.float16)
+ """Per-expert global scale as one fp32 value per output row (a scalar per expert is broadcast)."""
+ flat = piece.reshape(piece.shape[0], -1).to(torch.float32)
if flat.shape[1] == rows:
return flat
assert flat.shape[1] == 1, (flat.shape, rows)
diff --git a/python/freetoken/layers/quantization/moe/nvfp4.py b/python/freetoken/layers/quantization/moe/nvfp4.py
index 0dce3995..2a5e8bac 100644
--- a/python/freetoken/layers/quantization/moe/nvfp4.py
+++ b/python/freetoken/layers/quantization/moe/nvfp4.py
@@ -52,10 +52,10 @@ class TritonNvfp4MoEKernel(MoEKernel):
return {
"gate_up": BankSpec((2 * i, h // 2), torch.uint8),
"gate_up_scale": BankSpec((2 * i, h // GROUP), FP8),
- "gate_up_global": BankSpec((2 * i,), torch.float16),
+ "gate_up_global": BankSpec((2 * i,), torch.float32),
"down": BankSpec((h, i // 2), torch.uint8),
"down_scale": BankSpec((h, i // GROUP), FP8),
- "down_global": BankSpec((h,), torch.float16),
+ "down_global": BankSpec((h,), torch.float32),
}
def pack(self, pieces, cfg: MoEConfig, out):
@@ -183,7 +183,7 @@ def _marlin_pack_proj(
"""Pack one expert projection (on GPU) into the Marlin layout.
``packed`` ``[N, K//2]`` uint8 e2m1 pairs, ``scale`` ``[N, K//16]`` e4m3,
- ``row_global`` ``[N]`` fp16 per-row global (rows of a merged gate_up may carry
+ ``row_global`` ``[N]`` fp32 per-row global (rows of a merged gate_up may carry
w1's and w3's different globals). Returns ``(qweight [K//16, 2N] int32,
scales [K//16, N] e4m3-coded, global scalar bf16)``.
diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.py
index 08293c7e..7797c784 100644
--- a/python/freetoken/models/gemma4/weight.py
+++ b/python/freetoken/models/gemma4/weight.py
@@ -66,15 +66,15 @@ _NVFP4_DENSE_MLP_RE = re.compile(r"\.mlp\.(gate_proj|up_proj|down_proj)\.weight$
def _nvfp4_dense_parts(reader: ShardReader, raw_base: str):
- """Load an NVFP4 dense weight as the NVFP4 linear method's buffers: weight uint8 [O, IN//2], weight_scale fp8-e4m3 block [O, IN//16], weight_global fp16 [O] (the per-tensor weight_scale_2 per output row), input_scale fp32 scalar or None when the export has none."""
+ """Load an NVFP4 dense weight as the NVFP4 linear method's buffers: weight uint8 [O, IN//2], weight_scale fp8-e4m3 block [O, IN//16], weight_global fp32 [O] (the per-tensor weight_scale_2 per output row), input_scale fp32 scalar or None when the export has none."""
w = reader.get_tensor(raw_base + ".weight")
s = reader.get_tensor(raw_base + ".weight_scale")
- g = reader.get_tensor(raw_base + ".weight_scale_2").reshape(1).to(torch.float16)
+ g = reader.get_tensor(raw_base + ".weight_scale_2").reshape(1).to(torch.float32)
g = g.expand(w.shape[0]).contiguous()
assert (
w.dtype is torch.uint8
and s.dtype is torch.float8_e4m3fn
- and g.dtype is torch.float16
+ and g.dtype is torch.float32
), f"unexpected NVFP4 dense dtypes at {raw_base}: {w.dtype}/{s.dtype}/{g.dtype}"
a = reader.get_tensor(raw_base + ".input_scale").reshape(()).to(torch.float32) if reader.has(raw_base + ".input_scale") else None
return w, s, g, a
diff --git a/python/freetoken/models/glm4_moe/weight.py b/python/freetoken/models/glm4_moe/weight.py
index aa6538dc..0ca79efb 100644
--- a/python/freetoken/models/glm4_moe/weight.py
+++ b/python/freetoken/models/glm4_moe/weight.py
@@ -78,12 +78,12 @@ def _iter_nvfp4_resident(
"""Yield native NVFP4 buffers for an always-resident Linear.
``weight`` (packed fp4) + ``weight_scale`` (fp8 block scale) verbatim; per-tensor
- ``weight_scale_2`` broadcast to per-row fp16 ``weight_global`` as the NVFP4 linear
+ ``weight_scale_2`` broadcast to per-row fp32 ``weight_global`` as the NVFP4 linear
method / dequant_nvfp4 expect. lossless vs checkpoint; same dequant math as routed experts.
"""
packed = reader.get(f"{src_prefix}.weight") # [OUT, IN//2] uint8
scale = reader.get(f"{src_prefix}.weight_scale") # [OUT, IN//16] fp8-e4m3
- g = reader.get(f"{src_prefix}.weight_scale_2").reshape(()).to(torch.float16) # scalar
+ g = reader.get(f"{src_prefix}.weight_scale_2").reshape(()).to(torch.float32) # scalar
yield f"{dst_prefix}.weight", packed
yield f"{dst_prefix}.weight_scale", scale
yield f"{dst_prefix}.weight_global", g.expand(packed.shape[0]).contiguous()
diff --git a/python/freetoken/models/loader.py b/python/freetoken/models/loader.py
index 0f310ccb..4a9ac94c 100644
--- a/python/freetoken/models/loader.py
+++ b/python/freetoken/models/loader.py
@@ -208,11 +208,11 @@ class ShardReader:
def nvfp4_parts_ct(f, raw_base: str) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
- """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16], per-output-row global fp16 [O], dequant-side input_scale fp32 scalar or None)`` for the NVFP4 linear method."""
+ """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16], per-output-row global fp32 [O], dequant-side input_scale fp32 scalar or None)`` for the NVFP4 linear method."""
w = f.get_tensor(raw_base + ".weight_packed")
s = f.get_tensor(raw_base + ".weight_scale")
wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)
- g = (1.0 / wg).to(torch.float16).expand(w.shape[0]).contiguous()
+ g = (1.0 / wg).expand(w.shape[0]).contiguous()
a = None
if f.has(raw_base + ".input_global_scale"):
a = (1.0 / f.get_tensor(raw_base + ".input_global_scale").reshape(()).to(torch.float32))
diff --git a/python/freetoken/models/muse_glimmer/weight.py b/python/freetoken/models/muse_glimmer/weight.py
index 3b8eb6d2..11a18172 100644
--- a/python/freetoken/models/muse_glimmer/weight.py
+++ b/python/freetoken/models/muse_glimmer/weight.py
@@ -131,7 +131,7 @@ def _iter_weights_compressed_tensors(
Every text Linear (q/k/v/o, the attention gate, the MLP) is kept native NVFP4
(W4A16): ``.weight`` (uint8 packed) + ``.weight_scale`` (fp8 block) + ``.weight_global``
- (fp16 per-row, the reciprocal of the stored quant-side global). q/k/v fuse into
+ (fp32 per-row, the reciprocal of the stored quant-side global). q/k/v fuse into
``qkv_proj`` and gate/up into ``gate_up_proj`` on the output dim, each part keeping
its own scales, so the fused FP4 weights are exact. Embeddings, norms, lm_head and the
vision tower are bf16 (the checkpoint's ignore list). Scale lookups go through the
diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.py
index e30429c0..53822945 100644
--- a/python/freetoken/models/nvfp4_banks.py
+++ b/python/freetoken/models/nvfp4_banks.py
@@ -36,7 +36,7 @@ def _canon_kind(spec: "Nvfp4ExpertSourceSpec", kind: str) -> str:
def _ingest_global(spec: "Nvfp4ExpertSourceSpec", tensor: torch.Tensor) -> torch.Tensor:
if spec.global_reciprocal:
tensor = 1.0 / tensor.float()
- return tensor.to(torch.float16)
+ return tensor.to(torch.float32)
def _activation_scale():
@@ -99,7 +99,7 @@ def iter_nvfp4_expert_pieces(
):
"""One piece per routed expert: ``gate`` / ``up`` / ``down`` codes plus their ``_scale``
(fp8 block scales) and ``_global`` (the per-tensor scale, reciprocal for quant-side dialects,
- fp16) companions, straight from the safetensors shards. When every expert also stores the
+ fp32) companions, straight from the safetensors shards. When every expert also stores the
dialect's static activation scale, the pieces carry it as fp32 ``_input`` (dequant side).
Serial reads walk the shards in order; ``parallel`` uses the chunked O_DIRECT reader. Either
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index 724ad153..bf3cecf0 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -227,7 +227,7 @@ class _DenseReader:
g = part["weight_global"].reshape(-1).to(torch.float32)
if g.numel() != 1:
raise ValueError(f"{target}: weight_global has {g.numel()} elements, expected one per-tensor scale")
- out["weight_global"] = g.to(torch.float16).expand(rows).contiguous()
+ out["weight_global"] = g.expand(rows).contiguous()
if "input_scale" in part:
out["input_scale"] = part["input_scale"].reshape(()).to(torch.float32)
return out
diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.py
index 057b86fd..0225cc4f 100644
--- a/python/freetoken/moe/benchbw.py
+++ b/python/freetoken/moe/benchbw.py
@@ -309,7 +309,7 @@ def _gguf_bank_shapes(fmt: str, opts: dict, H: int, I: int) -> dict[str, tuple[t
def _offload_bank_specs(fmt: str, H: int, I: int, opts: dict | None = None) -> dict[str, tuple[int, torch.dtype]]:
"""Per-bank (elems_per_expert, dtype) for the flat offload host banks (the gather)."""
- u8, bf16, f16, f8 = torch.uint8, torch.bfloat16, torch.float16, torch.float8_e4m3fn
+ u8, bf16, f32, f8 = torch.uint8, torch.bfloat16, torch.float32, torch.float8_e4m3fn
if fmt in ("gguf", "gguf_scaled"):
return {name: (math.prod(shape), dtype) for name, (shape, dtype) in _gguf_bank_shapes(fmt, opts, H, I).items()}
if fmt == "bf16":
@@ -322,8 +322,8 @@ def _offload_bank_specs(fmt: str, H: int, I: int, opts: dict | None = None) -> d
if fmt == "nvfp4":
return {
"gate_up_packed": (2 * I * (H // 2), u8), "gate_up_scale": (2 * I * (H // 16), u8),
- "gate_up_global": (2 * I, f16), "down_packed": (H * (I // 2), u8),
- "down_scale": (H * (I // 16), u8), "down_global": (H, f16),
+ "gate_up_global": (2 * I, f32), "down_packed": (H * (I // 2), u8),
+ "down_scale": (H * (I // 16), u8), "down_global": (H, f32),
}
if fmt == "mxfp4_triton":
return {
@@ -403,13 +403,13 @@ def _cpu_moe_bank_sources(fmt: str, H: int, I: int, E: int, opts: dict | None =
b = {
"gate_up_packed": pin(E, 2 * I, H // 2, dtype=torch.uint8),
"gate_up_scale": pin(E, 2 * I, H // 16, dtype=torch.uint8),
- "gate_up_global": pin(E, 2 * I, dtype=torch.float16),
+ "gate_up_global": pin(E, 2 * I, dtype=torch.float32),
"down_packed": pin(E, H, I // 2, dtype=torch.uint8),
"down_scale": pin(E, H, I // 16, dtype=torch.uint8),
- "down_global": pin(E, H, dtype=torch.float16),
+ "down_global": pin(E, H, dtype=torch.float32),
}
b["gate_up_global"].fill_(1.0)
- b["down_global"].fill_(1.0) # e4m3 scales decode to normal float32; globals are fp16
+ b["down_global"].fill_(1.0) # e4m3 scales decode to normal float32
return b
if fmt == "mxfp4_triton": # transposed split-K layout
b = {
diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py
index bb65146b..1f275749 100644
--- a/python/freetoken/moe/cpu_executor.py
+++ b/python/freetoken/moe/cpu_executor.py
@@ -448,12 +448,12 @@ class CpuMoeExecutor:
if fmt == "fp8_block":
return self._resolve_fp8_block_banks(banks, rep)
- # nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.
+ # nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp32 row globals.
gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]
dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]
assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype)
assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte"
- assert gug[rep].dtype == torch.float16 and dng[rep].dtype == torch.float16, (gug[rep].dtype, dng[rep].dtype)
+ assert gug[rep].dtype == torch.float32 and dng[rep].dtype == torch.float32, (gug[rep].dtype, dng[rep].dtype)
I = int(gup[rep].shape[1] // 2)
H = int(gup[rep].shape[2] * 2)
assert gup[rep].shape[1] == 2 * I
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index 6b428c41..0d702d8a 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -51,7 +51,7 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {
# GGUF experts whose NVFP4 tensors carry per-expert scales: global [L*E, 4] fp32 (gate, up, down, 1) weight_scale_2
"gguf_scaled": ("gate_up", "down", "global"),
# native ModelOpt rows for the Triton inline-dequant kernels: packed e2m1 codes +
- # fp8-e4m3 per-16 block scales + per-output-row fp16 globals (w1/w3 carry distinct
+ # fp8-e4m3 per-16 block scales + per-output-row fp32 globals (w1/w3 carry distinct
# globals, and folding them into the e4m3 block scales would underflow)
"nvfp4": (
"gate_up_packed",
@@ -94,7 +94,7 @@ _BANK_BYTES_PER_EXPERT = {
(2 * I // 128) * fp8_block_scale_pad(2 * I // 128, H // 128)
+ (H // 128) * fp8_block_scale_pad(H // 128, I // 128)
) * 2,
- "nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2),
+ "nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 4) + H * (I // 2 + I // 16 + 4),
"mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2),
"ds_fp4": lambda H, I: 2 * I * (H // 2 + H // 32) + H * (I // 2 + I // 32),
}
diff --git a/scripts/ftw_hotfix.py b/scripts/ftw_hotfix.py
index 12574651..7b4d2257 100644
--- a/scripts/ftw_hotfix.py
+++ b/scripts/ftw_hotfix.py
@@ -38,7 +38,7 @@ import tempfile
import torch
-from freetoken.checkpoint.ftw import ALIGN, FORMAT_TAG, INDEX_NAME, _align_up, _dtype_of, _dtype_str, _SHARD_FMT
+from freetoken.checkpoint.ftw import ALIGN, FORMAT_TAG, INDEX_NAME, _align_up, _dtype_of, _dtype_str, _is_nvfp4_global, _SHARD_FMT
from freetoken.models.config import VISION_KEY_PREFIXES
from freetoken.distributed.info import set_tp_info, try_get_tp_info
from freetoken.engine.config import EngineConfig
@@ -340,6 +340,10 @@ def validate_ftw(ftw_dir: str, index: dict, expected: dict, renames: dict[str, s
# the loader casts between float types; anything else has to match exactly
if dt != exp[1] and not (dt in _FLOAT_DTYPES and exp[1] in _FLOAT_DTYPES):
problems.append(f"{name}: FTW dtype {e['dtype']} is not loadable as {exp[1]}")
+ fp16_globals = [e["name"] for e in index["tensors"] if e["dtype"] == "float16" and _is_nvfp4_global(e["name"])]
+ if fp16_globals:
+ problems.append(f"{len(fp16_globals)} NVFP4 global scale entries are fp16 (e.g. {fp16_globals[0]}) but FreeToken reads them as fp32; "
+ "this script cannot repair that, reconvert the FTW with `ft checkpoint`")
blocks = sorted((e["global_off"], e["global_off"] + _align_up(e["nbytes"]), e["name"]) for e in index["tensors"])
for (_, a1, an), (b0, _, bn) in zip(blocks, blocks[1:]):
if b0 < a1:
diff --git a/tests/kernels/test_e4m3_compat.py b/tests/kernels/test_e4m3_compat.py
index fa8a1f1d..1532feb4 100644
--- a/tests/kernels/test_e4m3_compat.py
+++ b/tests/kernels/test_e4m3_compat.py
@@ -217,7 +217,7 @@ def _emit_all(path: str) -> None:
N, K = 128, 256
pk = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device=dev)
sc = (torch.rand(N, K // 16, device=dev) * 2 + 0.1).to(FP8)
- g = (torch.rand(N, device=dev) * 0.05 + 0.01).to(torch.float16)
+ g = torch.rand(N, device=dev) * 0.05 + 0.01
xa = torch.randn(1, K, dtype=torch.bfloat16, device=dev)
xb = torch.randn(8, K, dtype=torch.bfloat16, device=dev)
xc = torch.randn(200, K, dtype=torch.bfloat16, device=dev)
@@ -231,7 +231,7 @@ def _emit_all(path: str) -> None:
# nvfp4 bank dequant
pkc = torch.randint(0, 256, (E, 64, 128), dtype=torch.uint8, device=dev)
scc = (torch.rand(E, 64, 16, device=dev) + 0.1).to(FP8)
- gc = (torch.rand(E, 64, device=dev) * 0.05 + 0.01).to(torch.float16)
+ gc = torch.rand(E, 64, device=dev) * 0.05 + 0.01
out["nv_dequant"] = f32(dequant_nvfp4(
pkc, scc, gc, torch.tensor([1, 3], dtype=torch.int32, device=dev)))
@@ -239,10 +239,10 @@ def _emit_all(path: str) -> None:
S, H4, I4 = 4, 256, 128
gup = torch.randint(0, 256, (S, 2 * I4, H4 // 2), dtype=torch.uint8, device=dev)
gus4 = (torch.rand(S, 2 * I4, H4 // 16, device=dev) + 0.1).to(FP8)
- gug = (torch.rand(S, 2 * I4, device=dev) * 0.05 + 0.01).to(torch.float16)
+ gug = torch.rand(S, 2 * I4, device=dev) * 0.05 + 0.01
dnp = torch.randint(0, 256, (S, H4, I4 // 2), dtype=torch.uint8, device=dev)
dns4 = (torch.rand(S, H4, I4 // 16, device=dev) + 0.1).to(FP8)
- dng = (torch.rand(S, H4, device=dev) * 0.05 + 0.01).to(torch.float16)
+ dng = torch.rand(S, H4, device=dev) * 0.05 + 0.01
hid4 = torch.randn(2, H4, dtype=torch.bfloat16, device=dev)
out["nv_moe_marlin"] = f32(fused_experts_decode_nvfp4_marlin(
hid4, gup, gus4, gug, dnp, dns4, dng, tw, tids))
diff --git a/tests/kernels/test_nvfp4_w4a4_linear.py b/tests/kernels/test_nvfp4_w4a4_linear.py
index 1840d30f..03fd2c7e 100644
--- a/tests/kernels/test_nvfp4_w4a4_linear.py
+++ b/tests/kernels/test_nvfp4_w4a4_linear.py
@@ -86,7 +86,7 @@ def test_w4a4_linear_matches_nvfp4_activation_reference(kernel_name, M, N, K, pa
g = torch.Generator(device="cuda").manual_seed(0)
weight = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g)
scale = (torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn)
- row_global = torch.full((N,), 1e-3, dtype=torch.float16, device="cuda")
+ row_global = torch.full((N,), 1e-3, dtype=torch.float32, device="cuda")
if parts_differ:
row_global[N // 2:] = 4e-3
bias = torch.randn(N, device="cuda", dtype=torch.bfloat16)
@@ -149,7 +149,7 @@ def test_w4a4_linear_takes_row_strided_and_empty_inputs(kernel_name, M):
g = torch.Generator(device="cuda").manual_seed(0)
layer = SimpleNamespace(weight=torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g),
weight_scale=(torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn),
- weight_global=torch.full((N,), 1e-3, dtype=torch.float16, device="cuda"),
+ weight_global=torch.full((N,), 1e-3, dtype=torch.float32, device="cuda"),
input_scale=torch.tensor(0.01, device="cuda"), bias=None)
kernel.finalize(layer)
buf = torch.randn(M, K + 64, device="cuda", dtype=torch.bfloat16, generator=g)
diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.py
index 605c40f1..4395a863 100644
--- a/tests/models/qwen4_exp/test_weight_ckpt.py
+++ b/tests/models/qwen4_exp/test_weight_ckpt.py
@@ -263,7 +263,7 @@ def test_ple_table_loads_pinned_and_matches_the_checkpoint(reader):
# The two pinned host allocations the engine must budget for.
total = PLE_BYTES + NUM_LAYERS * EXPERT_LAYER_BYTES
- assert abs(total / 2**30 - 111.14) < 0.05
+ assert abs(total / 2**30 - 111.32) < 0.05
@pytest.fixture(scope="module")
@@ -312,14 +312,14 @@ def test_sampled_experts_match_the_checkpoint(layer0_expert_banks, reader):
scale = reader.get(f"{base}.{proj}.weight_scale")
assert torch.equal(banks[bank][0][expert][rows].reshape(-1).view(torch.uint8),
scale.reshape(-1).view(torch.uint8))
- gate_g = reader.get(f"{base}.gate_proj.weight_scale_2").to(torch.float16)
+ gate_g = reader.get(f"{base}.gate_proj.weight_scale_2").to(torch.float32)
assert torch.equal(banks["gate_up_global"][0][expert, :I], gate_g.reshape(1).expand(I))
def test_expert_bank_bytes_match_the_aot_row_table(layer0_expert_banks):
measured = sum(t[0].numel() * t[0].element_size() for t in layer0_expert_banks.values())
assert measured == EXPERT_LAYER_BYTES
- assert abs(NUM_LAYERS * EXPERT_LAYER_BYTES / 2**30 - 63.46) < 0.05
+ assert abs(NUM_LAYERS * EXPERT_LAYER_BYTES / 2**30 - 63.63) < 0.05
@pytest.mark.skipif(not torch.cuda.is_available(), reason="dummy banks are pinned")
diff --git a/tests/models/test_qwen3_5_moe_weight.py b/tests/models/test_qwen3_5_moe_weight.py
index 16f439a9..c502067e 100644
--- a/tests/models/test_qwen3_5_moe_weight.py
+++ b/tests/models/test_qwen3_5_moe_weight.py
@@ -431,7 +431,7 @@ def test_modelopt_fp8_scales_broadcast_per_part_and_input_scale_is_the_max(check
assert _same(loaded[f"{fused}.weight"][:I], raw[f"{shared}.gate_proj.weight"])
assert _same(loaded[f"{fused}.weight_scale"][I:], raw[f"{shared}.up_proj.weight_scale"])
glob = loaded[f"{fused}.weight_global"]
- assert glob.dtype is torch.float16 and torch.equal(glob[:I], raw[f"{shared}.gate_proj.weight_scale_2"].to(torch.float16).expand(I))
+ assert glob.dtype is torch.float32 and torch.equal(glob[:I], raw[f"{shared}.gate_proj.weight_scale_2"].float().expand(I))
# W4A16: no activation quantizer, so the layer has no input_scale even when the export ships one
assert f"{fused}.input_scale" not in loaded and "lm_head.input_scale" not in loaded
assert loaded["lm_head.weight"].dtype is torch.uint8 and loaded["lm_head.weight_global"].shape == (V,)
@@ -444,8 +444,8 @@ def test_compressed_tensors_nvfp4_global_is_the_reciprocal(checkpoint):
loaded = _load(folder)
attn = f"{LM}.layers.1.self_attn"
glob = loaded["model.layers.1.self_attn.qkv_proj.weight_global"]
- expected = torch.cat([(1.0 / raw[f"{attn}.{p}_proj.weight_global_scale"]).to(torch.float16).expand(raw[f"{attn}.{p}_proj.weight_packed"].shape[0]) for p in "qkv"])
- assert glob.dtype is torch.float16 and torch.equal(glob, expected)
+ expected = torch.cat([(1.0 / raw[f"{attn}.{p}_proj.weight_global_scale"].float()).expand(raw[f"{attn}.{p}_proj.weight_packed"].shape[0]) for p in "qkv"])
+ assert glob.dtype is torch.float32 and torch.equal(glob, expected)
assert _same(loaded["model.layers.1.self_attn.o_proj.weight_scale"], raw[f"{attn}.o_proj.weight_scale"])
# ``dynamic: local`` ships the quant-side activation global too; it lands as the dequant-side input_scale
assert torch.equal(loaded["model.layers.1.self_attn.o_proj.input_scale"], (1.0 / raw[f"{attn}.o_proj.input_global_scale"]).reshape(()))
@@ -514,11 +514,11 @@ def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):
base = f"{LM}.layers.1.mlp.experts.2.gate_proj"
if name.startswith("modelopt_mixed"):
assert _same(piece["gate"][0], raw[f"{base}.weight"])
- assert torch.equal(piece["gate_global"].reshape(-1), raw[f"{base}.weight_scale_2"].reshape(-1).to(torch.float16))
+ assert torch.equal(piece["gate_global"].reshape(-1), raw[f"{base}.weight_scale_2"].reshape(-1).float())
else:
assert _same(piece["gate"][0], raw[f"{base}.weight_packed"])
assert _same(piece["down_scale"][0], raw[f"{LM}.layers.1.mlp.experts.2.down_proj.weight_scale"])
- assert torch.equal(piece["gate_global"].reshape(-1), (1.0 / raw[f"{base}.weight_global_scale"]).to(torch.float16))
+ assert torch.equal(piece["gate_global"].reshape(-1), 1.0 / raw[f"{base}.weight_global_scale"].float())
@pytest.mark.parametrize("parallel", [False, True])
@@ -563,7 +563,7 @@ def test_a_quantized_router_is_dequantized_because_the_family_serves_it_bf16(tmp
for layer, gate in enumerate(gates):
got = loaded[f"model.layers.{layer}.mlp.gate.weight"]
assert got.dtype is torch.bfloat16 and got.shape == (E, H)
- expected = _nvfp4_reference(raw[f"{gate}.weight_packed"], raw[f"{gate}.weight_scale"], (1.0 / raw[f"{gate}.weight_global_scale"]).to(torch.float16).expand(E))
+ expected = _nvfp4_reference(raw[f"{gate}.weight_packed"], raw[f"{gate}.weight_scale"], (1.0 / raw[f"{gate}.weight_global_scale"].float()).expand(E))
assert torch.equal(got, expected)
diff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.py
index 56c6c18a..edd9ddc9 100644
--- a/tests/moe/test_cpu_moe.py
+++ b/tests/moe/test_cpu_moe.py
@@ -90,9 +90,9 @@ def _make_nvfp4_cache(L, E, H, I, seed=0):
def rows(OUT, IN):
codes = torch.randint(0, 16, (S, OUT, IN), dtype=torch.uint8)
packed = _pack_nvfp4(codes) # [S, OUT, IN//2]
- # block scales in fp8-e4m3 (1 byte), ~[0.5, 1.5]; row globals in fp16, small.
+ # block scales in fp8-e4m3 (1 byte), ~[0.5, 1.5]; row globals in fp32, small.
scale = (0.5 + torch.rand(S, OUT, IN // 16)).to(torch.float8_e4m3fn)
- glob = (0.02 + 0.01 * torch.rand(S, OUT)).to(torch.float16)
+ glob = 0.02 + 0.01 * torch.rand(S, OUT)
return packed, scale.contiguous(), glob.contiguous()
gup, gus, gug = rows(2 * I, H)
diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.py
index fd98c2ee..58a55bfa 100644
--- a/tests/moe/test_nvfp4_backends.py
+++ b/tests/moe/test_nvfp4_backends.py
@@ -64,9 +64,9 @@ def _make_native_sources(device: torch.device, seed: int = 0) -> dict[str, list[
def rand_scale(*shape):
return (torch.rand(*shape, generator=g) * 1.5 + 0.25).to(torch.float8_e4m3fn)
- gate_up_global = torch.full((total, 2 * I), 1.0, dtype=torch.float16)
+ gate_up_global = torch.full((total, 2 * I), 1.0, dtype=torch.float32)
gate_up_global[:, I:] = 0.5 # w3 global != w1 global: exercises the alpha fold
- down_global = torch.full((total, H), 0.75, dtype=torch.float16)
+ down_global = torch.full((total, H), 0.75, dtype=torch.float32)
flat = {
"gate_up_packed": rand_u8(total, 2 * I, H // 2),
"gate_up_scale": rand_scale(total, 2 * I, H // 16),
@@ -392,7 +392,7 @@ def test_triton_prefill_routes_to_gluon_on_sm120(monkeypatch, hidden_size, inter
def bank(n, k):
packed = torch.randint(0, 256, (E, n, k // 2), dtype=torch.uint8, device=device, generator=g)
scale = (torch.rand(E, n, k // 16, device=device, generator=g) + 0.25).to(torch.float8_e4m3fn)
- return packed, scale, (torch.rand(E, n, device=device, generator=g) + 0.5).to(torch.float16)
+ return packed, scale, torch.rand(E, n, device=device, generator=g) + 0.5
banks = (*bank(2 * inter, hidden_size), *bank(hidden_size, inter))
M = 1024
@@ -471,7 +471,7 @@ def test_w4a4_prefill_matches_nvfp4_activation_reference(hidden_size, inter, act
def bank(n, k):
packed = torch.randint(0, 256, (E, n, k // 2), dtype=torch.uint8, device=device, generator=g)
scale = (torch.rand(E, n, k // 16, device=device, generator=g) + 0.25).to(torch.float8_e4m3fn)
- return packed, scale, (torch.rand(E, n, device=device, generator=g) * 0.02 + 0.01).to(torch.float16)
+ return packed, scale, torch.rand(E, n, device=device, generator=g) * 0.02 + 0.01
gu, gus, gug = bank(2 * inter, hidden_size)
dn, dns, dng = bank(hidden_size, inter)
@@ -621,10 +621,10 @@ def test_dummy_nvfp4_banks_match_the_triton_layout():
expected = {
"gate_up": ((E, 2 * I, H // 2), torch.uint8),
"gate_up_scale": ((E, 2 * I, H // 16), torch.float8_e4m3fn),
- "gate_up_global": ((E, 2 * I), torch.float16),
+ "gate_up_global": ((E, 2 * I), torch.float32),
"down": ((E, H, I // 2), torch.uint8),
"down_scale": ((E, H, I // 16), torch.float8_e4m3fn),
- "down_global": ((E, H), torch.float16),
+ "down_global": ((E, H), torch.float32),
}
assert banks.sources.keys() == expected.keys()
for name, (shape, dtype) in expected.items():