FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(quant)!: keep nvfp4 global scales in fp32 (#739)
Xiaoze Fan0db5023b60ec
diff --git a/docs/ftw-hotfix.md b/docs/ftw-hotfix.mdindex 299d522d..a88c0066 100644--- a/docs/ftw-hotfix.md+++ b/docs/ftw-hotfix.md@@ -32,6 +32,7 @@ byte range, never the whole checkpoint; `--revision` pins the Hub revision to re| `KeyError: 'model.embed.weight'` | deepseek-ai/DeepSeek-V4-Flash-0731 | rename the index entries | none || `RuntimeError: Unexpected keys ... .weight_scale` | nvidia/GLM-5.2-NVFP4 | dequantize the old runtime-fp8 weights back to bf16 | none || `PLE shard indices are not contiguous 0..N-1: []` | Qwen3.8-Flash-Next (FTWs converted before #420) | write the PLE table as `ple-table-*.safetensors` | 47.7 GiB |+| `ValueError: ... stores NVFP4 global scales as fp16` | NVFP4 FTWs converted by older builds | none: the fp16 rounding cannot be undone, so the script refuses; reconvert with `ft checkpoint` | - || `ValueError: ... holds no vision encoder tensors` | image-input families (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next, Gemma-4, GLM-5.3-Flash, Muse-Glimmer, MiniMax-M3) converted before the family served images | add the vision encoder: the tower's checkpoint tensors are fetched and the family's own weight reader emits them as the converter would | 1 to 4 GiB |The script decides by itself: it builds the current model from the FTW's `config.json`, comparesdiff --git a/docs/models.md b/docs/models.mdindex fd124ea3..9c35ece2 100644--- a/docs/models.md+++ b/docs/models.md@@ -86,6 +86,8 @@ unset on PCIe GPUs: wider grids add no bandwidth there.see [ftw-hotfix.md](ftw-hotfix.md) for the affected checkpoints and the repair tool.- Muse-Glimmer keeps the attention gate as its own linear next to `qkv_proj`; an FTW convertedwhen the gate was fused into `qkvg_proj` no longer loads, reconvert it with `ft checkpoint`.+- NVFP4 global scales are kept in fp32. An NVFP4 FTW that stores them as fp16 (converted by an+ older build) is refused at load; reconvert it with `ft checkpoint`.- An FTW converted before its family served images holds no vision encoder: `ft serve`refuses it unless started with `--text-model-only` (or `--mm-disable vision`); reconvert itwith `ft checkpoint`, or add the encoder in place with [scripts/ftw_hotfix.py](ftw-hotfix.md).diff --git a/python/freetoken/checkpoint/ftw.py b/python/freetoken/checkpoint/ftw.pyindex ae97a98d..ca9c33dc 100644--- a/python/freetoken/checkpoint/ftw.py+++ b/python/freetoken/checkpoint/ftw.py@@ -102,6 +102,11 @@ def _elsize(dt: torch.dtype) -> int:return torch.empty((), dtype=dt).element_size()+def _is_nvfp4_global(name: str) -> bool:+ m = _LAYER_ENTRY_RE.match(name)+ return (m["base"] if m else name).endswith(("gate_up_global", "down_global", ".weight_global"))++def is_ftw_checkpoint(path: str) -> bool:"""True if ``path`` is a directory holding a FreeToken Weight (FTW) index."""return os.path.isfile(os.path.join(path, INDEX_NAME))@@ -221,6 +226,12 @@ class FTWReader:self.dir = pathself.shards = sorted(self.index["shards"], key=lambda s: s["global_off"])self.tensors = {t["name"]: t for t in self.index["tensors"]}+ fp16_globals = [n for n, t in self.tensors.items() if t["dtype"] == "float16" and _is_nvfp4_global(n)]+ if fp16_globals:+ raise ValueError(+ f"{path!r} stores NVFP4 global scales as fp16 (e.g. {fp16_globals[0]!r}) but FreeToken reads them as fp32; "+ "reconvert it from the source checkpoint with `ft checkpoint`"+ )self._fds: dict[str, int] = {}self._maps: dict[str, tuple[mmap.mmap, memoryview]] = {}# O_DIRECT (DMA straight from disk, bypassing the page cache) is the fast path but adiff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex 1c367aa4..fd67d565 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -92,7 +92,7 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int"down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2,}if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"):- # models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp16+ # models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp32# per-row globals; marlin/b12x repacks are byte-identical with the globals# folded into GPU-resident alphas (layers/quantization/moe/nvfp4.py), so no global banks.banks = {@@ -102,8 +102,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int"down_scale": H * (I // 16),}if fmt == "nvfp4":- banks["gate_up_global"] = 2 * I * 2- banks["down_global"] = H * 2+ banks["gate_up_global"] = 2 * I * 4+ banks["down_global"] = H * 4return banksif fmt == "mxfp4_triton":# gpt-oss mxfp4 expert banks: transposed split-K blocks/scales + bf16 biasdiff --git a/python/freetoken/kernel/csrc/cpu_moe/common.h b/python/freetoken/kernel/csrc/cpu_moe/common.hindex adf7e145..0edee015 100644--- a/python/freetoken/kernel/csrc/cpu_moe/common.h+++ b/python/freetoken/kernel/csrc/cpu_moe/common.h@@ -92,33 +92,6 @@ constexpr int PF_AHEAD = 512;const float kE2M1[16] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f,-0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f};-inline float fp16_to_f32(uint16_t h) {- const uint32_t sign = (uint32_t)(h & 0x8000u) << 16;- uint32_t exp = (h >> 10) & 0x1Fu;- uint32_t man = h & 0x3FFu;- uint32_t f;- if (exp == 0) {- if (man == 0) {- f = sign;- } else {- exp = 127 - 15 + 1;- while ((man & 0x400u) == 0) {- man <<= 1;- --exp;- }- man &= 0x3FFu;- f = sign | (exp << 23) | (man << 13);- }- } else if (exp == 0x1Fu) {- f = sign | 0x7F800000u | (man << 13);- } else {- f = sign | ((exp + (127 - 15)) << 23) | (man << 13);- }- float out;- std::memcpy(&out, &f, sizeof(out));- return out;-}-// e4m3 (OCP "fn": finite, max-normal 448, exp bias 7). Decoded into a 256-entry LUT.inline float e4m3_decode(uint8_t v) {const float sign = (v & 0x80u) ? -1.0f : 1.0f;diff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cppindex 429955fa..3c5a56d5 100644--- a/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/nvfp4.cpp@@ -7,7 +7,7 @@ namespace {// --------------------------- NVFP4 (W4A16) dequant ---------------------------// Weights: e2m1 4-bit codes (2/byte, low nibble first), per-16 block scale in-// fp8-e4m3, per-output-row global scale in fp16. Dequant matches the GPU kernels+// fp8-e4m3, per-output-row global scale in fp32. Dequant matches the GPU kernels// (freetoken/kernel/triton/nvfp4_dequant.py): w = E2M1[code] * e4m3(scale) * global.// Activations stay bf16 (W4A16); the GEMV dequantizes weights inside the K-loop.@@ -405,7 +405,7 @@ class Nvfp4Format final : public MoeFormat {const int H = p.H;const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));- const uint16_t* global = static_cast<const uint16_t*>(tbl_at(gu_global_, layer));+ const float* global = static_cast<const float*>(tbl_at(gu_global_, layer));const float* xe = xe_.data() + (size_t)tok * (H / 2);const float* xo = xo_.data() + (size_t)tok * (H / 2);const int8_t* xi8 = nvi8dot_ != nullptr ? xi8_.data() + (size_t)tok * H : nullptr;@@ -414,9 +414,9 @@ class Nvfp4Format final : public MoeFormat {const size_t r = (size_t)e * (2 * p.I) + row;if (nvi8dot_ != nullptr)return nvi8dot_(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16),- fp16_to_f32(global[r]), xi8, H, e4m3_lut_, xas);+ global[r], xi8, H, e4m3_lut_, xas);return nvdot_(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16),- fp16_to_f32(global[r]), xe, xo, H, e2m1_lut_, e4m3_lut_);+ global[r], xe, xo, H, e2m1_lut_, e4m3_lut_);});}@@ -424,15 +424,15 @@ class Nvfp4Format final : public MoeFormat {const int I = p.I;const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(down_, layer));const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));- const uint16_t* global = static_cast<const uint16_t*>(tbl_at(dn_global_, layer));+ const float* global = static_cast<const float*>(tbl_at(dn_global_, layer));for (int h = h0; h < h1; ++h) {for (int j = 0; j < n; ++j) {const size_t r = (size_t)routes[j].r;const size_t row = (size_t)routes[j].e * p.H + h;out[j * kRowBlock + h - h0] = nvi8dot_ != nullptr- ? nvi8dot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), fp16_to_f32(global[row]),+ ? nvi8dot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), global[row],gi8_.data() + r * I, I, e4m3_lut_, gas_.data() + r * (I / 16))- : nvdot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), fp16_to_f32(global[row]),+ : nvdot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), global[row],ge_.data() + r * (I / 2), go_.data() + r * (I / 2), I, e2m1_lut_, e4m3_lut_);}}@@ -441,7 +441,7 @@ class Nvfp4Format final : public MoeFormat {private:nvdot_fn nvdot_;nvi8dot_fn nvi8dot_; // AVX-VNNI W4A8 nvfp4 dot (nullptr -> use fp32 nvdot)- // ModelOpt rows: packed e2m1 [E,2I,H/2], e4m3 scales per 16 along K [E,2I,H/16], fp16 row globals [E,2I]; down likewise with K = I.+ // ModelOpt rows: packed e2m1 [E,2I,H/2], e4m3 scales per 16 along K [E,2I,H/16], fp32 row globals [E,2I]; down likewise with K = I.const uint64_t* gate_up_ = nullptr;const uint64_t* gu_scale_ = nullptr;const uint64_t* gu_global_ = nullptr;diff --git a/python/freetoken/kernel/triton/nvfp4_dequant.py b/python/freetoken/kernel/triton/nvfp4_dequant.pyindex a949e7fa..214e6ab7 100644--- a/python/freetoken/kernel/triton/nvfp4_dequant.py+++ b/python/freetoken/kernel/triton/nvfp4_dequant.py@@ -26,7 +26,7 @@ def _e2m1_lut(device_index: int) -> torch.Tensor:def _dequant_nvfp4_kernel(packed_ptr, # [S, OUT, IN // 2] uint8scale_ptr, # [S, OUT, IN // 16] fp8-e4m3 per-block scale (checkpoint native)- global_ptr, # [S, OUT] fp16 per-output-row global scale (weight_scale_2)+ global_ptr, # [S, OUT] fp32 per-output-row global scale (weight_scale_2)slots_ptr, # [N] int32 -> cache slot for each output expertout_ptr, # [N, OUT, IN] compute dtypelut_ptr, # [16] float32 E2M1 valuesdiff --git a/python/freetoken/kernel/triton/nvfp4_fused_moe.py b/python/freetoken/kernel/triton/nvfp4_fused_moe.pyindex 61e867b1..08f2f777 100644--- a/python/freetoken/kernel/triton/nvfp4_fused_moe.py+++ b/python/freetoken/kernel/triton/nvfp4_fused_moe.py@@ -1,14 +1,14 @@"""Inline-dequant NVFP4 fused-MoE Triton kernels.These kernels read the NVFP4 expert cache (packed FP4 codes + fp8 block scales +-fp16 per-row global scale) *directly* and dequantize inside the GEMM K-loop, so the+fp32 per-row global scale) *directly* and dequantize inside the GEMM K-loop, so thegrouped MoE GEMM never materializes a BF16 copy of the experts (no separate dequantpass, no HBM round-trip of the 4x larger BF16 weights).Layout, for an expert weight ``W[N, K]``:- ``packed[slot, n, k//2]`` uint8: two FP4 codes per byte, low nibble = even k.- ``scale[slot, n, k//16]`` fp8-e4m3: per 16-wide block scale.- - ``global[slot, n]`` fp16: per-output-row scale (``weight_scale_2``).+ - ``global[slot, n]`` fp32: per-output-row scale (``weight_scale_2``).- ``W[n, k] = E2M1[code] * scale[n, k//16] * global[n]``.The global scale is constant along K, so it is applied once after the K-loop.@@ -52,7 +52,7 @@ def _decode_nvfp4_moe_kernel(a_ptr, # [M, K] activations (compute dtype)packed_ptr, # [S, N, K // 2] uint8scale_ptr, # [S, N, K // 16] fp8-e4m3- global_ptr, # [S, N] fp16+ global_ptr, # [S, N] fp32c_ptr, # [M, TOP_K, N] output (compute dtype)topk_weights_ptr, # [M, TOP_K] fp32topk_ids_ptr, # [M, TOP_K] int32 -> cache slot@@ -163,7 +163,7 @@ def _decode_nvfp4_marlin_kernel(a_ptr, # [M, K] activations (compute dtype)packed_ptr, # [S, N, K // 8] int32 (8 fp4 codes per word, nibble j -> k=8*w+j)scale_ptr, # [S, N, K // 16] fp8-e4m3- global_ptr, # [S, N] fp16+ global_ptr, # [S, N] fp32c_ptr, # [M, TOP_K, N] output (compute dtype)topk_weights_ptr, # [M, TOP_K] fp32topk_ids_ptr, # [M, TOP_K] int32 -> cache slot@@ -357,7 +357,7 @@ def _prefill_nvfp4_moe_kernel(a_ptr, # [M, K] activationspacked_ptr, # [S, N, K // 2] uint8scale_ptr, # [S, N, K // 16] fp8-e4m3 (uint8 view before sm_89)- global_ptr, # [S, N] fp16+ global_ptr, # [S, N] fp32c_ptr, # [num_valid_tokens, N] output (flat over M*top_k)topk_weights_ptr,sorted_token_ids_ptr,diff --git a/python/freetoken/kernel/triton/nvfp4_linear.py b/python/freetoken/kernel/triton/nvfp4_linear.pyindex 53abdedc..38f148f0 100644--- a/python/freetoken/kernel/triton/nvfp4_linear.py+++ b/python/freetoken/kernel/triton/nvfp4_linear.py@@ -158,7 +158,7 @@ def _nvfp4_gemv_kernel(a_ptr, # [K] activation (compute dtype), contiguouspacked_ptr, # [N, K // 8] int32 (8 fp4 codes per word, nibble j -> k = 8*w + j)scale_ptr, # [N, K // 16] fp8-e4m3 per-16 block scale- gscale_ptr, # [N] fp16 per-output-row global scale (weight_scale_2)+ gscale_ptr, # [N] fp32 per-output-row global scale (weight_scale_2)out_ptr, # [N] out dtypeN, K,stride_pn, stride_pkw,@@ -226,7 +226,7 @@ def _nvfp4_gemv_splitk_kernel(a_ptr, # [K] activation (compute dtype), contiguouspacked_ptr, # [N, K // 8] int32scale_ptr, # [N, K // 16] fp8-e4m3- gscale_ptr, # [N] fp16 per-output-row global scale+ gscale_ptr, # [N] fp32 per-output-row global scalepart_ptr, # [SPLIT_K, N] fp32 partial sums (pre-global-scale)counter_ptr, # [cdiv(N, BLOCK_N)] int32 arrival counters, zeroed (self-resetting)out_ptr, # [N] out dtype@@ -311,7 +311,7 @@ def _gemv(a: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tensor,gscale: torch.Tensor, out_dtype: torch.dtype, transposed: bool) -> torch.Tensor:"""M==1 W4A16 GEMV. ``a`` [K] compute-dtype; ``packed_i32`` logical [N, K//8] int32(row-major or a K-major transposed view); ``scale`` logical [N, K//16] fp8; ``gscale``- [N] fp16. Picks split-K so small ``N`` (shared expert) still fills the SMs; large ``N``+ [N] fp32. Picks split-K so small ``N`` (shared expert) still fills the SMs; large ``N``(lm_head) stays single-pass (split_k=1)."""out_tl = _TL_DTYPE[out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16]N = packed_i32.shape[0]@@ -389,7 +389,7 @@ def _nvfp4_gemm_kernel(a_ptr, # [M, K] activationspacked_ptr, # logical [N, K // 8] int32 (row- or K-major via strides)scale_ptr, # logical [N, K // 16] fp8-e4m3- gscale_ptr, # [N] fp16+ gscale_ptr, # [N] fp32c_ptr, # [M, N] output (written when SPLIT_K == 1)part_ptr, # [SPLIT_K, M, N] fp32 partials (written when SPLIT_K > 1)M, N, K, K_WORDS, tiles_per,@@ -612,7 +612,7 @@ def _gemm_inkernel(a: torch.Tensor, packed_i32: torch.Tensor, scale: torch.Tensodef _nvfp4_dequant_rows_kernel(packed_ptr, # [N, K // 8] int32scale_ptr, # [N, K // 16] fp8-e4m3- gscale_ptr, # [N] fp16+ gscale_ptr, # [N] fp32out_ptr, # [N, K] compute dtypeK_WORDS,stride_pn, stride_pkw,@@ -646,7 +646,7 @@ def _nvfp4_dequant_rows_kernel(def _nvfp4_dequant_cols_kernel(packed_ptr, # K-major: element (n, kw) at kw * stride_pkw + n (stride_pn == 1)scale_ptr,- gscale_ptr, # [N] fp16+ gscale_ptr, # [N] fp32out_ptr, # [n_rows, K] compute dtype, row-majorN, K_WORDS,stride_pn, stride_pkw,@@ -781,7 +781,7 @@ def nvfp4_dense_linear(weight_global: torch.Tensor, bias: torch.Tensor | None = None,) -> torch.Tensor:"""``y = x @ dequant(weight)^T`` on the checkpoint-native row-major layout.- ``weight`` [N, K//2] uint8, ``weight_scale`` [N, K//16] fp8-e4m3, ``weight_global`` [N] fp16."""+ ``weight`` [N, K//2] uint8, ``weight_scale`` [N, K//16] fp8-e4m3, ``weight_global`` [N] fp32."""return _linear_impl(x, weight.view(torch.int32), weight_scale, weight_global, bias, x.dtype, transposed=False)diff --git a/python/freetoken/layers/quantization/linear/nvfp4.py b/python/freetoken/layers/quantization/linear/nvfp4.pyindex 6a131653..1002a70b 100644--- a/python/freetoken/layers/quantization/linear/nvfp4.py+++ b/python/freetoken/layers/quantization/linear/nvfp4.py@@ -1,4 +1,4 @@-"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp16 global per output row."""+"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp32 global per output row."""from __future__ import annotations@@ -256,7 +256,7 @@ class MarlinNvfp4LinearKernel(LinearKernel):dev = layer.weight.devicelayer.weight = torch.empty(0, dtype=torch.uint8, device=dev)layer.weight_scale = torch.empty(0, dtype=FP8, device=dev)- layer.weight_global = torch.empty(0, dtype=torch.float16, device=dev)+ layer.weight_global = torch.empty(0, dtype=torch.float32, device=dev)def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import apply_fp4_marlin_linear@@ -301,6 +301,6 @@ class Nvfp4LinearMethod(LinearMethod):raise ValueError(f"nvfp4 needs in_features divisible by {GROUP}, got {g.in_features}")layer.weight = torch.empty(g.out_features, g.in_features // 2, dtype=torch.uint8)layer.weight_scale = torch.empty(g.out_features, g.in_features // GROUP, dtype=FP8)- layer.weight_global = torch.empty(g.out_features, dtype=torch.float16)+ layer.weight_global = torch.empty(g.out_features, dtype=torch.float32)# no W4A16 kernel reads it; declared so a W4A4 checkpoint loads complete and a W4A4 kernel finds it in placelayer.input_scale = torch.empty((), dtype=torch.float32) if self.scheme.has("input_scale") else Nonediff --git a/python/freetoken/layers/quantization/moe/base.py b/python/freetoken/layers/quantization/moe/base.pyindex 5ae89c97..b17a9135 100644--- a/python/freetoken/layers/quantization/moe/base.py+++ b/python/freetoken/layers/quantization/moe/base.py@@ -96,8 +96,8 @@ def fused_piece(pieces: dict[str, torch.Tensor], role: str) -> torch.Tensor:def global_rows(piece: torch.Tensor, rows: int) -> torch.Tensor:- """Per-expert global scale as one fp16 value per output row (a scalar per expert is broadcast)."""- flat = piece.reshape(piece.shape[0], -1).to(torch.float16)+ """Per-expert global scale as one fp32 value per output row (a scalar per expert is broadcast)."""+ flat = piece.reshape(piece.shape[0], -1).to(torch.float32)if flat.shape[1] == rows:return flatassert flat.shape[1] == 1, (flat.shape, rows)diff --git a/python/freetoken/layers/quantization/moe/nvfp4.py b/python/freetoken/layers/quantization/moe/nvfp4.pyindex 0dce3995..2a5e8bac 100644--- a/python/freetoken/layers/quantization/moe/nvfp4.py+++ b/python/freetoken/layers/quantization/moe/nvfp4.py@@ -52,10 +52,10 @@ class TritonNvfp4MoEKernel(MoEKernel):return {"gate_up": BankSpec((2 * i, h // 2), torch.uint8),"gate_up_scale": BankSpec((2 * i, h // GROUP), FP8),- "gate_up_global": BankSpec((2 * i,), torch.float16),+ "gate_up_global": BankSpec((2 * i,), torch.float32),"down": BankSpec((h, i // 2), torch.uint8),"down_scale": BankSpec((h, i // GROUP), FP8),- "down_global": BankSpec((h,), torch.float16),+ "down_global": BankSpec((h,), torch.float32),}def pack(self, pieces, cfg: MoEConfig, out):@@ -183,7 +183,7 @@ def _marlin_pack_proj("""Pack one expert projection (on GPU) into the Marlin layout.``packed`` ``[N, K//2]`` uint8 e2m1 pairs, ``scale`` ``[N, K//16]`` e4m3,- ``row_global`` ``[N]`` fp16 per-row global (rows of a merged gate_up may carry+ ``row_global`` ``[N]`` fp32 per-row global (rows of a merged gate_up may carryw1's and w3's different globals). Returns ``(qweight [K//16, 2N] int32,scales [K//16, N] e4m3-coded, global scalar bf16)``.diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.pyindex 08293c7e..7797c784 100644--- a/python/freetoken/models/gemma4/weight.py+++ b/python/freetoken/models/gemma4/weight.py@@ -66,15 +66,15 @@ _NVFP4_DENSE_MLP_RE = re.compile(r"\.mlp\.(gate_proj|up_proj|down_proj)\.weight$def _nvfp4_dense_parts(reader: ShardReader, raw_base: str):- """Load an NVFP4 dense weight as the NVFP4 linear method's buffers: weight uint8 [O, IN//2], weight_scale fp8-e4m3 block [O, IN//16], weight_global fp16 [O] (the per-tensor weight_scale_2 per output row), input_scale fp32 scalar or None when the export has none."""+ """Load an NVFP4 dense weight as the NVFP4 linear method's buffers: weight uint8 [O, IN//2], weight_scale fp8-e4m3 block [O, IN//16], weight_global fp32 [O] (the per-tensor weight_scale_2 per output row), input_scale fp32 scalar or None when the export has none."""w = reader.get_tensor(raw_base + ".weight")s = reader.get_tensor(raw_base + ".weight_scale")- g = reader.get_tensor(raw_base + ".weight_scale_2").reshape(1).to(torch.float16)+ g = reader.get_tensor(raw_base + ".weight_scale_2").reshape(1).to(torch.float32)g = g.expand(w.shape[0]).contiguous()assert (w.dtype is torch.uint8and s.dtype is torch.float8_e4m3fn- and g.dtype is torch.float16+ and g.dtype is torch.float32), f"unexpected NVFP4 dense dtypes at {raw_base}: {w.dtype}/{s.dtype}/{g.dtype}"a = reader.get_tensor(raw_base + ".input_scale").reshape(()).to(torch.float32) if reader.has(raw_base + ".input_scale") else Nonereturn w, s, g, adiff --git a/python/freetoken/models/glm4_moe/weight.py b/python/freetoken/models/glm4_moe/weight.pyindex aa6538dc..0ca79efb 100644--- a/python/freetoken/models/glm4_moe/weight.py+++ b/python/freetoken/models/glm4_moe/weight.py@@ -78,12 +78,12 @@ def _iter_nvfp4_resident("""Yield native NVFP4 buffers for an always-resident Linear.``weight`` (packed fp4) + ``weight_scale`` (fp8 block scale) verbatim; per-tensor- ``weight_scale_2`` broadcast to per-row fp16 ``weight_global`` as the NVFP4 linear+ ``weight_scale_2`` broadcast to per-row fp32 ``weight_global`` as the NVFP4 linearmethod / dequant_nvfp4 expect. lossless vs checkpoint; same dequant math as routed experts."""packed = reader.get(f"{src_prefix}.weight") # [OUT, IN//2] uint8scale = reader.get(f"{src_prefix}.weight_scale") # [OUT, IN//16] fp8-e4m3- g = reader.get(f"{src_prefix}.weight_scale_2").reshape(()).to(torch.float16) # scalar+ g = reader.get(f"{src_prefix}.weight_scale_2").reshape(()).to(torch.float32) # scalaryield f"{dst_prefix}.weight", packedyield f"{dst_prefix}.weight_scale", scaleyield f"{dst_prefix}.weight_global", g.expand(packed.shape[0]).contiguous()diff --git a/python/freetoken/models/loader.py b/python/freetoken/models/loader.pyindex 0f310ccb..4a9ac94c 100644--- a/python/freetoken/models/loader.py+++ b/python/freetoken/models/loader.py@@ -208,11 +208,11 @@ class ShardReader:def nvfp4_parts_ct(f, raw_base: str) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:- """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16], per-output-row global fp16 [O], dequant-side input_scale fp32 scalar or None)`` for the NVFP4 linear method."""+ """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16], per-output-row global fp32 [O], dequant-side input_scale fp32 scalar or None)`` for the NVFP4 linear method."""w = f.get_tensor(raw_base + ".weight_packed")s = f.get_tensor(raw_base + ".weight_scale")wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)- g = (1.0 / wg).to(torch.float16).expand(w.shape[0]).contiguous()+ g = (1.0 / wg).expand(w.shape[0]).contiguous()a = Noneif f.has(raw_base + ".input_global_scale"):a = (1.0 / f.get_tensor(raw_base + ".input_global_scale").reshape(()).to(torch.float32))diff --git a/python/freetoken/models/muse_glimmer/weight.py b/python/freetoken/models/muse_glimmer/weight.pyindex 3b8eb6d2..11a18172 100644--- a/python/freetoken/models/muse_glimmer/weight.py+++ b/python/freetoken/models/muse_glimmer/weight.py@@ -131,7 +131,7 @@ def _iter_weights_compressed_tensors(Every text Linear (q/k/v/o, the attention gate, the MLP) is kept native NVFP4(W4A16): ``.weight`` (uint8 packed) + ``.weight_scale`` (fp8 block) + ``.weight_global``- (fp16 per-row, the reciprocal of the stored quant-side global). q/k/v fuse into+ (fp32 per-row, the reciprocal of the stored quant-side global). q/k/v fuse into``qkv_proj`` and gate/up into ``gate_up_proj`` on the output dim, each part keepingits own scales, so the fused FP4 weights are exact. Embeddings, norms, lm_head and thevision tower are bf16 (the checkpoint's ignore list). Scale lookups go through thediff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.pyindex e30429c0..53822945 100644--- a/python/freetoken/models/nvfp4_banks.py+++ b/python/freetoken/models/nvfp4_banks.py@@ -36,7 +36,7 @@ def _canon_kind(spec: "Nvfp4ExpertSourceSpec", kind: str) -> str:def _ingest_global(spec: "Nvfp4ExpertSourceSpec", tensor: torch.Tensor) -> torch.Tensor:if spec.global_reciprocal:tensor = 1.0 / tensor.float()- return tensor.to(torch.float16)+ return tensor.to(torch.float32)def _activation_scale():@@ -99,7 +99,7 @@ def iter_nvfp4_expert_pieces():"""One piece per routed expert: ``gate`` / ``up`` / ``down`` codes plus their ``_scale``(fp8 block scales) and ``_global`` (the per-tensor scale, reciprocal for quant-side dialects,- fp16) companions, straight from the safetensors shards. When every expert also stores the+ fp32) companions, straight from the safetensors shards. When every expert also stores thedialect's static activation scale, the pieces carry it as fp32 ``_input`` (dequant side).Serial reads walk the shards in order; ``parallel`` uses the chunked O_DIRECT reader. Eitherdiff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex 724ad153..bf3cecf0 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -227,7 +227,7 @@ class _DenseReader:g = part["weight_global"].reshape(-1).to(torch.float32)if g.numel() != 1:raise ValueError(f"{target}: weight_global has {g.numel()} elements, expected one per-tensor scale")- out["weight_global"] = g.to(torch.float16).expand(rows).contiguous()+ out["weight_global"] = g.expand(rows).contiguous()if "input_scale" in part:out["input_scale"] = part["input_scale"].reshape(()).to(torch.float32)return outdiff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex 057b86fd..0225cc4f 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -309,7 +309,7 @@ def _gguf_bank_shapes(fmt: str, opts: dict, H: int, I: int) -> dict[str, tuple[tdef _offload_bank_specs(fmt: str, H: int, I: int, opts: dict | None = None) -> dict[str, tuple[int, torch.dtype]]:"""Per-bank (elems_per_expert, dtype) for the flat offload host banks (the gather)."""- u8, bf16, f16, f8 = torch.uint8, torch.bfloat16, torch.float16, torch.float8_e4m3fn+ u8, bf16, f32, f8 = torch.uint8, torch.bfloat16, torch.float32, torch.float8_e4m3fnif fmt in ("gguf", "gguf_scaled"):return {name: (math.prod(shape), dtype) for name, (shape, dtype) in _gguf_bank_shapes(fmt, opts, H, I).items()}if fmt == "bf16":@@ -322,8 +322,8 @@ def _offload_bank_specs(fmt: str, H: int, I: int, opts: dict | None = None) -> dif fmt == "nvfp4":return {"gate_up_packed": (2 * I * (H // 2), u8), "gate_up_scale": (2 * I * (H // 16), u8),- "gate_up_global": (2 * I, f16), "down_packed": (H * (I // 2), u8),- "down_scale": (H * (I // 16), u8), "down_global": (H, f16),+ "gate_up_global": (2 * I, f32), "down_packed": (H * (I // 2), u8),+ "down_scale": (H * (I // 16), u8), "down_global": (H, f32),}if fmt == "mxfp4_triton":return {@@ -403,13 +403,13 @@ def _cpu_moe_bank_sources(fmt: str, H: int, I: int, E: int, opts: dict | None =b = {"gate_up_packed": pin(E, 2 * I, H // 2, dtype=torch.uint8),"gate_up_scale": pin(E, 2 * I, H // 16, dtype=torch.uint8),- "gate_up_global": pin(E, 2 * I, dtype=torch.float16),+ "gate_up_global": pin(E, 2 * I, dtype=torch.float32),"down_packed": pin(E, H, I // 2, dtype=torch.uint8),"down_scale": pin(E, H, I // 16, dtype=torch.uint8),- "down_global": pin(E, H, dtype=torch.float16),+ "down_global": pin(E, H, dtype=torch.float32),}b["gate_up_global"].fill_(1.0)- b["down_global"].fill_(1.0) # e4m3 scales decode to normal float32; globals are fp16+ b["down_global"].fill_(1.0) # e4m3 scales decode to normal float32return bif fmt == "mxfp4_triton": # transposed split-K layoutb = {diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex bb65146b..1f275749 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -448,12 +448,12 @@ class CpuMoeExecutor:if fmt == "fp8_block":return self._resolve_fp8_block_banks(banks, rep)- # nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp16 row globals.+ # nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp32 row globals.gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"]dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"]assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype)assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte"- assert gug[rep].dtype == torch.float16 and dng[rep].dtype == torch.float16, (gug[rep].dtype, dng[rep].dtype)+ assert gug[rep].dtype == torch.float32 and dng[rep].dtype == torch.float32, (gug[rep].dtype, dng[rep].dtype)I = int(gup[rep].shape[1] // 2)H = int(gup[rep].shape[2] * 2)assert gup[rep].shape[1] == 2 * Idiff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 6b428c41..0d702d8a 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -51,7 +51,7 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {# GGUF experts whose NVFP4 tensors carry per-expert scales: global [L*E, 4] fp32 (gate, up, down, 1) weight_scale_2"gguf_scaled": ("gate_up", "down", "global"),# native ModelOpt rows for the Triton inline-dequant kernels: packed e2m1 codes +- # fp8-e4m3 per-16 block scales + per-output-row fp16 globals (w1/w3 carry distinct+ # fp8-e4m3 per-16 block scales + per-output-row fp32 globals (w1/w3 carry distinct# globals, and folding them into the e4m3 block scales would underflow)"nvfp4": ("gate_up_packed",@@ -94,7 +94,7 @@ _BANK_BYTES_PER_EXPERT = {(2 * I // 128) * fp8_block_scale_pad(2 * I // 128, H // 128)+ (H // 128) * fp8_block_scale_pad(H // 128, I // 128)) * 2,- "nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2),+ "nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 4) + H * (I // 2 + I // 16 + 4),"mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2),"ds_fp4": lambda H, I: 2 * I * (H // 2 + H // 32) + H * (I // 2 + I // 32),}diff --git a/scripts/ftw_hotfix.py b/scripts/ftw_hotfix.pyindex 12574651..7b4d2257 100644--- a/scripts/ftw_hotfix.py+++ b/scripts/ftw_hotfix.py@@ -38,7 +38,7 @@ import tempfileimport torch-from freetoken.checkpoint.ftw import ALIGN, FORMAT_TAG, INDEX_NAME, _align_up, _dtype_of, _dtype_str, _SHARD_FMT+from freetoken.checkpoint.ftw import ALIGN, FORMAT_TAG, INDEX_NAME, _align_up, _dtype_of, _dtype_str, _is_nvfp4_global, _SHARD_FMTfrom freetoken.models.config import VISION_KEY_PREFIXESfrom freetoken.distributed.info import set_tp_info, try_get_tp_infofrom freetoken.engine.config import EngineConfig@@ -340,6 +340,10 @@ def validate_ftw(ftw_dir: str, index: dict, expected: dict, renames: dict[str, s# the loader casts between float types; anything else has to match exactlyif dt != exp[1] and not (dt in _FLOAT_DTYPES and exp[1] in _FLOAT_DTYPES):problems.append(f"{name}: FTW dtype {e['dtype']} is not loadable as {exp[1]}")+ fp16_globals = [e["name"] for e in index["tensors"] if e["dtype"] == "float16" and _is_nvfp4_global(e["name"])]+ if fp16_globals:+ problems.append(f"{len(fp16_globals)} NVFP4 global scale entries are fp16 (e.g. {fp16_globals[0]}) but FreeToken reads them as fp32; "+ "this script cannot repair that, reconvert the FTW with `ft checkpoint`")blocks = sorted((e["global_off"], e["global_off"] + _align_up(e["nbytes"]), e["name"]) for e in index["tensors"])for (_, a1, an), (b0, _, bn) in zip(blocks, blocks[1:]):if b0 < a1:diff --git a/tests/kernels/test_e4m3_compat.py b/tests/kernels/test_e4m3_compat.pyindex fa8a1f1d..1532feb4 100644--- a/tests/kernels/test_e4m3_compat.py+++ b/tests/kernels/test_e4m3_compat.py@@ -217,7 +217,7 @@ def _emit_all(path: str) -> None:N, K = 128, 256pk = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device=dev)sc = (torch.rand(N, K // 16, device=dev) * 2 + 0.1).to(FP8)- g = (torch.rand(N, device=dev) * 0.05 + 0.01).to(torch.float16)+ g = torch.rand(N, device=dev) * 0.05 + 0.01xa = torch.randn(1, K, dtype=torch.bfloat16, device=dev)xb = torch.randn(8, K, dtype=torch.bfloat16, device=dev)xc = torch.randn(200, K, dtype=torch.bfloat16, device=dev)@@ -231,7 +231,7 @@ def _emit_all(path: str) -> None:# nvfp4 bank dequantpkc = torch.randint(0, 256, (E, 64, 128), dtype=torch.uint8, device=dev)scc = (torch.rand(E, 64, 16, device=dev) + 0.1).to(FP8)- gc = (torch.rand(E, 64, device=dev) * 0.05 + 0.01).to(torch.float16)+ gc = torch.rand(E, 64, device=dev) * 0.05 + 0.01out["nv_dequant"] = f32(dequant_nvfp4(pkc, scc, gc, torch.tensor([1, 3], dtype=torch.int32, device=dev)))@@ -239,10 +239,10 @@ def _emit_all(path: str) -> None:S, H4, I4 = 4, 256, 128gup = torch.randint(0, 256, (S, 2 * I4, H4 // 2), dtype=torch.uint8, device=dev)gus4 = (torch.rand(S, 2 * I4, H4 // 16, device=dev) + 0.1).to(FP8)- gug = (torch.rand(S, 2 * I4, device=dev) * 0.05 + 0.01).to(torch.float16)+ gug = torch.rand(S, 2 * I4, device=dev) * 0.05 + 0.01dnp = torch.randint(0, 256, (S, H4, I4 // 2), dtype=torch.uint8, device=dev)dns4 = (torch.rand(S, H4, I4 // 16, device=dev) + 0.1).to(FP8)- dng = (torch.rand(S, H4, device=dev) * 0.05 + 0.01).to(torch.float16)+ dng = torch.rand(S, H4, device=dev) * 0.05 + 0.01hid4 = torch.randn(2, H4, dtype=torch.bfloat16, device=dev)out["nv_moe_marlin"] = f32(fused_experts_decode_nvfp4_marlin(hid4, gup, gus4, gug, dnp, dns4, dng, tw, tids))diff --git a/tests/kernels/test_nvfp4_w4a4_linear.py b/tests/kernels/test_nvfp4_w4a4_linear.pyindex 1840d30f..03fd2c7e 100644--- a/tests/kernels/test_nvfp4_w4a4_linear.py+++ b/tests/kernels/test_nvfp4_w4a4_linear.py@@ -86,7 +86,7 @@ def test_w4a4_linear_matches_nvfp4_activation_reference(kernel_name, M, N, K, pag = torch.Generator(device="cuda").manual_seed(0)weight = torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g)scale = (torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn)- row_global = torch.full((N,), 1e-3, dtype=torch.float16, device="cuda")+ row_global = torch.full((N,), 1e-3, dtype=torch.float32, device="cuda")if parts_differ:row_global[N // 2:] = 4e-3bias = torch.randn(N, device="cuda", dtype=torch.bfloat16)@@ -149,7 +149,7 @@ def test_w4a4_linear_takes_row_strided_and_empty_inputs(kernel_name, M):g = torch.Generator(device="cuda").manual_seed(0)layer = SimpleNamespace(weight=torch.randint(0, 256, (N, K // 2), dtype=torch.uint8, device="cuda", generator=g),weight_scale=(torch.rand(N, K // 16, device="cuda", generator=g) + 0.25).to(torch.float8_e4m3fn),- weight_global=torch.full((N,), 1e-3, dtype=torch.float16, device="cuda"),+ weight_global=torch.full((N,), 1e-3, dtype=torch.float32, device="cuda"),input_scale=torch.tensor(0.01, device="cuda"), bias=None)kernel.finalize(layer)buf = torch.randn(M, K + 64, device="cuda", dtype=torch.bfloat16, generator=g)diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.pyindex 605c40f1..4395a863 100644--- a/tests/models/qwen4_exp/test_weight_ckpt.py+++ b/tests/models/qwen4_exp/test_weight_ckpt.py@@ -263,7 +263,7 @@ def test_ple_table_loads_pinned_and_matches_the_checkpoint(reader):# The two pinned host allocations the engine must budget for.total = PLE_BYTES + NUM_LAYERS * EXPERT_LAYER_BYTES- assert abs(total / 2**30 - 111.14) < 0.05+ assert abs(total / 2**30 - 111.32) < 0.05@pytest.fixture(scope="module")@@ -312,14 +312,14 @@ def test_sampled_experts_match_the_checkpoint(layer0_expert_banks, reader):scale = reader.get(f"{base}.{proj}.weight_scale")assert torch.equal(banks[bank][0][expert][rows].reshape(-1).view(torch.uint8),scale.reshape(-1).view(torch.uint8))- gate_g = reader.get(f"{base}.gate_proj.weight_scale_2").to(torch.float16)+ gate_g = reader.get(f"{base}.gate_proj.weight_scale_2").to(torch.float32)assert torch.equal(banks["gate_up_global"][0][expert, :I], gate_g.reshape(1).expand(I))def test_expert_bank_bytes_match_the_aot_row_table(layer0_expert_banks):measured = sum(t[0].numel() * t[0].element_size() for t in layer0_expert_banks.values())assert measured == EXPERT_LAYER_BYTES- assert abs(NUM_LAYERS * EXPERT_LAYER_BYTES / 2**30 - 63.46) < 0.05+ assert abs(NUM_LAYERS * EXPERT_LAYER_BYTES / 2**30 - 63.63) < 0.05@pytest.mark.skipif(not torch.cuda.is_available(), reason="dummy banks are pinned")diff --git a/tests/models/test_qwen3_5_moe_weight.py b/tests/models/test_qwen3_5_moe_weight.pyindex 16f439a9..c502067e 100644--- a/tests/models/test_qwen3_5_moe_weight.py+++ b/tests/models/test_qwen3_5_moe_weight.py@@ -431,7 +431,7 @@ def test_modelopt_fp8_scales_broadcast_per_part_and_input_scale_is_the_max(checkassert _same(loaded[f"{fused}.weight"][:I], raw[f"{shared}.gate_proj.weight"])assert _same(loaded[f"{fused}.weight_scale"][I:], raw[f"{shared}.up_proj.weight_scale"])glob = loaded[f"{fused}.weight_global"]- assert glob.dtype is torch.float16 and torch.equal(glob[:I], raw[f"{shared}.gate_proj.weight_scale_2"].to(torch.float16).expand(I))+ assert glob.dtype is torch.float32 and torch.equal(glob[:I], raw[f"{shared}.gate_proj.weight_scale_2"].float().expand(I))# W4A16: no activation quantizer, so the layer has no input_scale even when the export ships oneassert f"{fused}.input_scale" not in loaded and "lm_head.input_scale" not in loadedassert loaded["lm_head.weight"].dtype is torch.uint8 and loaded["lm_head.weight_global"].shape == (V,)@@ -444,8 +444,8 @@ def test_compressed_tensors_nvfp4_global_is_the_reciprocal(checkpoint):loaded = _load(folder)attn = f"{LM}.layers.1.self_attn"glob = loaded["model.layers.1.self_attn.qkv_proj.weight_global"]- expected = torch.cat([(1.0 / raw[f"{attn}.{p}_proj.weight_global_scale"]).to(torch.float16).expand(raw[f"{attn}.{p}_proj.weight_packed"].shape[0]) for p in "qkv"])- assert glob.dtype is torch.float16 and torch.equal(glob, expected)+ expected = torch.cat([(1.0 / raw[f"{attn}.{p}_proj.weight_global_scale"].float()).expand(raw[f"{attn}.{p}_proj.weight_packed"].shape[0]) for p in "qkv"])+ assert glob.dtype is torch.float32 and torch.equal(glob, expected)assert _same(loaded["model.layers.1.self_attn.o_proj.weight_scale"], raw[f"{attn}.o_proj.weight_scale"])# ``dynamic: local`` ships the quant-side activation global too; it lands as the dequant-side input_scaleassert torch.equal(loaded["model.layers.1.self_attn.o_proj.input_scale"], (1.0 / raw[f"{attn}.o_proj.input_global_scale"]).reshape(()))@@ -514,11 +514,11 @@ def test_nvfp4_expert_pieces_read_either_dialect_from_a_single_file(checkpoint):base = f"{LM}.layers.1.mlp.experts.2.gate_proj"if name.startswith("modelopt_mixed"):assert _same(piece["gate"][0], raw[f"{base}.weight"])- assert torch.equal(piece["gate_global"].reshape(-1), raw[f"{base}.weight_scale_2"].reshape(-1).to(torch.float16))+ assert torch.equal(piece["gate_global"].reshape(-1), raw[f"{base}.weight_scale_2"].reshape(-1).float())else:assert _same(piece["gate"][0], raw[f"{base}.weight_packed"])assert _same(piece["down_scale"][0], raw[f"{LM}.layers.1.mlp.experts.2.down_proj.weight_scale"])- assert torch.equal(piece["gate_global"].reshape(-1), (1.0 / raw[f"{base}.weight_global_scale"]).to(torch.float16))+ assert torch.equal(piece["gate_global"].reshape(-1), 1.0 / raw[f"{base}.weight_global_scale"].float())@pytest.mark.parametrize("parallel", [False, True])@@ -563,7 +563,7 @@ def test_a_quantized_router_is_dequantized_because_the_family_serves_it_bf16(tmpfor layer, gate in enumerate(gates):got = loaded[f"model.layers.{layer}.mlp.gate.weight"]assert got.dtype is torch.bfloat16 and got.shape == (E, H)- expected = _nvfp4_reference(raw[f"{gate}.weight_packed"], raw[f"{gate}.weight_scale"], (1.0 / raw[f"{gate}.weight_global_scale"]).to(torch.float16).expand(E))+ expected = _nvfp4_reference(raw[f"{gate}.weight_packed"], raw[f"{gate}.weight_scale"], (1.0 / raw[f"{gate}.weight_global_scale"].float()).expand(E))assert torch.equal(got, expected)diff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.pyindex 56c6c18a..edd9ddc9 100644--- a/tests/moe/test_cpu_moe.py+++ b/tests/moe/test_cpu_moe.py@@ -90,9 +90,9 @@ def _make_nvfp4_cache(L, E, H, I, seed=0):def rows(OUT, IN):codes = torch.randint(0, 16, (S, OUT, IN), dtype=torch.uint8)packed = _pack_nvfp4(codes) # [S, OUT, IN//2]- # block scales in fp8-e4m3 (1 byte), ~[0.5, 1.5]; row globals in fp16, small.+ # block scales in fp8-e4m3 (1 byte), ~[0.5, 1.5]; row globals in fp32, small.scale = (0.5 + torch.rand(S, OUT, IN // 16)).to(torch.float8_e4m3fn)- glob = (0.02 + 0.01 * torch.rand(S, OUT)).to(torch.float16)+ glob = 0.02 + 0.01 * torch.rand(S, OUT)return packed, scale.contiguous(), glob.contiguous()gup, gus, gug = rows(2 * I, H)diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.pyindex fd98c2ee..58a55bfa 100644--- a/tests/moe/test_nvfp4_backends.py+++ b/tests/moe/test_nvfp4_backends.py@@ -64,9 +64,9 @@ def _make_native_sources(device: torch.device, seed: int = 0) -> dict[str, list[def rand_scale(*shape):return (torch.rand(*shape, generator=g) * 1.5 + 0.25).to(torch.float8_e4m3fn)- gate_up_global = torch.full((total, 2 * I), 1.0, dtype=torch.float16)+ gate_up_global = torch.full((total, 2 * I), 1.0, dtype=torch.float32)gate_up_global[:, I:] = 0.5 # w3 global != w1 global: exercises the alpha fold- down_global = torch.full((total, H), 0.75, dtype=torch.float16)+ down_global = torch.full((total, H), 0.75, dtype=torch.float32)flat = {"gate_up_packed": rand_u8(total, 2 * I, H // 2),"gate_up_scale": rand_scale(total, 2 * I, H // 16),@@ -392,7 +392,7 @@ def test_triton_prefill_routes_to_gluon_on_sm120(monkeypatch, hidden_size, interdef bank(n, k):packed = torch.randint(0, 256, (E, n, k // 2), dtype=torch.uint8, device=device, generator=g)scale = (torch.rand(E, n, k // 16, device=device, generator=g) + 0.25).to(torch.float8_e4m3fn)- return packed, scale, (torch.rand(E, n, device=device, generator=g) + 0.5).to(torch.float16)+ return packed, scale, torch.rand(E, n, device=device, generator=g) + 0.5banks = (*bank(2 * inter, hidden_size), *bank(hidden_size, inter))M = 1024@@ -471,7 +471,7 @@ def test_w4a4_prefill_matches_nvfp4_activation_reference(hidden_size, inter, actdef bank(n, k):packed = torch.randint(0, 256, (E, n, k // 2), dtype=torch.uint8, device=device, generator=g)scale = (torch.rand(E, n, k // 16, device=device, generator=g) + 0.25).to(torch.float8_e4m3fn)- return packed, scale, (torch.rand(E, n, device=device, generator=g) * 0.02 + 0.01).to(torch.float16)+ return packed, scale, torch.rand(E, n, device=device, generator=g) * 0.02 + 0.01gu, gus, gug = bank(2 * inter, hidden_size)dn, dns, dng = bank(hidden_size, inter)@@ -621,10 +621,10 @@ def test_dummy_nvfp4_banks_match_the_triton_layout():expected = {"gate_up": ((E, 2 * I, H // 2), torch.uint8),"gate_up_scale": ((E, 2 * I, H // 16), torch.float8_e4m3fn),- "gate_up_global": ((E, 2 * I), torch.float16),+ "gate_up_global": ((E, 2 * I), torch.float32),"down": ((E, H, I // 2), torch.uint8),"down_scale": ((E, H, I // 16), torch.float8_e4m3fn),- "down_global": ((E, H), torch.float16),+ "down_global": ((E, H), torch.float32),}assert banks.sources.keys() == expected.keys()for name, (shape, dtype) in expected.items():