FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): apply the weight scales of nvfp4 gguf tensors (#732)
Xiaoze Fan31c6ef4f7709
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 326c467d..ee655801 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -1770,6 +1770,9 @@ def _adjust_config(config: EngineConfig):"--dtype float16 with MXFP8 resident weights is unsupported (the ""W8A16 fold is only validated exact in bfloat16); use bfloat16.")+ if _dtype == torch.float16 and getattr(getattr(model_config, "quant", None), "weight_scales", False):+ # GGUF NVFP4 results are stored in the activation dtype before weight_scale_2 (~1e-4) multiplies them, which overflows fp16.+ raise ValueError("--dtype float16 with NVFP4 GGUF weight scales is unsupported (the unscaled results overflow float16); use bfloat16.")if config.attention_backend == "auto":override("attention_backend",diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex c4887efb..8162138c 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -48,6 +48,7 @@ std::unique_ptr<MoeFormat> make_mxfp4_format(const MoeParams& p, const FormatArgstd::unique_ptr<MoeFormat> make_ds_fp4_format(const MoeParams& p, const FormatArgs& a);std::unique_ptr<MoeFormat> make_fp8_block_format(const MoeParams& p, const FormatArgs& a);std::unique_ptr<MoeFormat> make_gguf_format(const MoeParams& p, const FormatArgs& a);+std::unique_ptr<MoeFormat> make_gguf_scaled_format(const MoeParams& p, const FormatArgs& a);void set_ggml_cpu(uintptr_t addr);namespace {@@ -97,6 +98,7 @@ inline MoeTask from_word(uint64_t w) {const std::pair<const char*, std::unique_ptr<MoeFormat> (*)(const MoeParams&, const FormatArgs&)> kFormats[] = {{"bf16", make_bf16_format}, {"nvfp4", make_nvfp4_format}, {"mxfp4_triton", make_mxfp4_format},{"ds_fp4", make_ds_fp4_format}, {"fp8_block", make_fp8_block_format}, {"gguf", make_gguf_format},+ {"gguf_scaled", make_gguf_scaled_format},};std::unique_ptr<MoeFormat> make_format(const std::string& name, const MoeParams& p, const FormatArgs& a) {diff --git a/python/freetoken/kernel/csrc/cpu_moe/kernels/gguf.cpp b/python/freetoken/kernel/csrc/cpu_moe/kernels/gguf.cppindex c1641af5..63c8107d 100644--- a/python/freetoken/kernel/csrc/cpu_moe/kernels/gguf.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/kernels/gguf.cpp@@ -31,10 +31,11 @@ void set_ggml_cpu(uintptr_t addr) {namespace {// Packed ggml rows of the layers whose experts have this gate_up type and this down type, dotted by ggml-cpu against the activation converted to each type's vec_dot input (q8_0, q8_1 or q8_K, or the float type itself).+// With scales, each expert's (gate, up, down, 1) NVFP4 weight_scale_2 multiplies its gate and up rows before the glu and its down rows before the routing weight, as llama.cpp's build_moe_ffn does.class GgufFormat final : public MoeFormat {public:- GgufFormat(const MoeParams& p, const FormatArgs& a)- : MoeFormat(p), ggml_(ggml_api()), gate_up_(a.bank("gate_up")), down_(a.bank("down")) {+ GgufFormat(const MoeParams& p, const FormatArgs& a, bool scaled)+ : MoeFormat(p), ggml_(ggml_api()), gate_up_(a.bank("gate_up")), down_(a.bank("down")), scales_(scaled ? a.bank("global") : nullptr) {static std::once_flag tables;std::call_once(tables, ggml_.init);gu_ = kernel(static_cast<int>(a.option("gate_up_type")), p.H, "gate_up");@@ -63,10 +64,11 @@ class GgufFormat final : public MoeFormat {void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float w_in, bf16_t* g) const override {const uint8_t* w = static_cast<const uint8_t*>(tbl_at(gate_up_, layer)) + static_cast<size_t>(e) * 2 * p.I * gu_.row_bytes;const uint8_t* xq = xq_.data() + static_cast<size_t>(tok) * gu_.act_row_bytes;+ const float* sc = expert_scales(layer, e);gated_rows(p, i0, i1, w_in, g, [&](int row) {float s;gu_.vec_dot(p.H, &s, 0, w + static_cast<size_t>(row) * gu_.row_bytes, 0, xq, 0, 1);- return s;+ return sc == nullptr ? s : s * sc[row < p.I ? 0 : 1];});}@@ -76,9 +78,19 @@ class GgufFormat final : public MoeFormat {for (int j = 0; j < n; ++j)dn_.vec_dot(p.I, &out[j * kRowBlock + h - h0], 0, w + (static_cast<size_t>(routes[j].e) * p.H + h) * dn_.row_bytes, 0,gq_.data() + static_cast<size_t>(routes[j].r) * dn_.act_row_bytes, 0, 1);+ if (scales_ == nullptr) return;+ for (int j = 0; j < n; ++j) {+ const float s = expert_scales(layer, routes[j].e)[2];+ for (int h = h0; h < h1; ++h) out[j * kRowBlock + h - h0] *= s;+ }}private:+ // the expert's (gate, up, down, 1) row of the layer's [E, 4] fp32 scale bank, or nullptr without scales+ const float* expert_scales(int layer, int e) const {+ return scales_ == nullptr ? nullptr : static_cast<const float*>(tbl_at(scales_, layer)) + static_cast<size_t>(e) * 4;+ }+ft_ggml_row_kernel kernel(int type, int k, const char* role) const {ft_ggml_row_kernel out{};const int rc = ggml_.row_kernel(type, k, &out);@@ -92,6 +104,7 @@ class GgufFormat final : public MoeFormat {const ft_ggml_cpu_api& ggml_;const uint64_t* gate_up_;const uint64_t* down_;+ const uint64_t* scales_;ft_ggml_row_kernel gu_{}, dn_{};std::vector<float> xf_;// the activation in the vec_dot input type: x per token, g per route@@ -101,7 +114,11 @@ class GgufFormat final : public MoeFormat {} // namespacestd::unique_ptr<MoeFormat> make_gguf_format(const MoeParams& p, const FormatArgs& a) {- return std::make_unique<GgufFormat>(p, a);+ return std::make_unique<GgufFormat>(p, a, false);+}++std::unique_ptr<MoeFormat> make_gguf_scaled_format(const MoeParams& p, const FormatArgs& a) {+ return std::make_unique<GgufFormat>(p, a, true);}} // namespace freetoken::cpu_moediff --git a/python/freetoken/kernel/csrc/ggml/VERSION b/python/freetoken/kernel/csrc/ggml/VERSIONindex 3c66f9b7..f7b5c017 100644--- a/python/freetoken/kernel/csrc/ggml/VERSION+++ b/python/freetoken/kernel/csrc/ggml/VERSION@@ -10,5 +10,6 @@ Local changes, kept in freetoken.patch (already applied):- mmvq.cu/.cuh: MMVQ takes bf16 or fp16 activations, can write bf16 or fp16 results, and can reuse activations already quantized to q8_1 (src1_q8_1_pre).- mmq.cu/.cuh, mmid.cu/.cuh: MoE MMQ runs a flat grid over only the (expert, column tile) pairs that hold tokens, listed by mm_ids_tile_map; ncols_max > 0 keeps the regular grid.- common.cuh: the native fp4 MMA (BLACKWELL_MMA_AVAILABLE, blackwell_mma_available) is off unless GGML_CUDA_FP4_MMA is defined; it needs an sm_120a/sm_120f build.+- mmvq.cu, common.cuh: the NVFP4 scale fusion (x_scale, gate_scale) reads strided scale vectors, e.g. columns of a per-expert scale bank.To update: copy the same files from a newer llama.cpp, apply freetoken.patch, and update this file.diff --git a/python/freetoken/kernel/csrc/ggml/freetoken.patch b/python/freetoken/kernel/csrc/ggml/freetoken.patchindex 3dc715fe..595498dd 100644--- a/python/freetoken/kernel/csrc/ggml/freetoken.patch+++ b/python/freetoken/kernel/csrc/ggml/freetoken.patch@@ -24,6 +24,16 @@}// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.+@@ -1582,6 +1588,9 @@+ const void * gate_bias = nullptr;+ const void * x_scale = nullptr;+ const void * gate_scale = nullptr;++ // FreeToken: element strides of x_scale / gate_scale, so they can be columns of a per-expert scale bank++ uint32_t x_scale_stride = 1;++ uint32_t gate_scale_stride = 1;+ ggml_glu_op glu_op;+ float glu_limit = 0.0f;+ const void * shared_up = nullptr;--- a/ggml/src/ggml-cuda/mmid.cu+++ b/ggml/src/ggml-cuda/mmid.cu@@ -177,3 +177,45 @@@@ -572,6 +582,19 @@stride_col_dst = fusion.shared_stride_col_dst;}+@@ -702,10 +706,10 @@+ }+ if constexpr (type == GGML_TYPE_NVFP4) {+ if (use_scale) {+- x_scales = x_scale[ids ? channel_x : 0];++ x_scales = x_scale[(ids ? channel_x : 0) * fusion.x_scale_stride];+ }+ if (use_gate_scale) {+- gate_scales = gate_scale[ids ? channel_x : 0];++ gate_scales = gate_scale[(ids ? channel_x : 0) * fusion.gate_scale_stride];+ }+ }+ }@@ -837,7 +841,7 @@}}@@ -615,6 +638,24 @@stride_col_dst = fusion.shared_stride_col_dst;}+@@ -963,7 +969,7 @@++ if constexpr (type == GGML_TYPE_NVFP4) {+ if (x_scale) {+- result *= x_scale[channel_x];++ result *= x_scale[channel_x * fusion.x_scale_stride];+ }+ }+ if (x_bias) {+@@ -973,7 +979,7 @@+ float gate_value = tmp_gate[threadIdx.x];+ if constexpr (type == GGML_TYPE_NVFP4) {+ if (gate_scale) {+- gate_value *= gate_scale[channel_x];++ gate_value *= gate_scale[channel_x * fusion.gate_scale_stride];+ }+ }+ if (gate_bias) {@@ -998,7 +1004,7 @@}}@@ -718,7 +759,30 @@GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.GGML_TENSOR_BINARY_OP_LOCALS;-@@ -1526,12 +1535,22 @@+@@ -1498,17 +1507,20 @@+ GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);+ fusion_local.gate_bias = fusion->gate_bias->data;+ }++ // FreeToken: a scale may be a strided vector, read with its element stride+ if (fusion->x_scale) {+ GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);+- GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));++ GGML_ASSERT(ggml_nelements(fusion->x_scale) == fusion->x_scale->ne[0] && fusion->x_scale->nb[0] % sizeof(float) == 0);+ GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));+ fusion_local.x_scale = fusion->x_scale->data;++ fusion_local.x_scale_stride = fusion->x_scale->nb[0] / sizeof(float);+ }+ if (fusion->gate_scale) {+ GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);+- GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));++ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == fusion->gate_scale->ne[0] && fusion->gate_scale->nb[0] % sizeof(float) == 0);+ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));+ fusion_local.gate_scale = fusion->gate_scale->data;++ fusion_local.gate_scale_stride = fusion->gate_scale->nb[0] / sizeof(float);+ }+ fusion_local.glu_op = fusion->glu_op;+ fusion_local.glu_limit = fusion->glu_limit;+@@ -1526,12 +1538,22 @@}const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);@@ -744,7 +808,7 @@}const int64_t s01 = src0->nb[1] / ts_src0;-@@ -1556,11 +1575,23 @@+@@ -1556,11 +1578,23 @@const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;diff --git a/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cuindex 69d93227..170cbcac 100644--- a/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu+++ b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu@@ -115,6 +115,14 @@ int64_t check_weight(TensorView w, ggml_type type, int64_t k) {return row_bytes;}+// one fp32 scale per expert, e.g. a column of a [E, n] scale bank; the MMVQ epilogue multiplies NVFP4 results by it+ggml_tensor describe_scale(TensorView s, ggml_type type, int64_t e) {+ RuntimeCheck(type == GGML_TYPE_NVFP4, "per-expert scales apply to NVFP4 weights, not ", ggml_type_name(type));+ RuntimeCheck(is_dtype(s.dtype(), kDLFloat, 32) && s.ndim() == 1 && s.size(0) == e && s.stride(0) > 0, "a per-expert scale must be an fp32 [E] view");+ const size_t nb0 = sizeof(float) * s.stride(0);+ return describe(GGML_TYPE_F32, s.data_ptr(), {e, 1, 1, 1}, {nb0, nb0 * e, nb0 * e, nb0 * e});+}+int64_t q8_1_row_bytes(int64_t k) { return GGML_PAD(k, MATRIX_ROW_PADDING) / QK8_1 * (int64_t) sizeof(block_q8_1); }// MMQ tiles read up to MATRIX_ROW_PADDING values past the last row; the caller keeps this many bytes after it@@ -234,7 +242,8 @@ void op_mul_mat(TensorView w, int64_t type_id, TensorView x, TensorView out, Opt}// out[T, k, N]: row (t, j) multiplies x[t, j or 0] by expert ids[t, j] of W[E, N, K]; the matrix kernel runs a flat grid over the tiles that hold routes-void op_mul_mat_id(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out) {+// scale[ids[t, j]] multiplies row (t, j); only the vector kernel takes it+void op_mul_mat_id(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out, Optional<TensorView> scale) {RuntimeCheck(w.ndim() == 3, "mul_mat_id takes w[E, N, row_bytes]");RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 2 && ids.IsContiguous(), "ids must be a contiguous int32 [T, k] tensor");const DeviceGuard guard(x.device());@@ -264,8 +273,15 @@ void op_mul_mat_id(TensorView w, int64_t type_id, TensorView x, TensorView ids,ggml_tensor src1 = describe(a.type, a.data, {k, ky, t, 1}, {a.esz, a.esz * k, a.esz * k * ky, a.esz * k * ky * t});const size_t oe = ggml_type_size(otype);ggml_tensor dst = describe(otype, out.data_ptr(), {n, top_k, t, 1}, {oe, oe * n, oe * n * top_k, oe * n * top_k * t});+ RuntimeCheck(mmvq || !scale.has_value(), "mul_mat_id takes a per-expert scale only on the vector kernel, T=", t);if (mmvq) {- ggml_cuda_mul_mat_vec_q(ctx, &src0, &src1, &tids, &dst);+ ggml_cuda_mm_fusion_args_host fusion{};+ ggml_tensor tscale;+ if (scale.has_value()) {+ tscale = describe_scale(scale.value(), type, e);+ fusion.x_scale = &tscale;+ }+ ggml_cuda_mul_mat_vec_q(ctx, &src0, &src1, &tids, &dst, scale.has_value() ? &fusion : nullptr);} else {ggml_cuda_mul_mat_q(ctx, &src0, &src1, &tids, &dst, 0);}@@ -291,7 +307,9 @@ ggml_glu_op fused_glu_op(const String & name) {} // namespace// out[T, k, I] = glu(x @ gate^T, x @ up^T) for small T in one MMVQ pass; w[E, 2I, row_bytes] holds each expert's gate rows then its up rows; limit is swiglu_clamp's-void op_mul_mat_id_glu(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out, String glu, double limit) {+// gate_scale / up_scale multiply the expert's gate / up results before the glu+void op_mul_mat_id_glu(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out, String glu, double limit,+ Optional<TensorView> gate_scale, Optional<TensorView> up_scale) {const ggml_glu_op glu_op = fused_glu_op(glu);RuntimeCheck(w.ndim() == 3 && w.size(1) % 2 == 0, "mul_mat_id_glu takes w[E, 2I, row_bytes]");RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 2 && ids.IsContiguous(), "ids must be a contiguous int32 [T, k] tensor");@@ -325,6 +343,15 @@ void op_mul_mat_id_glu(TensorView w, int64_t type_id, TensorView x, TensorView ifusion.gate = &gate;fusion.glu_op = glu_op;fusion.glu_limit = (float) limit;+ ggml_tensor tgate_scale, tup_scale;+ if (gate_scale.has_value()) {+ tgate_scale = describe_scale(gate_scale.value(), type, e);+ fusion.gate_scale = &tgate_scale;+ }+ if (up_scale.has_value()) {+ tup_scale = describe_scale(up_scale.value(), type, e);+ fusion.x_scale = &tup_scale;+ }ggml_cuda_mul_mat_vec_q(ctx, &up, &src1, &tids, &dst, &fusion);}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuhindex 36912ce4..a767c688 100644--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh@@ -1588,6 +1588,9 @@ struct ggml_cuda_mm_fusion_args_device {const void * gate_bias = nullptr;const void * x_scale = nullptr;const void * gate_scale = nullptr;+ // FreeToken: element strides of x_scale / gate_scale, so they can be columns of a per-expert scale bank+ uint32_t x_scale_stride = 1;+ uint32_t gate_scale_stride = 1;ggml_glu_op glu_op;float glu_limit = 0.0f;const void * shared_up = nullptr;diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cuindex f3a3ccc6..8487a035 100644--- a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu@@ -706,10 +706,10 @@ static __global__ void mul_mat_vec_q(}if constexpr (type == GGML_TYPE_NVFP4) {if (use_scale) {- x_scales = x_scale[ids ? channel_x : 0];+ x_scales = x_scale[(ids ? channel_x : 0) * fusion.x_scale_stride];}if (use_gate_scale) {- gate_scales = gate_scale[ids ? channel_x : 0];+ gate_scales = gate_scale[(ids ? channel_x : 0) * fusion.gate_scale_stride];}}}@@ -969,7 +969,7 @@ static __global__ void mul_mat_vec_q_moe(if constexpr (type == GGML_TYPE_NVFP4) {if (x_scale) {- result *= x_scale[channel_x];+ result *= x_scale[channel_x * fusion.x_scale_stride];}}if (x_bias) {@@ -979,7 +979,7 @@ static __global__ void mul_mat_vec_q_moe(float gate_value = tmp_gate[threadIdx.x];if constexpr (type == GGML_TYPE_NVFP4) {if (gate_scale) {- gate_value *= gate_scale[channel_x];+ gate_value *= gate_scale[channel_x * fusion.gate_scale_stride];}}if (gate_bias) {@@ -1507,17 +1507,20 @@ void ggml_cuda_mul_mat_vec_q(GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);fusion_local.gate_bias = fusion->gate_bias->data;}+ // FreeToken: a scale may be a strided vector, read with its element strideif (fusion->x_scale) {GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);- GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));+ GGML_ASSERT(ggml_nelements(fusion->x_scale) == fusion->x_scale->ne[0] && fusion->x_scale->nb[0] % sizeof(float) == 0);GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));fusion_local.x_scale = fusion->x_scale->data;+ fusion_local.x_scale_stride = fusion->x_scale->nb[0] / sizeof(float);}if (fusion->gate_scale) {GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);- GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));+ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == fusion->gate_scale->ne[0] && fusion->gate_scale->nb[0] % sizeof(float) == 0);GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));fusion_local.gate_scale = fusion->gate_scale->data;+ fusion_local.gate_scale_stride = fusion->gate_scale->nb[0] / sizeof(float);}fusion_local.glu_op = fusion->glu_op;fusion_local.glu_limit = fusion->glu_limit;diff --git a/python/freetoken/kernel/ggml.py b/python/freetoken/kernel/ggml.pyindex b672d58c..7a2765c5 100644--- a/python/freetoken/kernel/ggml.py+++ b/python/freetoken/kernel/ggml.py@@ -95,24 +95,37 @@ def uses_mmvq(ggml_type: int, m: int) -> bool:return _module().uses_mmvq(ggml_type, m)-def mul_mat_id(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor) -> torch.Tensor:- """Per-route expert GEMM over ``w[E, N, row_bytes]``: returns ``[T, k, N]`` in ``x``'s dtype; ``x`` is ``[T, K]`` (shared by the k routes) or ``[T, k, K]``."""+def mul_mat_id(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor, scale: torch.Tensor | None = None) -> torch.Tensor:+ """Per-route expert GEMM over ``w[E, N, row_bytes]``: returns ``[T, k, N]`` in ``x``'s dtype; ``x`` is ``[T, K]`` (shared by the k routes) or ``[T, k, K]``.++ ``scale`` is an fp32 ``[E]`` view (strided is fine) that multiplies each NVFP4 expert's rows; it needs ``T <= mmvq_max_tokens``.+ """t, top_k = ids.shapeout = torch.empty(t, top_k, w.shape[1], dtype=x.dtype, device=x.device)if t > mmvq_max_tokens(ggml_type):_check_mmq_tail(w, ggml_type, x.shape[-1])- _module().mul_mat_id(w, ggml_type, (x.unsqueeze(1) if x.dim() == 2 else x).contiguous(), ids, out)+ _module().mul_mat_id(w, ggml_type, (x.unsqueeze(1) if x.dim() == 2 else x).contiguous(), ids, out, scale)return out-def mul_mat_id_glu(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor, glu: str, limit: float = 0.0) -> torch.Tensor:+def mul_mat_id_glu(+ w: torch.Tensor,+ ggml_type: int,+ x: torch.Tensor,+ ids: torch.Tensor,+ glu: str,+ limit: float = 0.0,+ gate_scale: torch.Tensor | None = None,+ up_scale: torch.Tensor | None = None,+) -> torch.Tensor:"""``glu(gate, up)`` per route in one pass for small ``T``: ``w[E, 2I, row_bytes]`` holds gate rows then up rows; returns ``[T, k, I]`` in ``x``'s dtype.``glu`` is one of ``swiglu``, ``geglu`` (tanh GELU), ``swiglu_oai`` (alpha 1.702, limit 7) and ``swiglu_clamp`` (alpha 1, clamped at ``limit``).+ ``gate_scale`` / ``up_scale`` are fp32 ``[E]`` views that multiply each NVFP4 expert's gate / up results before the glu."""t, top_k = ids.shapeout = torch.empty(t, top_k, w.shape[1] // 2, dtype=x.dtype, device=x.device)- _module().mul_mat_id_glu(w, ggml_type, x.contiguous(), ids, out, glu, limit)+ _module().mul_mat_id_glu(w, ggml_type, x.contiguous(), ids, out, glu, limit, gate_scale, up_scale)return outdiff --git a/python/freetoken/layers/quantization/configs/gguf.py b/python/freetoken/layers/quantization/configs/gguf.pyindex bbc24f13..77859cd1 100644--- a/python/freetoken/layers/quantization/configs/gguf.py+++ b/python/freetoken/layers/quantization/configs/gguf.py@@ -4,7 +4,7 @@ from __future__ import annotationsfrom typing import Any, Callable, ClassVar-from ..ggml_types import is_quantized, type_name+from ..ggml_types import has_expert_scales, has_weight_scale, is_quantized, type_namefrom ..names import NameMapfrom ..quant_backend import get_quant_backendfrom ..registry import LayerKind, method_class, register_dialect@@ -29,19 +29,24 @@ class GGUFConfig(QuantConfig):super().__init__(name_map, unquantized)self.tensor_types = tensor_typesself.to_gguf = to_gguf+ self.expert_scales = has_expert_scales(tensor_types)+ self.weight_scales = any(n.endswith(".scale") and has_weight_scale(tensor_types, n.removesuffix(".scale")) for n in tensor_types)@classmethoddef claims(cls, q: dict[str, Any]) -> bool:return False- def _part_types(self, name: str) -> tuple[int, ...]:- types = []+ def _part_tensors(self, name: str) -> tuple[str, ...]:+ keys = []for tensor in self.to_gguf(name, self.tensor_types):key = f"{tensor}.weight" if f"{tensor}.weight" in self.tensor_types else tensorif key not in self.tensor_types:raise KeyError(f"GGUF has no tensor {key!r} for {name!r}")- types.append(self.tensor_types[key])- return tuple(types)+ keys.append(key)+ return tuple(keys)++ def _part_types(self, name: str) -> tuple[int, ...]:+ return tuple(self.tensor_types[key] for key in self._part_tensors(name))def scheme_for_name(self, name: str) -> QuantScheme | None:types = self._part_types(name)@@ -55,7 +60,8 @@ class GGUFConfig(QuantConfig):if prefix in self._schemes:return self._schemes[prefix]names = self.name_map.to_checkpoint(prefix)- parts = [() if self.unquantized(n) else self._part_types(n) for n in names]+ tensors = [() if self.unquantized(n) else self._part_tensors(n) for n in names]+ parts = [tuple(self.tensor_types[key] for key in ts) for ts in tensors]quantized = [any(is_quantized(t) for t in p) for p in parts]if not any(quantized):scheme = None@@ -63,7 +69,8 @@ class GGUFConfig(QuantConfig):found = {n: [type_name(t) for t in p] for n, p in zip(names, parts)}raise NotImplementedError(f"fused module {prefix!r} needs one GGUF tensor per part, got {found}")else:- scheme = gguf_scheme(p[0] for p in parts)+ scaled = any(has_weight_scale(self.tensor_types, ts[0]) for ts in tensors)+ scheme = gguf_scheme((p[0] for p in parts), scaled=scaled)self._schemes[prefix] = schemereturn scheme@@ -74,7 +81,8 @@ class GGUFConfig(QuantConfig):parts = [self._part_types(n) for n in self.name_map.to_checkpoint(prefix)]if any(len(p) != 1 for p in parts):raise NotImplementedError(f"experts {prefix!r} are stored as {parts} GGUF tensors per projection")- scheme = gguf_scheme(p[0] for p in parts)+ # the scale bank is per file, so every MoE layer keeps one bank set+ scheme = gguf_scheme((p[0] for p in parts), scaled=self.expert_scales)cfg = self.layer_config(layer, LayerKind.MOE, scheme)return method_class(QuantKind.GGUF, LayerKind.MOE)(cfg, get_quant_backend().select(LayerKind.MOE, QuantKind.GGUF))diff --git a/python/freetoken/layers/quantization/embedding/gguf.py b/python/freetoken/layers/quantization/embedding/gguf.pyindex f41b7b6a..56c72c2d 100644--- a/python/freetoken/layers/quantization/embedding/gguf.py+++ b/python/freetoken/layers/quantization/embedding/gguf.py@@ -7,7 +7,7 @@ from typing import Anyimport torch-from ..ggml_types import Segment, pad_rows, row_bytes+from ..ggml_types import NVFP4, Segment, pad_rows, row_bytesfrom ..linear.gguf import GgmlLinearKernelfrom ..registry import LayerKind, register_methodfrom ..scheme import QuantKind, gguf_types@@ -18,7 +18,10 @@ class GgmlEmbeddingKernel(EmbeddingKernel):name = "ggml"def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:- return "GGUF tables are not sharded over TP" if cfg.tp_size > 1 else None+ if cfg.tp_size > 1:+ return "GGUF tables are not sharded over TP"+ # the vendored get_rows has no NVFP4 case, and the table's weight_scale_2 would need applying too+ return "NVFP4 GGUF tables are not supported" if NVFP4 in gguf_types(cfg.scheme) else Nonedef embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:from freetoken.kernel import ggmldiff --git a/python/freetoken/layers/quantization/ggml_types.py b/python/freetoken/layers/quantization/ggml_types.pyindex 8b257cd8..262c5668 100644--- a/python/freetoken/layers/quantization/ggml_types.py+++ b/python/freetoken/layers/quantization/ggml_types.py@@ -2,6 +2,7 @@from __future__ import annotations+import refrom dataclasses import dataclassimport torch@@ -85,4 +86,33 @@ def segment_param(index: int) -> str:return "weight" if index == 0 else f"weight_{index}"-__all__ = ["MATRIX_ROW_PADDING", "Segment", "float_dtype", "is_quantized", "mmq_tail_bytes", "pad_rows", "row_bytes", "segment_param", "segments", "type_name"]+NVFP4 = int(gguf.GGMLQuantizationType.NVFP4)++# per-expert fp32 weight scales llama.cpp reads next to separate gate / up / down expert tensors (llama-model.cpp ffn_*_exps_s)+_EXPERT_SCALE_RE = re.compile(r"^blk\.\d+\.ffn_(gate|up|down)_exps\.scale$")+++def scale_name(tensor: str) -> str:+ """``<name>.scale`` of the GGUF tensor ``<name>`` or ``<name>.weight``."""+ return f"{tensor.removesuffix('.weight')}.scale"+++def has_weight_scale(tensor_types: dict[str, int], tensor: str) -> bool:+ """Whether llama.cpp's converter stored an NVFP4 tensor's weight_scale_2 as ``<name>.scale``; other types keep no weight scale."""+ weight = tensor if tensor.endswith(".weight") else f"{tensor}.weight"+ return tensor_types.get(weight, tensor_types.get(tensor)) == NVFP4 and scale_name(tensor) in tensor_types+++def has_expert_scales(tensor_types: dict[str, int]) -> bool:+ """Whether any NVFP4 expert tensor of the file carries per-expert scales, which then take a scale bank in every MoE layer."""+ merged = [n for n in tensor_types if n.endswith("ffn_gate_up_exps.scale") and tensor_types.get(n.replace(".scale", ".weight")) == NVFP4]+ if merged:+ # llama.cpp scales only separate gate / up expert tensors and would drop this one+ raise NotImplementedError(f"per-expert scales of merged gate_up NVFP4 experts are not supported (e.g. {merged[0]})")+ return any(_EXPERT_SCALE_RE.match(n) and has_weight_scale(tensor_types, n.removesuffix(".scale")) for n in tensor_types)+++__all__ = [+ "MATRIX_ROW_PADDING", "NVFP4", "Segment", "float_dtype", "has_expert_scales", "has_weight_scale", "is_quantized", "mmq_tail_bytes",+ "pad_rows", "row_bytes", "scale_name", "segment_param", "segments", "type_name",+]diff --git a/python/freetoken/layers/quantization/linear/gguf.py b/python/freetoken/layers/quantization/linear/gguf.pyindex ac08dbe2..41f61edf 100644--- a/python/freetoken/layers/quantization/linear/gguf.py+++ b/python/freetoken/layers/quantization/linear/gguf.py@@ -35,6 +35,9 @@ class GgmlLinearKernel(LinearKernel):ggml.mul_mat(w, s.ggml_type, x2, out=out, xq=xq)else:torch.mm(x2, w.t(), out=out)+ # llama.cpp multiplies an NVFP4 result by the tensor's weight_scale_2 (build_lora_mm)+ if getattr(layer, "ggml_scale", None) is not None:+ y.mul_(layer.ggml_scale)if layer.bias is not None:y = y + layer.biasreturn y.reshape(*x.shape[:-1], y.shape[-1])@@ -54,3 +57,6 @@ class GGUFLinearMethod(LinearMethod):setattr(layer, segment_param(i), torch.empty(s.rows + pad_rows(k), row_bytes(s.ggml_type, k), dtype=torch.uint8))else:setattr(layer, segment_param(i), torch.empty(s.rows, k))+ if g.scheme.weight.scale is not None:+ # one value per output row: each part's weight_scale_2, 1 for parts without one+ layer.ggml_scale = torch.empty(sum(s.rows for s in segs), dtype=torch.float32)diff --git a/python/freetoken/layers/quantization/moe/gguf.py b/python/freetoken/layers/quantization/moe/gguf.pyindex 416c2f4b..01e754d6 100644--- a/python/freetoken/layers/quantization/moe/gguf.py+++ b/python/freetoken/layers/quantization/moe/gguf.py@@ -6,7 +6,7 @@ import mathimport torch-from ..ggml_types import float_dtype, is_quantized, row_bytes, type_name+from ..ggml_types import NVFP4, float_dtype, is_quantized, row_bytes, type_namefrom ..registry import LayerKind, register_methodfrom ..scheme import QuantKind, gguf_typesfrom .base import BankSpec, ExpertView, MoEConfig, MoEKernel, MoEMethod, gated_epilogue_reason, is_resident, limit_or_inf@@ -80,7 +80,11 @@ class GgmlMoEKernel(MoEKernel):def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:gate_up, down = expert_types(cfg)i, h = cfg.local_intermediate, cfg.hidden- return {"gate_up": _bank_spec(gate_up, 2 * i, h), "down": _bank_spec(down, h, i)}+ banks = {"gate_up": _bank_spec(gate_up, 2 * i, h), "down": _bank_spec(down, h, i)}+ if cfg.scheme.weight.scale is not None:+ # each expert's NVFP4 weight_scale_2 as (gate, up, down, 1): a 16-byte row, which the slot cache's copies need+ banks["global"] = BankSpec((4,), torch.float32)+ return banksdef pack(self, pieces, cfg: MoEConfig, out):i = cfg.local_intermediate@@ -113,17 +117,29 @@ class GgmlMoEKernel(MoEKernel):return ggml.mul_mat_id(w, ggml_type, a, ids)return float_mul_mat_id(w, a, ids, topk_weights, config)+ # llama.cpp's build_moe_ffn: each NVFP4 GEMM result times the expert's weight_scale_2, gate and up before the glu, down before the routing weight+ scales = view.tensors.get("global")+ gate_up_scales = scales if scales is not None and gate_up_type == NVFP4 else None+ down_scales = scales if scales is not None and down_type == NVFP4 else Noneglu = fused_glu(layer)small = x.shape[0] <= min(ggml.mmvq_max_tokens(t) for t in (gate_up_type, down_type) if is_quantized(t))if glu is not None and small and is_quantized(gate_up_type):- inter = ggml.mul_mat_id_glu(gate_up, gate_up_type, x, ids, *glu)+ gu = (None, None) if gate_up_scales is None else (gate_up_scales[:, 0], gate_up_scales[:, 1])+ inter = ggml.mul_mat_id_glu(gate_up, gate_up_type, x, ids, *glu, *gu)else:h = mul_mat_id(gate_up, gate_up_type, x)t, k, two_i = h.shape+ if gate_up_scales is not None:+ h.view(t, k, 2, two_i // 2).mul_(gate_up_scales[ids, :2].unsqueeze(-1))inter = torch.empty(t, k, two_i // 2, dtype=h.dtype, device=h.device)gated_act_and_mul(layer.activation, h.view(-1, two_i), inter.view(-1, two_i // 2), alpha=float(layer.alpha), limit=limit_or_inf(layer))del h # free the gate_up output before the down GEMM allocates its own- y = mul_mat_id(view.tensors["down"], down_type, inter)+ if down_scales is not None and x.shape[0] <= ggml.mmvq_max_tokens(down_type):+ y = ggml.mul_mat_id(view.tensors["down"], down_type, inter, ids, scale=down_scales[:, 2])+ else:+ y = mul_mat_id(view.tensors["down"], down_type, inter)+ if down_scales is not None:+ topk_weights = topk_weights * down_scales[ids, 2]return ggml.moe_weighted_sum(y, topk_weights)@staticmethod@@ -147,4 +163,6 @@ class GGUFMoEMethod(MoEMethod):# BF16 experts keep the executor's bf16 kernels, which use AVX512-BF16 where the CPU has it; ggml-cpu is built for AVX2 onlyif float_dtype(gate_up) is torch.bfloat16 and float_dtype(down) is torch.bfloat16:return "bf16", {}- return "gguf", {"gate_up_type": gate_up, "down_type": down}+ # an NVFP4 layer of a file with weight scales reads the layout's global bank too+ scaled = self.cfg.scheme.weight.scale is not None and NVFP4 in (gate_up, down)+ return "gguf_scaled" if scaled else "gguf", {"gate_up_type": gate_up, "down_type": down}diff --git a/python/freetoken/layers/quantization/scheme.py b/python/freetoken/layers/quantization/scheme.pyindex ab43814e..a3aff0bc 100644--- a/python/freetoken/layers/quantization/scheme.py+++ b/python/freetoken/layers/quantization/scheme.py@@ -108,9 +108,11 @@ def mxfp4_scheme() -> QuantScheme:return QuantScheme(QuantKind.MXFP4, WeightDesc("e2m1", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale"})-def gguf_scheme(types: Iterable[int]) -> QuantScheme:- """Packed ggml blocks, one ggml type per fused part (``types`` follows the parts' order)."""- return QuantScheme(QuantKind.GGUF, WeightDesc("ggml:" + ",".join(str(int(t)) for t in types), None, None), {"weight"})+def gguf_scheme(types: Iterable[int], scaled: bool = False) -> QuantScheme:+ """Packed ggml blocks, one ggml type per fused part (``types`` follows the parts' order).++ ``scaled``: the NVFP4 tensors carry the fp32 ``<name>.scale`` (weight_scale_2) that llama.cpp's converter writes."""+ return QuantScheme(QuantKind.GGUF, WeightDesc("ggml:" + ",".join(str(int(t)) for t in types), None, "f32" if scaled else None), {"weight"})def gguf_types(scheme: QuantScheme) -> tuple[int, ...]:diff --git a/python/freetoken/models/gemma4/gguf.py b/python/freetoken/models/gemma4/gguf.pyindex 2b86cca1..3d4de920 100644--- a/python/freetoken/models/gemma4/gguf.py+++ b/python/freetoken/models/gemma4/gguf.py@@ -24,7 +24,7 @@ from freetoken.models.config import (SWAAttentionGroupConfig,)from freetoken.models.gguf.reader import gguf_tensor_index-from freetoken.models.gguf.tensors import float_tensor, fused, matrix+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix, weight_scaleif TYPE_CHECKING:from freetoken.models.gguf.config import GgufConfigShim@@ -48,6 +48,7 @@ def _full_rotary_dim(shim: "GgufConfigShim", full_head_dim: int) -> int:def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:+ check_scales(shim.model_path, known=("ffn_gate_inp.scale", "ffn_down_exps.scale"))m = shim.metadatadef g(key: str):@@ -246,6 +247,9 @@ def iter_gguf_weights(blk, base = f"blk.{layer}", f"model.layers.{layer}"for suffix, rel in scalar_map.items():tensor = float_tensor(index[f"{blk}.{suffix}"])+ if suffix == "ffn_down_exps.scale" and weight_scale(index[f"{blk}.ffn_down_exps.weight"]) is not None:+ # the converter folds per_expert_scale into the NVFP4 down weight_scale_2, which the expert kernel applies+ tensor = torch.ones_like(tensor)yield f"{base}.{rel}", tensor.reshape(1) if rel.endswith("layer_scalar") else tensorif f"{blk}.attn_qkv.weight" in index:# convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrixdiff --git a/python/freetoken/models/gguf/experts.py b/python/freetoken/models/gguf/experts.pyindex 0da56ceb..c21fff9f 100644--- a/python/freetoken/models/gguf/experts.py+++ b/python/freetoken/models/gguf/experts.py@@ -1,7 +1,8 @@"""Routed-expert banks straight from a GGUF's ``blk.N.ffn_{gate,up,down}_exps`` tensors, in their packed ggml rows: pinned host banks for the offload cache, or device banks for resident experts.Each layer keeps its own ggml types, so each layer gets its own bank shapes: ``gate_up`` [E, 2I, row_bytes] (expert e's gate rows, then its up rows) and ``down`` [E, H, row_bytes]; float experts keep their dtype ([E, 2I, H] and [E, H, I]).-A GGUF that stores ``ffn_gate_up_exps`` (gemma4) already holds the gate_up rows in that order."""+A GGUF that stores ``ffn_gate_up_exps`` (gemma4) already holds the gate_up rows in that order.+When NVFP4 experts carry per-expert ``.scale`` tensors, every layer also gets ``global`` [E, 4] fp32: the (gate, up, down, 1) weight_scale_2, 1 where a tensor has none."""from __future__ import annotations@@ -11,10 +12,11 @@ import osimport torchfrom freetoken.layers.quantization import QuantKind-from freetoken.layers.quantization.ggml_types import float_dtype, mmq_tail_bytes, pad_rows, type_name+from freetoken.layers.quantization.ggml_types import float_dtype, has_expert_scales, mmq_tail_bytes, pad_rows, type_namefrom freetoken.utils import init_loggerfrom .reader import GgufTensorInfo, gguf_tensor_index+from .tensors import weight_scalelogger = init_logger(__name__)@@ -46,14 +48,30 @@ def _device_bank(shape: tuple[int, ...], k: int, dtype: torch.dtype, device: torreturn flat[: experts * rows].view(shape)+def _expert_scales(gate_up: list[GgufTensorInfo], down: GgufTensorInfo, experts: int) -> torch.Tensor:+ """[E, 4] fp32 (gate, up, down, 1) weight_scale_2 of one layer's experts."""+ gate, up = gate_up if len(gate_up) == 2 else (gate_up[0], gate_up[0])+ out = torch.ones(experts, 4, dtype=torch.float32)+ for col, info in enumerate((gate, up, down)):+ s = weight_scale(info)+ if s is None:+ continue+ if s.numel() != experts:+ raise NotImplementedError(f"{info.name}: {s.numel()} scales for {experts} experts")+ out[:, col] = s+ return out++def load_gguf_expert_banks(model_path: str, num_layers: int, *, first_layer: int = 0, dummy: bool = False, device: torch.device | None = None):"""``ExpertBanks`` with one bank per (layer, role): pinned host banks for the offload cache, filled from the file and pinned while later layers load, or resident banks on ``device``."""from freetoken.moe.expert_banks import ExpertBanksfrom freetoken.moe.host_banks import HostBank, PinPipelineindex = gguf_tensor_index(model_path)- sources: dict[str, list[torch.Tensor]] = {"gate_up": [], "down": []}- tails: dict[str, list[int]] = {"gate_up": [], "down": []}+ scaled = has_expert_scales({name: info.ggml_type for name, info in index.items()})+ roles = ("gate_up", "down", "global") if scaled else ("gate_up", "down")+ sources: dict[str, list[torch.Tensor]] = {role: [] for role in roles}+ tails: dict[str, list[int]] = {role: [] for role in roles}tags = []with PinPipeline() if device is None else contextlib.nullcontext() as pins:for layer in range(num_layers):@@ -70,17 +88,26 @@ def load_gguf_expert_banks(model_path: str, num_layers: int, *, first_layer: inttags.append(("ggml", type_name(gate.ggml_type), type_name(down.ggml_type)))tails["gate_up"].append(mmq_tail_bytes(gate.ggml_type, gate.ne[0]))tails["down"].append(mmq_tail_bytes(down.ggml_type, down.ne[0]))+ if scaled:+ tails["global"].append(0)rows = [_expert_rows(info) for info in parts]d = _expert_rows(down)gate_up_shape = (rows[0].shape[0], sum(r.shape[1] for r in rows), rows[0].shape[2])+ experts = rows[0].shape[0]if device is None:host = {"gate_up": HostBank(gate_up_shape, rows[0].dtype), "down": HostBank(tuple(d.shape), d.dtype)}+ if scaled:+ host["global"] = HostBank((experts, 4), torch.float32)banks = {role: bank.tensor for role, bank in host.items()}else:banks = {"gate_up": _device_bank(gate_up_shape, gate.ne[0], rows[0].dtype, device),"down": _device_bank(tuple(d.shape), down.ne[0], d.dtype, device),}+ if scaled:+ banks["global"] = torch.empty(experts, 4, dtype=torch.float32, device=device)+ if scaled:+ banks["global"].copy_(_expert_scales(parts, down, experts))if dummy:banks["gate_up"].zero_()banks["down"].zero_()@@ -98,7 +125,7 @@ def load_gguf_expert_banks(model_path: str, num_layers: int, *, first_layer: intsources[role].append(bank)total = sum(t.numel() * t.element_size() for ts in sources.values() for t in ts)logger.info_rank0(f"GGUF expert banks: {num_layers} layers, {total / 2**30:.1f} GiB {'on ' + str(device) if device is not None else 'in host RAM'}")- return ExpertBanks("gguf", sources, kind=QuantKind.GGUF, kernel="ggml", layout=None, layer_tags=tags, row_tail_bytes=tails)+ return ExpertBanks("gguf_scaled" if scaled else "gguf", sources, kind=QuantKind.GGUF, kernel="ggml", layout=None, layer_tags=tags, row_tail_bytes=tails)__all__ = ["load_gguf_expert_banks"]diff --git a/python/freetoken/models/gguf/reader.py b/python/freetoken/models/gguf/reader.pyindex 2715f129..ff735094 100644--- a/python/freetoken/models/gguf/reader.py+++ b/python/freetoken/models/gguf/reader.py@@ -147,6 +147,8 @@ class GgufTensorInfo:nbytes: intggml_type: intne: tuple[int, ...]+ # the model path the index was built from, which resolves tensors in any split+ root: str = ""def data(self) -> np.ndarray:"""Zero-copy uint8 view of the bytes (memory-mapped)."""@@ -161,7 +163,7 @@ def gguf_tensor_index(model_path: str) -> dict[str, GgufTensorInfo]:for path in gguf_split_paths(model_path):for t in _reader(path).tensors:index[t.name] = GgufTensorInfo(- t.name, path, int(t.data_offset), int(t.n_bytes), int(t.tensor_type), tuple(int(x) for x in t.shape)+ t.name, path, int(t.data_offset), int(t.n_bytes), int(t.tensor_type), tuple(int(x) for x in t.shape), model_path)return indexdiff --git a/python/freetoken/models/gguf/tensors.py b/python/freetoken/models/gguf/tensors.pyindex 4cba1d08..83452d95 100644--- a/python/freetoken/models/gguf/tensors.py+++ b/python/freetoken/models/gguf/tensors.py@@ -7,9 +7,9 @@ from typing import Iteratorimport numpy as npimport torch-from freetoken.layers.quantization.ggml_types import is_quantized, pad_rows, segment_param, segments+from freetoken.layers.quantization.ggml_types import NVFP4, is_quantized, pad_rows, scale_name, segment_param, segments-from .reader import GgufTensorInfo+from .reader import GgufTensorInfo, gguf_tensor_index_FLOAT_DTYPES = {0: torch.float32, 1: torch.float16, 30: torch.bfloat16}@@ -29,8 +29,49 @@ def matrix(info: GgufTensorInfo) -> torch.Tensor:return packed_rows(info) if is_quantized(info.ggml_type) else float_tensor(info)+def check_scales(model_path: str, *, nvfp4: bool = True, known: tuple[str, ...] = ()) -> None:+ """Refuse ``.scale`` / ``.input_scale`` tensors that nothing reads.++ With ``nvfp4``, an NVFP4 tensor's ``.scale`` (its weight_scale_2) is read and its ``.input_scale`` goes unused, as in llama.cpp.+ ``known`` lists tensor-name suffixes the family's loader reads itself."""+ types = {name: info.ggml_type for name, info in gguf_tensor_index(model_path).items()}+ unread = []+ for name in types:+ weight = name.removesuffix(".input_scale") if name.endswith(".input_scale") else name.removesuffix(".scale")+ if weight == name or name.endswith(known):+ continue+ if not (nvfp4 and types.get(f"{weight}.weight", types.get(weight)) == NVFP4):+ unread.append(name)+ if unread:+ what = "scale tensors outside NVFP4 weights" if nvfp4 else "tensors with per-tensor scales"+ raise NotImplementedError(f"{model_path}: GGUF {what} are not supported (e.g. {sorted(unread)[0]})")+++def weight_scale(info: GgufTensorInfo) -> torch.Tensor | None:+ """The fp32 weight_scale_2 llama.cpp's converter writes next to an NVFP4 tensor as ``<name>.scale`` (one value, or one per expert); None if there is none."""+ if info.ggml_type != NVFP4:+ return None+ scale = gguf_tensor_index(info.root or info.path).get(scale_name(info.name))+ return None if scale is None else float_tensor(scale).float().reshape(-1)+++def _row_scales(prefix: str, infos: list[GgufTensorInfo], parts: list[torch.Tensor], pad: int) -> torch.Tensor | None:+ scales = [weight_scale(i) for i in infos]+ if all(s is None for s in scales):+ return None+ rows = []+ for info, part, s in zip(infos, parts, scales):+ # llama.cpp reads a dense tensor's scale as one value+ if s is not None and s.numel() != 1:+ raise NotImplementedError(f"{prefix}: {info.name} has {s.numel()} scales, expected one")+ rows.append(torch.ones(part.shape[0]) if s is None else s.expand(part.shape[0]))+ return torch.cat([*rows, torch.ones(pad)])++def fused(prefix: str, infos: list[GgufTensorInfo], parts: list[torch.Tensor], k: int, pad_to: int = 0) -> Iterator[tuple[str, torch.Tensor]]:- """One fused module: float parts concatenated into ``weight``; with any quantized part, one matrix per ggml-type run (float runs as bf16 rows, quantized runs packed)."""+ """One fused module: float parts concatenated into ``weight``; with any quantized part, one matrix per ggml-type run (float runs as bf16 rows, quantized runs packed).++ NVFP4 parts with a ``.scale`` add ``ggml_scale``: each part's scale on its output rows, fp32."""types = [i.ggml_type for i in infos]pad = (-sum(p.shape[0] for p in parts)) % pad_to if pad_to else 0if not any(is_quantized(t) for t in types):@@ -51,6 +92,9 @@ def fused(prefix: str, infos: list[GgufTensorInfo], parts: list[torch.Tensor], kdtype = torch.bfloat16tail = [torch.zeros(zeros, rows[0].shape[1], dtype=dtype)] if zeros else []yield f"{prefix}.{segment_param(i)}", torch.cat([*rows, *tail], dim=0)+ scales = _row_scales(prefix, infos, parts, pad)+ if scales is not None:+ yield f"{prefix}.ggml_scale", scales-__all__ = ["float_tensor", "fused", "matrix", "packed_rows"]+__all__ = ["check_scales", "float_tensor", "fused", "matrix", "packed_rows", "weight_scale"]diff --git a/python/freetoken/models/llama/gguf.py b/python/freetoken/models/llama/gguf.pyindex 09d02627..833a5d66 100644--- a/python/freetoken/models/llama/gguf.py+++ b/python/freetoken/models/llama/gguf.py@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Iteratorimport torchfrom freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index-from freetoken.models.gguf.tensors import float_tensor, fused, matrix+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrixfrom .config import parse_config@@ -53,10 +53,8 @@ def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:def _check_supported(shim: GgufConfigShim) -> None:if any(k.startswith("prism.hadamard.") for k in shim.metadata):raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")+ check_scales(shim.model_path)names = sorted(gguf_tensor_index(shim.model_path))- scaled = [n for n in names if n.endswith((".scale", ".input_scale"))]- if scaled:- raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")# the llama arch also carries Mixtral experts and optional biases, which the llama model does not haveextra = [n for n in names if n.endswith(".bias") or ".ffn_gate_inp." in n]if extra:diff --git a/python/freetoken/models/qwen3/gguf.py b/python/freetoken/models/qwen3/gguf.pyindex 06245ce4..8e0388fe 100644--- a/python/freetoken/models/qwen3/gguf.py+++ b/python/freetoken/models/qwen3/gguf.py@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Iteratorimport torchfrom freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index-from freetoken.models.gguf.tensors import float_tensor, fused, matrix+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrixfrom .config import parse_config@@ -52,9 +52,7 @@ def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:def _check_supported(shim: GgufConfigShim) -> None:if any(k.startswith("prism.hadamard.") for k in shim.metadata):raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")- scaled = sorted(n for n in gguf_tensor_index(shim.model_path) if n.endswith((".scale", ".input_scale")))- if scaled:- raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")+ check_scales(shim.model_path)def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:diff --git a/python/freetoken/models/qwen3_5_moe/gguf.py b/python/freetoken/models/qwen3_5_moe/gguf.pyindex 85af8ab8..7f72061e 100644--- a/python/freetoken/models/qwen3_5_moe/gguf.py+++ b/python/freetoken/models/qwen3_5_moe/gguf.py@@ -14,7 +14,7 @@ import torchfrom freetoken.layers.quantization.ggml_types import is_quantizedfrom freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index-from freetoken.models.gguf.tensors import float_tensor, fused, matrix+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrixfrom .config import parse_config@@ -67,9 +67,7 @@ def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:def _check_supported(shim: GgufConfigShim) -> None:if any(k.startswith("prism.hadamard.") for k in shim.metadata):raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")- scaled = sorted(n for n in gguf_tensor_index(shim.model_path) if n.endswith((".scale", ".input_scale")))- if scaled:- raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")+ check_scales(shim.model_path)def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:diff --git a/python/freetoken/models/qwen3_moe/gguf.py b/python/freetoken/models/qwen3_moe/gguf.pyindex 7bf01629..3a8ef98a 100644--- a/python/freetoken/models/qwen3_moe/gguf.py+++ b/python/freetoken/models/qwen3_moe/gguf.py@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Iteratorimport torchfrom freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index-from freetoken.models.gguf.tensors import float_tensor, fused, matrix+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrixfrom .config import parse_config@@ -56,9 +56,7 @@ def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:def _check_supported(shim: GgufConfigShim) -> None:if any(k.startswith("prism.hadamard.") for k in shim.metadata):raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")- scaled = sorted(n for n in gguf_tensor_index(shim.model_path) if n.endswith((".scale", ".input_scale")))- if scaled:- raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")+ check_scales(shim.model_path)def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:diff --git a/python/freetoken/models/qwen4_exp/gguf.py b/python/freetoken/models/qwen4_exp/gguf.pyindex 7909324b..0c1bf980 100644--- a/python/freetoken/models/qwen4_exp/gguf.py+++ b/python/freetoken/models/qwen4_exp/gguf.py@@ -14,7 +14,7 @@ import torchfrom freetoken.layers.quantization.ggml_types import is_quantizedfrom freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index-from freetoken.models.gguf.tensors import float_tensor, fused, matrix+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrixfrom .config import parse_config@@ -83,6 +83,8 @@ def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:+ # NVFP4 weight scales are unvalidated on this arch, so refuse them instead of risking wrong output+ check_scales(shim.model_path, nvfp4=False)md = shim.metadatadef kv(key: str):diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex 4c0aee7a..bb65146b 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -149,11 +149,11 @@ def resolve_threads_and_affinity(requested: int) -> tuple[int, list[int]]:def _format_label(name: str, opts: tuple) -> str:"""One format instance as the startup log names it, e.g. ``gguf(gate_up=IQ2_XXS, down=IQ4_NL)``."""- if name == "gguf":+ if name in ("gguf", "gguf_scaled"):from freetoken.layers.quantization.ggml_types import type_namegate_up, down = dict(opts)["gate_up_type"], dict(opts)["down_type"]- return f"gguf(gate_up={type_name(gate_up)}, down={type_name(down)})"+ return f"{name}(gate_up={type_name(gate_up)}, down={type_name(down)})"return name + (f"({', '.join(f'{k}={v}' for k, v in opts)})" if opts else "")@@ -192,7 +192,7 @@ class CpuMoeExecutor:f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are "f"{f!r}; use --moe-strategy offload (GPU-side dequant) instead.")- if f == "gguf":+ if f in ("gguf", "gguf_scaled"):from freetoken.kernel import ggml_cpuif (variant := ggml_cpu.select()) is None:@@ -423,8 +423,8 @@ class CpuMoeExecutor:(see ``_make_table``), not a single bank's -- the C++ side resolves``tbl[layer_id]`` per task."""- if fmt == "gguf":- return self._resolve_gguf_banks(banks, rep, int(opts["gate_up_type"]), int(opts["down_type"]))+ if fmt in ("gguf", "gguf_scaled"):+ return self._resolve_gguf_banks(banks, rep, int(opts["gate_up_type"]), int(opts["down_type"]), scaled=fmt == "gguf_scaled")if fmt == "bf16":gate_up = banks["gate_up"]down = banks["down"]@@ -465,8 +465,10 @@ class CpuMoeExecutor:banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, gate_up_global=gug, down_scale=dns, down_global=dng)return banks, {}, (H, I)- def _resolve_gguf_banks(self, banks: dict, rep: int, gate_up_type: int, down_type: int) -> tuple[dict, dict, tuple[int, int]]:- """GGUF rows read in place: gate_up [E, 2I, row_bytes(H)] (gate rows, then up rows) and down [E, H, row_bytes(I)], one ggml type each; float rows keep their dtype."""+ def _resolve_gguf_banks(self, banks: dict, rep: int, gate_up_type: int, down_type: int, scaled: bool = False) -> tuple[dict, dict, tuple[int, int]]:+ """GGUF rows read in place: gate_up [E, 2I, row_bytes(H)] (gate rows, then up rows) and down [E, H, row_bytes(I)], one ggml type each; float rows keep their dtype.++ ``scaled`` adds global [E, 4] fp32, each expert's (gate, up, down, 1) NVFP4 weight_scale_2."""from freetoken.layers.quantization.ggml_types import float_dtype, row_bytes, type_namegate_up, down = banks["gate_up"], banks["down"]@@ -475,8 +477,13 @@ class CpuMoeExecutor:dtype = float_dtype(ggml_type) if float_dtype(ggml_type) is not None else torch.uint8if t.dtype != dtype or t.shape[-1] * t.element_size() != row_bytes(ggml_type, k):raise ValueError(f"MoE layer {rep} {role} bank {tuple(t.shape)} {t.dtype} is not {type_name(ggml_type)} rows of {k} values")- banks = self._tables(gate_up=gate_up, down=down)- return banks, {"gate_up_type": gate_up_type, "down_type": down_type}, (H, I)+ types = {"gate_up_type": gate_up_type, "down_type": down_type}+ if not scaled:+ return self._tables(gate_up=gate_up, down=down), types, (H, I)+ scales = banks["global"]+ if scales[rep].dtype != torch.float32 or tuple(scales[rep].shape) != (gate_up[rep].shape[0], 4) or not scales[rep].is_contiguous():+ raise ValueError(f"MoE layer {rep} global bank {tuple(scales[rep].shape)} {scales[rep].dtype} is not contiguous [experts, 4] fp32")+ return self._tables(gate_up=gate_up, down=down, **{"global": scales}), types, (H, I)def _resolve_mxfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]:"""gpt-oss mxfp4 ``mxfp4_triton`` schema: transposed split-K blocks/scalesdiff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 23ba3251..5291e621 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -48,6 +48,8 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {"fp8_block": ("gate_up", "gate_up_scale", "down", "down_scale"),# GGUF experts in their own ggml type per layer: gate_up [L*E, 2I, row_bytes] (gate rows, then up rows), down [L*E, H, row_bytes]; float experts keep their dtype"gguf": ("gate_up", "down"),+ # GGUF experts whose NVFP4 tensors carry per-expert scales: global [L*E, 4] fp32 (gate, up, down, 1) weight_scale_2+ "gguf_scaled": ("gate_up", "down", "global"),# native ModelOpt rows for the Triton inline-dequant kernels: packed e2m1 codes +# fp8-e4m3 per-16 block scales + per-output-row fp16 globals (w1/w3 carry distinct# globals, and folding them into the e4m3 block scales would underflow)