Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

NVIDIA Qwen3.8-Flash-Next-FP4-Local: converter (CSF scale decode, MXFP8 to Q8_0/BF16, MTP, NVFP4 PLE table) and the NVFP4 PLE row format

Niko12219c6255cfcee9
diff --git a/include/strata/kernels/ngram.hpp b/include/strata/kernels/ngram.hpp
index 207707d0e..4452b4f6d 100644
--- a/include/strata/kernels/ngram.hpp
+++ b/include/strata/kernels/ngram.hpp
@@ -49,20 +49,21 @@ inline constexpr int PLE_ROW_BYTES_FP8 = PLE_HEAD_DIM; // 160:
inline constexpr int PLE_ROW_BYTES_Q5_1 = (PLE_HEAD_DIM / 32) * 24; // 120: a Q5_1 row (a Q5_K_M finetune)
inline constexpr int PLE_ROW_BYTES_Q8_0 = (PLE_HEAD_DIM / 32) * 34; // 170: a Q8_0 row (UD-Q6_K_XL, Swift-1.5 Q4_K_L)
inline constexpr int PLE_ROW_BYTES_BF16 = PLE_HEAD_DIM * 2; // 320: a BF16 row, two bytes a value
+inline constexpr int PLE_ROW_BYTES_NVFP4 = PLE_HEAD_DIM / 2 + PLE_HEAD_DIM / 16; // 90: 80 B of E2M1 codes + 10 E4M3 block scales
inline constexpr int PLE_ROW_BYTES_MAX = PLE_ROW_BYTES_BF16; // the widest row any format in `ple_formats()` has
/// THE FORMATS OF THE TABLE, ONE ROW EACH. A table type is one entry here (its GGUF type name, the bytes of one
/// 160-value row, how to turn a row into floats) and nothing else: `PleTable::open`, `format()`, the reader's row size,
/// the error string and the buffers' size all read this list, so a new format cannot be in one switch and missing from
/// another. A row is always `PLE_HEAD_DIM` = 160 values = 5 blocks of 32 for the block formats.
-enum class PleFormat : uint8_t { IQ4_NL, Q5_0, F8_E4M3, Q5_1, Q8_0, Q4_0, BF16 };
+enum class PleFormat : uint8_t { IQ4_NL, Q5_0, F8_E4M3, Q5_1, Q8_0, Q4_0, BF16, NVFP4 };
struct PleFormatInfo {
PleFormat id;
const char* name; ///< what `PleTable::format()` returns
const char* gguf_type; ///< `TensorInfo::type_name()` of the table tensor ("I8" for F8_E4M3: GGUF has no FP8 type)
uint32_t row_bytes; ///< bytes of one 160-value row in the file
- bool needs_scale; ///< the table has one scale in the GGUF metadata (`strata.ple.scale`), F8_E4M3 only
+ bool needs_scale; ///< the table has one scale in the GGUF metadata (`strata.ple.scale`): F8_E4M3 and NVFP4
void (*dequant)(const uint8_t* row, float scale, float* out160);
};
diff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cpp
index 91de37058..a0685bd52 100644
--- a/src/kernels/ngram.cpp
+++ b/src/kernels/ngram.cpp
@@ -152,6 +152,20 @@ void bf16_dequant_row(const uint8_t* row, float* out160) {
}
}
+// NVIDIA's FP4-Local table: a row of 160 values is 80 bytes of E2M1 codes (element 2i in the low nibble, 2i+1 in the
+// high one) followed by 10 E4M3 block scales (one per 16 values, sign bit always 0); one F32 scales the whole table.
+// value = e2m1(code) * e4m3(block scale) * table scale, exactly what vLLM's nvfp4 PLE lookup computes.
+void nvfp4_row_dequant(const uint8_t* row, float scale, float* out160) {
+ static const float kE2M1[8] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f};
+ for (int j = 0; j < PLE_HEAD_DIM; ++j) {
+ const uint8_t byte = row[j >> 1];
+ const uint8_t code = (j & 1) ? (uint8_t) (byte >> 4) : (uint8_t) (byte & 15);
+ const float v = kE2M1[code & 7];
+ const float s = kFp8.v[row[PLE_HEAD_DIM / 2 + (j >> 4)]];
+ out160[j] = ((code & 8) ? -v : v) * s * scale;
+ }
+}
+
// ---------------------------------------------------------------------------------------------------
// THE FORMAT TABLE (see PleFormatInfo). One entry per table type; the block formats decode 5 blocks of 32.
namespace {
@@ -162,6 +176,7 @@ void dequant_blocks(const uint8_t* row, float /*scale*/, float* out160) {
void dequant_iq4_nl(const uint8_t* row, float, float* out160) { iq4nl_dequant_row(row, out160); }
void dequant_fp8(const uint8_t* row, float scale, float* out160) { fp8_e4m3_dequant_row(row, scale, out160); }
void dequant_bf16(const uint8_t* row, float, float* out160) { bf16_dequant_row(row, out160); }
+void dequant_nvfp4(const uint8_t* row, float scale, float* out160) { nvfp4_row_dequant(row, scale, out160); }
const PleFormatInfo kPleFormats[] = {
{PleFormat::IQ4_NL, "IQ4_NL", "IQ4_NL", PLE_ROW_BYTES, false, dequant_iq4_nl},
@@ -175,6 +190,9 @@ const PleFormatInfo kPleFormats[] = {
// BF16 (#586): the checkpoint's own table at full precision, 320-byte rows; self-describing, so no scale and no
// metadata to trust (tools/ple_fp8_pack.py writes the FP8 form of the same table)
{PleFormat::BF16, "BF16", "BF16", PLE_ROW_BYTES_BF16, false, dequant_bf16},
+ // NVFP4 x16 (NVIDIA's Qwen3.8-Flash-Next-FP4-Local, tools/nvfp4_local_pack.py ple): 90-byte rows, I8 bytes marked
+ // strata.ple.format = nvfp4x16 with strata.ple.scale. The tensor is [90, rows]: ne0 is the row's byte count.
+ {PleFormat::NVFP4, "NVFP4", "I8", PLE_ROW_BYTES_NVFP4, true, dequant_nvfp4},
};
constexpr int kPleFormatCount = (int) (sizeof kPleFormats / sizeof kPleFormats[0]);
} // namespace
@@ -192,7 +210,7 @@ std::string ple_format_list() {
for (int i = 0; i < kPleFormatCount; ++i) {
const PleFormatInfo& f = kPleFormats[i];
if (i > 0) out += i + 1 == kPleFormatCount ? " or " : ", ";
- out += std::strcmp(f.gguf_type, "I8") == 0 ? "FP8 (I8)" : f.name;
+ out += std::strcmp(f.gguf_type, "I8") == 0 ? (f.id == PleFormat::NVFP4 ? "NVFP4 (I8)" : "FP8 (I8)") : f.name;
}
return out;
}
@@ -245,8 +263,12 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoO
close();
return false;
}
- // [160, 320001536]: ne0 = 160 is the FAST axis, so the ROW index is shape[1] and a row is contiguous.
- if (t->shape.size() != 2 || t->shape[0] != (uint64_t) PLE_HEAD_DIM) {
+ // [160, 320001536]: ne0 = 160 is the FAST axis, so the ROW index is shape[1] and a row is contiguous. The NVFP4 table
+ // is I8 bytes with ne0 = 90 (the row's byte count), marked strata.ple.format = nvfp4x16.
+ const MetaValue* fmt_key = impl_->file->get("strata.ple.format");
+ const bool nvfp4x16 = std::strcmp(t->type_name(), "I8") == 0 && fmt_key != nullptr && fmt_key->s == "nvfp4x16";
+ const uint64_t want_ne0 = nvfp4x16 ? (uint64_t) PLE_ROW_BYTES_NVFP4 : (uint64_t) PLE_HEAD_DIM;
+ if (t->shape.size() != 2 || t->shape[0] != want_ne0) {
err = "per_layer_token_embd.weight has an unexpected shape";
close();
return false;
@@ -254,7 +276,7 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoO
// The table's type is looked up in the format list (ple_formats()): IQ4_NL (ISTA-DASLab's shard 2, the original's
// own GGUF), Q5_0 (#296: OrcaRouter's GGUF), or the FP8 table as shipped: I8 bytes marked strata.ple.format =
// f8_e4m3 with strata.ple.scale (tools/ple_fp8_pack.py)
- const PleFormatInfo* fmt = ple_format_for_type(t->type_name());
+ const PleFormatInfo* fmt = nvfp4x16 ? &ple_format_info(PleFormat::NVFP4) : ple_format_for_type(t->type_name());
if (fmt == nullptr) {
err = std::string("per_layer_token_embd.weight is ") + t->type_name() + ", not " + ple_format_list();
close();
@@ -263,8 +285,9 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoO
if (fmt->needs_scale) {
const MetaValue* f = impl_->file->get("strata.ple.format");
const MetaValue* s = impl_->file->get("strata.ple.scale");
- if (f == nullptr || f->s != "f8_e4m3" || s == nullptr || !(s->num() > 0.0)) {
- err = "per_layer_token_embd.weight is I8 without strata.ple.format = f8_e4m3 and a positive strata.ple.scale";
+ const char* want = nvfp4x16 ? "nvfp4x16" : "f8_e4m3";
+ if (f == nullptr || f->s != want || s == nullptr || !(s->num() > 0.0)) {
+ err = std::string("per_layer_token_embd.weight is I8 without strata.ple.format = ") + want + " and a positive strata.ple.scale";
close();
return false;
}
diff --git a/src/kernels/ple_fp8_parity.cpp b/src/kernels/ple_fp8_parity.cpp
index 09dd199a8..bc54956ce 100644
--- a/src/kernels/ple_fp8_parity.cpp
+++ b/src/kernels/ple_fp8_parity.cpp
@@ -11,6 +11,7 @@
#include <cmath>
#include <cstdio>
+#include <cstdlib>
#include <cstring>
#include <fstream>
#include <string>
@@ -51,7 +52,7 @@ int main(int argc, char** argv) {
std::printf("%-6s %s, %llu rows: read_row %d/%zu values differ, gather_batch %s %d differ\n",
mode == k::PleIo::Direct ? "Direct" : "Mmap", t.format(), (unsigned long long) t.rows(), diff,
got.size(), ok ? "ok," : "FAILED,", bdiff);
- bad += diff + bdiff + (ok ? 0 : 1) + (std::strcmp(t.format(), "F8_E4M3") != 0);
+ bad += diff + bdiff + (ok ? 0 : 1) + (std::strcmp(t.format(), std::getenv("PLE_EXPECT_FORMAT") ? std::getenv("PLE_EXPECT_FORMAT") : "F8_E4M3") != 0);
}
if (argc > 3) {
k::PleTable q;
diff --git a/src/ngram/ple_reader_test.cpp b/src/ngram/ple_reader_test.cpp
index 4663455f7..569c04736 100644
--- a/src/ngram/ple_reader_test.cpp
+++ b/src/ngram/ple_reader_test.cpp
@@ -105,7 +105,7 @@ void bf16_widening_is_exact() {
/// and the type lookup all have to be right for every format, not only for IQ4_NL.
uint32_t gguf_type_id(const char* name) {
static const struct { const char* name; uint32_t id; } ids[] = {
- {"IQ4_NL", 20}, {"Q4_0", 2}, {"Q4_1", 3}, {"Q5_0", 6}, {"Q5_1", 7}, {"Q8_0", 8}, {"BF16", 30}, {"F8_E4M3", 24}};
+ {"IQ4_NL", 20}, {"Q4_0", 2}, {"Q4_1", 3}, {"Q5_0", 6}, {"Q5_1", 7}, {"Q8_0", 8}, {"BF16", 30}, {"F8_E4M3", 24}, {"NVFP4", 24}};
for (const auto& e : ids)
if (std::strcmp(e.name, name) == 0) return e.id;
return 0xFFFFFFFFu;
@@ -129,11 +129,11 @@ std::string write_format_table(const std::string& dir, const k::PleFormatInfo& f
const float scale = 0.75f;
uint32_t sb;
std::memcpy(&sb, &scale, 4);
- head += str("strata.ple.format") + le(8u, 4) + str("f8_e4m3");
+ head += str("strata.ple.format") + le(8u, 4) + str(f.id == k::PleFormat::NVFP4 ? "nvfp4x16" : "f8_e4m3");
head += str("strata.ple.scale") + le(6u, 4) + le(sb, 4); // 6 = FLOAT32
}
head += str("per_layer_token_embd.weight") + le(2u, 4);
- head += le((uint64_t) k::PLE_HEAD_DIM, 8) + le(claimed_rows != 0 ? claimed_rows : (uint64_t) rows, 8);
+ head += le(f.id == k::PleFormat::NVFP4 ? (uint64_t) f.row_bytes : (uint64_t) k::PLE_HEAD_DIM, 8) + le(claimed_rows != 0 ? claimed_rows : (uint64_t) rows, 8);
head += le(gguf_type_id(f.name), 4) + le(0ull, 8);
while (head.size() % 32) head += '\0';
out.write(head.data(), (std::streamsize) head.size());
diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.py
new file mode 100644
index 000000000..2d903fb03
--- /dev/null
+++ b/tools/nvfp4_local_pack.py
@@ -0,0 +1,469 @@
+"""tools/nvfp4_local_pack.py - NVIDIA's Qwen3.8-Flash-Next-FP4-Local (ModelOpt, QAD) -> files Strata can pack.
+
+ python tools/nvfp4_local_pack.py convert --model <nvidia/Qwen3.8-Flash-Next-FP4-Local dir> --outfile <model.gguf>
+ python tools/nvfp4_local_pack.py ple --model <same dir> --out <ple-nvfp4.gguf>
+ python tools/nvfp4_local_pack.py mtp --model <same dir> --out <mtp src dir> (then tools/mtp_pack.py, tools/mtp_rt.py)
+ python tools/nvfp4_local_pack.py verify --model <same dir> --gguf <model.gguf> [--experts 6]
+ python tools/iq_pack.py --gguf <model.gguf> --out <pack dir>
+
+Nothing of NVIDIA's weights is shipped: the user downloads the checkpoint from NVIDIA's repository and this runs
+locally. What the checkpoint stores, and what is done with it:
+
+ * routed experts: NVFP4 (E2M1 codes, E4M3 scale per 16 values, F32 weight_scale_2). The block scales are stored
+ in NVIDIA's "CSF" form (config `weight_scale_encoding: "csf"`, codec byte-window4-fixed-stream-u24-exceptions/1):
+ per tensor a `.nvfp4_csf_fixed` U8 stream and a `.nvfp4_csf_exceptions` U32 stream. They are decoded here to the
+ plain E4M3 matrix (the format is documented by vLLM's loader, PR 60646, and flashinfer's b12x decoder, both
+ Apache-2.0; this is a fresh implementation of that description), and from there the ordinary ModelOpt NVFP4
+ repack of tools/nvfp4_convert.py applies, bit for bit.
+ * attention, GDN projections, shared experts, indexer: MXFP8 (E4M3 values, E8M0 scale per 32 values). They are
+ dequantized exactly to BF16 and then take the type policy of tools/nvfp4_convert.py (Q8_0 for the big ones,
+ BF16 for the small projections the BF16 kernels read). Q8_0 has the same 32-value blocks; the added error is
+ ~0.6% RMS against MXFP8's own ~3%.
+ * the 26.8 GiB n-gram (PLE) table is NVFP4 too (160 values a row: 80 B of codes, 10 E4M3 scales, one F32 for the
+ whole table). `ple` copies it row by row into a one-tensor GGUF of 90-byte rows.
+ * the MTP head (NVFP4 experts, BF16 rest) is dequantized by `mtp` into the folder form tools/mtp_pack.py reads.
+ The vision tower is not written (text first).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import struct
+import sys
+import time
+
+import numpy as np
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+
+CSF_FIXED = ".nvfp4_csf_fixed"
+CSF_EXC = ".nvfp4_csf_exceptions"
+PLE_ROW_BYTES = 90 # 80 B of E2M1 codes + 10 E4M3 block scales, per 160-value row
+
+
+# --------------------------------------------------------------------------------------------- safetensors access
+class Checkpoint:
+ """Header-indexed, memory-mapped access to a checkpoint's safetensors shards (raw bytes, any dtype)."""
+
+ def __init__(self, folder, partial_ok=False):
+ self.dir = pathlib.Path(folder)
+ index = self.dir / "model.safetensors.index.json"
+ if index.exists():
+ wmap = json.loads(index.read_text(encoding="utf-8"))["weight_map"]
+ files = sorted(set(wmap.values()))
+ else:
+ files = sorted(p.name for p in self.dir.glob("*.safetensors"))
+ if not files:
+ raise FileNotFoundError("no .safetensors files in " + str(self.dir))
+ self.meta: dict[str, tuple[str, str, list[int], int, int]] = {}
+ self._mm: dict[str, np.memmap] = {}
+ for fn in files:
+ path = self.dir / fn
+ if not path.exists() and partial_ok:
+ continue # a subset of the shards (the MTP head is in two)
+ with open(path, "rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ hdr = json.loads(f.read(n))
+ for k, v in hdr.items():
+ if k != "__metadata__":
+ a, b = v["data_offsets"]
+ self.meta[k] = (fn, v["dtype"], v["shape"], 8 + n + a, 8 + n + b)
+
+ def __contains__(self, name):
+ return name in self.meta
+
+ def dtype(self, name):
+ return self.meta[name][1]
+
+ def shape(self, name):
+ return self.meta[name][2]
+
+ def raw(self, name) -> np.ndarray:
+ fn, _, _, a, b = self.meta[name]
+ if fn not in self._mm:
+ self._mm[fn] = np.memmap(self.dir / fn, dtype=np.uint8, mode="r")
+ return self._mm[fn][a:b]
+
+ def array(self, name, np_dtype) -> np.ndarray:
+ return self.raw(name).view(np_dtype).reshape(self.shape(name))
+
+
+# ----------------------------------------------------------------------------------------------------- CSF decoding
+def csf_decode(fixed: np.ndarray, exceptions: np.ndarray) -> np.ndarray:
+ """The CSF stream pair -> the plain E4M3 block-scale matrix as uint8 [rows, cols] (cols = K / 16).
+
+ fixed: uint8 [rows / 16, 16 * (1 + cols / 2)]. Per slab of 16 rows: the 16 rows' base bytes, then each
+ row's cols 4-bit codes (two to a byte, low nibble first). A scale byte is base + code.
+ exceptions: uint32 words, low 24 bits = flat row * cols + col, high 8 bits = the scale byte there.
+ """
+ if fixed.dtype != np.uint8 or fixed.ndim != 2 or fixed.shape[1] % 16 or fixed.shape[1] < 32:
+ raise ValueError("invalid CSF fixed stream %s %s" % (fixed.dtype, fixed.shape))
+ if exceptions.dtype != np.uint32:
+ raise ValueError("invalid CSF exceptions dtype %s" % exceptions.dtype)
+ rows = fixed.shape[0] * 16
+ cols = (fixed.shape[1] // 16 - 1) * 2
+ bases = fixed[:, :16].reshape(rows, 1).astype(np.int16)
+ if (bases > 240).any():
+ raise ValueError("CSF row bases must be in 0..240")
+ packed = fixed[:, 16:].reshape(rows, cols // 2)
+ codes = np.stack((packed & 0xF, packed >> 4), axis=-1).reshape(rows, cols)
+ out = (bases + codes).reshape(-1)
+ words = exceptions.reshape(-1).astype(np.int64)
+ pos = words & 0xFFFFFF
+ if pos.size and int(pos.max()) >= rows * cols:
+ raise ValueError("CSF exception position out of range")
+ out[pos] = words >> 24
+ return out.astype(np.uint8).reshape(rows, cols)
+
+
+def csf_encode(scales: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+ """The inverse, for the tests: uint8 [rows, cols] -> (fixed, exceptions). Base = the row's most common
+ window start; whatever falls outside base..base+15 is an exception."""
+ rows, cols = scales.shape
+ assert rows % 16 == 0 and cols % 2 == 0
+ base = np.zeros(rows, np.int16)
+ for r in range(rows):
+ v = scales[r].astype(np.int16)
+ best = (-1, 0)
+ for b in range(max(0, int(v.min()) - 0), int(v.max()) + 1):
+ c = int(((v >= b) & (v <= b + 15)).sum())
+ if c > best[0]:
+ best = (c, b)
+ base[r] = min(best[1], 240)
+ code = scales.astype(np.int16) - base[:, None]
+ exc = (code < 0) | (code > 15)
+ code = np.where(exc, 0, code).astype(np.uint8)
+ pk = (code[:, 0::2] | (code[:, 1::2] << 4)).astype(np.uint8)
+ fixed = np.empty((rows // 16, 16 * (1 + cols // 2)), np.uint8)
+ for s in range(rows // 16):
+ fixed[s, :16] = base[s * 16:(s + 1) * 16].astype(np.uint8)
+ fixed[s, 16:] = pk[s * 16:(s + 1) * 16].reshape(-1)
+ pos = np.flatnonzero(exc.reshape(-1))
+ words = (pos.astype(np.uint64) | (scales.reshape(-1)[pos].astype(np.uint64) << 24)).astype(np.uint32)
+ return fixed, words
+
+
+# --------------------------------------------------------------------------------------------- MXFP8 dequantizing
+def mxfp8_dequant(values_u8: np.ndarray, scales_u8: np.ndarray):
+ """MXFP8 (E4M3 values [R, K], E8M0 scale bytes [R, K/32]) -> float32 [R, K]. Exact (4-bit significands)."""
+ import torch
+ R, K = values_u8.shape
+ if scales_u8.shape != (R, K // 32) or K % 32:
+ raise ValueError("MXFP8 shapes %s / %s" % (values_u8.shape, scales_u8.shape))
+ if (scales_u8 == 255).any():
+ raise ValueError("MXFP8 scale byte 255 (NaN) in a weight")
+ v = torch.from_numpy(np.ascontiguousarray(values_u8)).view(torch.float8_e4m3fn).to(torch.float32)
+ s = torch.from_numpy(np.exp2(scales_u8.astype(np.float32) - 127.0))
+ return (v.reshape(R, K // 32, 32) * s[:, :, None]).reshape(R, K)
+
+
+# --------------------------------------------------------------------------- the overlay on llama.cpp's tensor index
+def _orig_name(ck: Checkpoint, name: str):
+ """The converter strips `language_model.`; find the checkpoint's own name."""
+ if name in ck:
+ return name
+ for cand in (name.replace("model.", "model.language_model.", 1), "model.language_model." + name):
+ if cand in ck:
+ return cand
+ return None
+
+
+def overlay(tensors: dict, ck: Checkpoint, stats: dict) -> dict:
+ """Replace the CSF stream pairs by plain `weight_scale` tensors, the MXFP8 weights by BF16, and drop what
+ the PLE writer and the MTP packer own."""
+ import torch
+ out = dict(tensors)
+ for name in list(out):
+ if name not in out:
+ continue
+ if name.endswith(CSF_FIXED):
+ base = name[: -len(CSF_FIXED)] # ...weight_scale
+ o_fix = _orig_name(ck, name)
+ o_exc = _orig_name(ck, base + CSF_EXC)
+ if o_fix is None or o_exc is None:
+ raise ValueError("CSF fixed stream without its exceptions: " + name)
+
+ def gen(o_fix=o_fix, o_exc=o_exc):
+ dec = csf_decode(ck.array(o_fix, np.uint8), ck.array(o_exc, np.uint32))
+ return torch.from_numpy(dec).view(torch.float8_e4m3fn)
+ del out[name]
+ out.pop(base + CSF_EXC, None)
+ out[base] = gen
+ stats["csf"] = stats.get("csf", 0) + 1
+ elif ".ngram_embedding." in name and (name.endswith(".weight_scale") or name.endswith(".weight_scale_2")):
+ del out[name] # the PLE table has its own writer (`ple`)
+ for name in list(out):
+ if not name.endswith(".weight") or ".ngram_embedding." in name:
+ continue
+ sname = name[: -len(".weight")] + ".weight_scale"
+ o_w, o_s = _orig_name(ck, name), _orig_name(ck, sname)
+ if o_w is None or o_s is None or sname not in out:
+ continue
+ if ck.dtype(o_w) == "F8_E4M3" and ck.dtype(o_s) == "U8": # MXFP8
+ def gen(o_w=o_w, o_s=o_s):
+ f = mxfp8_dequant(np.asarray(ck.raw(o_w)).reshape(ck.shape(o_w)), np.asarray(ck.raw(o_s)).reshape(ck.shape(o_s)))
+ return f.to(torch.bfloat16)
+ out[name] = gen
+ del out[sname]
+ stats["mxfp8"] = stats.get("mxfp8", 0) + 1
+ return out
+
+
+def install(ck: Checkpoint, stats: dict):
+ """Import the converter, give it nvfp4_convert's type policy, and hook the tensor index."""
+ import nvfp4_convert as base
+ base.install_policy()
+ from conversion.base import ModelBase
+ from conversion import qwen4exp as Q4
+ orig_index = ModelBase.index_tensors
+
+ def index_tensors(self, remote_hf_model_id=None):
+ return overlay(orig_index(self, remote_hf_model_id), ck, stats)
+ ModelBase.index_tensors = index_tensors
+
+ # NVIDIA's repo keeps the shards in a `tensors/` folder that its index names; the converter only lists the top level
+ orig_parts = ModelBase.get_model_part_names
+
+ def get_model_part_names(dir_model, prefix, suffix):
+ names = orig_parts(dir_model, prefix, suffix)
+ sub = pathlib.Path(dir_model) / "tensors"
+ if not names and suffix == ".safetensors" and prefix == "model" and sub.is_dir():
+ names = sorted("tensors/" + p.name for p in sub.glob("model*.safetensors"))
+ return names
+ ModelBase.get_model_part_names = staticmethod(get_model_part_names)
+
+ cls = Q4.Qwen4ExpTextModel
+
+ def place_ple_shard(self, data_torch, name):
+ # the shards are NVFP4 here: two values a byte, so a row of 160 values is 80 bytes
+ idx = int(name.rsplit("shard_", 1)[1].split(".", 1)[0])
+ self._ple_shards[idx] = name
+ self._ple_row_dim = int(data_torch.shape[-1]) * 2
+ base.dropped_ple.append(name)
+ return []
+ cls._place_ple_shard = place_ple_shard
+ return base
+
+
+def cmd_convert(a) -> int:
+ model = pathlib.Path(a.model)
+ ck = Checkpoint(model)
+ stats: dict = {}
+ base = install(ck, stats)
+ import convert_hf_to_gguf as C
+ out = pathlib.Path(a.outfile)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ argv = [str(model), "--outtype", "q8_0", "--outfile", str(out)] + (["--dry-run"] if a.dry_run else [])
+ sys.argv = ["convert_hf_to_gguf.py"] + argv
+ t0 = time.time()
+ rc = C.main()
+ took = time.time() - t0
+ print("nvfp4_local_pack: done in %.0f s; CSF scale tensors decoded %d, MXFP8 weights dequantized %d, PLE shards dropped %d"
+ % (took, stats.get("csf", 0), stats.get("mxfp8", 0), len(base.dropped_ple)))
+ if not a.dry_run:
+ out.with_suffix(".manifest.json").write_text(json.dumps({
+ "source": str(model), "csf_decoded": stats.get("csf", 0), "mxfp8_dequantized": stats.get("mxfp8", 0),
+ "ple_table": "written separately: nvfp4_local_pack.py ple", "mtp": "not written", "seconds": round(took, 1)},
+ indent=2) + "\n", encoding="utf-8")
+ return rc or 0
+
+
+# ------------------------------------------------------------------------------------------------- the PLE table
+GGUF_TYPE_STRING, GGUF_TYPE_F32 = 8, 6
+GGML_TYPE_I8 = 24
+ALIGN = 32
+
+
+def _gs(s):
+ b = s.encode("utf-8")
+ return struct.pack("<Q", len(b)) + b
+
+
+def interleave_rows(codes: np.ndarray, scales: np.ndarray) -> np.ndarray:
+ """[n, 80] codes + [n, 10] E4M3 scale bytes -> [n, 90]: the codes, then the scales."""
+ return np.concatenate([codes, scales], axis=1)
+
+
+def cmd_ple(a) -> int:
+ model, out = pathlib.Path(a.model), pathlib.Path(a.out)
+ ck = Checkpoint(model)
+ shards = {}
+ for name in ck.meta:
+ if ".ngram_embedding.shard_" in name and name.endswith(".weight"):
+ shards[int(name.rsplit(".shard_", 1)[1].split(".")[0])] = name
+ if not shards or sorted(shards) != list(range(len(shards))):
+ sys.exit("the n-gram shards are missing or not numbered 0..N-1")
+ g = [k for k in ck.meta if k.endswith(".ngram_embedding.weight_scale_2")]
+ if len(g) != 1:
+ sys.exit("expected one ngram_embedding.weight_scale_2")
+ scale2 = float(ck.array(g[0], np.float32).reshape(-1)[0])
+ rows, dim = 0, None
+ for k in range(len(shards)):
+ n = shards[k]
+ r, c = ck.shape(n)
+ if ck.dtype(n) != "U8" or c != 80 or ck.shape(n[:-6] + "weight_scale") != [r, 10]:
+ sys.exit("%s: not an NVFP4 160-value row table" % n)
+ rows += r
+ print("%d shards, %d rows x 160, global scale %.9g, %.2f GB" % (len(shards), rows, scale2, rows * PLE_ROW_BYTES / 1e9), flush=True)
+ kvs = (_gs("general.architecture") + struct.pack("<I", GGUF_TYPE_STRING) + _gs("strata-ple")
+ + _gs("general.name") + struct.pack("<I", GGUF_TYPE_STRING) + _gs("PLE n-gram table, NVFP4 as shipped")
+ + _gs("strata.ple.format") + struct.pack("<I", GGUF_TYPE_STRING) + _gs("nvfp4x16")
+ + _gs("strata.ple.scale") + struct.pack("<I", GGUF_TYPE_F32) + struct.pack("<f", scale2)
+ + _gs("strata.ple.source") + struct.pack("<I", GGUF_TYPE_STRING) + _gs(model.name))
+ # the tensor is [90, rows] I8 so one row is 90 contiguous bytes; the engine reads it by the format key
+ head = b"GGUF" + struct.pack("<IQQ", 3, 1, 5) + kvs
+ head += _gs("per_layer_token_embd.weight") + struct.pack("<I", 2) + struct.pack("<QQ", PLE_ROW_BYTES, rows)
+ head += struct.pack("<I", GGML_TYPE_I8) + struct.pack("<Q", 0)
+ head += b"\0" * ((-len(head)) % ALIGN)
+ out.parent.mkdir(parents=True, exist_ok=True)
+ tmp = out.with_suffix(out.suffix + ".part")
+ CH = 1 << 20
+ with open(tmp, "wb") as w:
+ w.write(head)
+ for k in range(len(shards)):
+ n = shards[k]
+ codes = ck.array(n, np.uint8)
+ sc = ck.array(n[:-6] + "weight_scale", np.uint8)
+ for i in range(0, codes.shape[0], CH):
+ w.write(np.ascontiguousarray(interleave_rows(codes[i:i + CH], sc[i:i + CH])).tobytes())
+ if k % 16 == 15 or k == len(shards) - 1:
+ print(" %3d/%d shards" % (k + 1, len(shards)), flush=True)
+ if tmp.stat().st_size != len(head) + rows * PLE_ROW_BYTES:
+ sys.exit("size check failed")
+ tmp.replace(out)
+ print("wrote %s (%d rows x %d B)" % (out, rows, PLE_ROW_BYTES))
+ return 0
+
+
+# ------------------------------------------------------------------------------------------------------- the MTP head
+E2M1 = np.array([0, .5, 1, 1.5, 2, 3, 4, 6, -0., -.5, -1, -1.5, -2, -3, -4, -6], np.float32)
+
+
+def e4m3_table() -> np.ndarray:
+ x = np.arange(256)
+ s, e, m = x >> 7, (x >> 3) & 15, x & 7
+ f = np.where(e == 0, np.ldexp(m / 8.0, -6), np.ldexp(1.0 + m / 8.0, e.astype(np.int32) - 7))
+ f = np.where((e == 15) & (m == 7), np.nan, f)
+ return np.where(s == 1, -f, f).astype(np.float32)
+
+
+def nvfp4_dequant(codes: np.ndarray, scales_u8: np.ndarray, scale2: float) -> np.ndarray:
+ """ModelOpt NVFP4 -> float32: codes [R, K/2] (low nibble first), E4M3 scale bytes [R, K/16], one F32."""
+ R = codes.shape[0]
+ lo, hi = codes & 15, codes >> 4
+ v = E2M1[np.stack([lo, hi], axis=-1).reshape(R, -1)]
+ sc = e4m3_table()[scales_u8]
+ return (v.reshape(R, -1, 16) * sc[:, :, None] * np.float32(scale2)).reshape(R, -1)
+
+
+def f32_to_bf16_bits(x: np.ndarray) -> np.ndarray:
+ u = np.ascontiguousarray(x, np.float32).view(np.uint32)
+ r = u + 0x7FFF + ((u >> 16) & 1) # round to nearest even
+ return (r >> 16).astype(np.uint16)
+
+
+def cmd_mtp(a) -> int:
+ """The MTP head in tools/mtp_fetch.py's folder form (raw BF16 files + mtp-manifest.json), so tools/mtp_pack.py and
+ tools/mtp_rt.py take it as they take Qwen's own: the 31 tensors, experts fused as gate_up_proj / down_proj."""
+ import hashlib
+ ck = Checkpoint(a.model, partial_ok=True)
+ out = pathlib.Path(a.out)
+ out.mkdir(parents=True, exist_ok=True)
+ manifest = []
+
+ def put(name, bits, shape):
+ fn = name.replace("/", "_") + ".bin"
+ data = np.ascontiguousarray(bits, np.uint16)
+ (out / fn).write_bytes(data.tobytes())
+ manifest.append({"name": name, "dtype": "BF16", "shape": list(shape), "file": fn,
+ "sha256": hashlib.sha256(data.tobytes()).hexdigest()})
+
+ for name in sorted(k for k in ck.meta if k.startswith("mtp.") and ".experts." not in k):
+ if ck.dtype(name) != "BF16":
+ sys.exit("%s is %s, expected BF16" % (name, ck.dtype(name)))
+ put(name, ck.array(name, np.uint16), ck.shape(name))
+ n_exp = 512
+ for fused, parts in (("mtp.layers.0.mlp.experts.gate_up_proj", ("gate_proj", "up_proj")),
+ ("mtp.layers.0.mlp.experts.down_proj", ("down_proj",))):
+ rows = [ck.shape("mtp.layers.0.mlp.experts.0.%s.weight" % p)[0] for p in parts]
+ cols = ck.shape("mtp.layers.0.mlp.experts.0.%s.weight" % parts[0])[1] * 2
+ shape = [n_exp, sum(rows), cols]
+ fn = fused + ".bin"
+ with open(out / fn, "wb") as f:
+ h = hashlib.sha256()
+ for e in range(n_exp):
+ mats = []
+ for p in parts:
+ b = "mtp.layers.0.mlp.experts.%d.%s." % (e, p)
+ mats.append(nvfp4_dequant(np.asarray(ck.array(b + "weight", np.uint8)),
+ np.asarray(ck.array(b + "weight_scale", np.uint8)),
+ float(ck.array(b + "weight_scale_2", np.float32).reshape(-1)[0])))
+ blob = f32_to_bf16_bits(np.concatenate(mats, axis=0)).tobytes()
+ f.write(blob)
+ h.update(blob)
+ manifest.append({"name": fused, "dtype": "BF16", "shape": shape, "file": fn, "sha256": h.hexdigest()})
+ print(" %s %s" % (fused, shape), flush=True)
+ (out / "mtp-manifest.json").write_text(json.dumps(manifest, indent=1), encoding="utf-8")
+ print("wrote %d MTP tensors to %s" % (len(manifest), out))
+ return 0
+
+
+# ------------------------------------------------------------------------------------------------------ verifying
+def cmd_verify(a) -> int:
+ """Experts: decode the checkpoint (CSF + ModelOpt definition) and compare with gguf-py's dequantization of the
+ GGUF's NVFP4 blocks times the scale tensor, bit for bit (zeros normalised: ggml has one zero)."""
+ import torch
+ import nvfp4_verify as V
+ from gguf import GGUFReader, GGMLQuantizationType as Q, quants
+ ck = Checkpoint(a.model)
+ rd = GGUFReader(a.gguf)
+ T = {t.name: t for t in rd.tensors}
+ bad = 0
+ rng = np.random.default_rng(1)
+ layers = sorted({int(k.split(".layers.")[1].split(".")[0]) for k in ck.meta if ".mlp.experts." in k and k.startswith("model.language_model")})
+ for _ in range(a.experts):
+ L = int(rng.choice(layers)); e = int(rng.integers(0, 512))
+ for proj, gname in (("gate_proj", "ffn_gate_exps"), ("up_proj", "ffn_up_exps"), ("down_proj", "ffn_down_exps")):
+ b = "model.language_model.layers.%d.mlp.experts.%d.%s." % (L, e, proj)
+ w = torch.from_numpy(np.array(ck.array(b + "weight", np.uint8)))
+ sc = csf_decode(ck.array(b + "weight_scale" + CSF_FIXED, np.uint8), ck.array(b + "weight_scale" + CSF_EXC, np.uint32))
+ s = torch.from_numpy(sc).view(torch.float8_e4m3fn)
+ s2 = torch.from_numpy(np.array(ck.array(b + "weight_scale_2", np.float32))).reshape(())
+ ref = V.modelopt_dequant(w, s, s2)
+ t = T["blk.%d.%s.weight" % (L, gname)]
+ blob = np.asarray(t.data) # [512, rows, bytes] of NVFP4 blocks
+ got = quants.dequantize(np.ascontiguousarray(blob[e]), Q.NVFP4).astype(np.float32)
+ sct = T["blk.%d.%s.scale" % (L, gname)]
+ got = got * np.float32(np.asarray(sct.data)[e])
+ same = np.array_equal((ref + np.float32(0)).view(np.uint32), (got.reshape(ref.shape) + np.float32(0)).view(np.uint32))
+ print(" layer %2d expert %3d %-9s bitwise %s" % (L, e, proj, "OK" if same else "NO"), flush=True)
+ bad += not same
+ print("RESULT:", "lossless" if not bad else "MISMATCH")
+ return 1 if bad else 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ sub = ap.add_subparsers(dest="cmd", required=True)
+ c = sub.add_parser("convert")
+ c.add_argument("--model", required=True)
+ c.add_argument("--outfile", required=True)
+ c.add_argument("--dry-run", action="store_true")
+ p = sub.add_parser("ple")
+ p.add_argument("--model", required=True)
+ p.add_argument("--out", required=True)
+ m = sub.add_parser("mtp")
+ m.add_argument("--model", required=True)
+ m.add_argument("--out", required=True)
+ v = sub.add_parser("verify")
+ v.add_argument("--model", required=True)
+ v.add_argument("--gguf", required=True)
+ v.add_argument("--experts", type=int, default=4)
+ a = ap.parse_args()
+ return {"convert": cmd_convert, "ple": cmd_ple, "mtp": cmd_mtp, "verify": cmd_verify}[a.cmd](a)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/test_nvfp4_local_pack.py b/tools/test_nvfp4_local_pack.py
new file mode 100644
index 000000000..a650eaa37
--- /dev/null
+++ b/tools/test_nvfp4_local_pack.py
@@ -0,0 +1,98 @@
+"""Tests for tools/nvfp4_local_pack.py: the CSF scale decode, the MXFP8 dequantization, the PLE rows."""
+import pathlib
+import sys
+
+import numpy as np
+import pytest
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
+import nvfp4_local_pack as P # noqa: E402
+
+
+def _scales(rows, cols, seed, outliers=0.01):
+ rng = np.random.default_rng(seed)
+ base = rng.integers(60, 100, size=(rows, 1))
+ s = base + rng.integers(0, 16, size=(rows, cols))
+ m = rng.random((rows, cols)) < outliers
+ s[m] = rng.integers(0, 127, size=int(m.sum()))
+ return s.astype(np.uint8)
+
+
[email protected]("rows,cols", [(32, 8), (640, 160), (2560, 40)])
+def test_csf_roundtrip_is_exact(rows, cols):
+ s = _scales(rows, cols, rows + cols)
+ fixed, exc = P.csf_encode(s)
+ assert fixed.shape == (rows // 16, 16 * (1 + cols // 2)) # the shapes of the real tensors
+ assert np.array_equal(P.csf_decode(fixed, exc), s)
+
+
+def test_csf_matches_the_spec_by_hand():
+ # one 16-row slab, 4 columns: base bytes 10..25, rows' codes = col index, one exception at row 3 col 2
+ rows, cols = 16, 4
+ fixed = np.zeros((1, 16 * (1 + cols // 2)), np.uint8)
+ fixed[0, :16] = np.arange(10, 26)
+ for r in range(rows):
+ fixed[0, 16 + r * 2] = 0 | (1 << 4) # cols 0 (low nibble), 1 (high)
+ fixed[0, 16 + r * 2 + 1] = 2 | (3 << 4) # cols 2, 3
+ exc = np.array([(3 * cols + 2) | (99 << 24)], np.uint32)
+ got = P.csf_decode(fixed, exc)
+ assert got[0].tolist() == [10, 11, 12, 13]
+ assert got[5].tolist() == [15, 16, 17, 18]
+ assert got[3].tolist() == [13, 14, 99, 16]
+
+
+def test_csf_rejects_bad_streams():
+ with pytest.raises(ValueError):
+ P.csf_decode(np.zeros((1, 33), np.uint8), np.zeros(0, np.uint32))
+ fixed = np.zeros((1, 32), np.uint8)
+ fixed[0, 0] = 250
+ with pytest.raises(ValueError):
+ P.csf_decode(fixed, np.zeros(0, np.uint32))
+ with pytest.raises(ValueError):
+ P.csf_decode(np.zeros((1, 32), np.uint8), np.array([16 * 4], np.uint32))
+
+
+def test_mxfp8_dequant_exact():
+ torch = pytest.importorskip("torch")
+ rng = np.random.default_rng(3)
+ vals = rng.integers(0, 126, size=(4, 64)).astype(np.uint8) | (rng.integers(0, 2, size=(4, 64)).astype(np.uint8) << 7)
+ sc = rng.integers(100, 130, size=(4, 2)).astype(np.uint8)
+ got = P.mxfp8_dequant(vals, sc).numpy()
+ ref = torch.from_numpy(vals).view(torch.float8_e4m3fn).to(torch.float32).numpy().reshape(4, 2, 32)
+ ref = ref * np.exp2(sc.astype(np.float64) - 127)[:, :, None]
+ assert np.array_equal(got.reshape(4, 2, 32), ref.astype(np.float32))
+ # exactly representable in BF16 (what the converter hands on)
+ bf = torch.from_numpy(got).to(torch.bfloat16).to(torch.float32).numpy()
+ assert np.array_equal(bf, got)
+
+
+def test_mxfp8_rejects_nan_scale():
+ pytest.importorskip("torch")
+ with pytest.raises(ValueError):
+ P.mxfp8_dequant(np.zeros((1, 32), np.uint8), np.full((1, 1), 255, np.uint8))
+
+
+def test_ple_rows_are_codes_then_scales():
+ c = np.arange(160, dtype=np.uint8).reshape(2, 80)
+ s = np.arange(20, dtype=np.uint8).reshape(2, 10) + 200
+ r = P.interleave_rows(c, s)
+ assert r.shape == (2, P.PLE_ROW_BYTES)
+ assert np.array_equal(r[1, :80], c[1]) and np.array_equal(r[1, 80:], s[1])
+
+
+def test_nvfp4_dequant_by_hand():
+ # element 2i in the low nibble: byte 0x21 = codes 1 (0.5) and 2 (1.0); 0xF7 = 7 (6.0) and 15 (-6.0)
+ codes = np.zeros((1, 8), np.uint8)
+ codes[0, 0], codes[0, 1] = 0x21, 0xF7
+ sc = np.array([[0x38]], np.uint8) # E4M3 0x38 = 1.0
+ got = P.nvfp4_dequant(codes, sc, 2.0)
+ assert got[0, :4].tolist() == [1.0, 2.0, 12.0, -12.0]
+
+
+def test_e4m3_table_and_bf16_rounding():
+ t = P.e4m3_table()
+ assert t[0x38] == 1.0 and t[0x7E] == 448.0 and np.isnan(t[0x7F]) and t[0x01] == 2.0 ** -9
+ x = np.array([1.0, 1.0 + 2.0 ** -8, 1.0 + 3 * 2.0 ** -9, -2.5], np.float32)
+ b = P.f32_to_bf16_bits(x)
+ back = (b.astype(np.uint32) << 16).view(np.float32)
+ assert back.tolist() == [1.0, 1.0, 1.0078125, -2.5] # ties to even; 1+3*2^-9 rounds up