Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
NVIDIA Qwen3.8-Flash-Next-FP4-Local: converter (CSF scale decode, MXFP8 to Q8_0/BF16, MTP, NVFP4 PLE table) and the NVFP4 PLE row format
Niko12219c6255cfcee9
diff --git a/include/strata/kernels/ngram.hpp b/include/strata/kernels/ngram.hppindex 207707d0e..4452b4f6d 100644--- a/include/strata/kernels/ngram.hpp+++ b/include/strata/kernels/ngram.hpp@@ -49,20 +49,21 @@ inline constexpr int PLE_ROW_BYTES_FP8 = PLE_HEAD_DIM; // 160:inline constexpr int PLE_ROW_BYTES_Q5_1 = (PLE_HEAD_DIM / 32) * 24; // 120: a Q5_1 row (a Q5_K_M finetune)inline constexpr int PLE_ROW_BYTES_Q8_0 = (PLE_HEAD_DIM / 32) * 34; // 170: a Q8_0 row (UD-Q6_K_XL, Swift-1.5 Q4_K_L)inline constexpr int PLE_ROW_BYTES_BF16 = PLE_HEAD_DIM * 2; // 320: a BF16 row, two bytes a value+inline constexpr int PLE_ROW_BYTES_NVFP4 = PLE_HEAD_DIM / 2 + PLE_HEAD_DIM / 16; // 90: 80 B of E2M1 codes + 10 E4M3 block scalesinline constexpr int PLE_ROW_BYTES_MAX = PLE_ROW_BYTES_BF16; // the widest row any format in `ple_formats()` has/// THE FORMATS OF THE TABLE, ONE ROW EACH. A table type is one entry here (its GGUF type name, the bytes of one/// 160-value row, how to turn a row into floats) and nothing else: `PleTable::open`, `format()`, the reader's row size,/// the error string and the buffers' size all read this list, so a new format cannot be in one switch and missing from/// another. A row is always `PLE_HEAD_DIM` = 160 values = 5 blocks of 32 for the block formats.-enum class PleFormat : uint8_t { IQ4_NL, Q5_0, F8_E4M3, Q5_1, Q8_0, Q4_0, BF16 };+enum class PleFormat : uint8_t { IQ4_NL, Q5_0, F8_E4M3, Q5_1, Q8_0, Q4_0, BF16, NVFP4 };struct PleFormatInfo {PleFormat id;const char* name; ///< what `PleTable::format()` returnsconst char* gguf_type; ///< `TensorInfo::type_name()` of the table tensor ("I8" for F8_E4M3: GGUF has no FP8 type)uint32_t row_bytes; ///< bytes of one 160-value row in the file- bool needs_scale; ///< the table has one scale in the GGUF metadata (`strata.ple.scale`), F8_E4M3 only+ bool needs_scale; ///< the table has one scale in the GGUF metadata (`strata.ple.scale`): F8_E4M3 and NVFP4void (*dequant)(const uint8_t* row, float scale, float* out160);};diff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cppindex 91de37058..a0685bd52 100644--- a/src/kernels/ngram.cpp+++ b/src/kernels/ngram.cpp@@ -152,6 +152,20 @@ void bf16_dequant_row(const uint8_t* row, float* out160) {}}+// NVIDIA's FP4-Local table: a row of 160 values is 80 bytes of E2M1 codes (element 2i in the low nibble, 2i+1 in the+// high one) followed by 10 E4M3 block scales (one per 16 values, sign bit always 0); one F32 scales the whole table.+// value = e2m1(code) * e4m3(block scale) * table scale, exactly what vLLM's nvfp4 PLE lookup computes.+void nvfp4_row_dequant(const uint8_t* row, float scale, float* out160) {+ static const float kE2M1[8] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f};+ for (int j = 0; j < PLE_HEAD_DIM; ++j) {+ const uint8_t byte = row[j >> 1];+ const uint8_t code = (j & 1) ? (uint8_t) (byte >> 4) : (uint8_t) (byte & 15);+ const float v = kE2M1[code & 7];+ const float s = kFp8.v[row[PLE_HEAD_DIM / 2 + (j >> 4)]];+ out160[j] = ((code & 8) ? -v : v) * s * scale;+ }+}+// ---------------------------------------------------------------------------------------------------// THE FORMAT TABLE (see PleFormatInfo). One entry per table type; the block formats decode 5 blocks of 32.namespace {@@ -162,6 +176,7 @@ void dequant_blocks(const uint8_t* row, float /*scale*/, float* out160) {void dequant_iq4_nl(const uint8_t* row, float, float* out160) { iq4nl_dequant_row(row, out160); }void dequant_fp8(const uint8_t* row, float scale, float* out160) { fp8_e4m3_dequant_row(row, scale, out160); }void dequant_bf16(const uint8_t* row, float, float* out160) { bf16_dequant_row(row, out160); }+void dequant_nvfp4(const uint8_t* row, float scale, float* out160) { nvfp4_row_dequant(row, scale, out160); }const PleFormatInfo kPleFormats[] = {{PleFormat::IQ4_NL, "IQ4_NL", "IQ4_NL", PLE_ROW_BYTES, false, dequant_iq4_nl},@@ -175,6 +190,9 @@ const PleFormatInfo kPleFormats[] = {// BF16 (#586): the checkpoint's own table at full precision, 320-byte rows; self-describing, so no scale and no// metadata to trust (tools/ple_fp8_pack.py writes the FP8 form of the same table){PleFormat::BF16, "BF16", "BF16", PLE_ROW_BYTES_BF16, false, dequant_bf16},+ // NVFP4 x16 (NVIDIA's Qwen3.8-Flash-Next-FP4-Local, tools/nvfp4_local_pack.py ple): 90-byte rows, I8 bytes marked+ // strata.ple.format = nvfp4x16 with strata.ple.scale. The tensor is [90, rows]: ne0 is the row's byte count.+ {PleFormat::NVFP4, "NVFP4", "I8", PLE_ROW_BYTES_NVFP4, true, dequant_nvfp4},};constexpr int kPleFormatCount = (int) (sizeof kPleFormats / sizeof kPleFormats[0]);} // namespace@@ -192,7 +210,7 @@ std::string ple_format_list() {for (int i = 0; i < kPleFormatCount; ++i) {const PleFormatInfo& f = kPleFormats[i];if (i > 0) out += i + 1 == kPleFormatCount ? " or " : ", ";- out += std::strcmp(f.gguf_type, "I8") == 0 ? "FP8 (I8)" : f.name;+ out += std::strcmp(f.gguf_type, "I8") == 0 ? (f.id == PleFormat::NVFP4 ? "NVFP4 (I8)" : "FP8 (I8)") : f.name;}return out;}@@ -245,8 +263,12 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoOclose();return false;}- // [160, 320001536]: ne0 = 160 is the FAST axis, so the ROW index is shape[1] and a row is contiguous.- if (t->shape.size() != 2 || t->shape[0] != (uint64_t) PLE_HEAD_DIM) {+ // [160, 320001536]: ne0 = 160 is the FAST axis, so the ROW index is shape[1] and a row is contiguous. The NVFP4 table+ // is I8 bytes with ne0 = 90 (the row's byte count), marked strata.ple.format = nvfp4x16.+ const MetaValue* fmt_key = impl_->file->get("strata.ple.format");+ const bool nvfp4x16 = std::strcmp(t->type_name(), "I8") == 0 && fmt_key != nullptr && fmt_key->s == "nvfp4x16";+ const uint64_t want_ne0 = nvfp4x16 ? (uint64_t) PLE_ROW_BYTES_NVFP4 : (uint64_t) PLE_HEAD_DIM;+ if (t->shape.size() != 2 || t->shape[0] != want_ne0) {err = "per_layer_token_embd.weight has an unexpected shape";close();return false;@@ -254,7 +276,7 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoO// The table's type is looked up in the format list (ple_formats()): IQ4_NL (ISTA-DASLab's shard 2, the original's// own GGUF), Q5_0 (#296: OrcaRouter's GGUF), or the FP8 table as shipped: I8 bytes marked strata.ple.format =// f8_e4m3 with strata.ple.scale (tools/ple_fp8_pack.py)- const PleFormatInfo* fmt = ple_format_for_type(t->type_name());+ const PleFormatInfo* fmt = nvfp4x16 ? &ple_format_info(PleFormat::NVFP4) : ple_format_for_type(t->type_name());if (fmt == nullptr) {err = std::string("per_layer_token_embd.weight is ") + t->type_name() + ", not " + ple_format_list();close();@@ -263,8 +285,9 @@ bool PleTable::open(const std::string& gguf_path, std::string& err, const PleIoOif (fmt->needs_scale) {const MetaValue* f = impl_->file->get("strata.ple.format");const MetaValue* s = impl_->file->get("strata.ple.scale");- if (f == nullptr || f->s != "f8_e4m3" || s == nullptr || !(s->num() > 0.0)) {- err = "per_layer_token_embd.weight is I8 without strata.ple.format = f8_e4m3 and a positive strata.ple.scale";+ const char* want = nvfp4x16 ? "nvfp4x16" : "f8_e4m3";+ if (f == nullptr || f->s != want || s == nullptr || !(s->num() > 0.0)) {+ err = std::string("per_layer_token_embd.weight is I8 without strata.ple.format = ") + want + " and a positive strata.ple.scale";close();return false;}diff --git a/src/kernels/ple_fp8_parity.cpp b/src/kernels/ple_fp8_parity.cppindex 09dd199a8..bc54956ce 100644--- a/src/kernels/ple_fp8_parity.cpp+++ b/src/kernels/ple_fp8_parity.cpp@@ -11,6 +11,7 @@#include <cmath>#include <cstdio>+#include <cstdlib>#include <cstring>#include <fstream>#include <string>@@ -51,7 +52,7 @@ int main(int argc, char** argv) {std::printf("%-6s %s, %llu rows: read_row %d/%zu values differ, gather_batch %s %d differ\n",mode == k::PleIo::Direct ? "Direct" : "Mmap", t.format(), (unsigned long long) t.rows(), diff,got.size(), ok ? "ok," : "FAILED,", bdiff);- bad += diff + bdiff + (ok ? 0 : 1) + (std::strcmp(t.format(), "F8_E4M3") != 0);+ bad += diff + bdiff + (ok ? 0 : 1) + (std::strcmp(t.format(), std::getenv("PLE_EXPECT_FORMAT") ? std::getenv("PLE_EXPECT_FORMAT") : "F8_E4M3") != 0);}if (argc > 3) {k::PleTable q;diff --git a/src/ngram/ple_reader_test.cpp b/src/ngram/ple_reader_test.cppindex 4663455f7..569c04736 100644--- a/src/ngram/ple_reader_test.cpp+++ b/src/ngram/ple_reader_test.cpp@@ -105,7 +105,7 @@ void bf16_widening_is_exact() {/// and the type lookup all have to be right for every format, not only for IQ4_NL.uint32_t gguf_type_id(const char* name) {static const struct { const char* name; uint32_t id; } ids[] = {- {"IQ4_NL", 20}, {"Q4_0", 2}, {"Q4_1", 3}, {"Q5_0", 6}, {"Q5_1", 7}, {"Q8_0", 8}, {"BF16", 30}, {"F8_E4M3", 24}};+ {"IQ4_NL", 20}, {"Q4_0", 2}, {"Q4_1", 3}, {"Q5_0", 6}, {"Q5_1", 7}, {"Q8_0", 8}, {"BF16", 30}, {"F8_E4M3", 24}, {"NVFP4", 24}};for (const auto& e : ids)if (std::strcmp(e.name, name) == 0) return e.id;return 0xFFFFFFFFu;@@ -129,11 +129,11 @@ std::string write_format_table(const std::string& dir, const k::PleFormatInfo& fconst float scale = 0.75f;uint32_t sb;std::memcpy(&sb, &scale, 4);- head += str("strata.ple.format") + le(8u, 4) + str("f8_e4m3");+ head += str("strata.ple.format") + le(8u, 4) + str(f.id == k::PleFormat::NVFP4 ? "nvfp4x16" : "f8_e4m3");head += str("strata.ple.scale") + le(6u, 4) + le(sb, 4); // 6 = FLOAT32}head += str("per_layer_token_embd.weight") + le(2u, 4);- head += le((uint64_t) k::PLE_HEAD_DIM, 8) + le(claimed_rows != 0 ? claimed_rows : (uint64_t) rows, 8);+ head += le(f.id == k::PleFormat::NVFP4 ? (uint64_t) f.row_bytes : (uint64_t) k::PLE_HEAD_DIM, 8) + le(claimed_rows != 0 ? claimed_rows : (uint64_t) rows, 8);head += le(gguf_type_id(f.name), 4) + le(0ull, 8);while (head.size() % 32) head += '\0';out.write(head.data(), (std::streamsize) head.size());diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.pynew file mode 100644index 000000000..2d903fb03--- /dev/null+++ b/tools/nvfp4_local_pack.py@@ -0,0 +1,469 @@+"""tools/nvfp4_local_pack.py - NVIDIA's Qwen3.8-Flash-Next-FP4-Local (ModelOpt, QAD) -> files Strata can pack.++ python tools/nvfp4_local_pack.py convert --model <nvidia/Qwen3.8-Flash-Next-FP4-Local dir> --outfile <model.gguf>+ python tools/nvfp4_local_pack.py ple --model <same dir> --out <ple-nvfp4.gguf>+ python tools/nvfp4_local_pack.py mtp --model <same dir> --out <mtp src dir> (then tools/mtp_pack.py, tools/mtp_rt.py)+ python tools/nvfp4_local_pack.py verify --model <same dir> --gguf <model.gguf> [--experts 6]+ python tools/iq_pack.py --gguf <model.gguf> --out <pack dir>++Nothing of NVIDIA's weights is shipped: the user downloads the checkpoint from NVIDIA's repository and this runs+locally. What the checkpoint stores, and what is done with it:++ * routed experts: NVFP4 (E2M1 codes, E4M3 scale per 16 values, F32 weight_scale_2). The block scales are stored+ in NVIDIA's "CSF" form (config `weight_scale_encoding: "csf"`, codec byte-window4-fixed-stream-u24-exceptions/1):+ per tensor a `.nvfp4_csf_fixed` U8 stream and a `.nvfp4_csf_exceptions` U32 stream. They are decoded here to the+ plain E4M3 matrix (the format is documented by vLLM's loader, PR 60646, and flashinfer's b12x decoder, both+ Apache-2.0; this is a fresh implementation of that description), and from there the ordinary ModelOpt NVFP4+ repack of tools/nvfp4_convert.py applies, bit for bit.+ * attention, GDN projections, shared experts, indexer: MXFP8 (E4M3 values, E8M0 scale per 32 values). They are+ dequantized exactly to BF16 and then take the type policy of tools/nvfp4_convert.py (Q8_0 for the big ones,+ BF16 for the small projections the BF16 kernels read). Q8_0 has the same 32-value blocks; the added error is+ ~0.6% RMS against MXFP8's own ~3%.+ * the 26.8 GiB n-gram (PLE) table is NVFP4 too (160 values a row: 80 B of codes, 10 E4M3 scales, one F32 for the+ whole table). `ple` copies it row by row into a one-tensor GGUF of 90-byte rows.+ * the MTP head (NVFP4 experts, BF16 rest) is dequantized by `mtp` into the folder form tools/mtp_pack.py reads.+ The vision tower is not written (text first).+"""+from __future__ import annotations++import argparse+import json+import pathlib+import struct+import sys+import time++import numpy as np++HERE = pathlib.Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))++CSF_FIXED = ".nvfp4_csf_fixed"+CSF_EXC = ".nvfp4_csf_exceptions"+PLE_ROW_BYTES = 90 # 80 B of E2M1 codes + 10 E4M3 block scales, per 160-value row+++# --------------------------------------------------------------------------------------------- safetensors access+class Checkpoint:+ """Header-indexed, memory-mapped access to a checkpoint's safetensors shards (raw bytes, any dtype)."""++ def __init__(self, folder, partial_ok=False):+ self.dir = pathlib.Path(folder)+ index = self.dir / "model.safetensors.index.json"+ if index.exists():+ wmap = json.loads(index.read_text(encoding="utf-8"))["weight_map"]+ files = sorted(set(wmap.values()))+ else:+ files = sorted(p.name for p in self.dir.glob("*.safetensors"))+ if not files:+ raise FileNotFoundError("no .safetensors files in " + str(self.dir))+ self.meta: dict[str, tuple[str, str, list[int], int, int]] = {}+ self._mm: dict[str, np.memmap] = {}+ for fn in files:+ path = self.dir / fn+ if not path.exists() and partial_ok:+ continue # a subset of the shards (the MTP head is in two)+ with open(path, "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ hdr = json.loads(f.read(n))+ for k, v in hdr.items():+ if k != "__metadata__":+ a, b = v["data_offsets"]+ self.meta[k] = (fn, v["dtype"], v["shape"], 8 + n + a, 8 + n + b)++ def __contains__(self, name):+ return name in self.meta++ def dtype(self, name):+ return self.meta[name][1]++ def shape(self, name):+ return self.meta[name][2]++ def raw(self, name) -> np.ndarray:+ fn, _, _, a, b = self.meta[name]+ if fn not in self._mm:+ self._mm[fn] = np.memmap(self.dir / fn, dtype=np.uint8, mode="r")+ return self._mm[fn][a:b]++ def array(self, name, np_dtype) -> np.ndarray:+ return self.raw(name).view(np_dtype).reshape(self.shape(name))+++# ----------------------------------------------------------------------------------------------------- CSF decoding+def csf_decode(fixed: np.ndarray, exceptions: np.ndarray) -> np.ndarray:+ """The CSF stream pair -> the plain E4M3 block-scale matrix as uint8 [rows, cols] (cols = K / 16).++ fixed: uint8 [rows / 16, 16 * (1 + cols / 2)]. Per slab of 16 rows: the 16 rows' base bytes, then each+ row's cols 4-bit codes (two to a byte, low nibble first). A scale byte is base + code.+ exceptions: uint32 words, low 24 bits = flat row * cols + col, high 8 bits = the scale byte there.+ """+ if fixed.dtype != np.uint8 or fixed.ndim != 2 or fixed.shape[1] % 16 or fixed.shape[1] < 32:+ raise ValueError("invalid CSF fixed stream %s %s" % (fixed.dtype, fixed.shape))+ if exceptions.dtype != np.uint32:+ raise ValueError("invalid CSF exceptions dtype %s" % exceptions.dtype)+ rows = fixed.shape[0] * 16+ cols = (fixed.shape[1] // 16 - 1) * 2+ bases = fixed[:, :16].reshape(rows, 1).astype(np.int16)+ if (bases > 240).any():+ raise ValueError("CSF row bases must be in 0..240")+ packed = fixed[:, 16:].reshape(rows, cols // 2)+ codes = np.stack((packed & 0xF, packed >> 4), axis=-1).reshape(rows, cols)+ out = (bases + codes).reshape(-1)+ words = exceptions.reshape(-1).astype(np.int64)+ pos = words & 0xFFFFFF+ if pos.size and int(pos.max()) >= rows * cols:+ raise ValueError("CSF exception position out of range")+ out[pos] = words >> 24+ return out.astype(np.uint8).reshape(rows, cols)+++def csf_encode(scales: np.ndarray) -> tuple[np.ndarray, np.ndarray]:+ """The inverse, for the tests: uint8 [rows, cols] -> (fixed, exceptions). Base = the row's most common+ window start; whatever falls outside base..base+15 is an exception."""+ rows, cols = scales.shape+ assert rows % 16 == 0 and cols % 2 == 0+ base = np.zeros(rows, np.int16)+ for r in range(rows):+ v = scales[r].astype(np.int16)+ best = (-1, 0)+ for b in range(max(0, int(v.min()) - 0), int(v.max()) + 1):+ c = int(((v >= b) & (v <= b + 15)).sum())+ if c > best[0]:+ best = (c, b)+ base[r] = min(best[1], 240)+ code = scales.astype(np.int16) - base[:, None]+ exc = (code < 0) | (code > 15)+ code = np.where(exc, 0, code).astype(np.uint8)+ pk = (code[:, 0::2] | (code[:, 1::2] << 4)).astype(np.uint8)+ fixed = np.empty((rows // 16, 16 * (1 + cols // 2)), np.uint8)+ for s in range(rows // 16):+ fixed[s, :16] = base[s * 16:(s + 1) * 16].astype(np.uint8)+ fixed[s, 16:] = pk[s * 16:(s + 1) * 16].reshape(-1)+ pos = np.flatnonzero(exc.reshape(-1))+ words = (pos.astype(np.uint64) | (scales.reshape(-1)[pos].astype(np.uint64) << 24)).astype(np.uint32)+ return fixed, words+++# --------------------------------------------------------------------------------------------- MXFP8 dequantizing+def mxfp8_dequant(values_u8: np.ndarray, scales_u8: np.ndarray):+ """MXFP8 (E4M3 values [R, K], E8M0 scale bytes [R, K/32]) -> float32 [R, K]. Exact (4-bit significands)."""+ import torch+ R, K = values_u8.shape+ if scales_u8.shape != (R, K // 32) or K % 32:+ raise ValueError("MXFP8 shapes %s / %s" % (values_u8.shape, scales_u8.shape))+ if (scales_u8 == 255).any():+ raise ValueError("MXFP8 scale byte 255 (NaN) in a weight")+ v = torch.from_numpy(np.ascontiguousarray(values_u8)).view(torch.float8_e4m3fn).to(torch.float32)+ s = torch.from_numpy(np.exp2(scales_u8.astype(np.float32) - 127.0))+ return (v.reshape(R, K // 32, 32) * s[:, :, None]).reshape(R, K)+++# --------------------------------------------------------------------------- the overlay on llama.cpp's tensor index+def _orig_name(ck: Checkpoint, name: str):+ """The converter strips `language_model.`; find the checkpoint's own name."""+ if name in ck:+ return name+ for cand in (name.replace("model.", "model.language_model.", 1), "model.language_model." + name):+ if cand in ck:+ return cand+ return None+++def overlay(tensors: dict, ck: Checkpoint, stats: dict) -> dict:+ """Replace the CSF stream pairs by plain `weight_scale` tensors, the MXFP8 weights by BF16, and drop what+ the PLE writer and the MTP packer own."""+ import torch+ out = dict(tensors)+ for name in list(out):+ if name not in out:+ continue+ if name.endswith(CSF_FIXED):+ base = name[: -len(CSF_FIXED)] # ...weight_scale+ o_fix = _orig_name(ck, name)+ o_exc = _orig_name(ck, base + CSF_EXC)+ if o_fix is None or o_exc is None:+ raise ValueError("CSF fixed stream without its exceptions: " + name)++ def gen(o_fix=o_fix, o_exc=o_exc):+ dec = csf_decode(ck.array(o_fix, np.uint8), ck.array(o_exc, np.uint32))+ return torch.from_numpy(dec).view(torch.float8_e4m3fn)+ del out[name]+ out.pop(base + CSF_EXC, None)+ out[base] = gen+ stats["csf"] = stats.get("csf", 0) + 1+ elif ".ngram_embedding." in name and (name.endswith(".weight_scale") or name.endswith(".weight_scale_2")):+ del out[name] # the PLE table has its own writer (`ple`)+ for name in list(out):+ if not name.endswith(".weight") or ".ngram_embedding." in name:+ continue+ sname = name[: -len(".weight")] + ".weight_scale"+ o_w, o_s = _orig_name(ck, name), _orig_name(ck, sname)+ if o_w is None or o_s is None or sname not in out:+ continue+ if ck.dtype(o_w) == "F8_E4M3" and ck.dtype(o_s) == "U8": # MXFP8+ def gen(o_w=o_w, o_s=o_s):+ f = mxfp8_dequant(np.asarray(ck.raw(o_w)).reshape(ck.shape(o_w)), np.asarray(ck.raw(o_s)).reshape(ck.shape(o_s)))+ return f.to(torch.bfloat16)+ out[name] = gen+ del out[sname]+ stats["mxfp8"] = stats.get("mxfp8", 0) + 1+ return out+++def install(ck: Checkpoint, stats: dict):+ """Import the converter, give it nvfp4_convert's type policy, and hook the tensor index."""+ import nvfp4_convert as base+ base.install_policy()+ from conversion.base import ModelBase+ from conversion import qwen4exp as Q4+ orig_index = ModelBase.index_tensors++ def index_tensors(self, remote_hf_model_id=None):+ return overlay(orig_index(self, remote_hf_model_id), ck, stats)+ ModelBase.index_tensors = index_tensors++ # NVIDIA's repo keeps the shards in a `tensors/` folder that its index names; the converter only lists the top level+ orig_parts = ModelBase.get_model_part_names++ def get_model_part_names(dir_model, prefix, suffix):+ names = orig_parts(dir_model, prefix, suffix)+ sub = pathlib.Path(dir_model) / "tensors"+ if not names and suffix == ".safetensors" and prefix == "model" and sub.is_dir():+ names = sorted("tensors/" + p.name for p in sub.glob("model*.safetensors"))+ return names+ ModelBase.get_model_part_names = staticmethod(get_model_part_names)++ cls = Q4.Qwen4ExpTextModel++ def place_ple_shard(self, data_torch, name):+ # the shards are NVFP4 here: two values a byte, so a row of 160 values is 80 bytes+ idx = int(name.rsplit("shard_", 1)[1].split(".", 1)[0])+ self._ple_shards[idx] = name+ self._ple_row_dim = int(data_torch.shape[-1]) * 2+ base.dropped_ple.append(name)+ return []+ cls._place_ple_shard = place_ple_shard+ return base+++def cmd_convert(a) -> int:+ model = pathlib.Path(a.model)+ ck = Checkpoint(model)+ stats: dict = {}+ base = install(ck, stats)+ import convert_hf_to_gguf as C+ out = pathlib.Path(a.outfile)+ out.parent.mkdir(parents=True, exist_ok=True)+ argv = [str(model), "--outtype", "q8_0", "--outfile", str(out)] + (["--dry-run"] if a.dry_run else [])+ sys.argv = ["convert_hf_to_gguf.py"] + argv+ t0 = time.time()+ rc = C.main()+ took = time.time() - t0+ print("nvfp4_local_pack: done in %.0f s; CSF scale tensors decoded %d, MXFP8 weights dequantized %d, PLE shards dropped %d"+ % (took, stats.get("csf", 0), stats.get("mxfp8", 0), len(base.dropped_ple)))+ if not a.dry_run:+ out.with_suffix(".manifest.json").write_text(json.dumps({+ "source": str(model), "csf_decoded": stats.get("csf", 0), "mxfp8_dequantized": stats.get("mxfp8", 0),+ "ple_table": "written separately: nvfp4_local_pack.py ple", "mtp": "not written", "seconds": round(took, 1)},+ indent=2) + "\n", encoding="utf-8")+ return rc or 0+++# ------------------------------------------------------------------------------------------------- the PLE table+GGUF_TYPE_STRING, GGUF_TYPE_F32 = 8, 6+GGML_TYPE_I8 = 24+ALIGN = 32+++def _gs(s):+ b = s.encode("utf-8")+ return struct.pack("<Q", len(b)) + b+++def interleave_rows(codes: np.ndarray, scales: np.ndarray) -> np.ndarray:+ """[n, 80] codes + [n, 10] E4M3 scale bytes -> [n, 90]: the codes, then the scales."""+ return np.concatenate([codes, scales], axis=1)+++def cmd_ple(a) -> int:+ model, out = pathlib.Path(a.model), pathlib.Path(a.out)+ ck = Checkpoint(model)+ shards = {}+ for name in ck.meta:+ if ".ngram_embedding.shard_" in name and name.endswith(".weight"):+ shards[int(name.rsplit(".shard_", 1)[1].split(".")[0])] = name+ if not shards or sorted(shards) != list(range(len(shards))):+ sys.exit("the n-gram shards are missing or not numbered 0..N-1")+ g = [k for k in ck.meta if k.endswith(".ngram_embedding.weight_scale_2")]+ if len(g) != 1:+ sys.exit("expected one ngram_embedding.weight_scale_2")+ scale2 = float(ck.array(g[0], np.float32).reshape(-1)[0])+ rows, dim = 0, None+ for k in range(len(shards)):+ n = shards[k]+ r, c = ck.shape(n)+ if ck.dtype(n) != "U8" or c != 80 or ck.shape(n[:-6] + "weight_scale") != [r, 10]:+ sys.exit("%s: not an NVFP4 160-value row table" % n)+ rows += r+ print("%d shards, %d rows x 160, global scale %.9g, %.2f GB" % (len(shards), rows, scale2, rows * PLE_ROW_BYTES / 1e9), flush=True)+ kvs = (_gs("general.architecture") + struct.pack("<I", GGUF_TYPE_STRING) + _gs("strata-ple")+ + _gs("general.name") + struct.pack("<I", GGUF_TYPE_STRING) + _gs("PLE n-gram table, NVFP4 as shipped")+ + _gs("strata.ple.format") + struct.pack("<I", GGUF_TYPE_STRING) + _gs("nvfp4x16")+ + _gs("strata.ple.scale") + struct.pack("<I", GGUF_TYPE_F32) + struct.pack("<f", scale2)+ + _gs("strata.ple.source") + struct.pack("<I", GGUF_TYPE_STRING) + _gs(model.name))+ # the tensor is [90, rows] I8 so one row is 90 contiguous bytes; the engine reads it by the format key+ head = b"GGUF" + struct.pack("<IQQ", 3, 1, 5) + kvs+ head += _gs("per_layer_token_embd.weight") + struct.pack("<I", 2) + struct.pack("<QQ", PLE_ROW_BYTES, rows)+ head += struct.pack("<I", GGML_TYPE_I8) + struct.pack("<Q", 0)+ head += b"\0" * ((-len(head)) % ALIGN)+ out.parent.mkdir(parents=True, exist_ok=True)+ tmp = out.with_suffix(out.suffix + ".part")+ CH = 1 << 20+ with open(tmp, "wb") as w:+ w.write(head)+ for k in range(len(shards)):+ n = shards[k]+ codes = ck.array(n, np.uint8)+ sc = ck.array(n[:-6] + "weight_scale", np.uint8)+ for i in range(0, codes.shape[0], CH):+ w.write(np.ascontiguousarray(interleave_rows(codes[i:i + CH], sc[i:i + CH])).tobytes())+ if k % 16 == 15 or k == len(shards) - 1:+ print(" %3d/%d shards" % (k + 1, len(shards)), flush=True)+ if tmp.stat().st_size != len(head) + rows * PLE_ROW_BYTES:+ sys.exit("size check failed")+ tmp.replace(out)+ print("wrote %s (%d rows x %d B)" % (out, rows, PLE_ROW_BYTES))+ return 0+++# ------------------------------------------------------------------------------------------------------- the MTP head+E2M1 = np.array([0, .5, 1, 1.5, 2, 3, 4, 6, -0., -.5, -1, -1.5, -2, -3, -4, -6], np.float32)+++def e4m3_table() -> np.ndarray:+ x = np.arange(256)+ s, e, m = x >> 7, (x >> 3) & 15, x & 7+ f = np.where(e == 0, np.ldexp(m / 8.0, -6), np.ldexp(1.0 + m / 8.0, e.astype(np.int32) - 7))+ f = np.where((e == 15) & (m == 7), np.nan, f)+ return np.where(s == 1, -f, f).astype(np.float32)+++def nvfp4_dequant(codes: np.ndarray, scales_u8: np.ndarray, scale2: float) -> np.ndarray:+ """ModelOpt NVFP4 -> float32: codes [R, K/2] (low nibble first), E4M3 scale bytes [R, K/16], one F32."""+ R = codes.shape[0]+ lo, hi = codes & 15, codes >> 4+ v = E2M1[np.stack([lo, hi], axis=-1).reshape(R, -1)]+ sc = e4m3_table()[scales_u8]+ return (v.reshape(R, -1, 16) * sc[:, :, None] * np.float32(scale2)).reshape(R, -1)+++def f32_to_bf16_bits(x: np.ndarray) -> np.ndarray:+ u = np.ascontiguousarray(x, np.float32).view(np.uint32)+ r = u + 0x7FFF + ((u >> 16) & 1) # round to nearest even+ return (r >> 16).astype(np.uint16)+++def cmd_mtp(a) -> int:+ """The MTP head in tools/mtp_fetch.py's folder form (raw BF16 files + mtp-manifest.json), so tools/mtp_pack.py and+ tools/mtp_rt.py take it as they take Qwen's own: the 31 tensors, experts fused as gate_up_proj / down_proj."""+ import hashlib+ ck = Checkpoint(a.model, partial_ok=True)+ out = pathlib.Path(a.out)+ out.mkdir(parents=True, exist_ok=True)+ manifest = []++ def put(name, bits, shape):+ fn = name.replace("/", "_") + ".bin"+ data = np.ascontiguousarray(bits, np.uint16)+ (out / fn).write_bytes(data.tobytes())+ manifest.append({"name": name, "dtype": "BF16", "shape": list(shape), "file": fn,+ "sha256": hashlib.sha256(data.tobytes()).hexdigest()})++ for name in sorted(k for k in ck.meta if k.startswith("mtp.") and ".experts." not in k):+ if ck.dtype(name) != "BF16":+ sys.exit("%s is %s, expected BF16" % (name, ck.dtype(name)))+ put(name, ck.array(name, np.uint16), ck.shape(name))+ n_exp = 512+ for fused, parts in (("mtp.layers.0.mlp.experts.gate_up_proj", ("gate_proj", "up_proj")),+ ("mtp.layers.0.mlp.experts.down_proj", ("down_proj",))):+ rows = [ck.shape("mtp.layers.0.mlp.experts.0.%s.weight" % p)[0] for p in parts]+ cols = ck.shape("mtp.layers.0.mlp.experts.0.%s.weight" % parts[0])[1] * 2+ shape = [n_exp, sum(rows), cols]+ fn = fused + ".bin"+ with open(out / fn, "wb") as f:+ h = hashlib.sha256()+ for e in range(n_exp):+ mats = []+ for p in parts:+ b = "mtp.layers.0.mlp.experts.%d.%s." % (e, p)+ mats.append(nvfp4_dequant(np.asarray(ck.array(b + "weight", np.uint8)),+ np.asarray(ck.array(b + "weight_scale", np.uint8)),+ float(ck.array(b + "weight_scale_2", np.float32).reshape(-1)[0])))+ blob = f32_to_bf16_bits(np.concatenate(mats, axis=0)).tobytes()+ f.write(blob)+ h.update(blob)+ manifest.append({"name": fused, "dtype": "BF16", "shape": shape, "file": fn, "sha256": h.hexdigest()})+ print(" %s %s" % (fused, shape), flush=True)+ (out / "mtp-manifest.json").write_text(json.dumps(manifest, indent=1), encoding="utf-8")+ print("wrote %d MTP tensors to %s" % (len(manifest), out))+ return 0+++# ------------------------------------------------------------------------------------------------------ verifying+def cmd_verify(a) -> int:+ """Experts: decode the checkpoint (CSF + ModelOpt definition) and compare with gguf-py's dequantization of the+ GGUF's NVFP4 blocks times the scale tensor, bit for bit (zeros normalised: ggml has one zero)."""+ import torch+ import nvfp4_verify as V+ from gguf import GGUFReader, GGMLQuantizationType as Q, quants+ ck = Checkpoint(a.model)+ rd = GGUFReader(a.gguf)+ T = {t.name: t for t in rd.tensors}+ bad = 0+ rng = np.random.default_rng(1)+ layers = sorted({int(k.split(".layers.")[1].split(".")[0]) for k in ck.meta if ".mlp.experts." in k and k.startswith("model.language_model")})+ for _ in range(a.experts):+ L = int(rng.choice(layers)); e = int(rng.integers(0, 512))+ for proj, gname in (("gate_proj", "ffn_gate_exps"), ("up_proj", "ffn_up_exps"), ("down_proj", "ffn_down_exps")):+ b = "model.language_model.layers.%d.mlp.experts.%d.%s." % (L, e, proj)+ w = torch.from_numpy(np.array(ck.array(b + "weight", np.uint8)))+ sc = csf_decode(ck.array(b + "weight_scale" + CSF_FIXED, np.uint8), ck.array(b + "weight_scale" + CSF_EXC, np.uint32))+ s = torch.from_numpy(sc).view(torch.float8_e4m3fn)+ s2 = torch.from_numpy(np.array(ck.array(b + "weight_scale_2", np.float32))).reshape(())+ ref = V.modelopt_dequant(w, s, s2)+ t = T["blk.%d.%s.weight" % (L, gname)]+ blob = np.asarray(t.data) # [512, rows, bytes] of NVFP4 blocks+ got = quants.dequantize(np.ascontiguousarray(blob[e]), Q.NVFP4).astype(np.float32)+ sct = T["blk.%d.%s.scale" % (L, gname)]+ got = got * np.float32(np.asarray(sct.data)[e])+ same = np.array_equal((ref + np.float32(0)).view(np.uint32), (got.reshape(ref.shape) + np.float32(0)).view(np.uint32))+ print(" layer %2d expert %3d %-9s bitwise %s" % (L, e, proj, "OK" if same else "NO"), flush=True)+ bad += not same+ print("RESULT:", "lossless" if not bad else "MISMATCH")+ return 1 if bad else 0+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])+ sub = ap.add_subparsers(dest="cmd", required=True)+ c = sub.add_parser("convert")+ c.add_argument("--model", required=True)+ c.add_argument("--outfile", required=True)+ c.add_argument("--dry-run", action="store_true")+ p = sub.add_parser("ple")+ p.add_argument("--model", required=True)+ p.add_argument("--out", required=True)+ m = sub.add_parser("mtp")+ m.add_argument("--model", required=True)+ m.add_argument("--out", required=True)+ v = sub.add_parser("verify")+ v.add_argument("--model", required=True)+ v.add_argument("--gguf", required=True)+ v.add_argument("--experts", type=int, default=4)+ a = ap.parse_args()+ return {"convert": cmd_convert, "ple": cmd_ple, "mtp": cmd_mtp, "verify": cmd_verify}[a.cmd](a)+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tools/test_nvfp4_local_pack.py b/tools/test_nvfp4_local_pack.pynew file mode 100644index 000000000..a650eaa37--- /dev/null+++ b/tools/test_nvfp4_local_pack.py@@ -0,0 +1,98 @@+"""Tests for tools/nvfp4_local_pack.py: the CSF scale decode, the MXFP8 dequantization, the PLE rows."""+import pathlib+import sys++import numpy as np+import pytest++sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))+import nvfp4_local_pack as P # noqa: E402+++def _scales(rows, cols, seed, outliers=0.01):+ rng = np.random.default_rng(seed)+ base = rng.integers(60, 100, size=(rows, 1))+ s = base + rng.integers(0, 16, size=(rows, cols))+ m = rng.random((rows, cols)) < outliers+ s[m] = rng.integers(0, 127, size=int(m.sum()))+ return s.astype(np.uint8)++[email protected]("rows,cols", [(32, 8), (640, 160), (2560, 40)])+def test_csf_roundtrip_is_exact(rows, cols):+ s = _scales(rows, cols, rows + cols)+ fixed, exc = P.csf_encode(s)+ assert fixed.shape == (rows // 16, 16 * (1 + cols // 2)) # the shapes of the real tensors+ assert np.array_equal(P.csf_decode(fixed, exc), s)+++def test_csf_matches_the_spec_by_hand():+ # one 16-row slab, 4 columns: base bytes 10..25, rows' codes = col index, one exception at row 3 col 2+ rows, cols = 16, 4+ fixed = np.zeros((1, 16 * (1 + cols // 2)), np.uint8)+ fixed[0, :16] = np.arange(10, 26)+ for r in range(rows):+ fixed[0, 16 + r * 2] = 0 | (1 << 4) # cols 0 (low nibble), 1 (high)+ fixed[0, 16 + r * 2 + 1] = 2 | (3 << 4) # cols 2, 3+ exc = np.array([(3 * cols + 2) | (99 << 24)], np.uint32)+ got = P.csf_decode(fixed, exc)+ assert got[0].tolist() == [10, 11, 12, 13]+ assert got[5].tolist() == [15, 16, 17, 18]+ assert got[3].tolist() == [13, 14, 99, 16]+++def test_csf_rejects_bad_streams():+ with pytest.raises(ValueError):+ P.csf_decode(np.zeros((1, 33), np.uint8), np.zeros(0, np.uint32))+ fixed = np.zeros((1, 32), np.uint8)+ fixed[0, 0] = 250+ with pytest.raises(ValueError):+ P.csf_decode(fixed, np.zeros(0, np.uint32))+ with pytest.raises(ValueError):+ P.csf_decode(np.zeros((1, 32), np.uint8), np.array([16 * 4], np.uint32))+++def test_mxfp8_dequant_exact():+ torch = pytest.importorskip("torch")+ rng = np.random.default_rng(3)+ vals = rng.integers(0, 126, size=(4, 64)).astype(np.uint8) | (rng.integers(0, 2, size=(4, 64)).astype(np.uint8) << 7)+ sc = rng.integers(100, 130, size=(4, 2)).astype(np.uint8)+ got = P.mxfp8_dequant(vals, sc).numpy()+ ref = torch.from_numpy(vals).view(torch.float8_e4m3fn).to(torch.float32).numpy().reshape(4, 2, 32)+ ref = ref * np.exp2(sc.astype(np.float64) - 127)[:, :, None]+ assert np.array_equal(got.reshape(4, 2, 32), ref.astype(np.float32))+ # exactly representable in BF16 (what the converter hands on)+ bf = torch.from_numpy(got).to(torch.bfloat16).to(torch.float32).numpy()+ assert np.array_equal(bf, got)+++def test_mxfp8_rejects_nan_scale():+ pytest.importorskip("torch")+ with pytest.raises(ValueError):+ P.mxfp8_dequant(np.zeros((1, 32), np.uint8), np.full((1, 1), 255, np.uint8))+++def test_ple_rows_are_codes_then_scales():+ c = np.arange(160, dtype=np.uint8).reshape(2, 80)+ s = np.arange(20, dtype=np.uint8).reshape(2, 10) + 200+ r = P.interleave_rows(c, s)+ assert r.shape == (2, P.PLE_ROW_BYTES)+ assert np.array_equal(r[1, :80], c[1]) and np.array_equal(r[1, 80:], s[1])+++def test_nvfp4_dequant_by_hand():+ # element 2i in the low nibble: byte 0x21 = codes 1 (0.5) and 2 (1.0); 0xF7 = 7 (6.0) and 15 (-6.0)+ codes = np.zeros((1, 8), np.uint8)+ codes[0, 0], codes[0, 1] = 0x21, 0xF7+ sc = np.array([[0x38]], np.uint8) # E4M3 0x38 = 1.0+ got = P.nvfp4_dequant(codes, sc, 2.0)+ assert got[0, :4].tolist() == [1.0, 2.0, 12.0, -12.0]+++def test_e4m3_table_and_bf16_rounding():+ t = P.e4m3_table()+ assert t[0x38] == 1.0 and t[0x7E] == 448.0 and np.isnan(t[0x7F]) and t[0x01] == 2.0 ** -9+ x = np.array([1.0, 1.0 + 2.0 ** -8, 1.0 + 3 * 2.0 ** -9, -2.5], np.float32)+ b = P.f32_to_bf16_bits(x)+ back = (b.astype(np.uint32) << 16).view(np.float32)+ assert back.tolist() == [1.0, 1.0, 1.0078125, -2.5] # ties to even; 1+3*2^-9 rounds up