Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
nvfp4_local_pack pack: the experts straight from the checkpoint through iq_pack (no 74 GB GGUF in between)
Niko12214cca90e757f9
diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.pyindex 7b527964c..72966a70b 100644--- a/tools/nvfp4_local_pack.py+++ b/tools/nvfp4_local_pack.py@@ -3,6 +3,7 @@python tools/nvfp4_local_pack.py convert --model <nvidia/Qwen3.8-Flash-Next-FP4-Local dir> --outfile <model.gguf>python tools/nvfp4_local_pack.py ple --model <same dir> --out <ple-nvfp4.gguf>python tools/nvfp4_local_pack.py mtp --model <same dir> --out <mtp src dir> (then tools/mtp_pack.py, tools/mtp_rt.py)+ python tools/nvfp4_local_pack.py pack --model <same dir> --dense-gguf <dense.gguf> --out <pack dir> (instead of iq_pack on a full GGUF)python tools/nvfp4_local_pack.py verify --model <same dir> --gguf <model.gguf> [--experts 6]python tools/iq_pack.py --gguf <model.gguf> --out <pack dir>@@ -449,6 +450,96 @@ def cmd_slim(a) -> int:return 0+# ------------------------------------------------------------------------------------------- the pack, without a big GGUF+def nvfp4_blocks(weight_u8: np.ndarray, scale_u8: np.ndarray) -> np.ndarray:+ """ModelOpt NVFP4 rows -> ggml's block_nvfp4 rows: weight [R, K/2] (two E2M1 codes a byte, element 2i low), scale [R, K/16]+ E4M3 bytes -> [R, K/64 * 36] bytes of super-blocks `[4 UE4M3 scales | 32 code bytes]`. The same bits as llama.cpp's+ ModelBase._nvfp4_pack (the scale's sign bit is dropped; it is 0 in every scale of this checkpoint)."""+ R, nb = scale_u8.shape+ w = weight_u8.reshape(R, nb, 8)+ vals = np.stack([w & 0x0F, w >> 4], axis=-1).reshape(R, nb, 16)+ qs = (vals[:, :, :8] | (vals[:, :, 8:] << 4)).astype(np.uint8)+ d = (scale_u8 & 0x7F).reshape(R, nb // 4, 4)+ qs = qs.reshape(R, nb // 4, 32)+ return np.concatenate([d, qs], axis=-1).reshape(R, (nb // 4) * 36)+++class _VTensor:+ """A tensor info for iq_pack: an expert tensor (or its scale) whose bytes come from the checkpoint."""++ def __init__(self, name, shape, type_id, type_name, nbytes):+ self.name, self.shape, self.type_id, self.type_name, self.offset = name, shape, type_id, type_name, 0+ self._n = nbytes++ def expected_bytes(self):+ return self._n+++class _VGguf:+ data_start = 0+++def experts_from_checkpoint(model, ck: Checkpoint, n_layers: int = 48, n_exp: int = 512):+ """Give an iq_pack.Model of the dense-only GGUF the routed experts of the checkpoint as virtual tensors."""+ from _paths import add_gguf_py+ add_gguf_py()+ from gguf import GGMLQuantizationType as Q+ src = next(iter(model.where.values()))[3]+ cache: dict = {}+ base = "model.language_model.layers.%d.mlp.experts.%d.%s_proj."++ def layer_role(l, role):+ key = (l, role)+ if key not in cache:+ cache.clear()+ rows = []+ scale2 = np.zeros(n_exp, np.float32)+ for e in range(n_exp):+ b = base % (l, e, role)+ dec = csf_decode(np.asarray(ck.array(b + "weight_scale" + CSF_FIXED, np.uint8)),+ np.asarray(ck.array(b + "weight_scale" + CSF_EXC, np.uint32)))+ rows.append(nvfp4_blocks(np.asarray(ck.array(b + "weight", np.uint8)), dec).reshape(-1))+ scale2[e] = ck.array(b + "weight_scale_2", np.float32).reshape(-1)[0]+ cache[key] = (np.stack(rows), scale2)+ return cache[key]++ # shapes in ggml order (ne0 = K, ne1 = rows, ne2 = experts)+ geo = {"gate": (2560, 640), "up": (2560, 640), "down": (640, 2560)}+ for l in range(n_layers):+ for role, (k, rows) in geo.items():+ n = "blk.%d.ffn_%s_exps.weight" % (l, role)+ vt = _VTensor(n, [k, rows, n_exp], Q.NVFP4.value, "NVFP4", n_exp * rows * (k // 64) * 36)+ model.where[n] = (_VGguf(), vt, None, src)+ sn = "blk.%d.ffn_%s_exps.scale" % (l, role)+ model.where[sn] = (_VGguf(), _VTensor(sn, [n_exp], Q.F32.value, "F32", n_exp * 4), None, src)+ orig_bytes = type(model).bytes++ def bytes_(name):+ if name.startswith("blk.") and "_exps." in name:+ l = int(name.split(".")[1]); role = name.split("ffn_")[1].split("_exps")[0]+ data, s2 = layer_role(l, role)+ return data.reshape(-1) if name.endswith(".weight") else s2.view(np.uint8)+ return orig_bytes(model, name)+ model.bytes = bytes_+ return model+++def cmd_pack(a) -> int:+ """tools/iq_pack.py on the dense-only GGUF, with the experts read from the checkpoint: experts.bin, native_experts.txt,+ dense.bin, index.txt and the tokenizer, with no 74 GB GGUF in between."""+ import iq_pack+ ck = Checkpoint(a.model)+ orig_model = iq_pack.Model++ class M(orig_model):+ def __init__(self, first):+ super().__init__(first)+ experts_from_checkpoint(self, ck)+ iq_pack.Model = M+ sys.argv = ["iq_pack.py", "--gguf", a.dense_gguf, "--out", a.out, "--experts-bin"]+ return iq_pack.main()++# ------------------------------------------------------------------------------------------------------ verifyingdef cmd_verify(a) -> int:"""Experts: decode the checkpoint (CSF + ModelOpt definition) and compare with gguf-py's dequantization of the@@ -498,6 +589,10 @@ def main() -> int:m = sub.add_parser("mtp")m.add_argument("--model", required=True)m.add_argument("--out", required=True)+ pk = sub.add_parser("pack")+ pk.add_argument("--model", required=True)+ pk.add_argument("--dense-gguf", required=True, help="the output of `convert --dense-only` (or `slim`)")+ pk.add_argument("--out", required=True)sl = sub.add_parser("slim")sl.add_argument("--gguf", required=True)sl.add_argument("--out", required=True)@@ -506,7 +601,7 @@ def main() -> int:v.add_argument("--gguf", required=True)v.add_argument("--experts", type=int, default=4)a = ap.parse_args()- return {"convert": cmd_convert, "ple": cmd_ple, "mtp": cmd_mtp, "slim": cmd_slim, "verify": cmd_verify}[a.cmd](a)+ return {"convert": cmd_convert, "ple": cmd_ple, "mtp": cmd_mtp, "slim": cmd_slim, "pack": cmd_pack, "verify": cmd_verify}[a.cmd](a)if __name__ == "__main__":diff --git a/tools/test_nvfp4_local_pack.py b/tools/test_nvfp4_local_pack.pyindex a650eaa37..6d85b619d 100644--- a/tools/test_nvfp4_local_pack.py+++ b/tools/test_nvfp4_local_pack.py@@ -96,3 +96,16 @@ def test_e4m3_table_and_bf16_rounding():b = P.f32_to_bf16_bits(x)back = (b.astype(np.uint32) << 16).view(np.float32)assert back.tolist() == [1.0, 1.0, 1.0078125, -2.5] # ties to even; 1+3*2^-9 rounds up+++def test_nvfp4_blocks_layout_by_hand():+ # one row of 64 values: codes 0..15 repeated; scales 0x38 0x39 0x3A 0x3B with a sign bit set on the first (dropped)+ codes = np.arange(64) % 16+ packed = (codes[0::2] | (codes[1::2] << 4)).astype(np.uint8).reshape(1, 32)+ sc = np.array([[0xB8, 0x39, 0x3A, 0x3B]], np.uint8)+ out = P.nvfp4_blocks(packed, sc)+ assert out.shape == (1, 36)+ assert out[0, :4].tolist() == [0x38, 0x39, 0x3A, 0x3B]+ # within a 16-block: element j in the low nibble, element j+8 in the high one+ blk0 = out[0, 4:12]+ assert [int(b & 15) for b in blk0] == list(range(8)) and [int(b >> 4) for b in blk0] == list(range(8, 16))