Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
FP4-Local prepare: separate step marks (the pack's own files were overwritten), scale tensors in the checkpoint-direct pack; docs section
Niko12210da73d9159de
diff --git a/docs/MODELS.md b/docs/MODELS.mdindex 9427d7b1e..9443dacbe 100644--- a/docs/MODELS.md+++ b/docs/MODELS.md@@ -164,6 +164,27 @@ NVIDIA, and AMD (experimental); not Intel Arc or the Mac. Images are off. DetailSTART-HERE.bat --setup --family gyro --model Gyro-S```+<a id="nvfp4-qad"></a>++### NVIDIA's retrained 4-bit model (NVFP4-QAD)++NVIDIA retrained Qwen3.8-Flash-Next for 4-bit experts (quantization-aware distillation) and published it as+[Qwen3.8-Flash-Next-FP4-Local](https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local). In setup's NVFP4 family it is+the size **NVFP4-QAD**. It is not a ready-made pack: setup downloads NVIDIA's files as they are (102 GB) and **turns them+into a Strata pack on your PC** (once, 20-40 minutes; about 110 GB more disk while it runs, and the 102 GB can be deleted+afterwards). Nothing of NVIDIA's weights is redistributed by Strata.++| Size | Download | Memory it needs | In plain words |+| --- | ---: | ---: | --- |+| **NVFP4-QAD** | 102 GB (+110 GB of disk to prepare) | like NVFP4: ~80 GB of RAM keeps every expert in RAM | NVIDIA's own 4-bit model; RTX 50 cards have FP4 tensor cores, other NVIDIA cards run it at 8-bit precision |++NVIDIA cards only. Images are off for now. Its licenses are the NVIDIA Open Model Agreement and Qwen's Community License.+Technical details and measurements: [NVFP4.md](NVFP4.md#nvidias-fp4-local-checkpoint).++```+START-HERE.bat --setup --family nvfp4 --model NVFP4-QAD+```+### OrcaRouter Uncensored IQ3_XXSFor **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](ORCA.md). It needs andiff --git a/setup.py b/setup.pyindex f2db84922..d7b8a12be 100644--- a/setup.py+++ b/setup.py@@ -1958,9 +1958,9 @@ def prepared_dir(model: str, models_dir: Path) -> Path:def prepared_done(model: str, models_dir: Path) -> bool:- """The last file tools/nvfp4_local_pack.py prepare writes is there (the draft layer's experts)."""- return (prepared_dir(model, models_dir) / "mtp" / "rt" / "experts.bin").is_file() and \- (prepared_dir(model, models_dir) / "pack" / "native_experts.txt").is_file()+ """Every step of tools/nvfp4_local_pack.py prepare is marked done (its marks, written when each step is whole)."""+ d = prepared_dir(model, models_dir)+ return all((d / m).is_file() for m in ("ple-nvfp4.gguf.done", "dense.gguf.done", "pack.done", "draft.done"))def ready_paths(model: str, models_dir: Path) -> dict:diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.pyindex 9bc661deb..5539cd432 100644--- a/tools/nvfp4_local_pack.py+++ b/tools/nvfp4_local_pack.py@@ -493,32 +493,39 @@ def experts_from_checkpoint(model, ck: Checkpoint, n_layers: int = 48, n_exp: inif key not in cache:cache.clear()rows = []- scale2 = np.zeros(n_exp, np.float32)for e in range(n_exp):b = base % (l, e, role)dec = csf_decode(np.asarray(ck.array(b + "weight_scale" + CSF_FIXED, np.uint8)),np.asarray(ck.array(b + "weight_scale" + CSF_EXC, np.uint32)))rows.append(nvfp4_blocks(np.asarray(ck.array(b + "weight", np.uint8)), dec).reshape(-1))- scale2[e] = ck.array(b + "weight_scale_2", np.float32).reshape(-1)[0]- cache[key] = (np.stack(rows), scale2)+ cache[key] = (np.stack(rows),)return cache[key]# shapes in ggml order (ne0 = K, ne1 = rows, ne2 = experts)geo = {"gate": (2560, 640), "up": (2560, 640), "down": (640, 2560)}++ class _Mem(_VGguf): # a tiny tensor held in memory: iq_pack reads it like any dense one+ pass+for l in range(n_layers):for role, (k, rows) in geo.items():n = "blk.%d.ffn_%s_exps.weight" % (l, role)vt = _VTensor(n, [k, rows, n_exp], Q.NVFP4.value, "NVFP4", n_exp * rows * (k // 64) * 36)model.where[n] = (_VGguf(), vt, None, src)- sn = "blk.%d.ffn_%s_exps.scale" % (l, role)- model.where[sn] = (_VGguf(), _VTensor(sn, [n_exp], Q.F32.value, "F32", n_exp * 4), None, src)+ # the per-expert F32 scales (weight_scale_2 -> .scale, input_scale -> .input_scale), as the converter writes them+ for suffix, key in ((".scale", "weight_scale_2"), (".input_scale", "input_scale")):+ vals = np.array([ck.array(base % (l, e, role) + key, np.float32).reshape(-1)[0] for e in range(n_exp)], np.float32)+ sn = "blk.%d.ffn_%s_exps%s" % (l, role, suffix)+ raw = vals.view(np.uint8)+ model.where[sn] = (_Mem(), _VTensor(sn, [n_exp], Q.F32.value, "F32", n_exp * 4), raw, src)orig_bytes = type(model).bytesdef bytes_(name):if name.startswith("blk.") and "_exps." in name:+ if not name.endswith(".weight"): # the small F32 scale tensors are in memory+ return model.where[name][2]l = int(name.split(".")[1]); role = name.split("ffn_")[1].split("_exps")[0]- data, s2 = layer_role(l, role)- return data.reshape(-1) if name.endswith(".weight") else s2.view(np.uint8)+ return layer_role(l, role)[0].reshape(-1)return orig_bytes(model, name)model.bytes = bytes_return model@@ -571,7 +578,7 @@ def cmd_prepare(a) -> int:lambda: run(me + ["ple", "--model", str(model), "--out", str(out / "ple-nvfp4.gguf")]))step("the dense weights", out / "dense.gguf.done",lambda: run(me + ["convert", "--dense-only", "--model", str(model), "--outfile", str(out / "dense.gguf")]))- step("the experts and the pack", out / "pack" / "native_experts.txt",+ step("the experts and the pack", out / "pack.done", # (native_experts.txt is the pack's own marker, not ours)lambda: run(me + ["pack", "--model", str(model), "--dense-gguf", str(out / "dense.gguf"), "--out", str(out / "pack")]))mtp = out / "mtp"@@ -584,7 +591,7 @@ def cmd_prepare(a) -> int:run([sys.executable, str(tools / "mtp_rt.py"), "--gguf", str(mtp / "mtp-q2_0.gguf"), "--out", str(mtp / "rt")])import shutilshutil.rmtree(src, ignore_errors=True)- step("the draft layer", mtp / "rt" / "experts.bin", draft_layer)+ step("the draft layer", out / "draft.done", draft_layer)print("prepare: done -> %s" % out)return 0diff --git a/tools/test_setup_nvfp4.py b/tools/test_setup_nvfp4.pyindex 2e0f3bee4..4909c9e68 100644--- a/tools/test_setup_nvfp4.py+++ b/tools/test_setup_nvfp4.py@@ -135,15 +135,16 @@ class PreparedModel(unittest.TestCase):self.assertEqual(p, {"pack": d / "pack", "dense": d / "dense.gguf", "ple": d / "ple-nvfp4.gguf", "mtp": d / "mtp"})self.assertNotIn("embd", p)- def test_prepared_done_needs_the_last_files(self):+ def test_prepared_done_needs_every_mark(self):with tempfile.TemporaryDirectory() as t:self.assertFalse(setup.prepared_done("NVFP4-QAD", Path(t)))d = setup.prepared_dir("NVFP4-QAD", Path(t))- (d / "pack").mkdir(parents=True)- (d / "pack" / "native_experts.txt").write_text("x")- self.assertFalse(setup.prepared_done("NVFP4-QAD", Path(t)))- (d / "mtp" / "rt").mkdir(parents=True)- (d / "mtp" / "rt" / "experts.bin").write_text("x")+ d.mkdir(parents=True)+ marks = ("ple-nvfp4.gguf.done", "dense.gguf.done", "pack.done", "draft.done")+ for m in marks[:-1]:+ (d / m).write_text("x")+ self.assertFalse(setup.prepared_done("NVFP4-QAD", Path(t)))+ (d / marks[-1]).write_text("x")self.assertTrue(setup.prepared_done("NVFP4-QAD", Path(t)))