Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

FP4-Local prepare: separate step marks (the pack's own files were overwritten), scale tensors in the checkpoint-direct pack; docs section

Niko12210da73d9159de
diff --git a/docs/MODELS.md b/docs/MODELS.md
index 9427d7b1e..9443dacbe 100644
--- a/docs/MODELS.md
+++ b/docs/MODELS.md
@@ -164,6 +164,27 @@ NVIDIA, and AMD (experimental); not Intel Arc or the Mac. Images are off. Detail
START-HERE.bat --setup --family gyro --model Gyro-S
```
+<a id="nvfp4-qad"></a>
+
+### NVIDIA's retrained 4-bit model (NVFP4-QAD)
+
+NVIDIA retrained Qwen3.8-Flash-Next for 4-bit experts (quantization-aware distillation) and published it as
+[Qwen3.8-Flash-Next-FP4-Local](https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local). In setup's NVFP4 family it is
+the size **NVFP4-QAD**. It is not a ready-made pack: setup downloads NVIDIA's files as they are (102 GB) and **turns them
+into a Strata pack on your PC** (once, 20-40 minutes; about 110 GB more disk while it runs, and the 102 GB can be deleted
+afterwards). Nothing of NVIDIA's weights is redistributed by Strata.
+
+| Size | Download | Memory it needs | In plain words |
+| --- | ---: | ---: | --- |
+| **NVFP4-QAD** | 102 GB (+110 GB of disk to prepare) | like NVFP4: ~80 GB of RAM keeps every expert in RAM | NVIDIA's own 4-bit model; RTX 50 cards have FP4 tensor cores, other NVIDIA cards run it at 8-bit precision |
+
+NVIDIA cards only. Images are off for now. Its licenses are the NVIDIA Open Model Agreement and Qwen's Community License.
+Technical details and measurements: [NVFP4.md](NVFP4.md#nvidias-fp4-local-checkpoint).
+
+```
+START-HERE.bat --setup --family nvfp4 --model NVFP4-QAD
+```
+
### OrcaRouter Uncensored IQ3_XXS
For **OrcaRouter's Flash-Next Uncensored IQ3_XXS**, see the [manual compatibility setup](ORCA.md). It needs an
diff --git a/setup.py b/setup.py
index f2db84922..d7b8a12be 100644
--- a/setup.py
+++ b/setup.py
@@ -1958,9 +1958,9 @@ def prepared_dir(model: str, models_dir: Path) -> Path:
def prepared_done(model: str, models_dir: Path) -> bool:
- """The last file tools/nvfp4_local_pack.py prepare writes is there (the draft layer's experts)."""
- return (prepared_dir(model, models_dir) / "mtp" / "rt" / "experts.bin").is_file() and \
- (prepared_dir(model, models_dir) / "pack" / "native_experts.txt").is_file()
+ """Every step of tools/nvfp4_local_pack.py prepare is marked done (its marks, written when each step is whole)."""
+ d = prepared_dir(model, models_dir)
+ return all((d / m).is_file() for m in ("ple-nvfp4.gguf.done", "dense.gguf.done", "pack.done", "draft.done"))
def ready_paths(model: str, models_dir: Path) -> dict:
diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.py
index 9bc661deb..5539cd432 100644
--- a/tools/nvfp4_local_pack.py
+++ b/tools/nvfp4_local_pack.py
@@ -493,32 +493,39 @@ def experts_from_checkpoint(model, ck: Checkpoint, n_layers: int = 48, n_exp: in
if key not in cache:
cache.clear()
rows = []
- scale2 = np.zeros(n_exp, np.float32)
for e in range(n_exp):
b = base % (l, e, role)
dec = csf_decode(np.asarray(ck.array(b + "weight_scale" + CSF_FIXED, np.uint8)),
np.asarray(ck.array(b + "weight_scale" + CSF_EXC, np.uint32)))
rows.append(nvfp4_blocks(np.asarray(ck.array(b + "weight", np.uint8)), dec).reshape(-1))
- scale2[e] = ck.array(b + "weight_scale_2", np.float32).reshape(-1)[0]
- cache[key] = (np.stack(rows), scale2)
+ cache[key] = (np.stack(rows),)
return cache[key]
# shapes in ggml order (ne0 = K, ne1 = rows, ne2 = experts)
geo = {"gate": (2560, 640), "up": (2560, 640), "down": (640, 2560)}
+
+ class _Mem(_VGguf): # a tiny tensor held in memory: iq_pack reads it like any dense one
+ pass
+
for l in range(n_layers):
for role, (k, rows) in geo.items():
n = "blk.%d.ffn_%s_exps.weight" % (l, role)
vt = _VTensor(n, [k, rows, n_exp], Q.NVFP4.value, "NVFP4", n_exp * rows * (k // 64) * 36)
model.where[n] = (_VGguf(), vt, None, src)
- sn = "blk.%d.ffn_%s_exps.scale" % (l, role)
- model.where[sn] = (_VGguf(), _VTensor(sn, [n_exp], Q.F32.value, "F32", n_exp * 4), None, src)
+ # the per-expert F32 scales (weight_scale_2 -> .scale, input_scale -> .input_scale), as the converter writes them
+ for suffix, key in ((".scale", "weight_scale_2"), (".input_scale", "input_scale")):
+ vals = np.array([ck.array(base % (l, e, role) + key, np.float32).reshape(-1)[0] for e in range(n_exp)], np.float32)
+ sn = "blk.%d.ffn_%s_exps%s" % (l, role, suffix)
+ raw = vals.view(np.uint8)
+ model.where[sn] = (_Mem(), _VTensor(sn, [n_exp], Q.F32.value, "F32", n_exp * 4), raw, src)
orig_bytes = type(model).bytes
def bytes_(name):
if name.startswith("blk.") and "_exps." in name:
+ if not name.endswith(".weight"): # the small F32 scale tensors are in memory
+ return model.where[name][2]
l = int(name.split(".")[1]); role = name.split("ffn_")[1].split("_exps")[0]
- data, s2 = layer_role(l, role)
- return data.reshape(-1) if name.endswith(".weight") else s2.view(np.uint8)
+ return layer_role(l, role)[0].reshape(-1)
return orig_bytes(model, name)
model.bytes = bytes_
return model
@@ -571,7 +578,7 @@ def cmd_prepare(a) -> int:
lambda: run(me + ["ple", "--model", str(model), "--out", str(out / "ple-nvfp4.gguf")]))
step("the dense weights", out / "dense.gguf.done",
lambda: run(me + ["convert", "--dense-only", "--model", str(model), "--outfile", str(out / "dense.gguf")]))
- step("the experts and the pack", out / "pack" / "native_experts.txt",
+ step("the experts and the pack", out / "pack.done", # (native_experts.txt is the pack's own marker, not ours)
lambda: run(me + ["pack", "--model", str(model), "--dense-gguf", str(out / "dense.gguf"), "--out", str(out / "pack")]))
mtp = out / "mtp"
@@ -584,7 +591,7 @@ def cmd_prepare(a) -> int:
run([sys.executable, str(tools / "mtp_rt.py"), "--gguf", str(mtp / "mtp-q2_0.gguf"), "--out", str(mtp / "rt")])
import shutil
shutil.rmtree(src, ignore_errors=True)
- step("the draft layer", mtp / "rt" / "experts.bin", draft_layer)
+ step("the draft layer", out / "draft.done", draft_layer)
print("prepare: done -> %s" % out)
return 0
diff --git a/tools/test_setup_nvfp4.py b/tools/test_setup_nvfp4.py
index 2e0f3bee4..4909c9e68 100644
--- a/tools/test_setup_nvfp4.py
+++ b/tools/test_setup_nvfp4.py
@@ -135,15 +135,16 @@ class PreparedModel(unittest.TestCase):
self.assertEqual(p, {"pack": d / "pack", "dense": d / "dense.gguf", "ple": d / "ple-nvfp4.gguf", "mtp": d / "mtp"})
self.assertNotIn("embd", p)
- def test_prepared_done_needs_the_last_files(self):
+ def test_prepared_done_needs_every_mark(self):
with tempfile.TemporaryDirectory() as t:
self.assertFalse(setup.prepared_done("NVFP4-QAD", Path(t)))
d = setup.prepared_dir("NVFP4-QAD", Path(t))
- (d / "pack").mkdir(parents=True)
- (d / "pack" / "native_experts.txt").write_text("x")
- self.assertFalse(setup.prepared_done("NVFP4-QAD", Path(t)))
- (d / "mtp" / "rt").mkdir(parents=True)
- (d / "mtp" / "rt" / "experts.bin").write_text("x")
+ d.mkdir(parents=True)
+ marks = ("ple-nvfp4.gguf.done", "dense.gguf.done", "pack.done", "draft.done")
+ for m in marks[:-1]:
+ (d / m).write_text("x")
+ self.assertFalse(setup.prepared_done("NVFP4-QAD", Path(t)))
+ (d / marks[-1]).write_text("x")
self.assertTrue(setup.prepared_done("NVFP4-QAD", Path(t)))