Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

setup: a prepared NVFP4-QAD model needs none of the checkpoint's safetensors; NVFP4-QAD reads pictures

Niko12212dbd302038fc
diff --git a/setup.py b/setup.py
index 4e0224aee..c3c0b71de 100644
--- a/setup.py
+++ b/setup.py
@@ -391,9 +391,9 @@ MODELS = {
"https://huggingface.co/nvidia/Qwen3.8-Flash-Next-NVFP4"},
"NVFP4-QAD": {"about": "NVIDIA's FP4, trained for FP4 (NVIDIA's Qwen3.8-Flash-Next-FP4-Local) - slightly faster on short prompts; "
"a 102 GB download that this PC turns into a Strata pack once (about 20-40 minutes, 110 GB more disk); "
- "RTX 50 cards have FP4 tensor cores, other NVIDIA cards run it at 8-bit precision. No images yet",
+ "RTX 50 cards have FP4 tensor cores, other NVIDIA cards run it at 8-bit precision. It reads pictures (the shared Qwen3.8-Flash-Next encoder)",
"download_gb": 102.2, "prepare_gb": 110.0, "ram_gb": 80, "arena_gb": 68.0,
- "families": ("nvfp4",), "nvidia_only": True, "vision": False, "repo": NVFP4_FP4LOCAL_REPO,
+ "families": ("nvfp4",), "nvidia_only": True, "vision": True, "repo": NVFP4_FP4LOCAL_REPO,
"prepare": "fp4-local", "engine": FP4LOCAL_ENGINE,
"license": "NVIDIA Open Model Agreement (Licensed by NVIDIA Corporation under the NVIDIA Open Model "
"Agreement) with Qwen3.8-Flash-Next's Qwen Community License 1.0: "
@@ -1957,6 +1957,15 @@ def ready_files(model: str) -> list:
return [(role, repo, t["revision"], path, size, sha) for path, (role, size, sha) in t["files"].items()]
+def ready_needed(model: str, models_dir: Path) -> list:
+ """ready_files without the checkpoint's tensors once a prepared model (MODELS[..]["prepare"]) is prepared: the
+ engine reads the prepared files only, so the ~95 GB of safetensors need not stay (or come back) on the disk."""
+ files = ready_files(model)
+ if MODELS[model].get("prepare") and prepared_done(model, models_dir):
+ files = [f for f in files if not f[3].startswith("tensors/")]
+ return files
+
+
def ready_where(models_dir: Path, f) -> Path:
"""Where one of them goes: <models>/<owner>--<repo>/<its path in the repository> (the repository's layout, so the
pack folder is whole as the engine reads it)."""
@@ -2004,7 +2013,7 @@ def ready_fetch(model: str, models_dir: Path) -> None:
ready-made pack, linked instead of downloaded (the PLE table and the embedding); or downloaded from its pinned
revision and checked against its size and SHA-256 (a wrong file is deleted and setup stops)."""
others = [(ready_where(models_dir, g), g[5]) for m in MODELS if MODELS[m].get("repo") for g in ready_files(m)]
- for f in ready_files(model):
+ for f in ready_needed(model, models_dir):
dst, size, sha = ready_where(models_dir, f), f[4], f[5]
if ready_verified(dst, sha):
ok(f"{f[3]} already downloaded")
@@ -7116,7 +7125,7 @@ def main() -> int:
if ready:
models_dir = Path(a.models_dir)
for r in elsewhere: # already downloaded and checked in a Strata folder elsewhere
- if all(ready_verified(ready_where(r / "models", f), f[5]) for f in ready_files(model)):
+ if all(ready_verified(ready_where(r / "models", f), f[5]) for f in ready_needed(model, r / "models")):
models_dir = r / "models"
ok(f"model files found in {models_dir}")
break
@@ -7141,12 +7150,15 @@ def main() -> int:
mark(s, "whole (checked against its own tensor directory)")
except OSError as e: # a read-only folder (--gguf-dir on a share): the file is still whole
warn(f"{s.name} is whole but its finish mark cannot be written ({e})")
- have_model = all(ready_verified(ready_where(models_dir, f), f[5]) for f in ready_files(model)) if ready else \
+ have_model = all(ready_verified(ready_where(models_dir, f), f[5]) for f in ready_needed(model, models_dir)) if ready else \
all(s.exists() and (done(s) or a.gguf_dir) for s in shards)
# #425 (jctaborda): a download that resumes needs room only for what is still missing - the finished shards and
# the .part files already on the disk count
on_disk = sum(f.stat().st_size for s in shards for f in (s, s.with_name(s.name + ".part")) if f.is_file()) / 1e9
to_fetch = 0 if a.gguf_dir or have_model else max(MODELS[model]["download_gb"] - on_disk, 0)
+ if ready and not have_model and MODELS[model].get("prepare") and prepared_done(model, models_dir):
+ to_fetch = sum(f[4] for f in ready_needed(model, models_dir)
+ if not ready_verified(ready_where(models_dir, f), f[5])) / 1e9 # the small files only
# count only what step 6 will still write: a pack whose experts.bin is already there (the AVX-512 Q2_0
# conversion, or the low-RAM mode's copy) and an existing MTP draft layer need no new room
pack_now = find_in(roots, f"packs/{tag.lower()}") or data / "packs" / tag.lower()
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 307bbc758..66d3d7277 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -682,7 +682,15 @@ class FlagsCommand(unittest.TestCase):
self.assertIn("Did you mean 'Office'", err)
def test_vision_is_turned_off_for_a_model_without_pictures(self):
- p = self.plan("--model", "nvfp4-qad", "--vision", "on")
+ # every catalog model reads pictures now (Gyro and NVFP4-QAD too): a row that does not stands in for a future one
+ import dataclasses
+ real_rows = Catalog.rows
+
+ def rows(self, *args, **kwargs):
+ return [dataclasses.replace(r, images=False) if r.model == "Gyro-S" else r for r in real_rows(self, *args, **kwargs)]
+
+ with mock.patch.object(Catalog, "rows", rows):
+ p = self.plan("--model", "gyro-s", "--vision", "on")
self.assertEqual(p["settings"]["vision"], "off")
self.assertTrue(any("cannot read pictures" in w for w in p["warnings"]))
diff --git a/tools/strata_mcp.py b/tools/strata_mcp.py
index 5fb540c79..274b74809 100644
--- a/tools/strata_mcp.py
+++ b/tools/strata_mcp.py
@@ -92,8 +92,8 @@ FALLBACK_MODELS = {
"and speed as NVFP4", "download_gb": 128.0, "ram_gb": 80, "arena_gb": 68.0,
"families": ("nvfp4",), "vision": True},
"NVFP4-QAD": {"about": "NVIDIA's FP4, trained for FP4 - slightly faster on short prompts; a 102 GB download "
- "that this PC turns into a Strata pack once; no images yet",
- "download_gb": 102.2, "ram_gb": 80, "arena_gb": 68.0, "families": ("nvfp4",), "vision": False},
+ "that this PC turns into a Strata pack once; it reads pictures",
+ "download_gb": 102.2, "ram_gb": 80, "arena_gb": 68.0, "families": ("nvfp4",), "vision": True},
}
FALLBACK_FAMILIES = {
"qwen": {"title": "Qwen3.8-Flash-Next", "about": "the original model", "tag": ""},
diff --git a/tools/test_setup_nvfp4.py b/tools/test_setup_nvfp4.py
index 4909c9e68..e7ca4024f 100644
--- a/tools/test_setup_nvfp4.py
+++ b/tools/test_setup_nvfp4.py
@@ -105,7 +105,7 @@ class Family(unittest.TestCase):
for m in names:
d = setup.MODELS[m]
self.assertTrue(d["nvidia_only"])
- self.assertEqual(d["vision"], m != "NVFP4-QAD") # the prepared QAD model: text first
+ self.assertIs(d["vision"], True) # the shared encoder reads pictures for every NVFP4 model (QAD: dense.gguf is the text model)
self.assertRegex(d["license"], "Licen[sc]e|Agreement")
self.assertEqual(d["engine"], setup.FP4LOCAL_ENGINE if m == "NVFP4-QAD" else setup.NVFP4_ENGINE)
@@ -148,6 +148,61 @@ class PreparedModel(unittest.TestCase):
self.assertTrue(setup.prepared_done("NVFP4-QAD", Path(t)))
+class PreparedWithoutCheckpoint(unittest.TestCase):
+ """A prepared NVFP4-QAD model needs none of the checkpoint's 95 GB of safetensors: the engine reads the prepared files."""
+
+ def prepare(self, models: Path) -> None:
+ d = setup.prepared_dir("NVFP4-QAD", models)
+ d.mkdir(parents=True)
+ for m in ("ple-nvfp4.gguf.done", "dense.gguf.done", "pack.done", "draft.done"):
+ (d / m).write_text("x")
+
+ def write_small_files(self, models: Path) -> None:
+ for f in setup.ready_files("NVFP4-QAD"):
+ if f[3].startswith("tensors/"):
+ continue
+ dst = setup.ready_where(models, f)
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ dst.write_bytes(b"x")
+ dst.with_name(dst.name + ".done").write_text(f"sha256 {f[5]}")
+
+ def test_prepared_model_does_not_need_the_tensors(self):
+ with tempfile.TemporaryDirectory() as t:
+ models = Path(t)
+ self.prepare(models)
+ needed = setup.ready_needed("NVFP4-QAD", models)
+ self.assertTrue(needed)
+ self.assertFalse([f for f in needed if f[3].startswith("tensors/")])
+ self.assertIn("config.json", {f[3] for f in needed})
+
+ def test_unprepared_model_still_needs_the_tensors(self):
+ with tempfile.TemporaryDirectory() as t:
+ needed = setup.ready_needed("NVFP4-QAD", Path(t))
+ self.assertEqual(len(needed), len(setup.ready_files("NVFP4-QAD")))
+ self.assertEqual(sum(1 for f in needed if f[3].startswith("tensors/model-")), 21)
+
+ def test_a_model_that_is_not_prepared_by_this_pc_keeps_all_its_files(self):
+ with tempfile.TemporaryDirectory() as t:
+ self.assertEqual(len(setup.ready_needed("NVFP4", Path(t))), len(setup.ready_files("NVFP4")))
+
+ def test_ready_fetch_downloads_nothing_for_a_prepared_model_without_tensors(self):
+ with tempfile.TemporaryDirectory() as t:
+ models = Path(t)
+ self.prepare(models)
+ self.write_small_files(models)
+ with mock.patch.object(setup, "download", side_effect=AssertionError("downloaded")) as dl:
+ setup.ready_fetch("NVFP4-QAD", models)
+ dl.assert_not_called()
+
+ def test_ready_fetch_wants_the_tensors_of_an_unprepared_model(self):
+ with tempfile.TemporaryDirectory() as t:
+ models = Path(t)
+ self.write_small_files(models)
+ with mock.patch.object(setup, "download", side_effect=RuntimeError("would download")):
+ with self.assertRaises(RuntimeError):
+ setup.ready_fetch("NVFP4-QAD", models)
+
+
class MockedInstall(unittest.TestCase):
"""setup.main() --family nvfp4 --yes on mocked PCs (test_setup_golden's harness): the run config's model flags."""
@@ -205,6 +260,15 @@ class MockedInstall(unittest.TestCase):
self.assertIn("NVIDIA Open Model Agreement", out)
self.assertIn("turned into a Strata pack on this PC", out)
+ def test_qad_reads_pictures_with_its_prepared_dense_gguf(self):
+ code, out, cfg, _ = self.run_install("128GB-1x24GB", "NVFP4-QAD", extra_argv=["--vision", "gpu"])
+ self.assertEqual(code, 0, out[-2500:])
+ v = cfg["vision"]
+ self.assertTrue(v["model"].endswith("nvidia--Qwen3.8-Flash-Next-FP4-Local/prepared/dense.gguf"))
+ self.assertTrue(v["mmproj"].endswith("/mmproj-Qwen3.8-Flash-Next-BF16.gguf"))
+ self.assertIn("--vision", cfg["args"])
+ self.assertNotIn("images are not available", out)
+
def test_96gb_keeps_every_expert_in_ram_64gb_takes_the_ram_budget(self):
code, out, cfg, _ = self.run_install("96GB-1x16GB") # 68 GB of experts + 10 GB beside them fit 95.8
self.assertEqual(code, 0, out[-2000:])