Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
setup: a prepared NVFP4-QAD model needs none of the checkpoint's safetensors; NVFP4-QAD reads pictures
Niko12212dbd302038fc
diff --git a/setup.py b/setup.pyindex 4e0224aee..c3c0b71de 100644--- a/setup.py+++ b/setup.py@@ -391,9 +391,9 @@ MODELS = {"https://huggingface.co/nvidia/Qwen3.8-Flash-Next-NVFP4"},"NVFP4-QAD": {"about": "NVIDIA's FP4, trained for FP4 (NVIDIA's Qwen3.8-Flash-Next-FP4-Local) - slightly faster on short prompts; ""a 102 GB download that this PC turns into a Strata pack once (about 20-40 minutes, 110 GB more disk); "- "RTX 50 cards have FP4 tensor cores, other NVIDIA cards run it at 8-bit precision. No images yet",+ "RTX 50 cards have FP4 tensor cores, other NVIDIA cards run it at 8-bit precision. It reads pictures (the shared Qwen3.8-Flash-Next encoder)","download_gb": 102.2, "prepare_gb": 110.0, "ram_gb": 80, "arena_gb": 68.0,- "families": ("nvfp4",), "nvidia_only": True, "vision": False, "repo": NVFP4_FP4LOCAL_REPO,+ "families": ("nvfp4",), "nvidia_only": True, "vision": True, "repo": NVFP4_FP4LOCAL_REPO,"prepare": "fp4-local", "engine": FP4LOCAL_ENGINE,"license": "NVIDIA Open Model Agreement (Licensed by NVIDIA Corporation under the NVIDIA Open Model ""Agreement) with Qwen3.8-Flash-Next's Qwen Community License 1.0: "@@ -1957,6 +1957,15 @@ def ready_files(model: str) -> list:return [(role, repo, t["revision"], path, size, sha) for path, (role, size, sha) in t["files"].items()]+def ready_needed(model: str, models_dir: Path) -> list:+ """ready_files without the checkpoint's tensors once a prepared model (MODELS[..]["prepare"]) is prepared: the+ engine reads the prepared files only, so the ~95 GB of safetensors need not stay (or come back) on the disk."""+ files = ready_files(model)+ if MODELS[model].get("prepare") and prepared_done(model, models_dir):+ files = [f for f in files if not f[3].startswith("tensors/")]+ return files++def ready_where(models_dir: Path, f) -> Path:"""Where one of them goes: <models>/<owner>--<repo>/<its path in the repository> (the repository's layout, so thepack folder is whole as the engine reads it)."""@@ -2004,7 +2013,7 @@ def ready_fetch(model: str, models_dir: Path) -> None:ready-made pack, linked instead of downloaded (the PLE table and the embedding); or downloaded from its pinnedrevision and checked against its size and SHA-256 (a wrong file is deleted and setup stops)."""others = [(ready_where(models_dir, g), g[5]) for m in MODELS if MODELS[m].get("repo") for g in ready_files(m)]- for f in ready_files(model):+ for f in ready_needed(model, models_dir):dst, size, sha = ready_where(models_dir, f), f[4], f[5]if ready_verified(dst, sha):ok(f"{f[3]} already downloaded")@@ -7116,7 +7125,7 @@ def main() -> int:if ready:models_dir = Path(a.models_dir)for r in elsewhere: # already downloaded and checked in a Strata folder elsewhere- if all(ready_verified(ready_where(r / "models", f), f[5]) for f in ready_files(model)):+ if all(ready_verified(ready_where(r / "models", f), f[5]) for f in ready_needed(model, r / "models")):models_dir = r / "models"ok(f"model files found in {models_dir}")break@@ -7141,12 +7150,15 @@ def main() -> int:mark(s, "whole (checked against its own tensor directory)")except OSError as e: # a read-only folder (--gguf-dir on a share): the file is still wholewarn(f"{s.name} is whole but its finish mark cannot be written ({e})")- have_model = all(ready_verified(ready_where(models_dir, f), f[5]) for f in ready_files(model)) if ready else \+ have_model = all(ready_verified(ready_where(models_dir, f), f[5]) for f in ready_needed(model, models_dir)) if ready else \all(s.exists() and (done(s) or a.gguf_dir) for s in shards)# #425 (jctaborda): a download that resumes needs room only for what is still missing - the finished shards and# the .part files already on the disk counton_disk = sum(f.stat().st_size for s in shards for f in (s, s.with_name(s.name + ".part")) if f.is_file()) / 1e9to_fetch = 0 if a.gguf_dir or have_model else max(MODELS[model]["download_gb"] - on_disk, 0)+ if ready and not have_model and MODELS[model].get("prepare") and prepared_done(model, models_dir):+ to_fetch = sum(f[4] for f in ready_needed(model, models_dir)+ if not ready_verified(ready_where(models_dir, f), f[5])) / 1e9 # the small files only# count only what step 6 will still write: a pack whose experts.bin is already there (the AVX-512 Q2_0# conversion, or the low-RAM mode's copy) and an existing MTP draft layer need no new roompack_now = find_in(roots, f"packs/{tag.lower()}") or data / "packs" / tag.lower()diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.pyindex 307bbc758..66d3d7277 100644--- a/tests/terminal/test_launch.py+++ b/tests/terminal/test_launch.py@@ -682,7 +682,15 @@ class FlagsCommand(unittest.TestCase):self.assertIn("Did you mean 'Office'", err)def test_vision_is_turned_off_for_a_model_without_pictures(self):- p = self.plan("--model", "nvfp4-qad", "--vision", "on")+ # every catalog model reads pictures now (Gyro and NVFP4-QAD too): a row that does not stands in for a future one+ import dataclasses+ real_rows = Catalog.rows++ def rows(self, *args, **kwargs):+ return [dataclasses.replace(r, images=False) if r.model == "Gyro-S" else r for r in real_rows(self, *args, **kwargs)]++ with mock.patch.object(Catalog, "rows", rows):+ p = self.plan("--model", "gyro-s", "--vision", "on")self.assertEqual(p["settings"]["vision"], "off")self.assertTrue(any("cannot read pictures" in w for w in p["warnings"]))diff --git a/tools/strata_mcp.py b/tools/strata_mcp.pyindex 5fb540c79..274b74809 100644--- a/tools/strata_mcp.py+++ b/tools/strata_mcp.py@@ -92,8 +92,8 @@ FALLBACK_MODELS = {"and speed as NVFP4", "download_gb": 128.0, "ram_gb": 80, "arena_gb": 68.0,"families": ("nvfp4",), "vision": True},"NVFP4-QAD": {"about": "NVIDIA's FP4, trained for FP4 - slightly faster on short prompts; a 102 GB download "- "that this PC turns into a Strata pack once; no images yet",- "download_gb": 102.2, "ram_gb": 80, "arena_gb": 68.0, "families": ("nvfp4",), "vision": False},+ "that this PC turns into a Strata pack once; it reads pictures",+ "download_gb": 102.2, "ram_gb": 80, "arena_gb": 68.0, "families": ("nvfp4",), "vision": True},}FALLBACK_FAMILIES = {"qwen": {"title": "Qwen3.8-Flash-Next", "about": "the original model", "tag": ""},diff --git a/tools/test_setup_nvfp4.py b/tools/test_setup_nvfp4.pyindex 4909c9e68..e7ca4024f 100644--- a/tools/test_setup_nvfp4.py+++ b/tools/test_setup_nvfp4.py@@ -105,7 +105,7 @@ class Family(unittest.TestCase):for m in names:d = setup.MODELS[m]self.assertTrue(d["nvidia_only"])- self.assertEqual(d["vision"], m != "NVFP4-QAD") # the prepared QAD model: text first+ self.assertIs(d["vision"], True) # the shared encoder reads pictures for every NVFP4 model (QAD: dense.gguf is the text model)self.assertRegex(d["license"], "Licen[sc]e|Agreement")self.assertEqual(d["engine"], setup.FP4LOCAL_ENGINE if m == "NVFP4-QAD" else setup.NVFP4_ENGINE)@@ -148,6 +148,61 @@ class PreparedModel(unittest.TestCase):self.assertTrue(setup.prepared_done("NVFP4-QAD", Path(t)))+class PreparedWithoutCheckpoint(unittest.TestCase):+ """A prepared NVFP4-QAD model needs none of the checkpoint's 95 GB of safetensors: the engine reads the prepared files."""++ def prepare(self, models: Path) -> None:+ d = setup.prepared_dir("NVFP4-QAD", models)+ d.mkdir(parents=True)+ for m in ("ple-nvfp4.gguf.done", "dense.gguf.done", "pack.done", "draft.done"):+ (d / m).write_text("x")++ def write_small_files(self, models: Path) -> None:+ for f in setup.ready_files("NVFP4-QAD"):+ if f[3].startswith("tensors/"):+ continue+ dst = setup.ready_where(models, f)+ dst.parent.mkdir(parents=True, exist_ok=True)+ dst.write_bytes(b"x")+ dst.with_name(dst.name + ".done").write_text(f"sha256 {f[5]}")++ def test_prepared_model_does_not_need_the_tensors(self):+ with tempfile.TemporaryDirectory() as t:+ models = Path(t)+ self.prepare(models)+ needed = setup.ready_needed("NVFP4-QAD", models)+ self.assertTrue(needed)+ self.assertFalse([f for f in needed if f[3].startswith("tensors/")])+ self.assertIn("config.json", {f[3] for f in needed})++ def test_unprepared_model_still_needs_the_tensors(self):+ with tempfile.TemporaryDirectory() as t:+ needed = setup.ready_needed("NVFP4-QAD", Path(t))+ self.assertEqual(len(needed), len(setup.ready_files("NVFP4-QAD")))+ self.assertEqual(sum(1 for f in needed if f[3].startswith("tensors/model-")), 21)++ def test_a_model_that_is_not_prepared_by_this_pc_keeps_all_its_files(self):+ with tempfile.TemporaryDirectory() as t:+ self.assertEqual(len(setup.ready_needed("NVFP4", Path(t))), len(setup.ready_files("NVFP4")))++ def test_ready_fetch_downloads_nothing_for_a_prepared_model_without_tensors(self):+ with tempfile.TemporaryDirectory() as t:+ models = Path(t)+ self.prepare(models)+ self.write_small_files(models)+ with mock.patch.object(setup, "download", side_effect=AssertionError("downloaded")) as dl:+ setup.ready_fetch("NVFP4-QAD", models)+ dl.assert_not_called()++ def test_ready_fetch_wants_the_tensors_of_an_unprepared_model(self):+ with tempfile.TemporaryDirectory() as t:+ models = Path(t)+ self.write_small_files(models)+ with mock.patch.object(setup, "download", side_effect=RuntimeError("would download")):+ with self.assertRaises(RuntimeError):+ setup.ready_fetch("NVFP4-QAD", models)++class MockedInstall(unittest.TestCase):"""setup.main() --family nvfp4 --yes on mocked PCs (test_setup_golden's harness): the run config's model flags."""@@ -205,6 +260,15 @@ class MockedInstall(unittest.TestCase):self.assertIn("NVIDIA Open Model Agreement", out)self.assertIn("turned into a Strata pack on this PC", out)+ def test_qad_reads_pictures_with_its_prepared_dense_gguf(self):+ code, out, cfg, _ = self.run_install("128GB-1x24GB", "NVFP4-QAD", extra_argv=["--vision", "gpu"])+ self.assertEqual(code, 0, out[-2500:])+ v = cfg["vision"]+ self.assertTrue(v["model"].endswith("nvidia--Qwen3.8-Flash-Next-FP4-Local/prepared/dense.gguf"))+ self.assertTrue(v["mmproj"].endswith("/mmproj-Qwen3.8-Flash-Next-BF16.gguf"))+ self.assertIn("--vision", cfg["args"])+ self.assertNotIn("images are not available", out)+def test_96gb_keeps_every_expert_in_ram_64gb_takes_the_ram_budget(self):code, out, cfg, _ = self.run_install("96GB-1x16GB") # 68 GB of experts + 10 GB beside them fit 95.8self.assertEqual(code, 0, out[-2000:])