Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

FP4-Local: setup installs the pinned torch, transformers and safetensors the converter imports (requirements-nvfp4-local.txt) right before a prepare that runs; the tool names a missing package up front instead of after 20 minutes (#3227)

Niko12211994f3acb2de
diff --git a/requirements-nvfp4-local.txt b/requirements-nvfp4-local.txt
new file mode 100644
index 000000000..f0af1999b
--- /dev/null
+++ b/requirements-nvfp4-local.txt
@@ -0,0 +1,7 @@
+# Packages only the NVIDIA FP4-Local (QAD) conversion needs (#3227): setup installs them right before tools/nvfp4_local_pack.py prepare runs,
+# and only then. llama.cpp's converter imports torch and transformers; safetensors reads NVIDIA's shards.
+# Licenses: torch BSD-3-Clause, transformers Apache-2.0, safetensors Apache-2.0 (all permissive; nothing of them is shipped with Strata).
+# Pinned to the versions of the PC the conversion is developed on (torch 2.9.1 is the CPU build on Windows; on Linux setup asks for the CPU wheel).
+torch==2.9.1
+transformers==4.57.6
+safetensors==0.7.0
diff --git a/setup.py b/setup.py
index 68d333172..60d641217 100644
--- a/setup.py
+++ b/setup.py
@@ -2813,7 +2813,7 @@ def control_missing_packages() -> list:
if importlib.util.find_spec(m) is None]
-def pip_install(packages, what):
+def pip_install(packages, what, extra=()):
"""pip install into .venv, skipped when the same list was installed before. An install from before the pinned
requirements (#214) recorded bare names: those packages are kept as they are (nothing is reinstalled), and the
pinned dependencies it already has count as installed."""
@@ -2833,11 +2833,44 @@ def pip_install(packages, what):
return
acquisition.require(what)
say(f" Installing {what} ...")
- run([sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", *need])
+ run([sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", *extra, *need])
stamp.write_text(json.dumps(sorted(set(have) | set(need)), indent=0))
ok(f"{what} installed")
+NVFP4_LOCAL_REQUIREMENTS = ROOT / "requirements-nvfp4-local.txt"
+NVFP4_LOCAL_MODULES = ("torch", "transformers", "safetensors")
+
+
+def nvfp4_local_packages() -> list:
+ """The pinned lines of requirements-nvfp4-local.txt (comments and options left out)."""
+ try:
+ lines = NVFP4_LOCAL_REQUIREMENTS.read_text(encoding="utf-8").splitlines()
+ except OSError:
+ return []
+ return [l.split("#")[0].strip() for l in lines if l.strip() and not l.lstrip().startswith(("#", "-"))]
+
+
+def ensure_nvfp4_local_packages() -> None:
+ """#3227: NVIDIA's FP4-Local checkpoint is converted on this PC with llama.cpp's converter, which imports torch and transformers
+ (the runtime requirements do not have them: nothing else in setup needs them). Installed only for a prepare that runs; what is
+ already there is kept. The app does not ship them (hundreds of MB): it says so and points at the source installer."""
+ import importlib.util
+ missing = [m for m in NVFP4_LOCAL_MODULES if importlib.util.find_spec(m) is None]
+ if not missing:
+ return
+ if CONTROL:
+ fail("preparing NVIDIA's FP4-Local checkpoint needs PyTorch and transformers, which the app does not include",
+ "install this model from the source download (SETUP.bat on Windows, ./setup.sh on Linux), or pick another version in the app")
+ pkgs = nvfp4_local_packages()
+ if not pkgs:
+ fail("requirements-nvfp4-local.txt is missing", "update Strata, or run: python -m pip install torch transformers safetensors")
+ say(" Converting NVIDIA's checkpoint needs PyTorch, transformers and safetensors (a few hundred MB, installed once).")
+ pip_install(pkgs, "PyTorch, transformers and safetensors (for the FP4-Local conversion)",
+ extra=[] if WIN else ["--extra-index-url", "https://download.pytorch.org/whl/cpu"]) # Linux: the CPU build, not 3 GB of CUDA libraries
+ importlib.invalidate_caches()
+
+
def cuda_lib_dirs(toolkit=13):
"""Where pip put NVIDIA's CUDA libraries (nvidia/cu13/bin/x86_64 on Windows, nvidia/cu13/lib on Linux).
toolkit 12: the CUDA 12 wheels' cuBLAS and runtime, in two folders (nvidia/cublas/bin, nvidia/cuda_runtime/bin)."""
@@ -8096,6 +8129,7 @@ def main() -> int:
if prepared and not prepared_done(model, models_dir):
say(" NVIDIA's checkpoint is turned into a Strata pack on this PC (once; 20-40 minutes, reads the 102 GB, writes about")
say(f" {MODELS[model]['prepare_gb']:.0f} GB). Nothing of NVIDIA's weights leaves this PC.")
+ ensure_nvfp4_local_packages()
run([sys.executable, str(ROOT / "tools" / "nvfp4_local_pack.py"), "prepare",
"--model", str(ready_where(models_dir, ("src", MODELS[model]["repo"], "", "", 0, ""))),
"--out", str(prepared_dir(model, models_dir))], env=env)
diff --git a/tools/nvfp4_local_pack.py b/tools/nvfp4_local_pack.py
index a00b1145e..03f5c8848 100644
--- a/tools/nvfp4_local_pack.py
+++ b/tools/nvfp4_local_pack.py
@@ -39,6 +39,23 @@ import numpy as np
HERE = pathlib.Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
+# #3227: llama.cpp's converter (conversion/base.py: `from transformers import AutoConfig`) and the MXFP8 / NVFP4 decoding below need
+# these; setup installs them from requirements-nvfp4-local.txt before a prepare, and a run by hand says which one is missing.
+NEEDED_PACKAGES = ("torch", "transformers", "safetensors")
+MARKS = ("ple-nvfp4.gguf.done", "dense.gguf.done", "pack.done", "draft.done")
+
+
+def require_packages() -> None:
+ import importlib.util
+ missing = [m for m in NEEDED_PACKAGES if importlib.util.find_spec(m) is None]
+ if missing:
+ req = HERE.parent / "requirements-nvfp4-local.txt"
+ sys.exit("tools/nvfp4_local_pack.py needs the Python package%s %s, which %s not installed for %s.\n"
+ "Install %s with: %s -m pip install -r %s" % ("s" if len(missing) > 1 else "", ", ".join(missing),
+ "are" if len(missing) > 1 else "is", sys.executable,
+ "them" if len(missing) > 1 else "it", sys.executable, req))
+
+
CSF_FIXED = ".nvfp4_csf_fixed"
CSF_EXC = ".nvfp4_csf_exceptions"
PLE_ROW_BYTES = 90 # 80 B of E2M1 codes + 10 E4M3 block scales, per 160-value row
@@ -251,6 +268,7 @@ def install(ck: Checkpoint, stats: dict, dense_only: bool = False):
def cmd_convert(a) -> int:
+ require_packages()
model = pathlib.Path(a.model)
ck = Checkpoint(model)
stats: dict = {}
@@ -556,6 +574,8 @@ def cmd_prepare(a) -> int:
The checkpoint is read, never changed; it can be deleted afterwards (the pack and the two GGUFs hold everything)."""
import subprocess
model, out = pathlib.Path(a.model), pathlib.Path(a.out)
+ if not all((out / m).is_file() for m in MARKS):
+ require_packages() # before the 20 minutes of the n-gram table, not after them (#3227)
out.mkdir(parents=True, exist_ok=True)
me = [sys.executable, str(pathlib.Path(__file__).resolve())]
tools = pathlib.Path(__file__).resolve().parent
diff --git a/tools/test_setup_dl_fix.py b/tools/test_setup_dl_fix.py
index 02ddee85c..4c01f36d6 100644
--- a/tools/test_setup_dl_fix.py
+++ b/tools/test_setup_dl_fix.py
@@ -1,4 +1,4 @@
-"""Setup's MTP download and the Gyro low-RAM pack (#3193 #3214 #3257): the draft layer's download is run again when it stops, every tool
+"""Setup's MTP download, the Gyro low-RAM pack and the FP4-Local conversion packages (#3193 #3214 #3257 #3227): the draft layer's download is run again when it stops, every tool
setup runs ends with setup, and Gyro's low-RAM pack gets --compat-bf16 like its first pack. Everything mocked; nothing downloaded.
python -m unittest tools.test_setup_dl_fix
@@ -76,5 +76,56 @@ class GyroLowRam(U.Base):
self.assertTrue(any("--experts-bin" in r for r in packs), "the low-RAM pack ran")
+class Fp4LocalPackages(unittest.TestCase):
+ """#3227: torch, transformers and safetensors come from a pinned requirements file, right before a prepare that runs."""
+
+ def test_the_requirements_file_pins_what_the_converter_imports(self):
+ pkgs = setup.nvfp4_local_packages()
+ self.assertEqual(sorted(p.split("==")[0] for p in pkgs), sorted(setup.NVFP4_LOCAL_MODULES))
+ self.assertTrue(all("==" in p for p in pkgs), pkgs)
+
+ def test_missing_packages_are_installed_once_with_the_pins(self):
+ calls = []
+ with mock.patch("importlib.util.find_spec", lambda name: None), mock.patch.object(setup, "CONTROL", False), \
+ mock.patch.object(setup, "say", lambda *a, **k: None), \
+ mock.patch.object(setup, "pip_install", lambda pkgs, what, extra=(): calls.append((list(pkgs), list(extra)))):
+ setup.ensure_nvfp4_local_packages()
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0][0], setup.nvfp4_local_packages())
+ self.assertEqual(bool(calls[0][1]), not setup.WIN) # the CPU wheel index only on Linux
+
+ def test_nothing_is_installed_when_they_are_there(self):
+ with mock.patch("importlib.util.find_spec", lambda name: object()), \
+ mock.patch.object(setup, "pip_install", side_effect=AssertionError("pip ran")):
+ setup.ensure_nvfp4_local_packages()
+
+ def test_the_app_says_so_instead_of_installing(self):
+ with mock.patch("importlib.util.find_spec", lambda name: None), mock.patch.object(setup, "CONTROL", True), \
+ mock.patch.object(setup, "say", lambda *a, **k: None), \
+ mock.patch.object(setup, "pip_install", side_effect=AssertionError("pip ran")), \
+ self.assertRaises(SystemExit):
+ setup.ensure_nvfp4_local_packages()
+
+ def test_the_tool_names_the_missing_package_when_run_by_hand(self):
+ import nvfp4_local_pack as P
+ with mock.patch("importlib.util.find_spec", lambda name: None if name == "transformers" else object()):
+ with self.assertRaises(SystemExit) as cm:
+ P.require_packages()
+ self.assertIn("transformers", str(cm.exception))
+ self.assertIn("requirements-nvfp4-local.txt", str(cm.exception))
+ self.assertNotIn("torch,", str(cm.exception))
+
+
+class Fp4LocalOrder(unittest.TestCase):
+ def test_the_packages_come_before_the_prepare_run(self):
+ import test_setup_golden as G
+ import test_setup_nvfp4 as N
+ events = []
+ with mock.patch.object(setup, "ensure_nvfp4_local_packages", lambda: events.append("packages")):
+ ram, found = G.PROFILES["128GB-1x24GB"]
+ _ = G.install(ram, found, ["--family", "nvfp4", "--no-start", "--model", "NVFP4-QAD"])
+ self.assertEqual(events, ["packages"])
+
+
if __name__ == "__main__":
unittest.main()