Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Say plainly why a CPU without AVX-512 stops the engine: the pack folder without native_experts.txt, not the CPU, is the cause; doctor warns before the start, the launcher gives the next step, the SYCL engine names the folder like the CUDA one does

Niko12217a80b3a96bf0
diff --git a/docs/INTEL_ARC.md b/docs/INTEL_ARC.md
index a38fab0d8..531523729 100644
--- a/docs/INTEL_ARC.md
+++ b/docs/INTEL_ARC.md
@@ -38,6 +38,11 @@ engine package. Otherwise [build the image yourself](DOCKER.md#build-it-yourself
- **Linux**, for example Ubuntu 24.04, with Intel's GPU driver: the `xe` or `i915` kernel driver plus the compute runtime and
Level Zero. On Ubuntu: `intel-opencl-icd libze1 libze-intel-gpu1`, or follow Intel's
[client GPU guide](https://dgpu-docs.intel.com/driver/client/overview.html).
+- **The CPU and the model folder.** The CPU needs no AVX-512: a Ryzen 3000-5000 or an Intel Core 12th-14th generation runs the experts that
+ do not fit on the card on AVX2 (measured on an A750 with a Ryzen 5 5600X, [INTEL.md](INTEL.md)). That holds for the model folders that
+ have a `native_experts.txt` (IQ3_XXS, IQ3_S, NVFP4, Unsloth). The older Q2_0 folder without that file has a CPU kernel that needs
+ AVX-512; the engine then stops with "this CPU cannot run the expert kernel", `strata doctor` says so before the start, and
+ building the model folder again with `tools/iq_pack.py` (or `strata setup`) fixes it.
- For a source build without Docker: **Intel oneAPI** with the DPC++ compiler (`icpx`, 2025.3 or newer) and **oneMKL**
(about 5 GB), `cmake` 3.24+, `ninja`, `git`, Python 3. The build fetches ggml unless you point `STRATA_GGML_DIR` at a
llama.cpp checkout.
diff --git a/src/kernels/cpu/expert_layout.cpp b/src/kernels/cpu/expert_layout.cpp
index bca01c0b8..f288c2464 100644
--- a/src/kernels/cpu/expert_layout.cpp
+++ b/src/kernels/cpu/expert_layout.cpp
@@ -390,7 +390,10 @@ void cpu_require_expert_support() {
if (f.usable()) return;
std::fprintf(stderr,
"strata: this CPU cannot run the expert kernel: %s.\n"
- " The engine needs AVX512-VNNI and AVX512-VBMI (Intel Ice Lake / AMD Zen 4 or newer).\n"
+ " The pack in --pack is the AVX-512 Q2_0 pack (its folder has no native_experts.txt): its CPU kernel needs\n"
+ " AVX512-VNNI and AVX512-VBMI (Intel Ice Lake / AMD Zen 4 or newer).\n"
+ " An i-quant (IQ3_XXS, IQ3_S, ...), NVFP4 or Unsloth pack has native_experts.txt in its folder and runs on an\n"
+ " AVX2 CPU as well: build that pack (setup does it on a CPU without AVX-512) and give its folder as --pack.\n"
" The scalar fallback exists for tests only and is far too slow to decode with.\n",
f.reason());
std::exit(1);
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index cd323caa6..3f197365f 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -2296,6 +2296,9 @@ HSA_START = re.compile(r"hsa_exception|HSA_STATUS_ERROR", re.I)
HSA_START_NEXT = ("The graphics card was not ready yet; wait a few seconds and run it again. If it keeps happening, restart the PC "
"and report the log.")
KNOWN_CAUSES = (
+ (re.compile(r"cannot run the expert kernel|needs AVX-?512", re.I),
+ "This model folder holds the pack for CPUs with AVX-512, and this PC's CPU does not have it. Run `strata setup` again: on a CPU "
+ "without AVX-512 it builds the pack that runs on AVX2 CPUs (Ryzen 3000-5000, Intel Core 12th-14th generation)."),
(re.compile(r"no device of requested type|device not found|no (gpu|sycl|level.zero) device|permission denied.*(dri|render)", re.I),
"The engine could not open the graphics card. On Linux the account must be allowed to use it: add it to the render group "
"(`sudo usermod -aG render $USER`), log in again and start it again."),
diff --git a/strata_control/doctor.py b/strata_control/doctor.py
index 0bd6bff7e..65cb599ba 100644
--- a/strata_control/doctor.py
+++ b/strata_control/doctor.py
@@ -229,6 +229,9 @@ def inspect(config_path: Path | None, *, profile_id: str | None = None, live: bo
access = _device_access_issue(cfg.get("backend"))
if access:
report["issues"].append(access)
+ cpu_pack = _cpu_pack_issue(cfg, config_path)
+ if cpu_pack:
+ report["issues"].append(cpu_pack)
if live:
report["live_gpu_probe"], live_issues = _live_probe(cfg, report["backend"])
report["issues"].extend(live_issues)
@@ -253,6 +256,41 @@ def _device_access_issue(backend: str | None, dev_root: str = "/dev") -> dict |
"Add the user to the `render` group (and `video` for some distributions), then log in again"}
+AVX512_FULL = {"avx512f", "avx512bw", "avx512vl", "avx512_vnni", "avx512vbmi"} # what the Q2_0 pack's CPU kernel uses
+
+
+def _cpu_flags(cpuinfo: str = "/proc/cpuinfo") -> set[str] | None:
+ """The CPU feature flags on Linux; None where they cannot be read (other OS, no /proc)."""
+ if not sys.platform.startswith("linux"):
+ return None
+ try:
+ for line in Path(cpuinfo).read_text(encoding="utf-8", errors="replace").splitlines():
+ if line.startswith("flags"):
+ return set(line.partition(":")[2].split())
+ except OSError:
+ return None
+ return None
+
+
+def _cpu_pack_issue(cfg: dict, config_path: Path, cpuinfo: str = "/proc/cpuinfo") -> dict | None:
+ """The canonical Q2_0 pack (a --pack folder with index.txt and no native_experts.txt) has an AVX-512 CPU kernel; the engine stops at
+ start on any other CPU with "this CPU cannot run the expert kernel". An i-quant, NVFP4 or Unsloth pack (native_experts.txt) runs on AVX2
+ CPUs. None when it is fine, cannot apply (a pack of another engine, a CPU whose flags cannot be read) or there is no such pack."""
+ args = cfg.get("args") or []
+ if "--pack" not in args or args.index("--pack") + 1 >= len(args):
+ return None
+ pack = _location(args[args.index("--pack") + 1], cfg, config_path)
+ if not (pack / "index.txt").is_file() or (pack / "native_experts.txt").exists() or (pack / "ds4.txt").exists():
+ return None
+ flags = _cpu_flags(cpuinfo)
+ if flags is None or AVX512_FULL <= flags:
+ return None
+ return {"code": "CPU_PACK_NEEDS_AVX512", "severity": "warning",
+ "message": f"{pack} is the pack for CPUs with AVX-512 (it has no native_experts.txt), and this CPU has no AVX-512 (Ryzen 3000-5000 and "
+ "Intel Core 12th-14th generation do not): the engine will stop at start. Run `strata setup` again; on this CPU it builds "
+ "the pack that runs on AVX2"}
+
+
def _monitor_component() -> dict:
component = {"name": "monitor_tui", "kind": "python_module", "required": False,
"expected_version": MONITOR_TEXTUAL_VERSION}
diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cpp
index 93514fc85..401e304bb 100644
--- a/sycl/src/program/generate.cpp
+++ b/sycl/src/program/generate.cpp
@@ -2730,8 +2730,26 @@ int main(int argc, char **argv) try {
}
}
// the canonical Q2_0 pack's CPU kernels are AVX-512 only; a native pack runs on AVX2 CPUs as well
- if (!native_pack) strata::kernels::cpu::cpu_require_expert_support();
- else if (!strata::kernels::cpu::cpu_avx512_ok())
+ if (!native_pack) {
+ // #2347 (the CUDA engine has this too): a --pack folder without native_experts.txt is read as the canonical Q2_0 pack, whose CPU
+ // kernel needs AVX-512; on an AVX2 CPU (Ryzen 5000, Intel 12th-14th gen) that reads as "this CPU is too old" when the folder is
+ // simply not the native pack (a half-built pack, the wrong folder). Say so before the refusal.
+ if (!strata::kernels::cpu::cpu_features().usable()) {
+ std::error_code ec;
+ const std::filesystem::path pk(o.pack);
+ std::fprintf(stderr, "strata generate: %s has no native_experts.txt, so it is read as the canonical Q2_0 "
+ "pack, whose CPU kernel needs AVX-512.\n"
+ " An NVFP4, i-quant or Unsloth pack has that file in the folder given as --pack "
+ "and runs on an AVX2 CPU as well;\n",
+ o.pack.c_str());
+ if (std::filesystem::exists(pk / "pack" / "native_experts.txt", ec))
+ std::fprintf(stderr, " it is in %s: use that folder as --pack.\n", (pk / "pack").generic_string().c_str());
+ else
+ std::fprintf(stderr, " if this is one of those packs, the folder is incomplete or the wrong one "
+ "(build it again with tools/iq_pack.py, or run setup again).\n");
+ }
+ strata::kernels::cpu::cpu_require_expert_support();
+ } else if (!strata::kernels::cpu::cpu_avx512_ok())
std::fprintf(stderr, "strata generate: this CPU has no AVX-512: the expert kernels run on %s "
"(multi-token for the i-quant gate/up rows)\n",
!strata::kernels::cpu::cpu_avx2_ok() ? "ggml-cpu vec_dot (no AVX2: the older-CPU build)"
diff --git a/tests/terminal/test_cpu_pack_avx512.py b/tests/terminal/test_cpu_pack_avx512.py
new file mode 100644
index 000000000..6d69d6bb5
--- /dev/null
+++ b/tests/terminal/test_cpu_pack_avx512.py
@@ -0,0 +1,82 @@
+"""A CPU without AVX-512 (Ryzen 3000-5000, Intel Core 12th-14th generation) and the pack for AVX-512 CPUs: the Terminal says so
+plainly (strata doctor before the start, the launcher after an engine stop), and does not blame the CPU for a pack that is simply
+not the native one."""
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from strata_cli.launch import runner as R
+from strata_control import doctor
+
+ZEN3 = {"fpu", "avx", "avx2", "fma", "bmi2"}
+ZEN4 = ZEN3 | doctor.AVX512_FULL
+
+
+class CpuPackIssue(unittest.TestCase):
+ def pack(self, d, *names):
+ p = Path(d) / "pack"
+ p.mkdir()
+ for n in names:
+ (p / n).write_text("x", encoding="utf-8")
+ return p
+
+ def cfg(self, p):
+ return {"args": ["--serve", "--pack", str(p), "--native", "x.gguf"], "cwd": str(p.parent)}
+
+ def test_the_avx512_pack_on_an_avx2_cpu_is_named_with_the_next_step(self):
+ with tempfile.TemporaryDirectory() as d, mock.patch.object(doctor, "_cpu_flags", return_value=ZEN3):
+ p = self.pack(d, "index.txt", "experts.bin", "dense.bin")
+ issue = doctor._cpu_pack_issue(self.cfg(p), Path(d) / "run.json")
+ self.assertEqual(issue["code"], "CPU_PACK_NEEDS_AVX512")
+ self.assertIn("no native_experts.txt", issue["message"])
+ self.assertIn("strata setup", issue["message"])
+ # a half-built pack (what stopped an Arc A750 on a Ryzen 5 5600X) reads the same way
+ q = Path(d) / "half"
+ q.mkdir()
+ (q / "index.txt").write_text("x", encoding="utf-8")
+ self.assertEqual(doctor._cpu_pack_issue(self.cfg(q), Path(d) / "run.json")["code"], "CPU_PACK_NEEDS_AVX512")
+
+ def test_nothing_is_said_when_the_cpu_has_avx512_the_pack_is_native_or_it_is_another_engine_pack(self):
+ with tempfile.TemporaryDirectory() as d:
+ p = self.pack(d, "index.txt", "experts.bin")
+ with mock.patch.object(doctor, "_cpu_flags", return_value=ZEN4):
+ self.assertIsNone(doctor._cpu_pack_issue(self.cfg(p), Path(d) / "run.json"))
+ with mock.patch.object(doctor, "_cpu_flags", return_value=ZEN3):
+ (p / "native_experts.txt").write_text("x", encoding="utf-8")
+ self.assertIsNone(doctor._cpu_pack_issue(self.cfg(p), Path(d) / "run.json"))
+ (p / "native_experts.txt").unlink()
+ (p / "ds4.txt").write_text("x", encoding="utf-8") # a DeepSeek pack: another engine, another kernel
+ self.assertIsNone(doctor._cpu_pack_issue(self.cfg(p), Path(d) / "run.json"))
+ (p / "ds4.txt").unlink()
+ self.assertIsNone(doctor._cpu_pack_issue({"args": ["--serve"]}, Path(d) / "run.json"))
+ with mock.patch.object(doctor, "_cpu_flags", return_value=None): # flags unreadable: no guess
+ self.assertIsNone(doctor._cpu_pack_issue(self.cfg(p), Path(d) / "run.json"))
+
+ def test_cpu_flags_reads_the_flags_line(self):
+ with tempfile.TemporaryDirectory() as d:
+ f = Path(d) / "cpuinfo"
+ f.write_text("processor\t: 0\nflags\t\t: fpu avx2 avx512f\nbogomips\t: 1\n", encoding="utf-8")
+ with mock.patch.object(doctor.sys, "platform", "linux"):
+ self.assertEqual(doctor._cpu_flags(str(f)), {"fpu", "avx2", "avx512f"})
+ self.assertIsNone(doctor._cpu_flags(str(Path(d) / "missing")))
+
+
+class EngineStopIsExplained(unittest.TestCase):
+ def test_the_engine_refusal_gets_a_plain_next_step(self):
+ with tempfile.TemporaryDirectory() as d:
+ log = Path(d) / "engine.log"
+ log.write_text("strata: this CPU cannot run the expert kernel: missing AVX512F AVX512BW.\n"
+ " The pack in --pack is the AVX-512 Q2_0 pack\n", encoding="utf-8")
+ cfg = Path(d) / "run.json"
+ cfg.write_text(json.dumps({"log": str(log)}), encoding="utf-8")
+ text, nxt = R.explain_engine_exit({"state": "failed", "error": "server_exited", "engine_exit_code": 1}, cfg)
+ self.assertIn("cannot run the expert kernel", text)
+ self.assertIn("strata setup", nxt)
+ self.assertIn("AVX2", nxt)
+
+
+if __name__ == "__main__":
+ unittest.main()