Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

setup: on an integrated GPU, Vulkan only where it was measured faster

JustVuggf882010a6eec
diff --git a/c/setup_flow.py b/c/setup_flow.py
index 82c8ba82..fdb0cbee 100644
--- a/c/setup_flow.py
+++ b/c/setup_flow.py
@@ -281,6 +281,16 @@ def package_hint(kinds, os_info=None, platform_name=None):
# ---------------------------------------------------------------- backend
+#: The engines whose Vulkan path was measured faster than their CPU run on an
+#: integrated GPU (a Radeon 780M, docs/vulkan.md "Measured on a Radeon 780M" and
+#: "The chain on a Radeon 780M"): Qwen3.6 writes at 6.0 tok/s on the CPU and 9.9
+#: with the GPU, Qwen3.8 Flash Next at 3.5 and 3.8. An integrated GPU has no memory
+#: of its own: it reads the same RAM as the CPU, through the same bandwidth, and on
+#: OLMoE that made it slower to write than the CPU alone (23 against 17 tok/s). So
+#: on an integrated GPU the other engines stay on the CPU unless asked
+#: (--backend vulkan). A discrete GPU, with its own VRAM, runs Vulkan for every one.
+VULKAN_IGPU_MEASURED = frozenset({"qwen36", "qwen38"})
+
def choose_backend(hw, family, tc, requested="auto"):
"""Which build to run the model with, and why.
@@ -288,8 +298,10 @@ def choose_backend(hw, family, tc, requested="auto"):
CUDA first for an NVIDIA card when this engine has a CUDA path and the
toolkit is here (its VRAM expert tier is the measured fast path); else
Vulkan when a Vulkan GPU answered and the build has its headers and glslc;
- else the CPU. Every GPU path the machine has but cannot build yet comes
- back in `missing` with the command that would enable it."""
+ else the CPU. On an integrated GPU, Vulkan only for the engines measured
+ faster there (VULKAN_IGPU_MEASURED) unless Vulkan was asked for by name.
+ Every GPU path the machine has but cannot build yet comes back in `missing`
+ with the command that would enable it."""
gpu = hw.get("gpu") or {}
vk, nvidia = gpu.get("vulkan"), gpu.get("nvidia") or []
decision = {"backend": "cpu", "reason": "", "missing": [], "gpu": None}
@@ -308,6 +320,12 @@ def choose_backend(hw, family, tc, requested="auto"):
elif requested == "cuda" and not cuda_engine:
decision["missing"].append(("cuda", f"{family.display_name} has no CUDA path here; "
"Vulkan or the CPU run it"))
+ if (vk and vk.get("type") == "integrated" and requested in ("auto", "cuda")
+ and family.id not in VULKAN_IGPU_MEASURED):
+ decision["reason"] = (f"{vk['name']} is an integrated GPU, which shares the CPU's RAM, "
+ f"and {family.display_name} was not measured faster on one: "
+ "the engine runs on the CPU (--backend vulkan uses the GPU anyway)")
+ return decision
if vk and requested in ("auto", "vulkan", "cuda"):
if tc.get("can_build_vulkan"):
kind = vk.get("type")
diff --git a/c/tests/test_setup_flow.py b/c/tests/test_setup_flow.py
index be4b82b3..2811b848 100644
--- a/c/tests/test_setup_flow.py
+++ b/c/tests/test_setup_flow.py
@@ -82,6 +82,8 @@ def hw_report(vulkan=None, nvidia=(), icd=None, os_id="ubuntu"):
IGPU = {"name": "Iris Xe", "type": "integrated", "api_version": "1.2.318",
"api_version_raw": (1 << 22) | (2 << 12) | 318}
+DGPU = {"name": "AMD Radeon RX 7800 XT", "type": "discrete", "api_version": "1.3.296",
+ "api_version_raw": (1 << 22) | (3 << 12) | 296}
RTX = {"index": 0, "name": "NVIDIA GeForce RTX 4070", "total_bytes": 12 * 2**30,
"free_bytes": 11 * 2**30, "driver": "560"}
TC_ALL = {"source_checkout": True, "make": "/usr/bin/make", "cc": "/usr/bin/gcc",
@@ -140,9 +142,33 @@ class BackendChoice(unittest.TestCase):
self.assertIn("nvidia-cuda-toolkit", decision["missing"][0][1])
def test_engine_without_cuda_path_uses_vulkan(self):
- decision = self.choose(hw_report(vulkan=IGPU, nvidia=[RTX]), family="mimo")
+ rtx_vk = {"name": RTX["name"], "type": "discrete", "api_version": "1.3.289",
+ "api_version_raw": (1 << 22) | (3 << 12) | 289}
+ decision = self.choose(hw_report(vulkan=rtx_vk, nvidia=[RTX]), family="mimo")
self.assertEqual(decision["backend"], "vulkan")
+ def test_integrated_gpu_only_for_engines_measured_faster_there(self):
+ for family in sorted(setup_flow.VULKAN_IGPU_MEASURED):
+ with self.subTest(family=family):
+ self.assertEqual(self.choose(hw_report(vulkan=IGPU), family=family)["backend"], "vulkan")
+ for family in ("mimo", "glm", "inkling", "kimi", "deepseek_v4"):
+ with self.subTest(family=family):
+ decision = self.choose(hw_report(vulkan=IGPU), family=family)
+ self.assertEqual(decision["backend"], "cpu")
+ self.assertIn("integrated GPU", decision["reason"])
+ self.assertIn("--backend vulkan", decision["reason"])
+ self.assertEqual(decision["missing"], []) # nothing to install: a choice, not a lack
+
+ def test_integrated_gpu_when_vulkan_is_asked_for(self):
+ decision = self.choose(hw_report(vulkan=IGPU), family="mimo", requested="vulkan")
+ self.assertEqual(decision["backend"], "vulkan")
+ self.assertEqual(decision["gpu"], "Iris Xe")
+
+ def test_discrete_gpu_runs_vulkan_for_every_engine(self):
+ for family in ("qwen36", "qwen38", "mimo", "glm", "inkling", "kimi", "deepseek_v4"):
+ with self.subTest(family=family):
+ self.assertEqual(self.choose(hw_report(vulkan=DGPU), family=family)["backend"], "vulkan")
+
def test_vulkan_gpu_without_headers_prints_the_package_command(self):
tc = dict(TC_ALL, vulkan_headers=False, glslc=None, can_build_vulkan=False)
decision = self.choose(hw_report(vulkan=IGPU), tc=tc)
diff --git a/docs/AI_SETUP.md b/docs/AI_SETUP.md
index 578de051..83aa5533 100644
--- a/docs/AI_SETUP.md
+++ b/docs/AI_SETUP.md
@@ -87,7 +87,10 @@ python3 c/coli setup --yes --model qwen36-35b --no-start
What it does, in order: detects the hardware, picks the engine build (CUDA
for an NVIDIA card when the CUDA toolkit is installed and the engine has a
CUDA path, else Vulkan when a Vulkan GPU answered and the Vulkan headers and
-`glslc` are installed, else the CPU), builds or fetches the engine, downloads
+`glslc` are installed, else the CPU; on an integrated GPU, which shares the
+CPU's RAM, Vulkan only for the engines measured faster there, today Qwen3.6,
+Qwen3-Coder and Qwen3.8, unless `--backend vulkan` asks for it), builds or
+fetches the engine, downloads
the model with resume, runs the planner on the files, and writes the run
configuration. It prints each step.