Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
setup: on an integrated GPU, Vulkan only where it was measured faster
JustVuggf882010a6eec
diff --git a/c/setup_flow.py b/c/setup_flow.pyindex 82c8ba82..fdb0cbee 100644--- a/c/setup_flow.py+++ b/c/setup_flow.py@@ -281,6 +281,16 @@ def package_hint(kinds, os_info=None, platform_name=None):# ---------------------------------------------------------------- backend+#: The engines whose Vulkan path was measured faster than their CPU run on an+#: integrated GPU (a Radeon 780M, docs/vulkan.md "Measured on a Radeon 780M" and+#: "The chain on a Radeon 780M"): Qwen3.6 writes at 6.0 tok/s on the CPU and 9.9+#: with the GPU, Qwen3.8 Flash Next at 3.5 and 3.8. An integrated GPU has no memory+#: of its own: it reads the same RAM as the CPU, through the same bandwidth, and on+#: OLMoE that made it slower to write than the CPU alone (23 against 17 tok/s). So+#: on an integrated GPU the other engines stay on the CPU unless asked+#: (--backend vulkan). A discrete GPU, with its own VRAM, runs Vulkan for every one.+VULKAN_IGPU_MEASURED = frozenset({"qwen36", "qwen38"})+def choose_backend(hw, family, tc, requested="auto"):"""Which build to run the model with, and why.@@ -288,8 +298,10 @@ def choose_backend(hw, family, tc, requested="auto"):CUDA first for an NVIDIA card when this engine has a CUDA path and thetoolkit is here (its VRAM expert tier is the measured fast path); elseVulkan when a Vulkan GPU answered and the build has its headers and glslc;- else the CPU. Every GPU path the machine has but cannot build yet comes- back in `missing` with the command that would enable it."""+ else the CPU. On an integrated GPU, Vulkan only for the engines measured+ faster there (VULKAN_IGPU_MEASURED) unless Vulkan was asked for by name.+ Every GPU path the machine has but cannot build yet comes back in `missing`+ with the command that would enable it."""gpu = hw.get("gpu") or {}vk, nvidia = gpu.get("vulkan"), gpu.get("nvidia") or []decision = {"backend": "cpu", "reason": "", "missing": [], "gpu": None}@@ -308,6 +320,12 @@ def choose_backend(hw, family, tc, requested="auto"):elif requested == "cuda" and not cuda_engine:decision["missing"].append(("cuda", f"{family.display_name} has no CUDA path here; ""Vulkan or the CPU run it"))+ if (vk and vk.get("type") == "integrated" and requested in ("auto", "cuda")+ and family.id not in VULKAN_IGPU_MEASURED):+ decision["reason"] = (f"{vk['name']} is an integrated GPU, which shares the CPU's RAM, "+ f"and {family.display_name} was not measured faster on one: "+ "the engine runs on the CPU (--backend vulkan uses the GPU anyway)")+ return decisionif vk and requested in ("auto", "vulkan", "cuda"):if tc.get("can_build_vulkan"):kind = vk.get("type")diff --git a/c/tests/test_setup_flow.py b/c/tests/test_setup_flow.pyindex be4b82b3..2811b848 100644--- a/c/tests/test_setup_flow.py+++ b/c/tests/test_setup_flow.py@@ -82,6 +82,8 @@ def hw_report(vulkan=None, nvidia=(), icd=None, os_id="ubuntu"):IGPU = {"name": "Iris Xe", "type": "integrated", "api_version": "1.2.318","api_version_raw": (1 << 22) | (2 << 12) | 318}+DGPU = {"name": "AMD Radeon RX 7800 XT", "type": "discrete", "api_version": "1.3.296",+ "api_version_raw": (1 << 22) | (3 << 12) | 296}RTX = {"index": 0, "name": "NVIDIA GeForce RTX 4070", "total_bytes": 12 * 2**30,"free_bytes": 11 * 2**30, "driver": "560"}TC_ALL = {"source_checkout": True, "make": "/usr/bin/make", "cc": "/usr/bin/gcc",@@ -140,9 +142,33 @@ class BackendChoice(unittest.TestCase):self.assertIn("nvidia-cuda-toolkit", decision["missing"][0][1])def test_engine_without_cuda_path_uses_vulkan(self):- decision = self.choose(hw_report(vulkan=IGPU, nvidia=[RTX]), family="mimo")+ rtx_vk = {"name": RTX["name"], "type": "discrete", "api_version": "1.3.289",+ "api_version_raw": (1 << 22) | (3 << 12) | 289}+ decision = self.choose(hw_report(vulkan=rtx_vk, nvidia=[RTX]), family="mimo")self.assertEqual(decision["backend"], "vulkan")+ def test_integrated_gpu_only_for_engines_measured_faster_there(self):+ for family in sorted(setup_flow.VULKAN_IGPU_MEASURED):+ with self.subTest(family=family):+ self.assertEqual(self.choose(hw_report(vulkan=IGPU), family=family)["backend"], "vulkan")+ for family in ("mimo", "glm", "inkling", "kimi", "deepseek_v4"):+ with self.subTest(family=family):+ decision = self.choose(hw_report(vulkan=IGPU), family=family)+ self.assertEqual(decision["backend"], "cpu")+ self.assertIn("integrated GPU", decision["reason"])+ self.assertIn("--backend vulkan", decision["reason"])+ self.assertEqual(decision["missing"], []) # nothing to install: a choice, not a lack++ def test_integrated_gpu_when_vulkan_is_asked_for(self):+ decision = self.choose(hw_report(vulkan=IGPU), family="mimo", requested="vulkan")+ self.assertEqual(decision["backend"], "vulkan")+ self.assertEqual(decision["gpu"], "Iris Xe")++ def test_discrete_gpu_runs_vulkan_for_every_engine(self):+ for family in ("qwen36", "qwen38", "mimo", "glm", "inkling", "kimi", "deepseek_v4"):+ with self.subTest(family=family):+ self.assertEqual(self.choose(hw_report(vulkan=DGPU), family=family)["backend"], "vulkan")+def test_vulkan_gpu_without_headers_prints_the_package_command(self):tc = dict(TC_ALL, vulkan_headers=False, glslc=None, can_build_vulkan=False)decision = self.choose(hw_report(vulkan=IGPU), tc=tc)diff --git a/docs/AI_SETUP.md b/docs/AI_SETUP.mdindex 578de051..83aa5533 100644--- a/docs/AI_SETUP.md+++ b/docs/AI_SETUP.md@@ -87,7 +87,10 @@ python3 c/coli setup --yes --model qwen36-35b --no-startWhat it does, in order: detects the hardware, picks the engine build (CUDAfor an NVIDIA card when the CUDA toolkit is installed and the engine has aCUDA path, else Vulkan when a Vulkan GPU answered and the Vulkan headers and-`glslc` are installed, else the CPU), builds or fetches the engine, downloads+`glslc` are installed, else the CPU; on an integrated GPU, which shares the+CPU's RAM, Vulkan only for the engines measured faster there, today Qwen3.6,+Qwen3-Coder and Qwen3.8, unless `--backend vulkan` asks for it), builds or+fetches the engine, downloadsthe model with resume, runs the planner on the files, and writes the runconfiguration. It prints each step.