Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
vulkan: warn when ReBAR leaves VRAM host-invisible; find shaders next to the binary
Steve Markgraf809a8476a26a
diff --git a/c/Makefile b/c/Makefileindex d306233e..0c4caac2 100644--- a/c/Makefile+++ b/c/Makefile@@ -290,7 +290,9 @@ endif# VK=1 adds an opt-in Vulkan compute backend (any Vulkan 1.2 driver, e.g. Mesa/RADV)# for resident expert tensors. Independent of CUDA/HIP: works on GPUs ROCm doesn't-# support (Polaris/RX 580) and measured ~35% faster than ROCm on RDNA4 for int4 experts.+# support (Polaris/RX 580); the VRAM-resident int4 expert-group PRIMITIVE measured ~35%+# faster than the ROCm/HIP one on an RX 9070 (per-call incl. readback β end-to-end tok/s+# depends on what else bounds the box; see docs/vulkan.md, benchmarking notes included).# Needs libvulkan + glslc (shaderc). Default build unchanged (all colibri.c hooks under# #ifdef COLI_VULKAN). The .comp shaders compile to .spv, loaded at runtime from# COLI_VK_SHADERS (default: alongside the binary / ./shaders).diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex a3de36dd..9d22d2df 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -364,6 +364,33 @@ int coli_vk_init(const char *spv_path) {G.memtype = (uint32_t)mt;G.memtype_cached = (uint32_t)pick_memtype_cached(G.phys);+ /* Resizable-BAR sanity (#523): on discrete cards the weight tiers want+ * HOST_VISIBLE|DEVICE_LOCAL. With ReBAR disabled that combination exists only in a+ * ~256 MB BAR window (or not at all), so tier allocations silently land in system+ * RAM and every access crosses PCIe β measurably SLOWER than the CPU path, while+ * the resident-experts log still reports an apparently healthy VRAM tier. Compare+ * the chosen type's heap against the largest DEVICE_LOCAL heap and say so up front. */+ {+ VkPhysicalDeviceMemoryProperties mp;+ vkGetPhysicalDeviceMemoryProperties(G.phys, &mp);+ VkDeviceSize dl_max = 0;+ for (uint32_t i = 0; i < mp.memoryHeapCount; i++)+ if ((mp.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) &&+ mp.memoryHeaps[i].size > dl_max) dl_max = mp.memoryHeaps[i].size;+ VkMemoryPropertyFlags cf = mp.memoryTypes[G.memtype].propertyFlags;+ VkDeviceSize hv_dl = mp.memoryHeaps[mp.memoryTypes[G.memtype].heapIndex].size;+ if (dl_max && !(cf & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))+ fprintf(stderr, "[VK] warning: no host-visible+device-local memory type β weight tiers "+ "will live in system RAM and every access crosses PCIe (expect slower than "+ "CPU-only). On a discrete card, enable Resizable BAR in the BIOS.\n");+ else if (dl_max && hv_dl * 4 < dl_max)+ fprintf(stderr, "[VK] warning: only %llu of %llu MB VRAM is host-visible (Resizable BAR "+ "appears disabled) β allocations beyond the %llu MB window fall back to system "+ "RAM and will be slow. Enable Resizable BAR / Smart Access Memory in the BIOS.\n",+ (unsigned long long)(hv_dl >> 20), (unsigned long long)(dl_max >> 20),+ (unsigned long long)(hv_dl >> 20));+ }+G.shader = load_spv(G.dev, spv_path);if (!G.shader) return 0;if (!build_pipeline(G.dev, 4, sizeof(struct PC), G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;diff --git a/c/colibri.c b/c/colibri.cindex 892e34eb..c24e659f 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -315,6 +315,30 @@ static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls* projections + shared expert) on Vulkan too */static int g_vk_budget2; /* COLI_VK_EXPERTS2: dev2 expert-tier cap (with COLI_VK_DEV2) */static int g_vk_reg_n2; /* experts resident on the dev2 tier */+/* Resolve the main shader path (#523): COLI_VK_SHADERS may be the qmatmul.spv file itself+ * OR a directory containing it; unset, look alongside the binary (<exedir>/shaders/, the+ * build layout) before the historical CWD-relative fallback, so launching from outside c/+ * works. The other shaders load as siblings of the returned path (backend derive_*). */+static const char *vk_resolve_spv(char *buf, size_t n){+ const char *env = getenv("COLI_VK_SHADERS");+ struct stat st;+ if(env && *env){+ if(!stat(env,&st) && S_ISDIR(st.st_mode)){ snprintf(buf,n,"%s/qmatmul.spv",env); return buf; }+ return env;+ }+#ifdef __linux__+ ssize_t k = readlink("/proc/self/exe", buf, n-1);+ if(k > 0){+ buf[k] = 0;+ char *sl = strrchr(buf, '/');+ if(sl && (size_t)(sl+1-buf) + sizeof("shaders/qmatmul.spv") <= n){+ strcpy(sl+1, "shaders/qmatmul.spv");+ if(!stat(buf,&st)) return buf;+ }+ }+#endif+ return "shaders/qmatmul.spv";+}/* PROF anatomy of the VK expert block (master-thread accumulated in moe(), printed by* profile_print): where a decode block's wall goes besides t_ecpu/t_ewait/t_egpu. */static double g_vkb_cls, g_vkb_issue, g_vkb_acc, g_vkb_wrk, g_vkb_join;@@ -7777,9 +7801,11 @@ int main(int argc, char **argv){#endif#ifdef COLI_VULKANif(getenv("COLI_VULKAN") && atoi(getenv("COLI_VULKAN"))){- const char *spv = getenv("COLI_VK_SHADERS"); if(!spv) spv = "shaders/qmatmul.spv";+ char spvbuf[512]; const char *spv = vk_resolve_spv(spvbuf, sizeof(spvbuf));g_vulkan = coli_vk_init(spv);- if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (need libvulkan + shaders/qmatmul{,_gate_up}.spv)\n"); return 2; }+ if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (tried %s; need libvulkan + "+ "the compiled shaders β point COLI_VK_SHADERS at the shader directory "+ "or the qmatmul.spv file, or run `make VK=1` to build them)\n", spv); return 2; }/* 320 = sweep optimum on a 16 GB card (256-384 measured flat, 320 best median;* ~6 GB tier + ~8 GB dense leaves headroom for the long-context KV mirror). */g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 320;