Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: warn when ReBAR leaves VRAM host-invisible; find shaders next to the binary

Steve Markgraf809a8476a26a
diff --git a/c/Makefile b/c/Makefile
index d306233e..0c4caac2 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -290,7 +290,9 @@ endif
# VK=1 adds an opt-in Vulkan compute backend (any Vulkan 1.2 driver, e.g. Mesa/RADV)
# for resident expert tensors. Independent of CUDA/HIP: works on GPUs ROCm doesn't
-# support (Polaris/RX 580) and measured ~35% faster than ROCm on RDNA4 for int4 experts.
+# support (Polaris/RX 580); the VRAM-resident int4 expert-group PRIMITIVE measured ~35%
+# faster than the ROCm/HIP one on an RX 9070 (per-call incl. readback β€” end-to-end tok/s
+# depends on what else bounds the box; see docs/vulkan.md, benchmarking notes included).
# Needs libvulkan + glslc (shaderc). Default build unchanged (all colibri.c hooks under
# #ifdef COLI_VULKAN). The .comp shaders compile to .spv, loaded at runtime from
# COLI_VK_SHADERS (default: alongside the binary / ./shaders).
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index a3de36dd..9d22d2df 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -364,6 +364,33 @@ int coli_vk_init(const char *spv_path) {
G.memtype = (uint32_t)mt;
G.memtype_cached = (uint32_t)pick_memtype_cached(G.phys);
+ /* Resizable-BAR sanity (#523): on discrete cards the weight tiers want
+ * HOST_VISIBLE|DEVICE_LOCAL. With ReBAR disabled that combination exists only in a
+ * ~256 MB BAR window (or not at all), so tier allocations silently land in system
+ * RAM and every access crosses PCIe β€” measurably SLOWER than the CPU path, while
+ * the resident-experts log still reports an apparently healthy VRAM tier. Compare
+ * the chosen type's heap against the largest DEVICE_LOCAL heap and say so up front. */
+ {
+ VkPhysicalDeviceMemoryProperties mp;
+ vkGetPhysicalDeviceMemoryProperties(G.phys, &mp);
+ VkDeviceSize dl_max = 0;
+ for (uint32_t i = 0; i < mp.memoryHeapCount; i++)
+ if ((mp.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) &&
+ mp.memoryHeaps[i].size > dl_max) dl_max = mp.memoryHeaps[i].size;
+ VkMemoryPropertyFlags cf = mp.memoryTypes[G.memtype].propertyFlags;
+ VkDeviceSize hv_dl = mp.memoryHeaps[mp.memoryTypes[G.memtype].heapIndex].size;
+ if (dl_max && !(cf & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
+ fprintf(stderr, "[VK] warning: no host-visible+device-local memory type β€” weight tiers "
+ "will live in system RAM and every access crosses PCIe (expect slower than "
+ "CPU-only). On a discrete card, enable Resizable BAR in the BIOS.\n");
+ else if (dl_max && hv_dl * 4 < dl_max)
+ fprintf(stderr, "[VK] warning: only %llu of %llu MB VRAM is host-visible (Resizable BAR "
+ "appears disabled) β€” allocations beyond the %llu MB window fall back to system "
+ "RAM and will be slow. Enable Resizable BAR / Smart Access Memory in the BIOS.\n",
+ (unsigned long long)(hv_dl >> 20), (unsigned long long)(dl_max >> 20),
+ (unsigned long long)(hv_dl >> 20));
+ }
+
G.shader = load_spv(G.dev, spv_path);
if (!G.shader) return 0;
if (!build_pipeline(G.dev, 4, sizeof(struct PC), G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;
diff --git a/c/colibri.c b/c/colibri.c
index 892e34eb..c24e659f 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -315,6 +315,30 @@ static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls
* projections + shared expert) on Vulkan too */
static int g_vk_budget2; /* COLI_VK_EXPERTS2: dev2 expert-tier cap (with COLI_VK_DEV2) */
static int g_vk_reg_n2; /* experts resident on the dev2 tier */
+/* Resolve the main shader path (#523): COLI_VK_SHADERS may be the qmatmul.spv file itself
+ * OR a directory containing it; unset, look alongside the binary (<exedir>/shaders/, the
+ * build layout) before the historical CWD-relative fallback, so launching from outside c/
+ * works. The other shaders load as siblings of the returned path (backend derive_*). */
+static const char *vk_resolve_spv(char *buf, size_t n){
+ const char *env = getenv("COLI_VK_SHADERS");
+ struct stat st;
+ if(env && *env){
+ if(!stat(env,&st) && S_ISDIR(st.st_mode)){ snprintf(buf,n,"%s/qmatmul.spv",env); return buf; }
+ return env;
+ }
+#ifdef __linux__
+ ssize_t k = readlink("/proc/self/exe", buf, n-1);
+ if(k > 0){
+ buf[k] = 0;
+ char *sl = strrchr(buf, '/');
+ if(sl && (size_t)(sl+1-buf) + sizeof("shaders/qmatmul.spv") <= n){
+ strcpy(sl+1, "shaders/qmatmul.spv");
+ if(!stat(buf,&st)) return buf;
+ }
+ }
+#endif
+ return "shaders/qmatmul.spv";
+}
/* PROF anatomy of the VK expert block (master-thread accumulated in moe(), printed by
* profile_print): where a decode block's wall goes besides t_ecpu/t_ewait/t_egpu. */
static double g_vkb_cls, g_vkb_issue, g_vkb_acc, g_vkb_wrk, g_vkb_join;
@@ -7777,9 +7801,11 @@ int main(int argc, char **argv){
#endif
#ifdef COLI_VULKAN
if(getenv("COLI_VULKAN") && atoi(getenv("COLI_VULKAN"))){
- const char *spv = getenv("COLI_VK_SHADERS"); if(!spv) spv = "shaders/qmatmul.spv";
+ char spvbuf[512]; const char *spv = vk_resolve_spv(spvbuf, sizeof(spvbuf));
g_vulkan = coli_vk_init(spv);
- if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (need libvulkan + shaders/qmatmul{,_gate_up}.spv)\n"); return 2; }
+ if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (tried %s; need libvulkan + "
+ "the compiled shaders β€” point COLI_VK_SHADERS at the shader directory "
+ "or the qmatmul.spv file, or run `make VK=1` to build them)\n", spv); return 2; }
/* 320 = sweep optimum on a 16 GB card (256-384 measured flat, 320 best median;
* ~6 GB tier + ~8 GB dense leaves headroom for the long-context KV mirror). */
g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 320;