Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
vulkan: VRAM pressure-proofing β priority-classed allocations + budget-capped tier fill
Steve Markgraf048ccbc91e81
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex c3fa9ac9..e53e3d15 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -87,6 +87,13 @@ static struct {ColiVkTensor *bound_tensor; int bound_S, bound_I, bound_O, cmd_ready;VkBuffer bound_xbuf, bound_ybuf;size_t used_bytes, tensor_count;+ /* VRAM pressure-proofing: with VK_EXT_memory_priority the attention working set+ * (KV mirror, scratches) outranks bulk expert weights, so an oversubscribed heap+ * evicts cold tier experts instead of thrashing the per-token attention submits+ * (measured: decode attention 7.8s at 7.6 GB resident -> 17.8s at 15.2 GB).+ * VK_EXT_memory_budget lets the tier fill stop at a reserve instead of guessing. */+ int has_prio, has_budget;+ float prio; /* priority applied to the NEXT allocations (class knob) */} G;struct PC { int fmt, S, I, O, rowWords; };@@ -131,11 +138,43 @@ static int alloc_hostvis_mt(size_t bytes, VkBuffer *buf, VkDeviceMemory *mem, vovkGetBufferMemoryRequirements(G.dev, *buf, &req);VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,.allocationSize = req.size, .memoryTypeIndex = memtype};+#ifdef VK_EXT_memory_priority+ VkMemoryPriorityAllocateInfoEXT pri = {.sType = VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT,+ .priority = G.prio};+ if (G.has_prio) ai.pNext = &pri;+#endifVKCHECK(vkAllocateMemory(G.dev, &ai, NULL, mem), "vkAllocateMemory");VKCHECK(vkBindBufferMemory(G.dev, *buf, *mem, 0), "vkBindBufferMemory");if (ptr) VKCHECK(vkMapMemory(G.dev, *mem, 0, bytes, 0, ptr), "vkMapMemory");return 1;}+/* Priority class of subsequent allocations (VK_EXT_memory_priority; no-op without it).+ * Scratches/KV force 1.0 internally; weight uploads take whatever is current β the+ * engine sets 0.4 around the bulk expert-tier fill, dense stays at the 0.75 default. */+void coli_vk_alloc_priority(float p) { G.prio = p < 0 ? 0 : p > 1 ? 1 : p; }++/* Device-local heap usage/budget in GB (VK_EXT_memory_budget). Returns 0 when the+ * extension is absent β callers then keep their count-based caps unchanged. */+int coli_vk_mem_budget(double *used_gb, double *budget_gb) {+#ifdef VK_EXT_memory_budget+ if (!G.has_budget || !G.phys) return 0;+ VkPhysicalDeviceMemoryBudgetPropertiesEXT bud = {+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};+ VkPhysicalDeviceMemoryProperties2 mp2 = {+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, .pNext = &bud};+ vkGetPhysicalDeviceMemoryProperties2(G.phys, &mp2);+ double u = 0, b = 0;+ for (uint32_t i = 0; i < mp2.memoryProperties.memoryHeapCount; i++)+ if (mp2.memoryProperties.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) {+ u += (double)bud.heapUsage[i]; b += (double)bud.heapBudget[i];+ }+ if (used_gb) *used_gb = u / 1e9;+ if (budget_gb) *budget_gb = b / 1e9;+ return b > 0;+#else+ (void)used_gb; (void)budget_gb; return 0;+#endif+}static int alloc_hostvis(size_t bytes, VkBuffer *buf, VkDeviceMemory *mem, void **ptr) {return alloc_hostvis_mt(bytes, buf, mem, ptr, G.memtype);}@@ -144,7 +183,10 @@ static int scratch_reserve_mt(Scratch *s, size_t bytes, uint32_t memtype) {if (s->cap >= bytes) return 1;if (s->buf) { vkDestroyBuffer(G.dev, s->buf, NULL); vkFreeMemory(G.dev, s->mem, NULL); }s->buf = VK_NULL_HANDLE; s->cap = 0; s->ptr = NULL;- if (!alloc_hostvis_mt(bytes, &s->buf, &s->mem, &s->ptr, memtype)) return 0;+ float p0 = G.prio; G.prio = 1.0f; /* scratches ride every submit: never evict */+ int ok = alloc_hostvis_mt(bytes, &s->buf, &s->mem, &s->ptr, memtype);+ G.prio = p0;+ if (!ok) return 0;s->cap = bytes;return 1;}@@ -266,10 +308,45 @@ int coli_vk_init(const char *spv_path) {float prio = 1.0f;VkDeviceQueueCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,.queueFamilyIndex = G.qfam, .queueCount = 1, .pQueuePriorities = &prio};+ /* Pressure-proofing extensions (both optional, detected at runtime):+ * memory_priority ranks allocations for the kernel's eviction order,+ * memory_budget exposes how much VRAM a new allocation can still take. */+ const char *dext[2]; uint32_t ndext = 0;+ {+ uint32_t ne = 0;+ vkEnumerateDeviceExtensionProperties(G.phys, NULL, &ne, NULL);+ VkExtensionProperties *ep = ne ? malloc(ne * sizeof(*ep)) : NULL;+ if (ep) {+ vkEnumerateDeviceExtensionProperties(G.phys, NULL, &ne, ep);+ for (uint32_t i = 0; i < ne; i++) {+#ifdef VK_EXT_memory_priority+ if (!strcmp(ep[i].extensionName, VK_EXT_MEMORY_PRIORITY_EXTENSION_NAME)) G.has_prio = 1;+#endif+#ifdef VK_EXT_memory_budget+ if (!strcmp(ep[i].extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) G.has_budget = 1;+#endif+ }+ free(ep);+ }+ }VkDeviceCreateInfo di = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,.queueCreateInfoCount = 1, .pQueueCreateInfos = &qi};+#ifdef VK_EXT_memory_priority+ VkPhysicalDeviceMemoryPriorityFeaturesEXT prif = {+ .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PRIORITY_FEATURES_EXT,+ .memoryPriority = VK_TRUE};+ if (G.has_prio) { dext[ndext++] = VK_EXT_MEMORY_PRIORITY_EXTENSION_NAME; prif.pNext = (void *)di.pNext; di.pNext = &prif; }+#endif+#ifdef VK_EXT_memory_budget+ if (G.has_budget) dext[ndext++] = VK_EXT_MEMORY_BUDGET_EXTENSION_NAME;+#endif+ di.enabledExtensionCount = ndext; di.ppEnabledExtensionNames = ndext ? dext : NULL;+ G.prio = 0.75f; /* default class: dense/resident weights */VKCHECK(vkCreateDevice(G.phys, &di, NULL, &G.dev), "vkCreateDevice");vkGetDeviceQueue(G.dev, G.qfam, 0, &G.queue);+ if (G.has_prio || G.has_budget)+ fprintf(stderr, "[VK] VRAM pressure-proofing: memory_priority %s, memory_budget %s\n",+ G.has_prio ? "on" : "absent", G.has_budget ? "on" : "absent");int mt = pick_memtype();if (mt < 0) { fprintf(stderr, "[VK] no host-visible memory\n"); return 0; }@@ -605,9 +682,12 @@ int coli_vk_kv_ensure(int layer, int max_rows, int K, int Rd) {if (!G.ready || layer < 0 || layer >= VK_KV_LAYERS || max_rows < 1 || K < 1 || Rd < 1) return 0;VkKvLayer *v = &G.kv[layer];if (v->bl) return v->rows >= max_rows && v->K == K && v->R == Rd; /* resize goes through coli_vk_kv_reset */- if (!alloc_hostvis((size_t)max_rows * K * 4, &v->bl, &v->ml, &v->pl)) return 0;- if (!alloc_hostvis((size_t)max_rows * Rd * 4, &v->br, &v->mr, &v->pr)) {- vkDestroyBuffer(G.dev, v->bl, NULL); vkFreeMemory(G.dev, v->ml, NULL);+ float p0 = G.prio; G.prio = 1.0f; /* KV mirror rides every attention submit */+ int ok1 = alloc_hostvis((size_t)max_rows * K * 4, &v->bl, &v->ml, &v->pl);+ int ok = ok1 && alloc_hostvis((size_t)max_rows * Rd * 4, &v->br, &v->mr, &v->pr);+ G.prio = p0;+ if (!ok) {+ if (ok1) { vkDestroyBuffer(G.dev, v->bl, NULL); vkFreeMemory(G.dev, v->ml, NULL); }memset(v, 0, sizeof(*v)); return 0;}v->rows = max_rows; v->K = K; v->R = Rd;diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex 20fb53a1..f1e0c271 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -21,6 +21,16 @@ void coli_vk_shutdown(void);int coli_vk_available(void);void coli_vk_mem_info(size_t *used_bytes, size_t *tensor_count);+/* VRAM pressure-proofing (both no-ops when the extension is absent):+ * alloc_priority sets the eviction-priority class of SUBSEQUENT weight uploads+ * (VK_EXT_memory_priority; scratches and the KV mirror pin themselves at 1.0) β+ * the engine brackets the bulk expert-tier fill at 0.4 so an oversubscribed heap+ * evicts cold experts, never the per-token attention working set.+ * mem_budget reports device-local usage/budget in GB (VK_EXT_memory_budget);+ * returns 0 if unavailable. */+void coli_vk_alloc_priority(float p);+int coli_vk_mem_budget(double *used_gb, double *budget_gb);+/* y[S,O] = (x[S,I] @ dequant(W[O,I])^T) * scale[O].* fmt matches QT in glm.c: 1=int8, 2=int4. (0=f32,3=int2 fall back to CPU.)* First call uploads W+scales; later calls reuse the resident copy.diff --git a/c/colibri.c b/c/colibri.cindex 1bc860b9..c61d7754 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -6575,7 +6575,18 @@ static void vk_registry_fill(Model *m){if(!g_vk_reg){ free(cand); return; }ESlot tmp; memset(&tmp,0,sizeof(tmp)); tmp.eid=-1;double t0=now_s(); int64_t bytes=0; int tried=0, loadfail=0;+ /* Pressure-proofing: tier weights are the EVICTABLE class (0.4 vs scratch/KV 1.0,+ * dense 0.75) so an oversubscribed heap sheds cold experts instead of thrashing+ * the per-token attention submits; and the fill STOPS while the device-local+ * budget still holds COLI_VK_RESERVE_GB (default 3) for the lazily-allocated+ * dense weights + KV mirror + staging (measured ~1.7 GB at 4k ctx, growing with+ * max_t). Without the budget extension the count cap alone applies, as before. */+ double vkr_reserve = getenv("COLI_VK_RESERVE_GB")?atof(getenv("COLI_VK_RESERVE_GB")):3.0;+ int vkr_stopped=0; double vkr_used=0, vkr_budget=0;+ coli_vk_alloc_priority(0.4f);for(int64_t i2=0;i2<n && g_vk_reg_n<g_vk_budget;i2++){+ if(vkr_reserve>0 && (g_vk_reg_n&7)==0 && coli_vk_mem_budget(&vkr_used,&vkr_budget)+ && vkr_budget-vkr_used < vkr_reserve){ vkr_stopped=1; break; }int layer=cand[i2].layer, eid=cand[i2].eid; tried++;ESlot *src=NULL, *P=m->pin[layer];for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid && P[z].slab){ src=&P[z]; break; }@@ -6601,10 +6612,14 @@ static void vk_registry_fill(Model *m){bytes+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);g_vk_reg_n++;}+ coli_vk_alloc_priority(0.75f); /* back to the dense/default class */if(tmp.slab){ compat_aligned_free(tmp.slab); free(tmp.fslab); }free(cand);fprintf(stderr,"[VK] expert tier: %d hot experts resident (%.2f GB VRAM, %.1fs, top-%d of history)\n",g_vk_reg_n,bytes/1e9,now_s()-t0,tried);+ if(vkr_stopped)+ fprintf(stderr,"[VK] expert tier: budget stop at %d experts β %.1f of %.1f GB device-local used, %.1f GB reserved (COLI_VK_RESERVE_GB)\n",+ g_vk_reg_n,vkr_used,vkr_budget,vkr_reserve);}#endif