Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: host memory the device reads in place, measured
JustVugg7b347c854bbb
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex c3fc74f7..a645eef2 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -132,7 +132,7 @@ static struct {* evicts cold tier experts instead of thrashing the per-token attention submits* (measured: decode attention 7.8s at 7.6 GB resident -> 17.8s at 15.2 GB).* VK_EXT_memory_budget lets the tier fill stop at a reserve instead of guessing. */- int has_prio, has_budget;+ int has_prio, has_budget, has_hostmem;float prio; /* priority applied to the NEXT allocations (class knob) *//* the routed-expert tier's queue (coli_vk_xb_*): family, index, whether it is the* main queue after all, and its timestamp bits; device limits its batches respect */@@ -509,7 +509,7 @@ int coli_vk_init(const char *spv_path) {/* Pressure-proofing extensions (both optional, detected at runtime):* memory_priority ranks allocations for the kernel's eviction order,* memory_budget exposes how much VRAM a new allocation can still take. */- const char *dext[4]; uint32_t ndext = 0;+ const char *dext[6]; uint32_t ndext = 0;int has_cm = 0, has_ssc = 0;{uint32_t ne = 0;@@ -524,6 +524,9 @@ int coli_vk_init(const char *spv_path) {#ifdef VK_EXT_memory_budgetif (!strcmp(ep[i].extensionName, VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) G.has_budget = 1;#endif+#ifdef VK_EXT_external_memory_host+ if (!strcmp(ep[i].extensionName, VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME)) G.has_hostmem = 1;+#endif#ifdef VK_KHR_cooperative_matrixif (!strcmp(ep[i].extensionName, VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME)) has_cm = 1;if (!strcmp(ep[i].extensionName, VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME)) has_ssc = 1;@@ -599,13 +602,18 @@ int coli_vk_init(const char *spv_path) {#endif#ifdef VK_EXT_memory_budgetif (G.has_budget) dext[ndext++] = VK_EXT_MEMORY_BUDGET_EXTENSION_NAME;+#endif+#ifdef VK_EXT_external_memory_host+ /* host memory the device reads in place (no copy): measured by the harness+ * (COLI_VK_TEST_HOSTMEM), see docs/vulkan.md on integrated GPUs */+ if (G.has_hostmem) dext[ndext++] = VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME;#endifdi.enabledExtensionCount = ndext; di.ppEnabledExtensionNames = ndext ? dext : NULL;G.prio = 0.75f; /* default class: dense/resident weights */#ifdef VK_KHR_cooperative_matrixif (G.has_coop) { /* a device that refuses the extra features still comes up without them */VkDeviceCreateInfo dc = di;- const char *cext[4]; uint32_t nc = 0;+ const char *cext[8]; uint32_t nc = 0;for (uint32_t i = 0; i < ndext; i++) cext[nc++] = dext[i];cext[nc++] = VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME;cext[nc++] = VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME;@@ -3434,6 +3442,136 @@ static int run_xbatch_all(void) {return bad;}+/* COLI_VK_TEST_HOSTMEM=1: the expert batch reading its experts from the tier pool+ * (device memory the tier fills by a copy) against host memory the device reads in+ * place (VK_EXT_external_memory_host, no copy). Batches of 10 experts of Qwen3.8's+ * shape (hidden 2560, inter 640, int4-g64) cycle through 48 distinct experts so no+ * expert is in a cache; the device time comes from the batch's timestamps. Also+ * what the copy into the tier pool costs per expert, against a copy into malloc'd+ * memory, and whether both memories give the same bits. */+typedef struct { void *host[2]; VkDeviceMemory mem[2]; } HostImp;+static int host_import(size_t bytes, void **host, VkDeviceMemory *mem, VkBuffer *buf) {+#ifdef VK_EXT_external_memory_host+ VkPhysicalDeviceExternalMemoryHostPropertiesEXT hp = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT};+ VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &hp};+ vkGetPhysicalDeviceProperties2(G.phys, &p2);+ size_t al = hp.minImportedHostPointerAlignment ? hp.minImportedHostPointerAlignment : 4096;+ size_t sz = (bytes + al - 1) / al * al;+ if (posix_memalign(host, al, sz)) return 0;+ memset(*host, 0, sz);+ PFN_vkGetMemoryHostPointerPropertiesEXT gp = (PFN_vkGetMemoryHostPointerPropertiesEXT)+ vkGetDeviceProcAddr(G.dev, "vkGetMemoryHostPointerPropertiesEXT");+ VkMemoryHostPointerPropertiesEXT mp = {.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};+ if (!gp || gp(G.dev, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, *host, &mp) != VK_SUCCESS || !mp.memoryTypeBits) return 0;+ uint32_t mt = 0; while (!(mp.memoryTypeBits & (1u << mt))) mt++;+ VkImportMemoryHostPointerInfoEXT imp = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,+ .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, .pHostPointer = *host};+ VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &imp, .allocationSize = sz, .memoryTypeIndex = mt};+ if (vkAllocateMemory(G.dev, &ai, NULL, mem) != VK_SUCCESS) return 0;+ VkExternalMemoryBufferCreateInfo eb = {.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,+ .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT};+ VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = &eb, .size = bytes,+ .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE};+ if (vkCreateBuffer(G.dev, &bi, NULL, buf) != VK_SUCCESS) return 0;+ if (vkBindBufferMemory(G.dev, *buf, *mem, 0) != VK_SUCCESS) return 0;+ static int said;+ if (!said++) printf("hostmem: imported with alignment %zu, memory type %u\n", al, mt);+ return 1;+#else+ (void)bytes; (void)host; (void)mem; (void)buf; return 0;+#endif+}+static ColiVkTensor *host_tensor(int fmt, int I, int O, int gs, HostImp *hi, uint8_t **rows, float **sc) {+ ColiVkTensor *t = calloc(1, sizeof(*t));+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = gs;+ t->wbytes = (size_t)t->rowWords * 4 * O;+ size_t sb = scale_floats(fmt, I, O, gs) * 4;+ if (!host_import(t->wbytes, &hi->host[0], &hi->mem[0], &t->wbuf) || !host_import(sb, &hi->host[1], &hi->mem[1], &t->sbuf)) {+ free(t); return NULL;+ }+ *rows = hi->host[0]; *sc = hi->host[1];+ return t;+}+static void bench_hostmem(void) {+ enum { N = 48, B = 10, R = 60 };+ const int D = 2560, I = 640, gs = 64;+ if (!G.has_hostmem) { printf("hostmem: VK_EXT_external_memory_host absent on this device\n"); return; }+ xb_shutdown();+ if (!coli_vk_xb_init(D, I, COLI_VK_ACT_SWIGLU, 0.f, 0.f, 0.f)) { printf("hostmem: no expert batch\n"); return; }+ coli_vk_tier_pool_limit(0);+ size_t rbg = (size_t)(D / 2), rbd = (size_t)(I / 2);+ size_t sg = (size_t)I * (D / gs), sd = (size_t)D * (I / gs);+ size_t ebytes = 2 * (rbg * I + sg * 4) + rbd * D + sd * 4;+ uint8_t *src = malloc(2 * rbg * I + rbd * D); float *ssrc = malloc((2 * sg + sd) * 4);+ for (size_t i = 0; i < 2 * rbg * I + rbd * D; i++) src[i] = (uint8_t)rand();+ for (size_t i = 0; i < 2 * sg + sd; i++) ssrc[i] = 0.001f * (1 + rand() % 7);+ ColiVkExpert *ex[2][N]; HostImp hi[N][3];+ double copy_ms = 0, malloc_ms = 0;+ uint8_t *mbuf = malloc(ebytes);+ for (int kind = 0; kind < 2; kind++)+ for (int e = 0; e < N; e++) {+ ColiVkTensor *t[3]; uint8_t *rows; float *sc; size_t stride;+ for (int k = 0; k < 3; k++) {+ int In = k < 2 ? D : I, On = k < 2 ? I : D;+ const uint8_t *cs = src + (k < 2 ? k * rbg * I : 2 * rbg * I);+ const float *ss = ssrc + (k < 2 ? k * sg : 2 * sg);+ size_t rb = k < 2 ? rbg : rbd, ns = k < 2 ? sg : sd;+ if (kind == 0) {+ if (!coli_vk_tier_tensor(&t[k], 4, In, On, gs, &rows, &stride, &sc)) { printf("hostmem: tier pool full\n"); return; }+ } else {+ t[k] = host_tensor(4, In, On, gs, &hi[e][k], &rows, &sc); stride = rb;+ if (!t[k]) { printf("hostmem: import failed\n"); return; }+ }+ double c0 = vk_now();+ for (int o = 0; o < On; o++) memcpy(rows + (size_t)o * stride, cs + (size_t)o * rb, rb);+ memcpy(sc, ss, ns * 4);+ if (kind == 0) copy_ms += vk_now() - c0;+ }+ if (kind == 0) { double c0 = vk_now(); memcpy(mbuf, src, 2 * rbg * I + rbd * D); memcpy(mbuf + 2 * rbg * I + rbd * D, ssrc, (2 * sg + sd) * 4);+ __asm__ volatile("" :: "r"(mbuf) : "memory"); /* the copy is kept: nothing reads mbuf */+ malloc_ms += vk_now() - c0; }+ ex[kind][e] = coli_vk_xb_expert(t[0], t[1], t[2]);+ }+ float *x = malloc((size_t)D * 4); for (int i = 0; i < D; i++) x[i] = 0.01f * (i % 13) - 0.06f;+ const float *xr[B], *yr[B]; int rows1[B];+ for (int j = 0; j < B; j++) { xr[j] = x; rows1[j] = 1; }+ double ms[2] = {0, 0}, wall[2] = {0, 0};+ float *y0 = malloc((size_t)D * 4 * B);+ int same = 1;+ for (int kind = 0; kind < 2; kind++)+ for (int r = 0; r < R; r++) {+ ColiVkExpert *b[B];+ for (int j = 0; j < B; j++) b[j] = ex[kind][(r * B + j) % N];+ double t0 = vk_now(), dm = 0;+ coli_vk_xb_issue(b, rows1, B, xr); coli_vk_xb_join(yr, &dm);+ if (r) { ms[kind] += dm; wall[kind] += vk_now() - t0; } /* the first batch warms the pipelines */+ if (r == 0) for (int j = 0; j < B; j++) {+ if (kind == 0) memcpy(y0 + (size_t)j * D, yr[j], (size_t)D * 4);+ else same &= !memcmp(y0 + (size_t)j * D, yr[j], (size_t)D * 4);+ }+ }+ double gb = (double)ebytes * B * (R - 1) / 1e9;+ printf("hostmem: %d batches of %d experts (%.2f MB each): device memory %.2f ms/batch = %.1f GB/s (wall %.2f ms), "+ "imported host memory %.2f ms/batch = %.1f GB/s (wall %.2f ms); same bits: %s\n",+ R - 1, B, ebytes / 1e6, ms[0] / (R - 1), gb / (ms[0] / 1e3), wall[0] / (R - 1),+ ms[1] / (R - 1), gb / (ms[1] / 1e3), wall[1] / (R - 1), same ? "yes" : "NO");+ printf("hostmem: one expert copied into the tier pool %.3f ms (%.1f GB/s), into malloc'd memory %.3f ms\n",+ copy_ms / N, ebytes / 1e6 / copy_ms * N, malloc_ms / N);+ for (int e = 0; e < N; e++) {+ coli_vk_xb_expert_free(ex[0][e]);+ ColiVkExpert *h = ex[1][e];+ ColiVkTensor *ts[3] = {h->g, h->u, h->d};+ xb_expert_unlink(h); free(h);+ for (int k = 0; k < 3; k++) {+ vkDestroyBuffer(G.dev, ts[k]->wbuf, NULL); vkDestroyBuffer(G.dev, ts[k]->sbuf, NULL);+ vkFreeMemory(G.dev, hi[e][k].mem[0], NULL); vkFreeMemory(G.dev, hi[e][k].mem[1], NULL);+ free(hi[e][k].host[0]); free(hi[e][k].host[1]); free(ts[k]);+ }+ }+ free(src); free(ssrc); free(mbuf); free(x); free(y0);+ xb_shutdown();+}+int main(int argc, char **argv) {const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";if (!coli_vk_init(spv)) { printf("vk init failed\n"); return 1; }@@ -3457,6 +3595,11 @@ int main(int argc, char **argv) {coli_vk_shutdown();return 0;}+ if (getenv("COLI_VK_TEST_HOSTMEM") && atoi(getenv("COLI_VK_TEST_HOSTMEM"))) {+ bench_hostmem();+ coli_vk_shutdown();+ return 0;+ }G.gemm_min_s = 0; /* every case below is the GEMV's, but for run_gemm_case's */bad |= run_case(1, 1, 6144, 1536, 50); // int8 expert gate/up shape (S=1 decode)bad |= run_case(2, 1, 6144, 1536, 50); // int4 expert