Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Vulkan compute backend (from PR #84), M1: builds + correct on RX 9070

Steve Markgraf38af749a6eee
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
new file mode 100644
index 00000000..fcf7a623
--- /dev/null
+++ b/c/backend_vulkan.c
@@ -0,0 +1,400 @@
+// Vulkan compute backend for colibri's quantized matmul, targeting the
+// Strix Halo iGPU (RADV gfx1151). Mirrors backend_cuda.c's contract but
+// exploits unified memory: weight "uploads" write into HOST_VISIBLE|
+// DEVICE_LOCAL memory β€” the same physical RAM the iGPU reads β€” so there is
+// no PCIe copy. That is what makes offloading *streamed experts* profitable
+// here, which the discrete-CUDA path deliberately avoids.
+//
+// M2 scope: correctness + a standalone GPU-vs-CPU test harness. Synchronous
+// submit/wait per call; async queues and zero-copy import come in M4.
+#include "backend_vulkan.h"
+#include <vulkan/vulkan.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define VKCHECK(x, what) do { VkResult _r = (x); if (_r != VK_SUCCESS) { \
+ fprintf(stderr, "[VK] %s failed: %d\n", what, _r); return 0; } } while (0)
+
+struct ColiVkTensor {
+ VkBuffer wbuf, sbuf;
+ VkDeviceMemory wmem, smem;
+ size_t wbytes;
+ int fmt, I, O, rowWords;
+};
+
+typedef struct {
+ VkBuffer buf; VkDeviceMemory mem; void *ptr; size_t cap;
+} Scratch;
+
+static struct {
+ int ready;
+ VkInstance inst;
+ VkPhysicalDevice phys;
+ VkDevice dev;
+ VkQueue queue;
+ uint32_t qfam;
+ uint32_t memtype; // HOST_VISIBLE|HOST_COHERENT (prefer DEVICE_LOCAL)
+ VkDescriptorSetLayout dsl;
+ VkPipelineLayout plyt;
+ VkPipeline pipe;
+ VkShaderModule shader;
+ VkDescriptorPool dpool;
+ VkDescriptorSet dset;
+ VkCommandPool cpool;
+ VkCommandBuffer cmd;
+ VkFence fence;
+ Scratch x, y;
+ size_t used_bytes, tensor_count;
+} G;
+
+struct PC { int fmt, S, I, O, rowWords; };
+
+static int pick_memtype(void) {
+ VkPhysicalDeviceMemoryProperties m;
+ vkGetPhysicalDeviceMemoryProperties(G.phys, &m);
+ int best = -1;
+ for (uint32_t i = 0; i < m.memoryTypeCount; i++) {
+ VkMemoryPropertyFlags f = m.memoryTypes[i].propertyFlags;
+ if ((f & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) &&
+ (f & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
+ if (f & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) return (int)i; // ideal on APU
+ if (best < 0) best = (int)i;
+ }
+ }
+ return best;
+}
+
+static int alloc_hostvis(size_t bytes, VkBuffer *buf, VkDeviceMemory *mem, void **ptr) {
+ VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
+ .size = bytes, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
+ .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
+ VKCHECK(vkCreateBuffer(G.dev, &bi, NULL, buf), "vkCreateBuffer");
+ VkMemoryRequirements req;
+ vkGetBufferMemoryRequirements(G.dev, *buf, &req);
+ VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
+ .allocationSize = req.size, .memoryTypeIndex = G.memtype};
+ VKCHECK(vkAllocateMemory(G.dev, &ai, NULL, mem), "vkAllocateMemory");
+ VKCHECK(vkBindBufferMemory(G.dev, *buf, *mem, 0), "vkBindBufferMemory");
+ if (ptr) VKCHECK(vkMapMemory(G.dev, *mem, 0, bytes, 0, ptr), "vkMapMemory");
+ return 1;
+}
+
+static int scratch_reserve(Scratch *s, size_t bytes) {
+ if (s->cap >= bytes) return 1;
+ if (s->buf) { vkDestroyBuffer(G.dev, s->buf, NULL); vkFreeMemory(G.dev, s->mem, NULL); }
+ s->buf = VK_NULL_HANDLE; s->cap = 0; s->ptr = NULL;
+ if (!alloc_hostvis(bytes, &s->buf, &s->mem, &s->ptr)) return 0;
+ s->cap = bytes;
+ return 1;
+}
+
+static int rowwords(int fmt, int I) {
+ size_t rb = fmt == 1 ? (size_t)I : (size_t)(I + 1) / 2; // bytes/row on CPU side
+ return (int)((rb + 3) / 4); // padded to uint32
+}
+
+static VkShaderModule load_spv(const char *path) {
+ FILE *f = fopen(path, "rb");
+ if (!f) { fprintf(stderr, "[VK] cannot open %s\n", path); return VK_NULL_HANDLE; }
+ fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);
+ if (n <= 0 || n % 4 != 0) { // SPIR-V is a stream of uint32; empty/non-seekable/odd size is invalid
+ fprintf(stderr, "[VK] bad SPIR-V size %ld in %s\n", n, path); fclose(f); return VK_NULL_HANDLE; }
+ uint32_t *code = malloc((size_t)n);
+ if (!code) { fclose(f); return VK_NULL_HANDLE; }
+ if (fread(code, 1, n, f) != (size_t)n) { fclose(f); free(code); return VK_NULL_HANDLE; }
+ fclose(f);
+ VkShaderModuleCreateInfo si = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
+ .codeSize = n, .pCode = code};
+ VkShaderModule m;
+ VkResult r = vkCreateShaderModule(G.dev, &si, NULL, &m);
+ free(code);
+ return r == VK_SUCCESS ? m : VK_NULL_HANDLE;
+}
+
+int coli_vk_init(const char *spv_path) {
+ if (G.ready) return 1;
+ VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
+ .apiVersion = VK_API_VERSION_1_2};
+ VkInstanceCreateInfo ici = {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
+ .pApplicationInfo = &app};
+ VKCHECK(vkCreateInstance(&ici, NULL, &G.inst), "vkCreateInstance");
+
+ uint32_t nd = 0;
+ vkEnumeratePhysicalDevices(G.inst, &nd, NULL);
+ if (!nd) { fprintf(stderr, "[VK] no devices\n"); return 0; }
+ VkPhysicalDevice devs[8]; if (nd > 8) nd = 8;
+ vkEnumeratePhysicalDevices(G.inst, &nd, devs);
+ // Prefer a real GPU over a CPU/software device (llvmpipe) on multi-adapter hosts:
+ // discrete > integrated > virtual > other/cpu. Falls back to devs[0] if all equal.
+ G.phys = devs[0];
+ int bestrank = -1;
+ for (uint32_t i = 0; i < nd; i++) {
+ VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(devs[i], &p);
+ int rank = p.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 4 :
+ p.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 3 :
+ p.deviceType == VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU ? 2 :
+ p.deviceType == VK_PHYSICAL_DEVICE_TYPE_OTHER ? 1 : 0; // CPU last
+ if (rank > bestrank) { bestrank = rank; G.phys = devs[i]; }
+ }
+
+ uint32_t nq = 0;
+ vkGetPhysicalDeviceQueueFamilyProperties(G.phys, &nq, NULL);
+ VkQueueFamilyProperties qf[16]; if (nq > 16) nq = 16;
+ vkGetPhysicalDeviceQueueFamilyProperties(G.phys, &nq, qf);
+ G.qfam = UINT32_MAX;
+ for (uint32_t i = 0; i < nq; i++)
+ if (qf[i].queueFlags & VK_QUEUE_COMPUTE_BIT) { G.qfam = i; break; }
+ if (G.qfam == UINT32_MAX) { fprintf(stderr, "[VK] no compute queue\n"); return 0; }
+
+ float prio = 1.0f;
+ VkDeviceQueueCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
+ .queueFamilyIndex = G.qfam, .queueCount = 1, .pQueuePriorities = &prio};
+ VkDeviceCreateInfo di = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
+ .queueCreateInfoCount = 1, .pQueueCreateInfos = &qi};
+ VKCHECK(vkCreateDevice(G.phys, &di, NULL, &G.dev), "vkCreateDevice");
+ vkGetDeviceQueue(G.dev, G.qfam, 0, &G.queue);
+
+ int mt = pick_memtype();
+ if (mt < 0) { fprintf(stderr, "[VK] no host-visible memory\n"); return 0; }
+ G.memtype = (uint32_t)mt;
+
+ G.shader = load_spv(spv_path);
+ if (!G.shader) return 0;
+
+ VkDescriptorSetLayoutBinding b[4];
+ for (int i = 0; i < 4; i++) b[i] = (VkDescriptorSetLayoutBinding){
+ .binding = i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
+ .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT};
+ VkDescriptorSetLayoutCreateInfo dsli = {
+ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
+ .bindingCount = 4, .pBindings = b};
+ VKCHECK(vkCreateDescriptorSetLayout(G.dev, &dsli, NULL, &G.dsl), "descSetLayout");
+
+ VkPushConstantRange pcr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
+ .offset = 0, .size = sizeof(struct PC)};
+ VkPipelineLayoutCreateInfo pli = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
+ .setLayoutCount = 1, .pSetLayouts = &G.dsl,
+ .pushConstantRangeCount = 1, .pPushConstantRanges = &pcr};
+ VKCHECK(vkCreatePipelineLayout(G.dev, &pli, NULL, &G.plyt), "pipelineLayout");
+
+ VkComputePipelineCreateInfo cpi = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = G.shader, .pName = "main"},
+ .layout = G.plyt};
+ VKCHECK(vkCreateComputePipelines(G.dev, VK_NULL_HANDLE, 1, &cpi, NULL, &G.pipe), "pipeline");
+
+ VkDescriptorPoolSize ps = {.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 4};
+ VkDescriptorPoolCreateInfo dpi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
+ .maxSets = 1, .poolSizeCount = 1, .pPoolSizes = &ps};
+ VKCHECK(vkCreateDescriptorPool(G.dev, &dpi, NULL, &G.dpool), "descPool");
+ VkDescriptorSetAllocateInfo dsa = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
+ .descriptorPool = G.dpool, .descriptorSetCount = 1, .pSetLayouts = &G.dsl};
+ VKCHECK(vkAllocateDescriptorSets(G.dev, &dsa, &G.dset), "allocDescSet");
+
+ VkCommandPoolCreateInfo cpci = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
+ .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = G.qfam};
+ VKCHECK(vkCreateCommandPool(G.dev, &cpci, NULL, &G.cpool), "cmdPool");
+ VkCommandBufferAllocateInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
+ .commandPool = G.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
+ VKCHECK(vkAllocateCommandBuffers(G.dev, &cbi, &G.cmd), "cmdBuf");
+ VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
+ VKCHECK(vkCreateFence(G.dev, &fi, NULL, &G.fence), "fence");
+
+ G.ready = 1;
+ VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(G.phys, &p);
+ fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u\n", p.deviceName, G.qfam, G.memtype);
+ return 1;
+}
+
+int coli_vk_available(void) { return G.ready; }
+
+void coli_vk_mem_info(size_t *used, size_t *count) {
+ if (used) *used = G.used_bytes;
+ if (count) *count = G.tensor_count;
+}
+
+static int upload_tensor(ColiVkTensor **out, const void *weights, const float *scales,
+ int fmt, int I, int O) {
+ if (*out) return (*out)->fmt == fmt && (*out)->I == I && (*out)->O == O;
+ if (fmt != 1 && fmt != 2) return 0;
+ ColiVkTensor *t = calloc(1, sizeof(*t));
+ if (!t) return 0;
+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I);
+ size_t stride = (size_t)t->rowWords * 4; // padded row bytes
+ size_t cpu_rb = fmt == 1 ? (size_t)I : (size_t)(I + 1) / 2;
+ t->wbytes = stride * (size_t)O;
+ void *wptr;
+ if (!alloc_hostvis(t->wbytes, &t->wbuf, &t->wmem, &wptr)) { free(t); return 0; }
+ memset(wptr, 0, t->wbytes);
+ for (int o = 0; o < O; o++) // copy row-by-row into padded layout
+ memcpy((uint8_t *)wptr + (size_t)o * stride,
+ (const uint8_t *)weights + (size_t)o * cpu_rb, cpu_rb);
+ void *sptr;
+ if (!alloc_hostvis((size_t)O * sizeof(float), &t->sbuf, &t->smem, &sptr)) {
+ vkDestroyBuffer(G.dev, t->wbuf, NULL); vkFreeMemory(G.dev, t->wmem, NULL); free(t); return 0;
+ }
+ memcpy(sptr, scales, (size_t)O * sizeof(float));
+ // Counters are touched concurrently: frees run from expert_load under
+ // `#pragma omp parallel`, so RMW them atomically (torn counts otherwise).
+ __atomic_add_fetch(&G.used_bytes, t->wbytes + (size_t)O * sizeof(float), __ATOMIC_RELAXED);
+ __atomic_add_fetch(&G.tensor_count, 1, __ATOMIC_RELAXED);
+ *out = t;
+ return 1;
+}
+
+int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
+ const void *weights, const float *scales,
+ int fmt, int S, int I, int O) {
+ if (!G.ready || S < 1 || !upload_tensor(tensor, weights, scales, fmt, I, O)) return 0;
+ ColiVkTensor *t = *tensor;
+ size_t xb = (size_t)S * I * sizeof(float), yb = (size_t)S * O * sizeof(float);
+ if (!scratch_reserve(&G.x, xb) || !scratch_reserve(&G.y, yb)) return 0;
+ memcpy(G.x.ptr, x, xb);
+
+ VkDescriptorBufferInfo bi[4] = {
+ {.buffer = G.x.buf, .range = VK_WHOLE_SIZE},
+ {.buffer = t->wbuf, .range = VK_WHOLE_SIZE},
+ {.buffer = t->sbuf, .range = VK_WHOLE_SIZE},
+ {.buffer = G.y.buf, .range = VK_WHOLE_SIZE}};
+ VkWriteDescriptorSet w[4];
+ for (int i = 0; i < 4; i++) w[i] = (VkWriteDescriptorSet){
+ .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = G.dset,
+ .dstBinding = i, .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[i]};
+ vkUpdateDescriptorSets(G.dev, 4, w, 0, NULL);
+
+ VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");
+ VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
+ .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
+ VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");
+ vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);
+ vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.dset, 0, NULL);
+ struct PC pc = {fmt, S, I, O, t->rowWords};
+ vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+ vkCmdDispatch(G.cmd, (uint32_t)O, (uint32_t)S, 1);
+ VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");
+
+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
+ .commandBufferCount = 1, .pCommandBuffers = &G.cmd};
+ VKCHECK(vkResetFences(G.dev, 1, &G.fence), "resetFence");
+ VKCHECK(vkQueueSubmit(G.queue, 1, &si, G.fence), "queueSubmit");
+ // Bounded wait: a GPU hang/TDR must never wedge the process. 10s is orders of
+ // magnitude over a single-GEMV dispatch; on timeout/device-loss disable VK for
+ // the rest of the run and fall back to CPU (the caller degrades on our 0 return).
+ VkResult wr = vkWaitForFences(G.dev, 1, &G.fence, VK_TRUE, 10000000000ULL);
+ if (wr != VK_SUCCESS) {
+ fprintf(stderr, "[VK] fence wait failed: %d β€” disabling GPU offload, staying on CPU\n", wr);
+ G.ready = 0;
+ return 0;
+ }
+ memcpy(y, G.y.ptr, yb);
+ return 1;
+}
+
+void coli_vk_tensor_free(ColiVkTensor *t) {
+ if (!t) return;
+ // If the device was lost/disabled (the fence-timeout path sets G.ready=0), a submission
+ // may still reference these buffers β€” do NOT vkDestroy into a dead device (GPU-side UAF).
+ // Leak the GPU handles (we're degrading to CPU for the rest of the run) and reclaim the
+ // host struct + counters only.
+ if (G.ready) {
+ if (t->wbuf) { vkDestroyBuffer(G.dev, t->wbuf, NULL); vkFreeMemory(G.dev, t->wmem, NULL); }
+ if (t->sbuf) { vkDestroyBuffer(G.dev, t->sbuf, NULL); vkFreeMemory(G.dev, t->smem, NULL); }
+ }
+ // Mirror upload_tensor exactly (weights + scales), atomically β€” otherwise
+ // used_bytes leaks the O*float scales buffer on every free and drifts upward.
+ __atomic_sub_fetch(&G.tensor_count, 1, __ATOMIC_RELAXED);
+ __atomic_sub_fetch(&G.used_bytes, t->wbytes + (size_t)t->O * sizeof(float), __ATOMIC_RELAXED);
+ free(t);
+}
+
+size_t coli_vk_tensor_bytes(const ColiVkTensor *t) { return t ? t->wbytes : 0; }
+
+void coli_vk_shutdown(void) {
+ if (!G.ready) return;
+ vkDeviceWaitIdle(G.dev);
+ if (G.x.buf) { vkDestroyBuffer(G.dev, G.x.buf, NULL); vkFreeMemory(G.dev, G.x.mem, NULL); }
+ if (G.y.buf) { vkDestroyBuffer(G.dev, G.y.buf, NULL); vkFreeMemory(G.dev, G.y.mem, NULL); }
+ vkDestroyFence(G.dev, G.fence, NULL);
+ vkDestroyCommandPool(G.dev, G.cpool, NULL);
+ vkDestroyDescriptorPool(G.dev, G.dpool, NULL);
+ vkDestroyPipeline(G.dev, G.pipe, NULL);
+ vkDestroyPipelineLayout(G.dev, G.plyt, NULL);
+ vkDestroyDescriptorSetLayout(G.dev, G.dsl, NULL);
+ vkDestroyShaderModule(G.dev, G.shader, NULL);
+ vkDestroyDevice(G.dev, NULL);
+ vkDestroyInstance(G.inst, NULL);
+ memset(&G, 0, sizeof(G));
+}
+
+#ifdef VK_TEST
+// ---- standalone GPU-vs-CPU validation + microbench --------------------------
+#include <math.h>
+#include <time.h>
+
+static double now(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec + ts.tv_nsec / 1e9; }
+
+static float deq(const uint8_t *row, int fmt, int i) {
+ if (fmt == 1) { int b = ((const int8_t *)row)[i]; return (float)b; }
+ uint8_t v = row[i >> 1]; int nib = (i & 1) ? (v >> 4) : (v & 15); return (float)(nib - 8);
+}
+
+static void cpu_ref(float *y, const float *x, const uint8_t *w, const float *sc,
+ int fmt, int S, int I, int O) {
+ size_t rb = fmt == 1 ? (size_t)I : (size_t)(I + 1) / 2;
+ for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {
+ double sum = 0; const uint8_t *row = w + (size_t)o * rb;
+ for (int i = 0; i < I; i++) sum += x[s * I + i] * deq(row, fmt, i);
+ y[s * O + o] = (float)(sum * sc[o]);
+ }
+}
+
+static int run_case(int fmt, int S, int I, int O, int iters) {
+ size_t rb = fmt == 1 ? (size_t)I : (size_t)(I + 1) / 2;
+ float *x = malloc((size_t)S * I * sizeof(float));
+ uint8_t *w = malloc(rb * O);
+ float *sc = malloc((size_t)O * sizeof(float));
+ float *yg = malloc((size_t)S * O * sizeof(float));
+ float *yc = malloc((size_t)S * O * sizeof(float));
+ for (int i = 0; i < S * I; i++) x[i] = (float)((rand() % 200 - 100) / 100.0);
+ for (size_t i = 0; i < rb * O; i++) w[i] = rand() & 0xff;
+ for (int o = 0; o < O; o++) sc[o] = 0.01f + (rand() % 100) / 10000.0f;
+
+ ColiVkTensor *t = NULL;
+ if (!coli_vk_matmul(&t, yg, x, w, sc, fmt, S, I, O)) { printf("matmul failed\n"); return 1; }
+ cpu_ref(yc, x, w, sc, fmt, S, I, O);
+ double maxerr = 0, maxrel = 0;
+ for (int i = 0; i < S * O; i++) {
+ double e = fabs(yg[i] - yc[i]); if (e > maxerr) maxerr = e;
+ if (fabs(yc[i]) > 1e-2) { double r = e / fabs(yc[i]); if (r > maxrel) maxrel = r; }
+ }
+ // microbench (GPU)
+ double t0 = now();
+ for (int k = 0; k < iters; k++) coli_vk_matmul(&t, yg, x, w, sc, fmt, S, I, O);
+ double gpu_ms = (now() - t0) * 1000 / iters;
+ // microbench (CPU ref, 1 iter β€” it's slow)
+ double c0 = now(); cpu_ref(yc, x, w, sc, fmt, S, I, O); double cpu_ms = (now() - c0) * 1000;
+ printf("fmt=%d S=%d I=%d O=%d | maxerr=%.4g maxrel=%.4g | gpu=%.3f ms cpu_ref=%.3f ms\n",
+ fmt, S, I, O, maxerr, maxrel, gpu_ms, cpu_ms);
+ coli_vk_tensor_free(t);
+ free(x); free(w); free(sc); free(yg); free(yc);
+ return maxrel > 1e-3 ? 1 : 0;
+}
+
+int main(int argc, char **argv) {
+ const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";
+ if (!coli_vk_init(spv)) { printf("vk init failed\n"); return 1; }
+ srand(1234);
+ int bad = 0;
+ bad |= run_case(1, 1, 6144, 1536, 50); // int8 expert gate/up shape (S=1 decode)
+ bad |= run_case(2, 1, 6144, 1536, 50); // int4 expert
+ bad |= run_case(1, 1, 1536, 6144, 50); // down proj shape
+ bad |= run_case(2, 8, 6144, 1536, 20); // prefill/MTP batch
+ bad |= run_case(1, 1, 512, 512, 100); // small
+ printf(bad ? "FAIL\n" : "PASS\n");
+ coli_vk_shutdown();
+ return bad;
+}
+#endif
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
new file mode 100644
index 00000000..2f6f6bc6
--- /dev/null
+++ b/c/backend_vulkan.h
@@ -0,0 +1,40 @@
+#ifndef COLIBRI_BACKEND_VULKAN_H
+#define COLIBRI_BACKEND_VULKAN_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Opaque persistent device copy of one resident quantized tensor,
+ * mirroring backend_cuda.h. On Strix Halo the "upload" writes into
+ * HOST_VISIBLE|DEVICE_LOCAL memory β€” same physical RAM the iGPU reads,
+ * so there is no PCIe copy, unlike the discrete-CUDA path. */
+typedef struct ColiVkTensor ColiVkTensor;
+
+/* Bring up instance/device/queue/pipeline. Returns 1 on success.
+ * spv_path points at the compiled qmatmul.spv. */
+int coli_vk_init(const char *spv_path);
+void coli_vk_shutdown(void);
+int coli_vk_available(void);
+void coli_vk_mem_info(size_t *used_bytes, size_t *tensor_count);
+
+/* y[S,O] = (x[S,I] @ dequant(W[O,I])^T) * scale[O].
+ * fmt matches QT in glm.c: 1=int8, 2=int4. (0=f32,3=int2 fall back to CPU.)
+ * First call uploads W+scales; later calls reuse the resident copy.
+ * Returns 1 on success, 0 if unavailable / unsupported fmt. */
+int coli_vk_matmul(ColiVkTensor **tensor,
+ float *y, const float *x,
+ const void *weights, const float *scales,
+ int fmt, int S, int I, int O);
+
+void coli_vk_tensor_free(ColiVkTensor *t);
+size_t coli_vk_tensor_bytes(const ColiVkTensor *t);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/c/shaders/qmatmul.comp b/c/shaders/qmatmul.comp
new file mode 100644
index 00000000..69fb7cb0
--- /dev/null
+++ b/c/shaders/qmatmul.comp
@@ -0,0 +1,74 @@
+#version 450
+// Quantized mat-vec/mat-mat mirroring colibri's CUDA quant_matmul:
+// y[s,o] = ( sum_i x[s,i] * dequant(W[o,i]) ) * scale[o]
+// fmt: 1=int8, 2=int4 (packed 2/byte, nibble-8). Weights are a byte buffer
+// exposed as uint32 words; we extract bytes/nibbles manually (no 8-bit-storage
+// extension dependency). One workgroup per (o,s); threads reduce over I.
+layout(local_size_x = 256) in;
+
+layout(std430, binding = 0) readonly buffer X { float x[]; };
+layout(std430, binding = 1) readonly buffer W { uint w[]; }; // row-major, row o at o*rowWords
+layout(std430, binding = 2) readonly buffer Sc { float scale[]; };
+layout(std430, binding = 3) writeonly buffer Y { float y[]; };
+
+layout(push_constant) uniform P {
+ int fmt; // 1=int8, 2=int4
+ int S, I, O;
+ int rowWords; // uint32 words per weight row
+} p;
+
+shared float partial[256];
+
+// signed int8 from a uint32 word's byte lane
+float i8(uint word, int lane) {
+ int b = int((word >> (uint(lane) * 8u)) & 0xffu);
+ if (b >= 128) b -= 256;
+ return float(b);
+}
+// int4 (value - 8) from a uint32 word's nibble lane (0..7)
+float i4(uint word, int lane) {
+ int nib = int((word >> (uint(lane) * 4u)) & 0xfu);
+ return float(nib - 8);
+}
+
+void main() {
+ int o = int(gl_WorkGroupID.x);
+ int s = int(gl_WorkGroupID.y);
+ int t = int(gl_LocalInvocationID.x);
+ int nth = int(gl_WorkGroupSize.x);
+
+ uint rowBase = uint(o) * uint(p.rowWords);
+ int xbase = s * p.I;
+ float sum = 0.0;
+
+ if (p.fmt == 1) {
+ // 4 int8 weights per uint word; stride threads over words.
+ int words = (p.I + 3) / 4;
+ for (int wi = t; wi < words; wi += nth) {
+ uint packed = w[rowBase + uint(wi)];
+ int i0 = wi * 4;
+ for (int k = 0; k < 4; k++) {
+ int i = i0 + k;
+ if (i < p.I) sum += x[xbase + i] * i8(packed, k);
+ }
+ }
+ } else { // fmt == 2, int4
+ int words = (p.I + 7) / 8;
+ for (int wi = t; wi < words; wi += nth) {
+ uint packed = w[rowBase + uint(wi)];
+ int i0 = wi * 8;
+ for (int k = 0; k < 8; k++) {
+ int i = i0 + k;
+ if (i < p.I) sum += x[xbase + i] * i4(packed, k);
+ }
+ }
+ }
+
+ partial[t] = sum;
+ barrier();
+ for (int n = nth >> 1; n > 0; n >>= 1) {
+ if (t < n) partial[t] += partial[t + n];
+ barrier();
+ }
+ if (t == 0) y[s * p.O + o] = partial[0] * scale[o];
+}