Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Vulkan backend: subgroup shader + overhead caching + batched probe

Steve Markgraf3e41761bba23
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index fcf7a623..8bc72882 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -45,6 +45,13 @@ static struct {
VkCommandBuffer cmd;
VkFence fence;
Scratch x, y;
+ /* resubmit cache: skip vkUpdateDescriptorSets + command re-record when the bound
+ * tensor / shape / scratch buffers are unchanged from the previous call (the hot-
+ * expert-called-repeatedly pattern). The synchronous fence wait each call means no
+ * submission is ever in flight, so rebinding/re-recording only when something
+ * actually changed is safe. */
+ ColiVkTensor *bound_tensor; int bound_S, bound_I, bound_O, cmd_ready;
+ VkBuffer bound_xbuf, bound_ybuf;
size_t used_bytes, tensor_count;
} G;
@@ -249,31 +256,46 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
if (!G.ready || S < 1 || !upload_tensor(tensor, weights, scales, fmt, I, O)) return 0;
ColiVkTensor *t = *tensor;
size_t xb = (size_t)S * I * sizeof(float), yb = (size_t)S * O * sizeof(float);
+ VkBuffer old_x = G.x.buf, old_y = G.y.buf;
if (!scratch_reserve(&G.x, xb) || !scratch_reserve(&G.y, yb)) return 0;
memcpy(G.x.ptr, x, xb);
- VkDescriptorBufferInfo bi[4] = {
- {.buffer = G.x.buf, .range = VK_WHOLE_SIZE},
- {.buffer = t->wbuf, .range = VK_WHOLE_SIZE},
- {.buffer = t->sbuf, .range = VK_WHOLE_SIZE},
- {.buffer = G.y.buf, .range = VK_WHOLE_SIZE}};
- VkWriteDescriptorSet w[4];
- for (int i = 0; i < 4; i++) w[i] = (VkWriteDescriptorSet){
- .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = G.dset,
- .dstBinding = i, .descriptorCount = 1,
- .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[i]};
- vkUpdateDescriptorSets(G.dev, 4, w, 0, NULL);
-
- VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");
- VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
- .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
- VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");
- vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);
- vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.dset, 0, NULL);
- struct PC pc = {fmt, S, I, O, t->rowWords};
- vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
- vkCmdDispatch(G.cmd, (uint32_t)O, (uint32_t)S, 1);
- VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");
+ /* Rebind descriptors only when the tensor or a scratch buffer changed (a realloc
+ * makes the old VkBuffer handle stale); otherwise the previous binding is still valid. */
+ int rebind = G.bound_tensor != t || G.x.buf != old_x || G.y.buf != old_y
+ || G.bound_xbuf != G.x.buf || G.bound_ybuf != G.y.buf;
+ if (rebind) {
+ VkDescriptorBufferInfo bi[4] = {
+ {.buffer = G.x.buf, .range = VK_WHOLE_SIZE},
+ {.buffer = t->wbuf, .range = VK_WHOLE_SIZE},
+ {.buffer = t->sbuf, .range = VK_WHOLE_SIZE},
+ {.buffer = G.y.buf, .range = VK_WHOLE_SIZE}};
+ VkWriteDescriptorSet w[4];
+ for (int i = 0; i < 4; i++) w[i] = (VkWriteDescriptorSet){
+ .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = G.dset,
+ .dstBinding = i, .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[i]};
+ vkUpdateDescriptorSets(G.dev, 4, w, 0, NULL);
+ G.bound_tensor = t; G.bound_xbuf = G.x.buf; G.bound_ybuf = G.y.buf;
+ }
+
+ /* Re-record the command buffer only when the binding or the dispatch shape changed.
+ * Recorded WITHOUT one-time-submit so the same buffer can be resubmitted verbatim β€”
+ * for repeated calls to the same expert this drops setup to a bare submit+wait. */
+ if (rebind || !G.cmd_ready || G.bound_S != S || G.bound_I != I || G.bound_O != O) {
+ VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");
+ VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
+ VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");
+ vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);
+ vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.dset, 0, NULL);
+ struct PC pc = {fmt, S, I, O, t->rowWords};
+ vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+ /* Grid-stride shader: one subgroup per output row (~8 rows/workgroup at wave32).
+ * Launch ~O/8 workgroups for occupancy; the shader loops to cover any O / wave width. */
+ vkCmdDispatch(G.cmd, (uint32_t)((O + 7) / 8), (uint32_t)S, 1);
+ VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");
+ G.cmd_ready = 1; G.bound_S = S; G.bound_I = I; G.bound_O = O;
+ }
VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.commandBufferCount = 1, .pCommandBuffers = &G.cmd};
@@ -294,6 +316,7 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
void coli_vk_tensor_free(ColiVkTensor *t) {
if (!t) return;
+ if (G.bound_tensor == t) { G.bound_tensor = NULL; G.cmd_ready = 0; } /* drop stale cache */
// If the device was lost/disabled (the fence-timeout path sets G.ready=0), a submission
// may still reference these buffers β€” do NOT vkDestroy into a dead device (GPU-side UAF).
// Leak the GPU handles (we're degrading to CPU for the rest of the run) and reclaim the
@@ -383,6 +406,42 @@ static int run_case(int fmt, int S, int I, int O, int iters) {
return maxrel > 1e-3 ? 1 : 0;
}
+/* Batched throughput: record N dispatches in ONE command buffer, one submit + one
+ * fence wait β€” the amortized per-matmul cost with the submit roundtrip spread across
+ * the batch (how the real expert tier would drive it). Reuses the descriptor binding
+ * left by a prior coli_vk_matmul call for this tensor/shape. */
+static double bench_batched(ColiVkTensor *t, const float *x, int fmt, int S, int I, int O, int N) {
+ size_t xb = (size_t)S * I * sizeof(float);
+ memcpy(G.x.ptr, x, xb);
+ vkResetCommandBuffer(G.cmd, 0);
+ VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
+ vkBeginCommandBuffer(G.cmd, &begin);
+ vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);
+ vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.dset, 0, NULL);
+ struct PC pc = {fmt, S, I, O, t->rowWords};
+ vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+ VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT};
+ for (int i = 0; i < N; i++) {
+ vkCmdDispatch(G.cmd, (uint32_t)((O + 7) / 8), (uint32_t)S, 1);
+ vkCmdPipelineBarrier(G.cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+ VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &mb, 0, NULL, 0, NULL);
+ }
+ vkEndCommandBuffer(G.cmd);
+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};
+ for (int warm = 0; warm < 2; warm++) {
+ vkResetFences(G.dev, 1, &G.fence); vkQueueSubmit(G.queue, 1, &si, G.fence);
+ vkWaitForFences(G.dev, 1, &G.fence, VK_TRUE, 10000000000ULL);
+ }
+ int iters = 10; double t0 = now();
+ for (int k = 0; k < iters; k++) {
+ vkResetFences(G.dev, 1, &G.fence); vkQueueSubmit(G.queue, 1, &si, G.fence);
+ vkWaitForFences(G.dev, 1, &G.fence, VK_TRUE, 10000000000ULL);
+ }
+ G.cmd_ready = 0; /* we clobbered the cached command buffer */
+ return (now() - t0) * 1000.0 / iters / N; /* ms per matmul, roundtrip amortized */
+}
+
int main(int argc, char **argv) {
const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";
if (!coli_vk_init(spv)) { printf("vk init failed\n"); return 1; }
@@ -393,6 +452,29 @@ int main(int argc, char **argv) {
bad |= run_case(1, 1, 1536, 6144, 50); // down proj shape
bad |= run_case(2, 8, 6144, 1536, 20); // prefill/MTP batch
bad |= run_case(1, 1, 512, 512, 100); // small
+ /* Batched (amortized) throughput on the int4 expert shapes β€” the real expert-tier pattern. */
+ {
+ int I = 6144, O = 2048; /* our gate/up dims */
+ float *x = malloc((size_t)I * 4); uint8_t *w = malloc((size_t)(I + 1) / 2 * O); float *sc = malloc((size_t)O * 4);
+ for (int i = 0; i < I; i++) x[i] = (rand() % 200 - 100) / 100.0f;
+ for (size_t i = 0; i < (size_t)(I + 1) / 2 * O; i++) w[i] = rand() & 0xff;
+ for (int o = 0; o < O; o++) sc[o] = 0.01f + (rand() % 100) / 10000.0f;
+ float *y = malloc((size_t)O * 4);
+ ColiVkTensor *t = NULL; coli_vk_matmul(&t, y, x, w, sc, 2, 1, I, O); /* bind */
+ printf("BATCHED int4 S=1 6144->2048 (our gate/up): %.4f ms/matmul (N=64, one submit)\n",
+ bench_batched(t, x, 2, 1, I, O, 64));
+ coli_vk_tensor_free(t); free(x); free(w); free(sc); free(y);
+ I = 2048; O = 6144; /* our down dims */
+ x = malloc((size_t)I * 4); w = malloc((size_t)(I + 1) / 2 * O); sc = malloc((size_t)O * 4);
+ for (int i = 0; i < I; i++) x[i] = (rand() % 200 - 100) / 100.0f;
+ for (size_t i = 0; i < (size_t)(I + 1) / 2 * O; i++) w[i] = rand() & 0xff;
+ for (int o = 0; o < O; o++) sc[o] = 0.01f + (rand() % 100) / 10000.0f;
+ y = malloc((size_t)O * 4);
+ t = NULL; coli_vk_matmul(&t, y, x, w, sc, 2, 1, I, O);
+ printf("BATCHED int4 S=1 2048->6144 (our down): %.4f ms/matmul (N=64, one submit)\n",
+ bench_batched(t, x, 2, 1, I, O, 64));
+ coli_vk_tensor_free(t); free(x); free(w); free(sc); free(y);
+ }
printf(bad ? "FAIL\n" : "PASS\n");
coli_vk_shutdown();
return bad;
diff --git a/c/shaders/qmatmul.comp b/c/shaders/qmatmul.comp
index 69fb7cb0..4f1df3b1 100644
--- a/c/shaders/qmatmul.comp
+++ b/c/shaders/qmatmul.comp
@@ -2,73 +2,76 @@
// Quantized mat-vec/mat-mat mirroring colibri's CUDA quant_matmul:
// y[s,o] = ( sum_i x[s,i] * dequant(W[o,i]) ) * scale[o]
// fmt: 1=int8, 2=int4 (packed 2/byte, nibble-8). Weights are a byte buffer
-// exposed as uint32 words; we extract bytes/nibbles manually (no 8-bit-storage
-// extension dependency). One workgroup per (o,s); threads reduce over I.
+// exposed as uint32 words; bytes/nibbles are extracted manually.
+//
+// Optimized decode (S=1) GEMV, techniques adapted from llama.cpp's mul_mat_vec
+// (MIT) to colibri's per-row int4/int8 layout:
+// - x[s,:] loaded ONCE into shared memory, reused across every output row in
+// the workgroup (x lives in host-visible scratch, so this avoids re-reading
+// it over the BAR for each row);
+// - one SUBGROUP per output row: its lanes stride the row's weight words with
+// full coalescing, then a single subgroupAdd replaces the barrier-heavy
+// shared-memory tree reduction;
+// - grid-stride over rows so a fixed, occupancy-friendly workgroup count
+// covers any O and any subgroup width (RADV wave32/64).
+#extension GL_KHR_shader_subgroup_arithmetic : require
layout(local_size_x = 256) in;
layout(std430, binding = 0) readonly buffer X { float x[]; };
-layout(std430, binding = 1) readonly buffer W { uint w[]; }; // row-major, row o at o*rowWords
+layout(std430, binding = 1) readonly buffer W { uint w[]; }; // row o at o*rowWords
layout(std430, binding = 2) readonly buffer Sc { float scale[]; };
layout(std430, binding = 3) writeonly buffer Y { float y[]; };
layout(push_constant) uniform P {
- int fmt; // 1=int8, 2=int4
+ int fmt; // 1=int8, 2=int4
int S, I, O;
- int rowWords; // uint32 words per weight row
+ int rowWords; // uint32 words per weight row
} p;
-shared float partial[256];
+// Max hidden dim across GLM expert matmuls is 6144 (24 KB LDS); down-proj uses 2048.
+shared float xsh[6144];
-// signed int8 from a uint32 word's byte lane
float i8(uint word, int lane) {
int b = int((word >> (uint(lane) * 8u)) & 0xffu);
if (b >= 128) b -= 256;
return float(b);
}
-// int4 (value - 8) from a uint32 word's nibble lane (0..7)
float i4(uint word, int lane) {
- int nib = int((word >> (uint(lane) * 4u)) & 0xfu);
- return float(nib - 8);
+ return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8);
}
void main() {
- int o = int(gl_WorkGroupID.x);
- int s = int(gl_WorkGroupID.y);
- int t = int(gl_LocalInvocationID.x);
+ int s = int(gl_WorkGroupID.y);
+ int tid = int(gl_LocalInvocationID.x);
int nth = int(gl_WorkGroupSize.x);
- uint rowBase = uint(o) * uint(p.rowWords);
- int xbase = s * p.I;
- float sum = 0.0;
+ // Stage x[s,:] into shared once; all rows this workgroup computes reuse it.
+ for (int i = tid; i < p.I; i += nth) xsh[i] = x[s * p.I + i];
+ barrier();
+
+ int sg = int(gl_SubgroupID);
+ int lane = int(gl_SubgroupInvocationID);
+ int sgsize = int(gl_SubgroupSize);
+ int nsg = nth / sgsize; // subgroups (= rows) per workgroup
+ int stride = int(gl_NumWorkGroups.x) * nsg; // grid-stride over output rows
- if (p.fmt == 1) {
- // 4 int8 weights per uint word; stride threads over words.
- int words = (p.I + 3) / 4;
- for (int wi = t; wi < words; wi += nth) {
- uint packed = w[rowBase + uint(wi)];
- int i0 = wi * 4;
- for (int k = 0; k < 4; k++) {
- int i = i0 + k;
- if (i < p.I) sum += x[xbase + i] * i8(packed, k);
+ for (int o = int(gl_WorkGroupID.x) * nsg + sg; o < p.O; o += stride) {
+ uint rowBase = uint(o) * uint(p.rowWords);
+ float sum = 0.0;
+ if (p.fmt == 1) {
+ int words = (p.I + 3) / 4;
+ for (int wi = lane; wi < words; wi += sgsize) {
+ uint pk = w[rowBase + uint(wi)]; int i0 = wi * 4;
+ for (int k = 0; k < 4; k++) { int i = i0 + k; if (i < p.I) sum += xsh[i] * i8(pk, k); }
}
- }
- } else { // fmt == 2, int4
- int words = (p.I + 7) / 8;
- for (int wi = t; wi < words; wi += nth) {
- uint packed = w[rowBase + uint(wi)];
- int i0 = wi * 8;
- for (int k = 0; k < 8; k++) {
- int i = i0 + k;
- if (i < p.I) sum += x[xbase + i] * i4(packed, k);
+ } else { // int4
+ int words = (p.I + 7) / 8;
+ for (int wi = lane; wi < words; wi += sgsize) {
+ uint pk = w[rowBase + uint(wi)]; int i0 = wi * 8;
+ for (int k = 0; k < 8; k++) { int i = i0 + k; if (i < p.I) sum += xsh[i] * i4(pk, k); }
}
}
+ float tot = subgroupAdd(sum);
+ if (lane == 0) y[s * p.O + o] = tot * scale[o];
}
-
- partial[t] = sum;
- barrier();
- for (int n = nth >> 1; n > 0; n >>= 1) {
- if (t < n) partial[t] += partial[t + n];
- barrier();
- }
- if (t == 0) y[s * p.O + o] = partial[0] * scale[o];
}