Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: a decode step's experts as one grouped GEMV per phase
huppiflupp39cc8c92c4f2
diff --git a/c/Makefile b/c/Makefileindex 7a8cb3d9..a53d0802 100644--- a/c/Makefile+++ b/c/Makefile@@ -709,7 +709,7 @@ endifVK_OBJ = backend_vulkan.o vk_tier.oVK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv shaders/expert_act.spv \- shaders/expert_act_v4.spv shaders/qmatmul_grp.spv shaders/qmatmul_grp_gate_up.spv shaders/expert_sum.spv+ shaders/expert_act_v4.spv shaders/qmatmul_grp.spv shaders/qmatmul_grp_gate_up.spv shaders/expert_sum.spv shaders/qmatmul_grp_gemv.spv shaders/qmatmul_grp_gemv_gate_up.spv# vk_chain.o: a layer's dense chain recorded into one submission (vk_chain.h), and# its shaders; an engine that does not call it links nothing from it.VK_OBJ += vk_chain.o@@ -882,6 +882,9 @@ shaders/qmatmul_grp_gate_up.spv: shaders/qmatmul_grp.comp@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }$(GLSLC) --target-env=vulkan1.2 -DGATE_UP $< -o $@ || { [ -f $@ ] && touch $@ && \echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }+shaders/qmatmul_grp_gemv_gate_up.spv: shaders/qmatmul_grp_gemv.comp+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }+ $(GLSLC) --target-env=vulkan1.2 -DGATE_UP $< -o $@vk_chain.o: vk_chain.c .build-config$(CC) $(CFLAGS) -c vk_chain.c -o $@shaders/%.spv: shaders/%.compdiff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex c873bb15..2f7e3521 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -3426,9 +3426,10 @@ typedef struct XbCtx {* batch, [1 + h] sub-batch half h. */VkShaderModule sh_grp[2]; VkDescriptorSetLayout dsl_grp; VkPipelineLayout pl_grp;VkPipeline p_grp[2]; VkDescriptorPool grp_pool; VkDescriptorSet grp_set[3][2];+ VkShaderModule sh_gv[2]; VkPipeline p_gv[2]; /* qmatmul_grp_gemv.comp: the same batches below grp_rows */XbBuf grp_it[3][2], grp_et[3][2], grp_map[3];int grp_rows;- unsigned long long grp_batches;+ unsigned long long grp_batches, gv_batches;/* a big step's whole-step buffers (coli_vk_xb_step_*): its token rows once (xt), every* assignment's output row in place (ys, s*K + k), the routed sum (out) and its inputs */struct {@@ -3442,7 +3443,8 @@ typedef struct XbCtx {static XbCtx g_xb[2];static int xb_dev_ready(const XbCtx *X) { return X->d ? G2.ready : G.ready; }struct PCV4 { int rows, n; };-struct PCGRP { int I, O; float limit; int ibase; int kgat; }; /* qmatmul_grp.comp */+struct PCGRP { int I, O; float limit; int ibase; int kgat; int rpw; }; /* qmatmul_grp.comp, qmatmul_grp_gemv.comp (rpw) */+#define XB_GV_RPW 64 /* outputs per grouped-GEMV workgroup */#define XB_GRP_TT 4 /* 16-row tiles of an expert per workgroup: 64 rows (2, 4 or 8) */static size_t xb_up(size_t v, size_t a) { return (v + a - 1) / a * a; }@@ -3711,6 +3713,23 @@ static void xb_grp_init(XbCtx *X) {X->p_grp[v] = VK_NULL_HANDLE; return;}}+ /* the grouped GEMV for the small batches (COLI_VK_XB_GEMV=0: per-expert GEMVs) */+ const char *ge = getenv("COLI_VK_XB_GEMV");+ const char *gnm[2] = {"qmatmul_grp_gemv.spv", "qmatmul_grp_gemv_gate_up.spv"};+ for (int v = 0; v < 2 && !(ge && *ge == '0'); v++) {+ char path[1100];+ derive_dir_file(X->spv, gnm[v], path, sizeof path);+ if (!(X->sh_gv[v] = load_spv(X->dev, path))) { X->p_gv[0] = VK_NULL_HANDLE; break; }+ VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = X->sh_gv[v], .pName = "main",+ .pSpecializationInfo = &si}, .layout = X->pl_grp};+ if (vkCreateComputePipelines(X->dev, VK_NULL_HANDLE, 1, &ci, NULL, &X->p_gv[v]) != VK_SUCCESS) {+ X->p_gv[v] = VK_NULL_HANDLE; X->p_gv[0] = VK_NULL_HANDLE; break;+ }+ }+ if (!X->p_gv[0] || !X->p_gv[1]) X->p_gv[0] = X->p_gv[1] = VK_NULL_HANDLE;VkDescriptorPoolSize ps = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 30};VkDescriptorPoolCreateInfo dp = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,.maxSets = 6, .poolSizeCount = 1, .pPoolSizes = &ps};@@ -3912,24 +3931,29 @@ static uint64_t vk_addr(VkBuffer b) {* which the sets bind as their offsets. `slot`: whose tables and sets (0 the single* batch, 1 + h a sub-batch half). 0 = not taken, nothing recorded. *//* Whether the grouped route takes this batch (and its row total). */+/* Whether the grouped route takes this batch (and its row total): 1 the grouped GEMM+ * (grp_rows rows or more), 2 the grouped GEMV (fewer), 0 neither. */static int xb_grp_ok(const XbCtx *X, ColiVkExpert *const *ex, const int *rows, int count, int *total_out) {const int D = X->D, I = X->I;const size_t dr = (size_t)D * 4, ir = (size_t)I * 4;int total = 0;for (int c = 0; c < count; c++) total += rows[c];if (total_out) *total_out = total;- if (!X->grp_rows || total < X->grp_rows || X->act != COLI_VK_ACT_SWIGLU) return 0;- if (D % 128 || I % 128 || dr % X->align || ir % X->align) return 0;+ if (!X->grp_rows || X->act != COLI_VK_ACT_SWIGLU || dr % X->align || ir % X->align) return 0;+ int mode = total >= X->grp_rows ? 1 : X->p_gv[0] ? 2 : 0;+ if (mode == 1 && (D % 128 || I % 128)) return 0;+ if (mode == 2 && (D % 32 || I % 32 || D > 8192 || I > 8192)) return 0;+ if (!mode) return 0;for (int c = 0; c < count; c++) {const ColiVkTensor *t[3] = {ex[c]->g, ex[c]->u, ex[c]->d};for (int k = 0; k < 3; k++) {int f = t[k]->fmt;if (t[k]->dev || !t[k]->pool || !(f == 1 || f == 2 || f == 4) || t[k]->rowWords % 4 ||- (f == 4 && (t[k]->gs < 8 || t[k]->gs % 8))) return 0;+ (f == 4 && (t[k]->gs < 8 || t[k]->gs % (mode == 2 ? 32 : 8)))) return 0;}if (ex[c]->u->fmt != ex[c]->g->fmt || ex[c]->u->gs != ex[c]->g->gs) return 0;}- return 1;+ return mode;}/* amap (a big step's sub-batch, X->st.on): packed row j is assignment amap[j]; x comes from* the step's token rows and y goes to the step's output rows (the scratch's x and y unused). */@@ -3937,8 +3961,8 @@ static int xb_grp_record(XbCtx *X, VkCommandBuffer cmd, int slot, ColiVkExpert *int count, const size_t *off, size_t x0, size_t i0, size_t y0, const int *amap) {const int D = X->D, I = X->I;const size_t dr = (size_t)D * 4, ir = (size_t)I * 4;- int total = 0;- if (!xb_grp_ok(X, ex, rows, count, &total)) return 0;+ int total = 0, mode = xb_grp_ok(X, ex, rows, count, &total);+ if (!mode || (mode == 2 && amap)) return 0;if (amap) {XbBuf *mb = &X->grp_map[slot];size_t nb = (size_t)total * 4;@@ -3947,11 +3971,12 @@ static int xb_grp_record(XbCtx *X, VkCommandBuffer cmd, int slot, ColiVkExpert *}/* items: (expert, first row, 128-row output block), 64 rows a workgroup; the tiles* of one (expert, block) adjacent, so workgroups in flight share weights */- const int tm = 16 * XB_GRP_TT;+ /* the GEMV's items: (expert, row, XB_GV_RPW outputs) */+ const int tm = mode == 1 ? 16 * XB_GRP_TT : 1, bo = mode == 1 ? 128 : XB_GV_RPW;int nit[2] = {0, 0};for (int c = 0; c < count; c++) {int nt = (rows[c] + tm - 1) / tm;- nit[0] += nt * (D / 128); nit[1] += nt * (I / 128);+ nit[0] += nt * ((D + bo - 1) / bo); nit[1] += nt * ((I + bo - 1) / bo);}for (int v = 0; v < 2; v++) {XbBuf *ib = &X->grp_it[slot][v], *eb = &X->grp_et[slot][v];@@ -3973,9 +3998,10 @@ static int xb_grp_record(XbCtx *X, VkCommandBuffer cmd, int slot, ColiVkExpert *et[8] = (uint32_t)rows[c]; et[9] = (uint32_t)((off[3 * c] - x0) / dr);et[10] = (uint32_t)a->fmt; et[11] = (uint32_t)a->rowWords; et[12] = (uint32_t)a->gs;memcpy((uint8_t *)X->grp_et[slot][v].ptr + (size_t)c * 64, et, sizeof et); /* written, never read back */- for (int ob = 0; ob < O / 128; ob++)+ for (int ob = 0; ob < (O + bo - 1) / bo; ob++)for (int t0 = 0; t0 < rows[c]; t0 += tm) {- uint32_t q[4] = {(uint32_t)c, (uint32_t)t0, (uint32_t)ob, 0};+ /* the GEMM takes a block index, the GEMV its first output */+ uint32_t q[4] = {(uint32_t)c, (uint32_t)t0, (uint32_t)(mode == 1 ? ob : ob * bo), 1};memcpy(it + (size_t)n * 4, q, sizeof q); n++;}}@@ -4000,16 +4026,16 @@ static int xb_grp_record(XbCtx *X, VkCommandBuffer cmd, int slot, ColiVkExpert *.dstBinding = (uint32_t)b, .descriptorCount = 1,.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[b]};vkUpdateDescriptorSets(X->dev, 5, w, 0, NULL);- vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_grp[v]);+ vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, mode == 1 ? X->p_grp[v] : X->p_gv[v]);vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl_grp, 0, 1, &X->grp_set[slot][v], 0, NULL);for (int i0n = 0; i0n < nit[v]; i0n += 65535) { /* the grid's x limit every device has */- struct PCGRP pc = {v ? D : I, v ? I : D, X->limit, i0n, amap ? X->st.K : 0};+ struct PCGRP pc = {v ? D : I, v ? I : D, X->limit, i0n, amap ? X->st.K : 0, XB_GV_RPW};vkCmdPushConstants(cmd, X->pl_grp, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);vkCmdDispatch(cmd, (uint32_t)(nit[v] - i0n < 65535 ? nit[v] - i0n : 65535), 1, 1);}if (!step) xb_barrier(cmd);}- X->grp_batches++;+ if (mode == 1) X->grp_batches++; else X->gv_batches++;return 1;}/* Record one batch laid out at off[] (its x rows already written) into cmd, with its@@ -4230,7 +4256,7 @@ static void xb_stats(const XbCtx *X, ColiVkXbStats *st) {st->scratch_bytes = 2 * (X->x.region + 3 * X->g.region + X->y.region);st->sub_batches = X->sub_batches; st->sub_experts = X->sub_experts; st->sub_rows = X->sub_rows;st->sub_gemm = X->sub_gemm; st->sub_ms = X->sub_ms;- st->cooperative_matmuls = X->cooperative_matmuls; st->grouped_batches = X->grp_batches;+ st->cooperative_matmuls = X->cooperative_matmuls; st->grouped_batches = X->grp_batches; st->gemv_batches = X->gv_batches;}void coli_vk_xb_stats(ColiVkXbStats *st) { xb_stats(&g_xb[0], st); }void coli_vk_xb_stats_dev(int dev, ColiVkXbStats *st) {@@ -4436,7 +4462,7 @@ int coli_vk_xb_sub_issue_step(int h, ColiVkExpert *const *ex, const int *rows, iif (!X->st.on || h < 0 || h > 1 || count < 1) return 0;for (int c = 0; c < count; c++) if (!ex[c] || ex[c]->X != X) return 0;int total = 0;- if (!xb_grp_ok(X, ex, rows, count, &total)) { /* packed as before, copied into ys at the join */+ if (xb_grp_ok(X, ex, rows, count, &total) != 1) { /* packed as before, copied into ys at the join */const float **xr = malloc((size_t)total * sizeof(*xr));if (!xr) return 0;for (int j = 0; j < total; j++) xr[j] = X->st.xh + (size_t)(assign[j] / X->st.K) * X->D;@@ -4537,6 +4563,10 @@ static void xb_shutdown(XbCtx *X) {if (X->st.pool) vkDestroyDescriptorPool(X->dev, X->st.pool, NULL);if (X->st.pl) vkDestroyPipelineLayout(X->dev, X->st.pl, NULL);if (X->st.dsl) vkDestroyDescriptorSetLayout(X->dev, X->st.dsl, NULL);+ for (int v = 0; v < 2; v++) {+ if (X->p_gv[v]) vkDestroyPipeline(X->dev, X->p_gv[v], NULL);+ if (X->sh_gv[v]) vkDestroyShaderModule(X->dev, X->sh_gv[v], NULL);+ }if (X->pl_grp) vkDestroyPipelineLayout(X->dev, X->pl_grp, NULL);if (X->dsl_grp) vkDestroyDescriptorSetLayout(X->dev, X->dsl_grp, NULL);for (int k = 0; k < X->ndpools; k++) vkDestroyDescriptorPool(X->dev, X->dpools[k], NULL);diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex 8df1d870..7eb807c2 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -256,6 +256,7 @@ int coli_vk_xb_join(const float **yrows, double *device_ms);typedef struct {unsigned long long batches, experts, rows, gemm_experts;unsigned long long grouped_batches; /* batches that took the grouped GEMM (qmatmul_grp.comp) */+ unsigned long long gemv_batches; /* batches that took the grouped GEMV (qmatmul_grp_gemv.comp) */double device_ms; /* summed batch device time (timestamps) */int timestamps, queue_shared, gemm_rows;size_t scratch_bytes;diff --git a/c/shaders/qmatmul_grp_gemv.comp b/c/shaders/qmatmul_grp_gemv.compnew file mode 100644index 00000000..88652d48--- /dev/null+++ b/c/shaders/qmatmul_grp_gemv.comp@@ -0,0 +1,197 @@+#version 450+// The expert batch's grouped GEMV (backend_vulkan.c, xb_issue below the grouped GEMM's+// rows: decode, an MTP verify): ONE dispatch covers every expert of a batch, each+// workgroup one (expert, row, RPW outputs) item with the expert's weights read through+// qmatmul_grp.comp's address table. The same bindings (but binding 4) and expert entries.+// y[r,o] = sum_i x[r,i] * dequant(W[o,i]) (default)+// hid[r,o] = silu(gate[r,o]) * up[r,o], clamped as expert_act.comp's act 0 (-DGATE_UP)+// x is staged in shared memory once per workgroup; each row's 16-byte weight words are+// spread over lpr lanes (a power of two up to 64), RPS rows in flight per lane group,+// partial sums joined with clustered subgroup adds. A row's arithmetic does not depend+// on the batch around it.+// Host: I % 32 == 0, I <= 8192, rowWords % 4 == 0; fmt 4: gs % 32 == 0.+#extension GL_KHR_shader_subgroup_arithmetic : require+#extension GL_KHR_shader_subgroup_basic : require+#extension GL_KHR_shader_subgroup_clustered : require+#extension GL_EXT_control_flow_attributes : enable+#extension GL_EXT_buffer_reference : require+#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require++layout(local_size_x = 256) in;+layout(constant_id = 0) const uint UNUSED = 0;+layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer WRef4 { uvec4 v[]; };+layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer SRef { float v[]; };+layout(std430, binding = 0) readonly buffer X { vec4 x4[]; };+layout(std430, binding = 1) readonly buffer IT { uvec4 items[]; };+layout(std430, binding = 2) readonly buffer ET { uint etab[]; };+layout(std430, binding = 3) writeonly buffer Y { float y[]; };+#ifdef GATE_UP+layout(std430, binding = 4) readonly buffer AM { int amap[]; }; // kgat > 0: as qmatmul_grp.comp's+#endif+layout(push_constant) uniform P { int I, O; float limit; int ibase; int kgat; int rpw; } p;++#ifndef RPW+#define RPW 64u+#endif+#ifndef RPS+#define RPS 4u+#endif+#ifndef XSHN+#define XSHN 2048+#endif+#define MAXNR 1u+shared vec4 xsh[XSHN];++float i8(uint w, uint k) { return float(int(w << (24u - 8u * k)) >> 24); }+float d8(uint w, vec4 a) {+ return a.x * i8(w, 0u) + a.y * i8(w, 1u) + a.z * i8(w, 2u) + a.w * i8(w, 3u);+}+float d4(uint w, vec4 a, vec4 b) {+ return a.x * float(int(w & 15u) - 8) + a.y * float(int((w >> 4u) & 15u) - 8) ++ a.z * float(int((w >> 8u) & 15u) - 8) + a.w * float(int((w >> 12u) & 15u) - 8) ++ b.x * float(int((w >> 16u) & 15u) - 8) + b.y * float(int((w >> 20u) & 15u) - 8) ++ b.z * float(int((w >> 24u) & 15u) - 8) + b.w * float(int(w >> 28u) - 8);+}++float red(float v, uint lpr) {+ if (lpr == gl_SubgroupSize) return subgroupAdd(v);+ if (lpr == 64u) return subgroupClusteredAdd(v, 64u);+ if (lpr == 32u) return subgroupClusteredAdd(v, 32u);+ if (lpr == 16u) return subgroupClusteredAdd(v, 16u);+ if (lpr == 8u) return subgroupClusteredAdd(v, 8u);+ if (lpr == 4u) return subgroupClusteredAdd(v, 4u);+ if (lpr == 2u) return subgroupClusteredAdd(v, 2u);+ return v;+}++// Called by each lane in the current word iteration, including padding rows.+// Pair sharing is enabled only+// for complete pairs of words: both lanes then have the same row, validity,+// scale index and loop participation. Odd W4 (a final half-group) uses the+// ordinary load throughout. The collective is outside the even-lane load branch.+float group_scale(SRef sc, uint o, uint ng, uint j, uint gs, bool ok, bool pair_scale) {+ return ok ? sc.v[o * ng + (j * 32u) / gs] : 0.0;+}++// One matrix, one RPS block of output rows. The word-major inner loop shares each+// input pair across the RPS rows.+void matrix_block(WRef4 w, SRef sc, int fmt, uint rowWords, uint gs, uint ng,+ uint o0, uint qbase, uint remaining, uint RW, uint rg,+ uint jj, uint lpr, uint W4, uint nr,+ out float acc[RPS * MAXNR]) {+ uint I4 = uint(p.I) / 4u;+ [[unroll]] for (uint a = 0u; a < RPS * MAXNR; a++) acc[a] = 0.0;+ bool pair_scale = gs == 64u && lpr >= 2u && (W4 & 1u) == 0u;+ for (uint j = jj; j < W4; j += lpr) {+ uvec4 v[RPS];+ float s[RPS];+ [[unroll]] for (uint t = 0u; t < RPS; t++) {+ uint q = qbase + t * rg, o = o0 + q;+ bool ok = t * rg < remaining && q < RW && o < uint(p.O);+ v[t] = ok ? w.v[o * (rowWords / 4u) + j] : uvec4(0u);+ }+ [[unroll]] for (uint t = 0u; t < RPS; t++) {+ uint q = qbase + t * rg, o = o0 + q;+ bool ok = t * rg < remaining && q < RW && o < uint(p.O);+ s[t] = 1.0;+ if (fmt == 4) s[t] = group_scale(sc, o, ng, j, gs, ok, pair_scale);+ }+ [[unroll]] for (uint k = 0u; k < MAXNR; k++) if (k < nr) {+ float d[RPS];+ [[unroll]] for (uint t = 0u; t < RPS; t++) d[t] = 0.0;+ if (fmt == 1) {+ uint x0 = k * I4 + j * 4u;+ [[unroll]] for (uint h = 0u; h < 4u; h++) {+ vec4 a = xsh[x0 + h];+ [[unroll]] for (uint t = 0u; t < RPS; t++) {+ float z = d8(v[t][h], a);+ d[t] = h == 0u ? z : d[t] + z;+ }+ }+ } else {+ uint x0 = k * I4 + j * 8u;+ [[unroll]] for (uint h = 0u; h < 4u; h++) {+ vec4 a = xsh[x0 + h * 2u], b = xsh[x0 + h * 2u + 1u];+ [[unroll]] for (uint t = 0u; t < RPS; t++) {+ float z = d4(v[t][h], a, b);+ d[t] = h == 0u ? z : d[t] + z;+ }+ }+ }+ [[unroll]] for (uint t = 0u; t < RPS; t++) acc[t * MAXNR + k] += s[t] * d[t];+ }+ }+ [[unroll]] for (uint t = 0u; t < RPS; t++) {+ uint q = qbase + t * rg, o = o0 + q;+ bool writer = jj == 0u && t * rg < remaining && q < RW && o < uint(p.O);+ // One per-row scale load, only in the lane which writes the result.+ float s = 1.0;+ if (writer && fmt != 4) s = sc.v[o];+ [[unroll]] for (uint k = 0u; k < MAXNR; k++) if (k < nr)+ acc[t * MAXNR + k] = red(acc[t * MAXNR + k], lpr) * s;+ }+}++void main() {+ uvec4 it = items[uint(p.ibase) + gl_WorkGroupID.x];+ uint eb = it.x * 16u;+ WRef4 wg = WRef4(packUint2x32(uvec2(etab[eb], etab[eb + 1u])));+ SRef gsc = SRef(packUint2x32(uvec2(etab[eb + 2u], etab[eb + 3u])));+#ifdef GATE_UP+ WRef4 wu = WRef4(packUint2x32(uvec2(etab[eb + 4u], etab[eb + 5u])));+ SRef usc = SRef(packUint2x32(uvec2(etab[eb + 6u], etab[eb + 7u])));+#endif+ uint xoff = etab[eb + 9u], rowWords = etab[eb + 11u];+ int fmt = int(etab[eb + 10u]);+ uint gs = max(etab[eb + 12u], 1u), I = uint(p.I), I4 = I / 4u;+ uint nr = 1u;+ for (uint k = 0u; k < nr; k++) {+ uint xr = xoff + it.y + k;+#ifdef GATE_UP+ if (p.kgat > 0) xr = uint(amap[xr]) / uint(p.kgat);+#endif+ for (uint i = gl_LocalInvocationID.x; i < I4; i += 256u)+ xsh[k * I4 + i] = x4[xr * I4 + i];+ }+ barrier();++ uint W4 = fmt == 1 ? I / 16u : I / 32u;+ uint cap = min(min(gl_SubgroupSize, W4), 64u);+ // Cluster sizes must be powers of two, even for non-power-of-two I.+ // I=512 int4 keeps 16 adjacent lanes per row (four rows per wave64).+ uint lpr = 1u;+ while (lpr * 2u <= cap) lpr *= 2u;+ uint rg = gl_SubgroupSize / lpr, rr = gl_SubgroupInvocationID / lpr;+ uint jj = gl_SubgroupInvocationID % lpr;+ uint RW = p.rpw > 0 ? uint(p.rpw) : RPW;+ uint rows_sg = (RW + gl_NumSubgroups - 1u) / gl_NumSubgroups;+ uint ng = (I + gs - 1u) / gs;+ for (uint base = 0u; base < rows_sg; base += RPS * rg) {+ uint qbase = gl_SubgroupID * rows_sg + base + rr;+ // Saturate before subtracting: the final cluster may lie past rows_sg.+ uint remaining = rows_sg - min(rows_sg, base + rr);+ float ag[RPS * MAXNR];+ matrix_block(wg, gsc, fmt, rowWords, gs, ng, it.z, qbase, remaining,+ RW, rg, jj, lpr, W4, nr, ag);+#ifdef GATE_UP+ float au[RPS * MAXNR];+ matrix_block(wu, usc, fmt, rowWords, gs, ng, it.z, qbase, remaining,+ RW, rg, jj, lpr, W4, nr, au);+#endif+ [[unroll]] for (uint t = 0u; t < RPS; t++) {+ uint q = qbase + t * rg, o = it.z + q;+ if (jj == 0u && t * rg < remaining && q < RW && o < uint(p.O)) {+ [[unroll]] for (uint k = 0u; k < MAXNR; k++) if (k < nr) {+ float g = ag[t * MAXNR + k];+#ifdef GATE_UP+ float u = au[t * MAXNR + k];+ if (p.limit > 0.0) { g = min(g, p.limit); u = clamp(u, -p.limit, p.limit); }+ y[(xoff + it.y + k) * uint(p.O) + o] = (g / (1.0 + exp(-g))) * u;+#else+ y[(xoff + it.y + k) * uint(p.O) + o] = g;+#endif+ }+ }+ }+ }+}diff --git a/c/shaders/qmatmul_grp_gemv.spv b/c/shaders/qmatmul_grp_gemv.spvnew file mode 100644index 00000000..6bd9e2aeBinary files /dev/null and b/c/shaders/qmatmul_grp_gemv.spv differdiff --git a/c/shaders/qmatmul_grp_gemv_gate_up.spv b/c/shaders/qmatmul_grp_gemv_gate_up.spvnew file mode 100644index 00000000..2a1fc0b0Binary files /dev/null and b/c/shaders/qmatmul_grp_gemv_gate_up.spv differdiff --git a/c/tests/test_vk_tier.c b/c/tests/test_vk_tier.cindex 56c1a399..88ae2730 100644--- a/c/tests/test_vk_tier.c+++ b/c/tests/test_vk_tier.c@@ -756,32 +756,32 @@ static void grouped(void) {{{VKT_SRC_I4U_PLANAR64, 64}, {VKT_SRC_I4U_PLANAR64, 64}, 0, "planar int4-g64"},};int h0 = H; H = 256;- int any_grouped = 0, any_summed = 0;+ int any_grouped = 0, any_summed = 0, any_gemv = 0;for (size_t c = 0; c < sizeof cs / sizeof *cs; c++) {- for (int mode = 0; mode < 3; mode++) { /* single, whole, mixed */+ for (int mode = 0; mode < 4; mode++) { /* single, whole, mixed, decode */model_make(cs[c].gu, cs[c].dn);g_act = VKT_ACT_SWIGLU; g_limit = cs[c].limit;set_budget(mode == 2 ? 4 : L * E, cs[c].gu, cs[c].dn);- setenv("COLI_VK_TIER_RATE", mode ? "0" : "64", 1);+ setenv("COLI_VK_TIER_RATE", mode == 1 || mode == 2 ? "0" : "64", 1);setenv("COLI_VK_TIER_SYNC", "1", 1);- if (mode) { setenv("COLI_VK_TIER_STREAM_SLOTS", "4", 1); setenv("COLI_VK_TIER_STREAM_ROWS", "6", 1);+ if (mode == 1 || mode == 2) { setenv("COLI_VK_TIER_STREAM_SLOTS", "4", 1); setenv("COLI_VK_TIER_STREAM_ROWS", "6", 1);setenv("COLI_VK_TIER_STREAM_HALF", "70", 1); }VktConfig vc = cfg_of(cs[c].gu, cs[c].dn, VKT_ACT_SWIGLU, cs[c].limit);- if (mode) { vc.load = load_cb; vc.release = release_cb; }+ if (mode == 1 || mode == 2) { vc.load = load_cb; vc.release = release_cb; }uint32_t hist[L][E], *hr[L];for (int l = 0; l < L; l++) { hr[l] = hist[l]; for (int e = 0; e < E; e++) hist[l][e] = 100 - e; }- int on = vkt_init(&vc, mode ? hr : NULL);+ int on = vkt_init(&vc, mode == 1 || mode == 2 || mode == 3 ? hr : NULL);unsetenv("COLI_VK_TIER_SYNC"); unsetenv("COLI_VK_TIER_STREAM_HALF");CHECK(on, "grouped %s: the tier did not start", cs[c].name);if (!on) { model_free(); continue; }- if (mode) { /* the warm start: whole = every expert, mixed = the four hottest */+ if (mode) { /* the warm start: whole and decode = every expert, mixed = the four hottest */int pl[L * E], pe[L * E], np = vkt_plan(pl, pe, L * E);for (int i = 0; i < np; i++) { VktExpertSrc s = src_of(&ex[pl[i]][pe[i]]); vkt_put(pl[i], pe[i], &s); }vkt_put_done();}ColiVkXbStats a, b; coli_vk_xb_stats(&a);Books bk = {0, 0}; double worst = 0; int summed = 0;- static const int Ss[3][4] = {{32, 32, 32, 32}, {40, 64, 40, 64}, {40, 64, 40, 64}};+ static const int Ss[4][4] = {{32, 32, 32, 32}, {40, 64, 40, 64}, {40, 64, 40, 64}, {1, 2, 1, 3}};for (int t = 0; t < 4; t++)for (int l = 0; l < L; l++) {int S = Ss[mode][t], idx[64 * K]; float w[64 * K];@@ -791,12 +791,13 @@ static void grouped(void) {if (r > worst) worst = r;}coli_vk_xb_stats(&b);- unsigned long long gb = b.grouped_batches - a.grouped_batches;- static const char *mn[3] = {"single", "whole", "mixed"};- printf(" %-30s %-6s device %3llu of %3llu, %3llu grouped batches, %d summed on the device, worst %.2e\n",- cs[c].name, mn[mode], bk.dev, bk.routed, gb, summed, worst);+ unsigned long long gb = b.grouped_batches - a.grouped_batches, gv = b.gemv_batches - a.gemv_batches;+ static const char *mn[4] = {"single", "whole", "mixed", "decode"};+ printf(" %-30s %-6s device %3llu of %3llu, %3llu grouped GEMM / %3llu GEMV batches, %d summed on the device, worst %.2e\n",+ cs[c].name, mn[mode], bk.dev, bk.routed, gb, gv, summed, worst);CHECK(bk.dev > 0, "grouped %s %s: nothing ran on the device", cs[c].name, mn[mode]);CHECK(worst < (gb ? 1e-2 : 2e-3), "grouped %s %s: off the reference (%.3g)", cs[c].name, mn[mode], worst);+ any_gemv |= gv > 0;any_grouped |= gb > 0; any_summed |= summed > 0;vkt_shutdown(); model_free();unsetenv("COLI_VK_TIER_STREAM_SLOTS"); unsetenv("COLI_VK_TIER_STREAM_ROWS");@@ -805,6 +806,8 @@ static void grouped(void) {printf(" grouped route %s, device sums %s\n", any_grouped ? "taken" : "not on this device",any_summed ? "taken" : "not on this device");CHECK(!any_grouped || any_summed, "grouped: the device took the grouped route but never summed a whole step");+ const char *gve = getenv("COLI_VK_XB_GEMV");+ CHECK(!any_grouped || any_gemv || (gve && *gve == '0'), "grouped: the device took the grouped GEMM but never the grouped GEMV");H = h0;}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex ced0b71e..1d936090 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -250,6 +250,7 @@ With `COLI_VULKAN=1` every MoE engine (qwen36, qwen38, inkling, olmoe, kimi_k3,| `COLI_VK_XB_GROUPED` | on where supported | A step's experts as one grouped GEMM (`qmatmul_grp.comp`): gate+up with the SwiGLU in one dispatch, down in another, the weights read by device address. Needs cooperative matrices at subgroup size 64, `bufferDeviceAddress`, hidden and expert widths that are multiples of 128, SwiGLU, and int8 or int4 experts (device fmt 1, 2, 4); otherwise the per-expert route. `0` keeps the per-expert route. || `COLI_VK_XB_GROUPED_ROWS` | `64` | Assignments (rows x top-k) in a batch from which the grouped GEMM takes it. Below it (decode, an MTP verify) the per-expert route keeps a row's bits independent of the batch. || `COLI_VK_XB_STEP` | on where supported | A big (prompt) step on the grouped GEMM: its token rows uploaded once, every expert's output written in place, and when every routed assignment ran on the device the routed sum per row computed there (`expert_sum.comp`, rank order). `0` keeps the packed rows and the host sum. |+| `COLI_VK_XB_GEMV` | on where supported | A batch below `COLI_VK_XB_GROUPED_ROWS` (decode, an MTP or lookup verify) as one grouped GEMV per phase (`qmatmul_grp_gemv.comp`: an (expert, row, 64 outputs) item a workgroup, the weights by address) instead of a GEMV dispatch per expert; a row's bits do not depend on the batch. Same device and expert conditions as the grouped GEMM, int4 groups a multiple of 32. `0` keeps the per-expert GEMVs. || `COLI_VK_BDA` | on where supported | `0`: the device is opened without `bufferDeviceAddress`, so the grouped GEMM is off. || `COLI_VK_TIER_COOP` | on where supported | Cooperative-matrix GEMM for resident and streamed expert batches. The same format and group-size checks as the per-matrix path apply to each gate, up and down projection independently; other projections keep the FP32 GEMM. `0` disables it; `COLI_VK_COOP=0` disables all cooperative pipelines. || `COLI_VK_TIER_COOP_ROWS` | `64` | Minimum rows of one expert for the cooperative path, also subject to `COLI_VK_TIER_GEMM_ROWS`. Small decode and draft-verification batches retain their GEMV arithmetic. |diff --git a/docs/vulkan.md b/docs/vulkan.mdindex 3725ea0e..19d60c8c 100644--- a/docs/vulkan.md+++ b/docs/vulkan.md@@ -479,6 +479,7 @@ the device, since no Vulkan tier runs.| `COLI_VK_XB_GROUPED` | on where supported | `0`: a step's experts keep their own dispatches where the grouped GEMM would take them (cooperative matrices at subgroup size 64, `bufferDeviceAddress`, widths multiples of 128, int8/int4, SwiGLU). || `COLI_VK_XB_GROUPED_ROWS` | `64` | Assignments in a batch from which the grouped GEMM takes it. || `COLI_VK_XB_STEP` | on where supported | `0`: a prompt step packs its rows per expert and sums on the host, as before the whole-step buffers. On a Radeon 8060S (Qwen3.6, 1011-token prompt) the grouped GEMM and the step buffers took the experts' device time from 762 to 344 ms and the first token from 2.08 to 1.49 s, perplexity unchanged (7.65). |+| `COLI_VK_XB_GEMV` | on where supported | `0`: a decode step's experts keep a GEMV dispatch each instead of the grouped GEMV (`qmatmul_grp_gemv.comp`). On a Radeon 8060S (Qwen3.6, a 256-token answer, tier 24 GB) the grouped GEMV took decode from 37.2 to 42.5 tok/s with the int8 trunk and from 45.0 to 53.5 with the int4 trunk. || `COLI_VK_TIER_QUEUE` | a second queue | `0`: the tier shares the main queue (its batches and the dense matmuls then serialize). || `COLI_VK_TIER_STREAM` | on | A prompt step's cold experts streamed to the device ([below](#big-prompt-chunks-and-expert-streaming)); `0` off. `_SLOTS`, `_ROWS`, `_HALF`, `_PAR` there. || `COLI_VK_DENSE` | on, but off on a device sharing the CPU's RAM while the tier is on | `0`: the dense trunk stays on the CPU and the device takes the routed experts only; `1`: the trunk on the device whatever the device. Unset: on a discrete GPU, or with the tier off, on the device; on an integrated GPU or Lavapipe with the tier on, on the CPU. The startup line says which and why. (The GLM engine reads it through the same rule with its own default, off.) |