Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: a step's experts as one grouped GEMM, and a big step summed on the device
huppiflupp0d3559b5f6d3
diff --git a/c/Makefile b/c/Makefileindex d883728c..7a8cb3d9 100644--- a/c/Makefile+++ b/c/Makefile@@ -709,7 +709,7 @@ endifVK_OBJ = backend_vulkan.o vk_tier.oVK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv shaders/expert_act.spv \- shaders/expert_act_v4.spv+ shaders/expert_act_v4.spv shaders/qmatmul_grp.spv shaders/qmatmul_grp_gate_up.spv shaders/expert_sum.spv# vk_chain.o: a layer's dense chain recorded into one submission (vk_chain.h), and# its shaders; an engine that does not call it links nothing from it.VK_OBJ += vk_chain.o@@ -871,6 +871,17 @@ backend_vulkan.o: backend_vulkan.c .build-config$(CC) $(CFLAGS) -c backend_vulkan.c -o $@vk_tier.o: vk_tier.c .build-config$(CC) $(CFLAGS) -c vk_tier.c -o $@+# The expert batch's grouped GEMM (qmatmul_grp.comp): down, and gate+up+SwiGLU with+# -DGATE_UP. Same glslc requirement and fallback as qmatmul_coop.spv; at run+# time the batch keeps its per-expert route without them.+shaders/qmatmul_grp.spv: shaders/qmatmul_grp.comp+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }+ $(GLSLC) --target-env=vulkan1.2 $< -o $@ || { [ -f $@ ] && touch $@ && \+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }+shaders/qmatmul_grp_gate_up.spv: shaders/qmatmul_grp.comp+ @command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }+ $(GLSLC) --target-env=vulkan1.2 -DGATE_UP $< -o $@ || { [ -f $@ ] && touch $@ && \+ echo "warning: $(GLSLC) cannot compile $<, keeping the committed $@" >&2; }vk_chain.o: vk_chain.c .build-config$(CC) $(CFLAGS) -c vk_chain.c -o $@shaders/%.spv: shaders/%.compdiff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex dcd52a5d..c873bb15 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -291,6 +291,7 @@ static struct {VkGemmTile gemm_t[VK_GEMM_SLOTS];VkCoopTile coop_t[VK_COOP_SLOTS];int gemm_min_s, gemm_min_so, has_coop, coop_sg, coop_off;+ int has_bda; /* bufferDeviceAddress on: the expert batch's grouped GEMM reads weights by address *//* fused dual gate+up+silu pipeline (6 bindings): x, Wg, gscale, Wu, uscale, hidden */VkShaderModule shader_gu; VkDescriptorSetLayout dsl_gu; VkPipelineLayout plyt_gu;VkPipeline pipe_gu; VkDescriptorPool dpool_gu; VkDescriptorSet dset_gu;@@ -980,6 +981,10 @@ int coli_vk_init(const char *spv_path) {sscf.subgroupSizeControl && sscf.computeFullSubgroups &&(ssp.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) && sgz >= 16) {G.has_coop = 1; G.coop_sg = sgz;+ /* the grouped expert GEMM (qmatmul_grp.comp) addresses the weights;+ * COLI_VK_BDA=0 leaves it off */+ const char *be = getenv("COLI_VK_BDA");+ G.has_bda = v12f.bufferDeviceAddress && !(be && *be == '0');}}}@@ -990,6 +995,7 @@ int coli_vk_init(const char *spv_path) {sscf.subgroupSizeControl = VK_TRUE; sscf.computeFullSubgroups = VK_TRUE;v12f.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; v12f.pNext = &sscf;v12f.shaderFloat16 = VK_TRUE; v12f.vulkanMemoryModel = VK_TRUE;+ v12f.bufferDeviceAddress = G.has_bda ? VK_TRUE : VK_FALSE;#endifVkDeviceCreateInfo di = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,.queueCreateInfoCount = nqi, .pQueueCreateInfos = qis};@@ -1018,7 +1024,7 @@ int coli_vk_init(const char *spv_path) {cext[nc++] = VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME;cmf.pNext = (void *)di.pNext; dc.pNext = &v12f;dc.enabledExtensionCount = nc; dc.ppEnabledExtensionNames = cext;- if (vkCreateDevice(G.phys, &dc, NULL, &G.dev) != VK_SUCCESS) { G.has_coop = 0; G.dev = VK_NULL_HANDLE; }+ if (vkCreateDevice(G.phys, &dc, NULL, &G.dev) != VK_SUCCESS) { G.has_coop = 0; G.has_bda = 0; G.dev = VK_NULL_HANDLE; }}if (!G.dev)#endif@@ -1479,6 +1485,9 @@ static void up_zero(int dev, VkDeviceMemory mem, uint64_t cap) {if (!lost) vkDestroyBuffer(u->dev, b, NULL); /* a failed wait: the fill may still run, leave it */}+/* The primary device's pools are addressable when bufferDeviceAddress is on (the+ * expert batch's grouped GEMM reads expert weights through an address table). */+static int pool_bda(const VkWPool *P) { return P->dev == 0 && G.has_bda; }/* Block memory for a pool (lock held). */static VkBlk *pool_new_block(VkWPool *P, uint64_t cap) {VkBlk *bk = calloc(1, sizeof(*bk));@@ -1490,6 +1499,9 @@ static VkBlk *pool_new_block(VkWPool *P, uint64_t cap) {.priority = P->prio >= 0.f ? P->prio : G.prio};if (pool_has_prio(P)) ai.pNext = &pri;#endif+ VkMemoryAllocateFlagsInfo afl = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,+ .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};+ if (pool_bda(P)) { afl.pNext = ai.pNext; ai.pNext = &afl; }VkDevice dev = pool_device(P);int staged = g_up[P->dev].on; /* device-local, never mapped: the uploader fills it */if (staged && up_fault("block")) { free(bk); return NULL; }@@ -1516,6 +1528,7 @@ static int pool_suballoc(VkWPool *P, size_t bytes, VkBuffer *buf, void **ptr, Vk.size = bytes ? bytes : 4, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,.sharingMode = VK_SHARING_MODE_EXCLUSIVE};if (g_up[P->dev].on) { bi.usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; up_sharing(&g_up[P->dev], &bi); }+ if (pool_bda(P)) bi.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;VKCHECK(vkCreateBuffer(dev, &bi, NULL, buf), "vkCreateBuffer");VkMemoryRequirements req;vkGetBufferMemoryRequirements(dev, *buf, &req);@@ -3406,10 +3419,31 @@ typedef struct XbCtx {size_t sub_x, sub_i, sub_y; /* a half's bytes of each scratch (0: not reserved) */unsigned long long sub_batches, sub_experts, sub_rows, sub_gemm;double sub_ms;+ /* The grouped GEMM (qmatmul_grp.comp, [0] down, [1] gate+up+SwiGLU): every expert of+ * a batch in one dispatch per phase, the weights read through an address table. The+ * primary device with cooperative matrices, subgroups of 64 and bufferDeviceAddress+ * (xb_grp_init). Tables and sets per batch that can be in flight: [0] the single+ * batch, [1 + h] sub-batch half h. */+ VkShaderModule sh_grp[2]; VkDescriptorSetLayout dsl_grp; VkPipelineLayout pl_grp;+ VkPipeline p_grp[2]; VkDescriptorPool grp_pool; VkDescriptorSet grp_set[3][2];+ XbBuf grp_it[3][2], grp_et[3][2], grp_map[3];+ int grp_rows;+ unsigned long long grp_batches;+ /* a big step's whole-step buffers (coli_vk_xb_step_*): its token rows once (xt), every+ * assignment's output row in place (ys, s*K + k), the routed sum (out) and its inputs */+ struct {+ int on, S, K; const float *xh;+ XbBuf xt, ys, out, w, use;+ VkShaderModule sh; VkDescriptorSetLayout dsl; VkPipelineLayout pl; VkPipeline pipe;+ VkDescriptorPool pool; VkDescriptorSet set;+ unsigned long long steps, sums; double sum_ms;+ } st;} XbCtx;static XbCtx g_xb[2];static int xb_dev_ready(const XbCtx *X) { return X->d ? G2.ready : G.ready; }struct PCV4 { int rows, n; };+struct PCGRP { int I, O; float limit; int ibase; int kgat; }; /* qmatmul_grp.comp */+#define XB_GRP_TT 4 /* 16-row tiles of an expert per workgroup: 64 rows (2, 4 or 8) */static size_t xb_up(size_t v, size_t a) { return (v + a - 1) / a * a; }@@ -3649,9 +3683,67 @@ static int xb_gemm_slot(const XbCtx *X, int S) {return k;}+/* The grouped GEMM's pipelines (qmatmul_grp*.spv beside the main shader); leaves+ * X->grp_rows at 0 when the device or the files do not allow it. */+static void xb_grp_init(XbCtx *X) {+ const char *e = getenv("COLI_VK_XB_GROUPED");+ if (X->d || !G.has_bda || !G.has_coop || G.coop_sg != 64 || (e && *e == '0')) return;+ const char *nm[2] = {"qmatmul_grp.spv", "qmatmul_grp_gate_up.spv"};+ for (int v = 0; v < 2; v++) {+ char path[1100];+ derive_dir_file(X->spv, nm[v], path, sizeof path);+ if (!(X->sh_grp[v] = load_spv(X->dev, path))) return;+ }+ if (!xb_layout(X, 5, 0u, sizeof(struct PCGRP), &X->dsl_grp, &X->pl_grp)) return;+ int32_t tt = XB_GRP_TT;+ VkSpecializationMapEntry me = {0, 0, sizeof(int32_t)};+ VkSpecializationInfo si = {1, &me, sizeof tt, &tt};+ VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT rss = {+ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,+ .requiredSubgroupSize = 64};+ for (int v = 0; v < 2; v++) {+ VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,+ .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = &rss,+ .flags = VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT,+ .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = X->sh_grp[v], .pName = "main",+ .pSpecializationInfo = &si}, .layout = X->pl_grp};+ if (vkCreateComputePipelines(X->dev, VK_NULL_HANDLE, 1, &ci, NULL, &X->p_grp[v]) != VK_SUCCESS) {+ X->p_grp[v] = VK_NULL_HANDLE; return;+ }+ }+ VkDescriptorPoolSize ps = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 30};+ VkDescriptorPoolCreateInfo dp = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,+ .maxSets = 6, .poolSizeCount = 1, .pPoolSizes = &ps};+ if (vkCreateDescriptorPool(X->dev, &dp, NULL, &X->grp_pool) != VK_SUCCESS) { X->grp_pool = VK_NULL_HANDLE; return; }+ VkDescriptorSetLayout l6[6] = {X->dsl_grp, X->dsl_grp, X->dsl_grp, X->dsl_grp, X->dsl_grp, X->dsl_grp};+ VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,+ .descriptorPool = X->grp_pool, .descriptorSetCount = 6, .pSetLayouts = l6};+ if (vkAllocateDescriptorSets(X->dev, &da, &X->grp_set[0][0]) != VK_SUCCESS) return;+ const char *r = getenv("COLI_VK_XB_GROUPED_ROWS");+ X->grp_rows = r && *r ? atoi(r) : 64;+ if (X->grp_rows < 1) X->grp_rows = 1;+ /* the step's routed sum (expert_sum.spv); without it big steps keep their row copies */+ char sp[1100];+ derive_dir_file(X->spv, "expert_sum.spv", sp, sizeof sp);+ const char *se = getenv("COLI_VK_XB_STEP");+ if ((se && *se == '0') || !(X->st.sh = load_spv(X->dev, sp))) return;+ if (!xb_layout(X, 4, 0u, 3 * sizeof(int), &X->st.dsl, &X->st.pl) ||+ !xb_pipeline(X, X->st.sh, X->st.pl, NULL, &X->st.pipe)) { X->st.pipe = VK_NULL_HANDLE; return; }+ VkDescriptorPoolSize ps2 = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4};+ VkDescriptorPoolCreateInfo dp2 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,+ .maxSets = 1, .poolSizeCount = 1, .pPoolSizes = &ps2};+ VkDescriptorSetAllocateInfo da2 = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,+ .descriptorSetCount = 1, .pSetLayouts = &X->st.dsl};+ if (vkCreateDescriptorPool(X->dev, &dp2, NULL, &X->st.pool) != VK_SUCCESS) { X->st.pool = VK_NULL_HANDLE; X->st.pipe = VK_NULL_HANDLE; return; }+ da2.descriptorPool = X->st.pool;+ if (vkAllocateDescriptorSets(X->dev, &da2, &X->st.set) != VK_SUCCESS) X->st.pipe = VK_NULL_HANDLE;+}+static int xb_init(XbCtx *X, int d, int D, int I, int act, float limit, float a, float b) {- if (X->ready) { /* the geometry stays; the activation rides the push constants */- if (X->D != D || X->I != I || X->inflight || X->sub[0].inflight || X->sub[1].inflight) return 0;+ if (X->ready) { /* the activation rides the push constants; the geometry changes+ * only while no expert is allocated (scratch grows by bytes) */+ if (X->inflight || X->sub[0].inflight || X->sub[1].inflight || ((X->D != D || X->I != I) && X->live)) return 0;+ X->D = D; X->I = I;if (act == COLI_VK_ACT_SWIGLU_V4 && !xb_v4_init(X)) return 0;X->act = act; X->limit = limit > 0.f ? limit : 0.f; X->a = a; X->b = b;return 1;@@ -3692,6 +3784,7 @@ static int xb_init(XbCtx *X, int d, int D, int I, int act, float limit, float a,X->gemm_rows = X->p_mm[0] && X->p_act ? (e && *e ? atoi(e) : 16) : 0;if (X->gemm_rows < 0) X->gemm_rows = 0;xb_coop_init(X);+ xb_grp_init(X);VkCommandPoolCreateInfo cp = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, .queueFamilyIndex = X->q_fam};VKCHECK(vkCreateCommandPool(X->dev, &cp, NULL, &X->cpool), "xb cmd pool");@@ -3808,11 +3901,123 @@ static int xb_plan(XbCtx *X, ColiVkExpert *const *ex, const int *rows, int count*xe = xo; *ie = io; *ye = yo; *total = t;return 1;}+static uint64_t vk_addr(VkBuffer b) {+ VkBufferDeviceAddressInfo ai = {.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = b};+ return (uint64_t)vkGetBufferDeviceAddress(G.dev, &ai);+}+/* The grouped route of a batch (X->p_grp) into cmd: gate+up+SwiGLU of every expert in+ * one dispatch, a barrier, down of every expert in another. The packed rows stay where+ * xb_plan put them: with row sizes on the buffers' alignment an expert's x, hidden and y+ * rows start at the same row index past the batch's bases x0, i0, y0 (its scratch half),+ * which the sets bind as their offsets. `slot`: whose tables and sets (0 the single+ * batch, 1 + h a sub-batch half). 0 = not taken, nothing recorded. */+/* Whether the grouped route takes this batch (and its row total). */+static int xb_grp_ok(const XbCtx *X, ColiVkExpert *const *ex, const int *rows, int count, int *total_out) {+ const int D = X->D, I = X->I;+ const size_t dr = (size_t)D * 4, ir = (size_t)I * 4;+ int total = 0;+ for (int c = 0; c < count; c++) total += rows[c];+ if (total_out) *total_out = total;+ if (!X->grp_rows || total < X->grp_rows || X->act != COLI_VK_ACT_SWIGLU) return 0;+ if (D % 128 || I % 128 || dr % X->align || ir % X->align) return 0;+ for (int c = 0; c < count; c++) {+ const ColiVkTensor *t[3] = {ex[c]->g, ex[c]->u, ex[c]->d};+ for (int k = 0; k < 3; k++) {+ int f = t[k]->fmt;+ if (t[k]->dev || !t[k]->pool || !(f == 1 || f == 2 || f == 4) || t[k]->rowWords % 4 ||+ (f == 4 && (t[k]->gs < 8 || t[k]->gs % 8))) return 0;+ }+ if (ex[c]->u->fmt != ex[c]->g->fmt || ex[c]->u->gs != ex[c]->g->gs) return 0;+ }+ return 1;+}+/* amap (a big step's sub-batch, X->st.on): packed row j is assignment amap[j]; x comes from+ * the step's token rows and y goes to the step's output rows (the scratch's x and y unused). */+static int xb_grp_record(XbCtx *X, VkCommandBuffer cmd, int slot, ColiVkExpert *const *ex, const int *rows,+ int count, const size_t *off, size_t x0, size_t i0, size_t y0, const int *amap) {+ const int D = X->D, I = X->I;+ const size_t dr = (size_t)D * 4, ir = (size_t)I * 4;+ int total = 0;+ if (!xb_grp_ok(X, ex, rows, count, &total)) return 0;+ if (amap) {+ XbBuf *mb = &X->grp_map[slot];+ size_t nb = (size_t)total * 4;+ if (nb > mb->region && !xb_buf(X, mb, xb_up(nb + nb / 2, 4096), X->mt_host, 1)) { X->grp_rows = 0; return 0; }+ memcpy(mb->ptr, amap, nb);+ }+ /* items: (expert, first row, 128-row output block), 64 rows a workgroup; the tiles+ * of one (expert, block) adjacent, so workgroups in flight share weights */+ const int tm = 16 * XB_GRP_TT;+ int nit[2] = {0, 0};+ for (int c = 0; c < count; c++) {+ int nt = (rows[c] + tm - 1) / tm;+ nit[0] += nt * (D / 128); nit[1] += nt * (I / 128);+ }+ for (int v = 0; v < 2; v++) {+ XbBuf *ib = &X->grp_it[slot][v], *eb = &X->grp_et[slot][v];+ size_t nb = (size_t)nit[v] * 16, ne = (size_t)count * 64;+ if ((nb > ib->region && !xb_buf(X, ib, xb_up(nb + nb / 2, 4096), X->mt_host, 1)) ||+ (ne > eb->region && !xb_buf(X, eb, xb_up(ne + ne / 2, 4096), X->mt_host, 1))) {+ X->grp_rows = 0; /* no room for the tables: the per-expert route from here on */+ return 0;+ }+ }+ for (int v = 0; v < 2; v++) { /* v 1: gate+up (out I), v 0: down (out D) */+ uint32_t et[16], *it = X->grp_it[slot][v].ptr;+ int n = 0, O = v ? I : D;+ for (int c = 0; c < count; c++) {+ const ColiVkTensor *a = v ? ex[c]->g : ex[c]->d, *b = v ? ex[c]->u : NULL;+ uint64_t ad[4] = {vk_addr(a->wbuf), vk_addr(a->sbuf), b ? vk_addr(b->wbuf) : 0, b ? vk_addr(b->sbuf) : 0};+ memset(et, 0, sizeof et);+ for (int k = 0; k < 4; k++) { et[2 * k] = (uint32_t)ad[k]; et[2 * k + 1] = (uint32_t)(ad[k] >> 32); }+ et[8] = (uint32_t)rows[c]; et[9] = (uint32_t)((off[3 * c] - x0) / dr);+ et[10] = (uint32_t)a->fmt; et[11] = (uint32_t)a->rowWords; et[12] = (uint32_t)a->gs;+ memcpy((uint8_t *)X->grp_et[slot][v].ptr + (size_t)c * 64, et, sizeof et); /* written, never read back */+ for (int ob = 0; ob < O / 128; ob++)+ for (int t0 = 0; t0 < rows[c]; t0 += tm) {+ uint32_t q[4] = {(uint32_t)c, (uint32_t)t0, (uint32_t)ob, 0};+ memcpy(it + (size_t)n * 4, q, sizeof q); n++;+ }+ }+ }+ /* windows from the bases: x rows, hidden rows, y rows of this batch */+ size_t total_d = (size_t)total * dr, total_i = (size_t)total * ir;+ VkDescriptorBufferInfo in[2] = {{X->h.buf, i0, total_i}, {X->x.buf, x0, total_d}};+ VkDescriptorBufferInfo out[2] = {{X->y.buf, y0, total_d}, {X->h.buf, i0, total_i}};+ VkDescriptorBufferInfo mp = {X->grp_it[slot][0].buf, 0, 16}; /* unread without amap */+ if (amap) {+ in[1] = (VkDescriptorBufferInfo){X->st.xt.buf, 0, (size_t)X->st.S * dr};+ out[0] = (VkDescriptorBufferInfo){X->st.ys.buf, 0, (size_t)X->st.S * X->st.K * dr};+ mp = (VkDescriptorBufferInfo){X->grp_map[slot].buf, 0, (size_t)total * 4};+ }+ for (int step = 0; step < 2; step++) {+ int v = 1 - step; /* gate+up first, then down */+ VkDescriptorBufferInfo bi[5] = {in[v], {X->grp_it[slot][v].buf, 0, X->grp_it[slot][v].region},+ {X->grp_et[slot][v].buf, 0, X->grp_et[slot][v].region}, out[v], mp};+ VkWriteDescriptorSet w[5];+ for (int b = 0; b < 5; b++)+ w[b] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = X->grp_set[slot][v],+ .dstBinding = (uint32_t)b, .descriptorCount = 1,+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[b]};+ vkUpdateDescriptorSets(X->dev, 5, w, 0, NULL);+ vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_grp[v]);+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl_grp, 0, 1, &X->grp_set[slot][v], 0, NULL);+ for (int i0n = 0; i0n < nit[v]; i0n += 65535) { /* the grid's x limit every device has */+ struct PCGRP pc = {v ? D : I, v ? I : D, X->limit, i0n, amap ? X->st.K : 0};+ vkCmdPushConstants(cmd, X->pl_grp, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);+ vkCmdDispatch(cmd, (uint32_t)(nit[v] - i0n < 65535 ? nit[v] - i0n : 65535), 1, 1);+ }+ if (!step) xb_barrier(cmd);+ }+ X->grp_batches++;+ return 1;+}/* Record one batch laid out at off[] (its x rows already written) into cmd, with its* timestamps in qp when the device has them, and submit it with fence on the tier queue.* *ngemm: how many experts took the GEMM route. 0 = nothing was submitted. */static int xb_record_submit(XbCtx *X, VkCommandBuffer cmd, VkFence fence, VkQueryPool qp, ColiVkExpert *const *ex,- const int *rows, int count, const size_t *off, int v4, int *ngemm_out) {+ const int *rows, int count, const size_t *off, int v4, int *ngemm_out,+ int slot, size_t x0, size_t i0, size_t y0, const int *amap) {const int D = X->D, I = X->I;const size_t a = X->align, dr = (size_t)D * 4;if (vkResetCommandBuffer(cmd, 0) != VK_SUCCESS) return 0;@@ -3826,83 +4031,86 @@ static int xb_record_submit(XbCtx *X, VkCommandBuffer cmd, VkFence fence, VkQuer/* the route of each expert: its GEMM slot, or -1 for the GEMVs */int *route = malloc((size_t)count * sizeof(int));if (!route) { vkEndCommandBuffer(cmd); return 0; }- int ngemm = 0, cooperative = 0;- for (int c = 0; c < count; c++) {- int k = X->gemm_rows && rows[c] >= X->gemm_rows ? xb_gemm_slot(X, rows[c]) : -1;- route[c] = k >= 0 && X->p_mm[k] ? k : -1;- ngemm += route[c] >= 0;- }- /* phase 1: gate and up of every expert (fused GEMV, or two GEMMs) */- vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_gu);- for (int c = 0; c < count; c++) {- if (route[c] >= 0) continue;- const ColiVkExpert *e = ex[c];- uint32_t dyn[2] = {(uint32_t)off[3 * c], (uint32_t)off[3 * c + 1]};- vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl6, 0, 1, &e->s_gu, 2, dyn);- struct PCGU pc = {e->g->fmt, rows[c], D, I, e->g->rowWords, e->g->gs, X->limit, X->act, X->a, X->b};- vkCmdPushConstants(cmd, X->pl6, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- vkCmdDispatch(cmd, (uint32_t)((I + 7) / 8), (uint32_t)rows[c], 1);- }- if (ngemm) {- VkPipeline bound = VK_NULL_HANDLE;+ int ngemm = 0, cooperative = 0, grouped = !v4 && xb_grp_record(X, cmd, slot, ex, rows, count, off, x0, i0, y0, amap);+ if (grouped) ngemm = count;+ else {for (int c = 0; c < count; c++) {- if (route[c] < 0) continue;+ int k = X->gemm_rows && rows[c] >= X->gemm_rows ? xb_gemm_slot(X, rows[c]) : -1;+ route[c] = k >= 0 && X->p_mm[k] ? k : -1;+ ngemm += route[c] >= 0;+ }+ /* phase 1: gate and up of every expert (fused GEMV, or two GEMMs) */+ vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_gu);+ for (int c = 0; c < count; c++) {+ if (route[c] >= 0) continue;const ColiVkExpert *e = ex[c];- int k = route[c];- const ColiVkTensor *gu[2] = {e->g, e->u};- VkDescriptorSet su[2] = {e->s_g, e->s_u};- for (int q = 0; q < 2; q++) {- int bm, bn;- VkPipeline pipe = xb_mm_pipeline(X, gu[q], rows[c], k, &bm, &bn, &cooperative);- if (pipe != bound) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); bound = pipe; }- uint32_t dyn[2] = {(uint32_t)off[3 * c], (uint32_t)off[3 * c + 1]};- vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl4, 0, 1, &su[q], 2, dyn);- struct PC pc = {gu[q]->fmt, rows[c], D, I, gu[q]->rowWords, gu[q]->gs};- vkCmdPushConstants(cmd, X->pl4, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- vkCmdDispatch(cmd, (uint32_t)((I + bm - 1) / bm), (uint32_t)((rows[c] + bn - 1) / bn), 1);+ uint32_t dyn[2] = {(uint32_t)off[3 * c], (uint32_t)off[3 * c + 1]};+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl6, 0, 1, &e->s_gu, 2, dyn);+ struct PCGU pc = {e->g->fmt, rows[c], D, I, e->g->rowWords, e->g->gs, X->limit, X->act, X->a, X->b};+ vkCmdPushConstants(cmd, X->pl6, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ vkCmdDispatch(cmd, (uint32_t)((I + 7) / 8), (uint32_t)rows[c], 1);+ }+ if (ngemm) {+ VkPipeline bound = VK_NULL_HANDLE;+ for (int c = 0; c < count; c++) {+ if (route[c] < 0) continue;+ const ColiVkExpert *e = ex[c];+ int k = route[c];+ const ColiVkTensor *gu[2] = {e->g, e->u};+ VkDescriptorSet su[2] = {e->s_g, e->s_u};+ for (int q = 0; q < 2; q++) {+ int bm, bn;+ VkPipeline pipe = xb_mm_pipeline(X, gu[q], rows[c], k, &bm, &bn, &cooperative);+ if (pipe != bound) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); bound = pipe; }+ uint32_t dyn[2] = {(uint32_t)off[3 * c], (uint32_t)off[3 * c + 1]};+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl4, 0, 1, &su[q], 2, dyn);+ struct PC pc = {gu[q]->fmt, rows[c], D, I, gu[q]->rowWords, gu[q]->gs};+ vkCmdPushConstants(cmd, X->pl4, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ vkCmdDispatch(cmd, (uint32_t)((I + bm - 1) / bm), (uint32_t)((rows[c] + bn - 1) / bn), 1);+ }+ }+ xb_barrier(cmd);+ /* phase 2: the activation of the GEMM experts */+ vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_act);+ for (int c = 0; c < count; c++) {+ if (route[c] < 0) continue;+ uint32_t dyn[3] = {(uint32_t)off[3 * c + 1], (uint32_t)off[3 * c + 1], (uint32_t)off[3 * c + 1]};+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl3, 0, 1, &X->act_set, 3, dyn);+ int n = rows[c] * I;+ struct PCAct pc = {n, X->act, X->limit, X->a, X->b};+ vkCmdPushConstants(cmd, X->pl3, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ uint32_t groups = (uint32_t)((n + 255) / 256), gx = groups < 65535 ? groups : 65535;+ vkCmdDispatch(cmd, gx, (groups + gx - 1) / gx, 1);}}xb_barrier(cmd);- /* phase 2: the activation of the GEMM experts */- vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_act);- for (int c = 0; c < count; c++) {- if (route[c] < 0) continue;- uint32_t dyn[3] = {(uint32_t)off[3 * c + 1], (uint32_t)off[3 * c + 1], (uint32_t)off[3 * c + 1]};- vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl3, 0, 1, &X->act_set, 3, dyn);- int n = rows[c] * I;- struct PCAct pc = {n, X->act, X->limit, X->a, X->b};- vkCmdPushConstants(cmd, X->pl3, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- uint32_t groups = (uint32_t)((n + 255) / 256), gx = groups < 65535 ? groups : 65535;- vkCmdDispatch(cmd, gx, (groups + gx - 1) / gx, 1);+ if (v4) { /* DeepSeek V4: route weight, bf16 and E4M3 per 128 on the hidden rows */+ vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_v4);+ for (int c = 0; c < count; c++) {+ uint32_t dyn[2] = {(uint32_t)off[3 * c + 1], (uint32_t)(off[3 * c] + xb_up((size_t)rows[c] * dr, a))};+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl_v4, 0, 1, &X->v4_set, 2, dyn);+ struct PCV4 pc = {rows[c], I};+ vkCmdPushConstants(cmd, X->pl_v4, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ vkCmdDispatch(cmd, (uint32_t)((I + 127) / 128), (uint32_t)rows[c], 1);+ }+ xb_barrier(cmd);}- }- xb_barrier(cmd);- if (v4) { /* DeepSeek V4: route weight, bf16 and E4M3 per 128 on the hidden rows */- vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->p_v4);+ /* phase 3: down of every expert */+ VkPipeline bound = VK_NULL_HANDLE;for (int c = 0; c < count; c++) {- uint32_t dyn[2] = {(uint32_t)off[3 * c + 1], (uint32_t)(off[3 * c] + xb_up((size_t)rows[c] * dr, a))};- vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl_v4, 0, 1, &X->v4_set, 2, dyn);- struct PCV4 pc = {rows[c], I};- vkCmdPushConstants(cmd, X->pl_v4, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- vkCmdDispatch(cmd, (uint32_t)((I + 127) / 128), (uint32_t)rows[c], 1);+ const ColiVkExpert *e = ex[c];+ int k = route[c], gemm = k >= 0;+ int bm = 8, bn = 1;+ VkPipeline pipe = gemm ? xb_mm_pipeline(X, e->d, rows[c], k, &bm, &bn, &cooperative) : X->p_mv;+ if (pipe != bound) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); bound = pipe; }+ uint32_t dyn[2] = {(uint32_t)off[3 * c + 1], (uint32_t)off[3 * c + 2]};+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl4, 0, 1, &e->s_dn, 2, dyn);+ struct PC pc = {e->d->fmt, rows[c], I, D, e->d->rowWords, e->d->gs};+ vkCmdPushConstants(cmd, X->pl4, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ if (gemm) {+ vkCmdDispatch(cmd, (uint32_t)((D + bm - 1) / bm), (uint32_t)((rows[c] + bn - 1) / bn), 1);+ } else vkCmdDispatch(cmd, (uint32_t)((D + 7) / 8), (uint32_t)rows[c], 1);}- xb_barrier(cmd);- }- /* phase 3: down of every expert */- VkPipeline bound = VK_NULL_HANDLE;- for (int c = 0; c < count; c++) {- const ColiVkExpert *e = ex[c];- int k = route[c], gemm = k >= 0;- int bm = 8, bn = 1;- VkPipeline pipe = gemm ? xb_mm_pipeline(X, e->d, rows[c], k, &bm, &bn, &cooperative) : X->p_mv;- if (pipe != bound) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipe); bound = pipe; }- uint32_t dyn[2] = {(uint32_t)off[3 * c + 1], (uint32_t)off[3 * c + 2]};- vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->pl4, 0, 1, &e->s_dn, 2, dyn);- struct PC pc = {e->d->fmt, rows[c], I, D, e->d->rowWords, e->d->gs};- vkCmdPushConstants(cmd, X->pl4, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- if (gemm) {- vkCmdDispatch(cmd, (uint32_t)((D + bm - 1) / bm), (uint32_t)((rows[c] + bn - 1) / bn), 1);- } else vkCmdDispatch(cmd, (uint32_t)((D + 7) / 8), (uint32_t)rows[c], 1);}/* the host reads y after the fence */VkMemoryBarrier hb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,@@ -3951,7 +4159,7 @@ static int xb_issue(XbCtx *X, ColiVkExpert *const *ex, const int *rows, int coun}X->nrows = total;int ngemm = 0;- int ok = xb_record_submit(X, X->cmd, X->fence, X->qpool, ex, rows, count, off, v4, &ngemm);+ int ok = xb_record_submit(X, X->cmd, X->fence, X->qpool, ex, rows, count, off, v4, &ngemm, 0, 0, 0, 0, NULL);free(off);if (!ok) return 0;X->inflight = 1; async_begin(X->d);@@ -4022,7 +4230,7 @@ static void xb_stats(const XbCtx *X, ColiVkXbStats *st) {st->scratch_bytes = 2 * (X->x.region + 3 * X->g.region + X->y.region);st->sub_batches = X->sub_batches; st->sub_experts = X->sub_experts; st->sub_rows = X->sub_rows;st->sub_gemm = X->sub_gemm; st->sub_ms = X->sub_ms;- st->cooperative_matmuls = X->cooperative_matmuls;+ st->cooperative_matmuls = X->cooperative_matmuls; st->grouped_batches = X->grp_batches;}void coli_vk_xb_stats(ColiVkXbStats *st) { xb_stats(&g_xb[0], st); }void coli_vk_xb_stats_dev(int dev, ColiVkXbStats *st) {@@ -4152,7 +4360,7 @@ int coli_vk_xb_sub_issue(int h, ColiVkExpert *const *ex, const int *rows, int cofree(first);X->sub[h].nrows = total;int ngemm = 0;- int ok = xb_record_submit(X, X->sub[h].cmd, X->sub[h].fence, X->sub[h].qp, ex, rows, count, off, v4, &ngemm);+ int ok = xb_record_submit(X, X->sub[h].cmd, X->sub[h].fence, X->sub[h].qp, ex, rows, count, off, v4, &ngemm, 1 + h, x0, i0, y0, NULL);free(off);if (!ok) return 0;X->sub[h].inflight = 1; async_begin(X->d);@@ -4190,6 +4398,119 @@ int coli_vk_xb_sub_join(int h, float *const *yout, double *device_ms) {return 1;}+/* ---- a big step's whole-step buffers (the tier's prompt steps on the primary device) ----+ * coli_vk_xb_step_begin uploads the step's S token rows once and hands out ys, S*K output+ * rows in assignment order (s*K + k). Sub-batches issued with coli_vk_xb_sub_issue_step+ * then take the grouped route reading their x rows through the assignment map and+ * writing their y rows straight to ys: no packed x rows (K copies of each token row) and+ * no copy back at the join. A batch the grouped route does not take is packed from the+ * caller's rows as before and copied into ys at its join. coli_vk_xb_step_sum adds every+ * used row of a token in rank order on the device, so the host reads S rows, not S*K. */+static int st_buf(XbCtx *X, XbBuf *b, size_t bytes, uint32_t memtype) {+ if (bytes <= b->region) return 1;+ if (b->buf) { vkDestroyBuffer(X->dev, b->buf, NULL); vkFreeMemory(X->dev, b->mem, NULL); }+ memset(b, 0, sizeof(*b));+ size_t r = xb_up(bytes + bytes / 4, 65536);+ if (!alloc_hostvis_mt(r, &b->buf, &b->mem, &b->ptr, memtype)) { memset(b, 0, sizeof(*b)); return 0; }+ b->region = r;+ return 1;+}+float *coli_vk_xb_step_begin(int S, int K, const float *x) {+ XbCtx *X = &g_xb[0];+ if (!X->ready || !X->st.pipe || !X->grp_rows || xb_busy(X) || S < 1 || K < 1) return NULL;+ const size_t dr = (size_t)X->D * 4;+ if ((size_t)S * K * dr > X->ssbo_range) return NULL;+ if (!st_buf(X, &X->st.xt, (size_t)S * dr, X->mt_host) || !st_buf(X, &X->st.ys, (size_t)S * K * dr, X->mt_cached) ||+ !st_buf(X, &X->st.out, (size_t)S * dr, X->mt_cached) || !st_buf(X, &X->st.w, (size_t)S * K * 4, X->mt_host) ||+ !st_buf(X, &X->st.use, (size_t)S * K * 4, X->mt_host)) return NULL;+ uint8_t *xt = X->st.xt.ptr;+ #pragma omp parallel for schedule(static) if (S >= 256)+ for (int s = 0; s < S; s++) memcpy(xt + (size_t)s * dr, x + (size_t)s * X->D, dr);+ X->st.on = 1; X->st.S = S; X->st.K = K; X->st.xh = x; X->st.steps++;+ return X->st.ys.ptr;+}+void coli_vk_xb_step_end(void) { g_xb[0].st.on = 0; }+int coli_vk_xb_sub_issue_step(int h, ColiVkExpert *const *ex, const int *rows, int count, const int *assign,+ const float *wrows) {+ XbCtx *X = &g_xb[0];+ if (!X->st.on || h < 0 || h > 1 || count < 1) return 0;+ for (int c = 0; c < count; c++) if (!ex[c] || ex[c]->X != X) return 0;+ int total = 0;+ if (!xb_grp_ok(X, ex, rows, count, &total)) { /* packed as before, copied into ys at the join */+ const float **xr = malloc((size_t)total * sizeof(*xr));+ if (!xr) return 0;+ for (int j = 0; j < total; j++) xr[j] = X->st.xh + (size_t)(assign[j] / X->st.K) * X->D;+ int ok = coli_vk_xb_sub_issue(h, ex, rows, count, xr, wrows);+ free(xr);+ return ok;+ }+ if (!X->ready || X->inflight || X->sub[h].inflight || !X->sub_x) return 0;+ size_t *off = malloc((size_t)count * 3 * sizeof(size_t));+ if (!off) return 0;+ size_t x0 = h * X->sub_x, i0 = h * X->sub_i, y0 = h * X->sub_y, xe, ie, ye; int t2;+ if (!xb_plan(X, ex, rows, count, 0, x0, i0, y0, off, &xe, &ie, &ye, &t2) ||+ xe > x0 + X->sub_x || ie > i0 + X->sub_i || ye > y0 + X->sub_y) { free(off); return 0; }+ X->sub[h].nrows = 0; /* the rows are in ys already: nothing to copy at the join */+ int ngemm = 0;+ int ok = xb_record_submit(X, X->sub[h].cmd, X->sub[h].fence, X->sub[h].qp, ex, rows, count, off, 0, &ngemm,+ 1 + h, x0, i0, y0, assign);+ free(off);+ if (!ok) return 0;+ X->sub[h].inflight = 1; async_begin(X->d);+ X->sub_batches++; X->sub_experts += (unsigned long long)count; X->sub_rows += (unsigned long long)total;+ X->sub_gemm += (unsigned long long)ngemm;+ return 1;+}+const float *coli_vk_xb_step_sum(const float *w, const uint8_t *use, double *device_ms) {+ XbCtx *X = &g_xb[0];+ if (device_ms) *device_ms = 0;+ if (!X->st.on || xb_busy(X)) return NULL;+ const int S = X->st.S, K = X->st.K, D = X->D, n = S * K;+ float *wd = X->st.w.ptr; uint32_t *ud = X->st.use.ptr;+ for (int i = 0; i < n; i++) { wd[i] = w ? w[i] : 1.0f; ud[i] = use[i] != 0; }+ VkDescriptorBufferInfo bi[4] = {{X->st.ys.buf, 0, (size_t)n * D * 4}, {X->st.w.buf, 0, (size_t)n * 4},+ {X->st.use.buf, 0, (size_t)n * 4}, {X->st.out.buf, 0, (size_t)S * D * 4}};+ VkWriteDescriptorSet wr[4];+ for (int b = 0; b < 4; b++)+ wr[b] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = X->st.set,+ .dstBinding = (uint32_t)b, .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,+ .pBufferInfo = &bi[b]};+ vkUpdateDescriptorSets(X->dev, 4, wr, 0, NULL);+ VkCommandBuffer cmd = X->cmd;+ if (vkResetCommandBuffer(cmd, 0) != VK_SUCCESS) return NULL;+ VkCommandBufferBeginInfo cb = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,+ .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};+ vkBeginCommandBuffer(cmd, &cb);+ if (X->has_ts) { vkCmdResetQueryPool(cmd, X->qpool, 0, 2); vkCmdWriteTimestamp(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, X->qpool, 0); }+ vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->st.pipe);+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, X->st.pl, 0, 1, &X->st.set, 0, NULL);+ int pc[3] = {S, K, D};+ vkCmdPushConstants(cmd, X->st.pl, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);+ vkCmdDispatch(cmd, (uint32_t)((D + 255) / 256), (uint32_t)S, 1);+ VkMemoryBarrier hb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, .dstAccessMask = VK_ACCESS_HOST_READ_BIT};+ vkCmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &hb, 0, NULL, 0, NULL);+ if (X->has_ts) vkCmdWriteTimestamp(cmd, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, X->qpool, 1);+ if (vkEndCommandBuffer(cmd) != VK_SUCCESS) return NULL;+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &cmd};+ if (vkResetFences(X->dev, 1, &X->fence) != VK_SUCCESS || vk_submit(X->d, X->q, &si, X->fence) != VK_SUCCESS) return NULL;+ async_begin(X->d);+ VkResult r = vk_fence_wait_gemm(X->dev, X->fence);+ async_end(X->d);+ if (r != VK_SUCCESS) { fprintf(stderr, "[VK] expert batch: the step's sum failed (%d)\n", r); X->ready = 0; return NULL; }+ double ms = 0;+ if (X->has_ts) {+ uint64_t ts[2];+ if (vkGetQueryPoolResults(X->dev, X->qpool, 0, 2, sizeof ts, ts, sizeof ts[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) {+ uint64_t mask = X->ts_bits >= 64 ? ~0ull : ((1ull << X->ts_bits) - 1);+ ms = (double)((ts[1] - ts[0]) & mask) * X->ts_period / 1e6;+ }+ }+ X->st.sums++; X->st.sum_ms += ms;+ if (device_ms) *device_ms = ms;+ return X->st.out.ptr;+}+static void xb_shutdown(XbCtx *X) {if (!X->dev) return;if (X->inflight) { vk_fence_wait(X->dev, X->fence); X->inflight = 0; async_end(X->d); }@@ -4200,9 +4521,24 @@ static void xb_shutdown(XbCtx *X) {free(X->sub[h].yoff);}while (X->live) coli_vk_xb_expert_free(X->live);- XbBuf *bufs[5] = {&X->x, &X->g, &X->u, &X->h, &X->y};- for (int k = 0; k < 5; k++)+ XbBuf *bufs[17] = {&X->x, &X->g, &X->u, &X->h, &X->y};+ for (int k = 0; k < 6; k++) bufs[5 + k] = &X->grp_it[k / 2][k % 2], bufs[11 + k] = &X->grp_et[k / 2][k % 2];+ for (int k = 0; k < 17; k++)if (bufs[k]->buf) { vkDestroyBuffer(X->dev, bufs[k]->buf, NULL); vkFreeMemory(X->dev, bufs[k]->mem, NULL); }+ for (int v = 0; v < 2; v++) {+ if (X->p_grp[v]) vkDestroyPipeline(X->dev, X->p_grp[v], NULL);+ if (X->sh_grp[v]) vkDestroyShaderModule(X->dev, X->sh_grp[v], NULL);+ }+ if (X->grp_pool) vkDestroyDescriptorPool(X->dev, X->grp_pool, NULL);+ XbBuf *sb[8] = {&X->st.xt, &X->st.ys, &X->st.out, &X->st.w, &X->st.use, &X->grp_map[0], &X->grp_map[1], &X->grp_map[2]};+ for (int k = 0; k < 8; k++) if (sb[k]->buf) { vkDestroyBuffer(X->dev, sb[k]->buf, NULL); vkFreeMemory(X->dev, sb[k]->mem, NULL); }+ if (X->st.pipe) vkDestroyPipeline(X->dev, X->st.pipe, NULL);+ if (X->st.sh) vkDestroyShaderModule(X->dev, X->st.sh, NULL);+ if (X->st.pool) vkDestroyDescriptorPool(X->dev, X->st.pool, NULL);+ if (X->st.pl) vkDestroyPipelineLayout(X->dev, X->st.pl, NULL);+ if (X->st.dsl) vkDestroyDescriptorSetLayout(X->dev, X->st.dsl, NULL);+ if (X->pl_grp) vkDestroyPipelineLayout(X->dev, X->pl_grp, NULL);+ if (X->dsl_grp) vkDestroyDescriptorSetLayout(X->dev, X->dsl_grp, NULL);for (int k = 0; k < X->ndpools; k++) vkDestroyDescriptorPool(X->dev, X->dpools[k], NULL);free(X->dpools);if (X->act_pool) vkDestroyDescriptorPool(X->dev, X->act_pool, NULL);diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex 882fb7d1..8df1d870 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -255,6 +255,7 @@ int coli_vk_xb_issue_w(ColiVkExpert *const *ex, const int *rows, int count, conint coli_vk_xb_join(const float **yrows, double *device_ms);typedef struct {unsigned long long batches, experts, rows, gemm_experts;+ unsigned long long grouped_batches; /* batches that took the grouped GEMM (qmatmul_grp.comp) */double device_ms; /* summed batch device time (timestamps) */int timestamps, queue_shared, gemm_rows;size_t scratch_bytes;@@ -263,6 +264,18 @@ typedef struct {unsigned long long cooperative_matmuls; /* all projections, including sub-batches */} ColiVkXbStats;void coli_vk_xb_stats(ColiVkXbStats *st);+/* A big step's whole-step buffers on the primary device (the grouped GEMM): the step's+ * S token rows x[S*D] uploaded once; returns its S*K output rows (s*K + k, host-readable)+ * or NULL when the device cannot (the tier then packs and copies as before). Sub-batches+ * of the step go through coli_vk_xb_sub_issue_step (assign[j]: packed row j's s*K + k)+ * and land in those rows; coli_vk_xb_step_sum then adds a token's used rows in rank order+ * (w[i] * row i as an fma chain from 0) on the device and returns S*D rows, valid until+ * the next step. coli_vk_xb_step_end closes the step either way. */+float *coli_vk_xb_step_begin(int S, int K, const float *x);+int coli_vk_xb_sub_issue_step(int h, ColiVkExpert *const *ex, const int *rows, int count, const int *assign,+ const float *wrows); /* wrows: as coli_vk_xb_sub_issue's (a batch off the grouped route) */+const float *coli_vk_xb_step_sum(const float *w, const uint8_t *use, double *device_ms);+void coli_vk_xb_step_end(void);/* Sub-batches, for a prefill step too big for one batch (vk_tier.c's streaming): batch k* of a step goes to half k % 2 of the scratch, so two run at once and the host fills one* while the device computes the other. _reserve sizes each half for a batch of up todiff --git a/c/qwen36.c b/c/qwen36.cindex 93bb0739..88be6cbb 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -3252,13 +3252,22 @@ static void moe_vk_run(Model *m, Layer *l, int layer, const float *x, int S, floif (xf) moe_vk_xf_cpu(m, layer, xb, rows, ib, want, ctb);else moe_vk_i8_cpu(m, layer, xb, rows, ib, want, ctb, g, u);if (shb) { memset(shb, 0, (size_t)rows * D * sizeof(float)); qwen_shared_experts_cpu(m, l, xb, rows, shb, g, u, hh); }- if (ndev && !vkt_join(dev)) { /* the batch failed (the tier stops): those pairs here */+ const float *dsum = NULL; /* a step wholly on the device: its routed rows summed there */+ if (ndev && !vkt_join_sum(dev, vb, &dsum)) { /* the batch failed (the tier stops): those pairs here */if (xf) moe_vk_xf_cpu(m, layer, xb, rows, ib, taken, ctb);else moe_vk_i8_cpu(m, layer, xb, rows, ib, taken, ctb, g, u);memset(taken, 0, (size_t)n);}+ /* rows apart (a prompt step reads tens of MB here); each row in rank order */+ #pragma omp parallel for schedule(static) if (rows >= 64)for (int s = 0; s < rows; s++) {float *os = out + (int64_t)(s0 + s) * D;+ if (dsum) { /* the device's sum is this loop's for a zeroed row */+ const float *ds = dsum + (int64_t)s * D;+ for (int d = 0; d < D; d++) os[d] += ds[d];+ if (shb) { const float *sp = shb + (int64_t)s * D; for (int d = 0; d < D; d++) os[d] += sp[d]; }+ continue;+ }for (int k = 0; k < K; k++) {int i = s * K + k;if (ib[i] < 0) continue;diff --git a/c/shaders/expert_sum.comp b/c/shaders/expert_sum.compnew file mode 100644index 00000000..85525f52--- /dev/null+++ b/c/shaders/expert_sum.comp@@ -0,0 +1,22 @@+#version 450+// A big step's routed sum on the device (backend_vulkan.c, coli_vk_xb_step_sum):+// out[s][d] = sum over k in rank order with use[s*K+k] of w[s*K+k] * y[s*K+k][d]+// as an fma chain from 0 -- the engines' host loop (os[d] += w * row[d] from a zeroed+// row, contracted to fma), so the rows come back as that loop would make them.+// Grid: x = ceil(D / 256), y = S.+layout(local_size_x = 256) in;+layout(std430, binding = 0) readonly buffer Y { float y[]; };+layout(std430, binding = 1) readonly buffer W { float w[]; };+layout(std430, binding = 2) readonly buffer U { uint use[]; };+layout(std430, binding = 3) writeonly buffer O { float o[]; };+layout(push_constant) uniform P { int S, K, D; } p;+void main() {+ uint d = gl_GlobalInvocationID.x, s = gl_WorkGroupID.y;+ if (d >= uint(p.D)) return;+ float acc = 0.0;+ for (uint k = 0u; k < uint(p.K); k++) {+ uint i = s * uint(p.K) + k;+ if (use[i] != 0u) acc = fma(w[i], y[i * uint(p.D) + d], acc);+ }+ o[s * uint(p.D) + d] = acc;+}diff --git a/c/shaders/expert_sum.spv b/c/shaders/expert_sum.spvnew file mode 100644index 00000000..7686340aBinary files /dev/null and b/c/shaders/expert_sum.spv differdiff --git a/c/shaders/qmatmul_grp.comp b/c/shaders/qmatmul_grp.compnew file mode 100644index 00000000..ad3e7513--- /dev/null+++ b/c/shaders/qmatmul_grp.comp@@ -0,0 +1,281 @@+#version 450+// The expert batch's grouped GEMM (backend_vulkan.c, xb_issue): ONE dispatch covers+// every expert of a batch. Each workgroup reads its job from an item list (expert,+// first row, 128-row output block) and the expert's weights and scales by device+// address from an expert table (VK_KHR_buffer_device_address), so a prefill step's+// experts run as one grid instead of a few small dispatches per expert.+// y[s,o] = sum_i x[s,i] * dequant(W[o,i]) (default)+// hid[s,o] = silu(gate[s,o]) * up[s,o], clamped as expert_act.comp's act 0 (-DGATE_UP)+// 256 invocations = 4 subgroups of 64 (the host requires subgroup size 64), a block of+// 128 output rows x 16*TT rows of the expert. Per 64-input step two lanes dequantise+// one weight row (32 values) to f16 in shared memory and x is converted beside it;+// the next step's raw words, scales and x values are fetched into registers while+// the matrix units work on the current one. Shared memory holds every 16x16 operand+// tile contiguously (stride 16): strided cooperative-matrix loads out of a wide array+// measured about 2x slower on RDNA3.5. A partial last row tile goes through a+// per-subgroup 16x16 patch, so rows past the expert's are never written.+// Inputs are rounded to f16 (as qmatmul_coop.comp; not bit-identical to the GEMVs).+// fmt: 1 = int8 per row, 2 = int4 per row, 4 = int4 per (row, gs); int4 nibbles as+// v+8, element k at bits 4k of its word (qmatmul.comp's packing).+// Host gates: I % 64 == 0, O % 128 == 0, rowWords % 4 == 0, fmt 4: gs % 8 == 0.+#extension GL_KHR_cooperative_matrix : require+#extension GL_KHR_memory_scope_semantics : require+#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require+#extension GL_EXT_control_flow_attributes : enable+#extension GL_KHR_shader_subgroup_basic : require+#extension GL_EXT_buffer_reference : require+#extension GL_EXT_shader_explicit_arithmetic_types_int64 : require+layout(local_size_x = 256) in;+layout(constant_id = 0) const uint TT = 4; // 16-row token tiles per workgroup++layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer WRef { uint v[]; };+// the same weights as 16-byte words: a lane's 4 (int4) or 8 (int8) words of a step+// are 16-byte aligned (rowWords and the step offset are multiples of 4), and with+// 4-byte references the compiler issued one global_load_dword per word+layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer WRef4 { uvec4 v[]; };+layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer SRef { float v[]; };+layout(std430, binding = 0) readonly buffer X { vec4 x4[]; }; // packed rows of all experts (read as vec4)+layout(std430, binding = 1) readonly buffer IT { uvec4 items[]; }; // (expert, first row, output block, -)+layout(std430, binding = 2) readonly buffer ET { uint etab[]; }; // 16 uints per expert, see below+layout(std430, binding = 3) writeonly buffer Y { float y[]; }; // packed output rows+// kgat > 0 (a big step's whole-step buffers, backend_vulkan.c's coli_vk_xb_step_*): packed+// row j belongs to assignment amap[j] = s*K + k; gate+up reads x row amap[j] / kgat (the+// step's token rows, no packing), down writes y row amap[j] (the step's output rows)+layout(std430, binding = 4) readonly buffer AM { int amap[]; };+// expert entry: [0..1] gate/W address, [2..3] its scales, [4..7] up + scales (GATE_UP),+// [8] rows, [9] first packed row, [10] fmt, [11] rowWords, [12] gs+layout(push_constant) uniform P { int I, O; float limit; int ibase; int kgat; } p;+WRef4 wgr; SRef gsr;+#ifdef GATE_UP+WRef4 wur; SRef usr;+#endif+struct Job { int fmt, S, rowWords, gs; uint xoff; } J;+#define gscale(i) gsr.v[i]+#define uscale(i) usr.v[i]++#define BO 128u // output rows per workgroup (8 tiles)+#define TM (16u * TT) // token rows per workgroup+shared uvec4 sx[TM * 8u]; // x slice, A tiles row-major, 8 f16 per uvec4+shared uvec4 sg[BO * 8u]; // weights, B tiles COLUMN-major (= W[n][k], k contiguous)+#ifdef GATE_UP+shared uvec4 su[BO * 8u];+#endif+shared float spatch[4u * 256u]; // per-subgroup 16x16 patch for partial token tiles (4 x wave64)++// Shared memory holds every 16x16 operand tile CONTIGUOUSLY (256 values, loaded+// with stride 16): strided cooperative-matrix loads out of a wide row-major+// array are what made the first version slow (RADV, cf. Kernel-Lab E020/E030).+// x: A tile (token tile t, k tile kt), element (row r, col k) -> r*16 + k+// W: B tile (out tile q, k tile kt), B(k, n) = W[n][k], row-major -> k*16 + n+uint wix(uint n, uint k) { return ((n >> 4u) * 4u + (k >> 4u)) * 256u + (n & 15u) * 16u + (k & 15u); }+uint xix(uint r, uint k) { return ((r >> 4u) * 4u + (k >> 4u)) * 256u + (r & 15u) * 16u + (k & 15u); }++// Register double buffering: the raw words, scales and x values of the NEXT 64-input+// step are fetched into registers while the matrix units work on the current step+// out of shared memory; they are dequantised into shared memory after a barrier.+#define XPL (TM * 64u / 256u) // x values per lane per step; TT must be 2, 4 or 8 (16 would exceed 64 KB shared memory on current hardware)+uint rwg[8];+float rsg[4];+#ifdef GATE_UP+uint rwu[8];+float rsu[4];+#endif+float rx[XPL];+void fetch_raw(uint s0, uint o0, uint k0, uint tid) {+ uint K = uint(p.I), S = uint(J.S);+ {+ uint r = tid / (64u / XPL), kk = (tid % (64u / XPL)) * XPL;+ bool ok = s0 + r < S;+ uint xr = J.xoff + s0 + r;+#ifdef GATE_UP+ if (ok && p.kgat > 0) xr = uint(amap[xr]) / uint(p.kgat);+#endif+ uint xb = (xr * K + k0 + kk) >> 2u; // K, k0, kk multiples of 4+ [[unroll]] for (uint i = 0u; i < XPL / 4u; i++) {+ vec4 v = ok ? x4[xb + i] : vec4(0.0);+ rx[4u*i] = v.x; rx[4u*i+1u] = v.y; rx[4u*i+2u] = v.z; rx[4u*i+3u] = v.w;+ }+ }+ uint n = tid >> 1u, hf = tid & 1u, o = o0 + n, kb = k0 + 32u * hf;+ int ng = (p.I + J.gs - 1) / max(J.gs, 1);+ if (J.fmt == 1) {+ uint rb = (o * uint(J.rowWords) + kb / 4u) >> 2u;+ uvec4 a0 = wgr.v[rb], a1 = wgr.v[rb + 1u];+ rwg[0] = a0.x; rwg[1] = a0.y; rwg[2] = a0.z; rwg[3] = a0.w; rwg[4] = a1.x; rwg[5] = a1.y; rwg[6] = a1.z; rwg[7] = a1.w;+ rsg[0] = gscale(o);+#ifdef GATE_UP+ uvec4 b0 = wur.v[rb], b1 = wur.v[rb + 1u];+ rwu[0] = b0.x; rwu[1] = b0.y; rwu[2] = b0.z; rwu[3] = b0.w; rwu[4] = b1.x; rwu[5] = b1.y; rwu[6] = b1.z; rwu[7] = b1.w;+ rsu[0] = uscale(o);+#endif+ } else {+ uint rb = (o * uint(J.rowWords) + kb / 8u) >> 2u;+ uvec4 a0 = wgr.v[rb];+ rwg[0] = a0.x; rwg[1] = a0.y; rwg[2] = a0.z; rwg[3] = a0.w;+#ifdef GATE_UP+ uvec4 b0 = wur.v[rb];+ rwu[0] = b0.x; rwu[1] = b0.y; rwu[2] = b0.z; rwu[3] = b0.w;+#endif+ [[unroll]] for (uint j = 0u; j < 4u; j++) {+ uint si = J.fmt == 4 ? o * uint(ng) + (kb + 8u * j) / uint(J.gs) : o;+ rsg[j] = gscale(si);+#ifdef GATE_UP+ rsu[j] = uscale(si);+#endif+ }+ }+}+uint pk(float a, float b) { return packFloat2x16(f16vec2(float16_t(a), float16_t(b))); }+uint i8(uint w, uint k) { return uint(int(w << (24u - 8u * k)) >> 24); }+void store_raw(uint tid) {+ {+ uint r = tid / (64u / XPL), kk = (tid % (64u / XPL)) * XPL;+ [[unroll]] for (uint c = 0u; c < XPL / 8u; c++) {+ uint k = kk + 8u * c;+ uint base = ((r >> 4u) * 4u + (k >> 4u)) * 32u + (r & 15u) * 2u + ((k & 15u) >> 3u);+ if (J.fmt == 1)+ sx[base] = uvec4(pk(rx[8u*c], rx[8u*c+1u]), pk(rx[8u*c+2u], rx[8u*c+3u]), pk(rx[8u*c+4u], rx[8u*c+5u]), pk(rx[8u*c+6u], rx[8u*c+7u]));+ else /* int4 weights come out as pairs (k, k+4): the same order of k here */+ sx[base] = uvec4(pk(rx[8u*c], rx[8u*c+4u]), pk(rx[8u*c+1u], rx[8u*c+5u]), pk(rx[8u*c+2u], rx[8u*c+6u]), pk(rx[8u*c+3u], rx[8u*c+7u]));+ }+ }+ uint n = tid >> 1u, hf = tid & 1u;+ // lane's 32 values = k tiles 2hf, 2hf+1; 2 uvec4 per tile row+ uint wb = ((n >> 4u) * 4u + 2u * hf) * 32u + (n & 15u) * 2u;+ if (J.fmt == 1) {+ [[unroll]] for (uint j = 0u; j < 8u; j += 2u) { // 2 words = 8 values = one uvec4+ float a = rsg[0];+ uvec4 v = uvec4(pk(float(int(i8(rwg[j],0u)))*a, float(int(i8(rwg[j],1u)))*a), pk(float(int(i8(rwg[j],2u)))*a, float(int(i8(rwg[j],3u)))*a),+ pk(float(int(i8(rwg[j+1u],0u)))*a, float(int(i8(rwg[j+1u],1u)))*a), pk(float(int(i8(rwg[j+1u],2u)))*a, float(int(i8(rwg[j+1u],3u)))*a));+ uint e = wb + (j >> 2u) * 32u + ((j >> 1u) & 1u);+ sg[e] = v;+#ifdef GATE_UP+ float b = rsu[0];+ su[e] = uvec4(pk(float(int(i8(rwu[j],0u)))*b, float(int(i8(rwu[j],1u)))*b), pk(float(int(i8(rwu[j],2u)))*b, float(int(i8(rwu[j],3u)))*b),+ pk(float(int(i8(rwu[j+1u],0u)))*b, float(int(i8(rwu[j+1u],1u)))*b), pk(float(int(i8(rwu[j+1u],2u)))*b, float(int(i8(rwu[j+1u],3u)))*b));+#endif+ }+ } else {+ // 1 word = 8 nibbles = one uvec4. Nibble m and m+4 (bits 4m, 4m+16) go into+ // one f16 pair via 0x6400 | n = 1024 + n, then (h - 1032) * scale in packed+ // f16: ~2.5 ops per value instead of ~6 (RDNA3 runs WMMA on the same VALUs,+ // so dequantisation time adds straight to the multiply time). k order within+ // 8 = 0,4,1,5,2,6,3,7 -- x is stored the same way.+ [[unroll]] for (uint j = 0u; j < 4u; j++) {+ f16vec2 sa = f16vec2(float16_t(rsg[j])); uint w = rwg[j];+ uvec4 v; [[unroll]] for (uint m = 0u; m < 4u; m++)+ v[m] = packFloat2x16((unpackFloat2x16(((w >> (4u*m)) & 0x000F000Fu) | 0x64006400u) - f16vec2(1032.0)) * sa);+ uint e = wb + (j >> 1u) * 32u + (j & 1u);+ sg[e] = v;+#ifdef GATE_UP+ f16vec2 sb = f16vec2(float16_t(rsu[j])); uint u = rwu[j];+ [[unroll]] for (uint m = 0u; m < 4u; m++)+ v[m] = packFloat2x16((unpackFloat2x16(((u >> (4u*m)) & 0x000F000Fu) | 0x64006400u) - f16vec2(1032.0)) * sb);+ su[e] = v;+#endif+ }+ }+}++// Each subgroup owns output tiles q = sgi, sgi + nsg, ... (8 per workgroup) for+// all TT token tiles: MAXQ tiles per subgroup at most (8 with one wave64 group of 4).+#define MAXQ 2u++void main() {+ uvec4 it = items[uint(p.ibase) + gl_WorkGroupID.x];+ uint eb = it.x * 16u;+ wgr = WRef4(packUint2x32(uvec2(etab[eb + 0u], etab[eb + 1u])));+ gsr = SRef(packUint2x32(uvec2(etab[eb + 2u], etab[eb + 3u])));+#ifdef GATE_UP+ wur = WRef4(packUint2x32(uvec2(etab[eb + 4u], etab[eb + 5u])));+ usr = SRef(packUint2x32(uvec2(etab[eb + 6u], etab[eb + 7u])));+#endif+ J.S = int(etab[eb + 8u]); J.xoff = etab[eb + 9u]; J.fmt = int(etab[eb + 10u]);+ J.rowWords = int(etab[eb + 11u]); J.gs = int(etab[eb + 12u]);+ uint s0 = it.y;+ uint o0 = it.z * BO;+ uint tid = gl_LocalInvocationID.x;+ uint nsg = gl_NumSubgroups, sgi = gl_SubgroupID;+ uint K = uint(p.I), S = uint(J.S);+ uint nq = (8u + nsg - 1u) / nsg; // output tiles per subgroup: 2 (wave64) or 1 (wave32)+ // token tiles that hold rows of this expert (workgroup-uniform): an expert gets+ // ~32 rows on average, the tiles past its last row would multiply zeros+ uint ntl = min(TT, (S - s0 + 15u) / 16u);++ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> G[MAXQ * TT];+#ifdef GATE_UP+ coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator> U[MAXQ * TT];+#endif+ [[unroll]] for (uint t = 0u; t < MAXQ * TT; t++) {+ G[t] = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);+#ifdef GATE_UP+ U[t] = coopmat<float, gl_ScopeSubgroup, 16, 16, gl_MatrixUseAccumulator>(0.0);+#endif+ }+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseA> A[TT];+ coopmat<float16_t, gl_ScopeSubgroup, 16, 16, gl_MatrixUseB> B;++ fetch_raw(s0, o0, 0u, tid);+ store_raw(tid);+ barrier();+ for (uint k0 = 0u; k0 < K; k0 += 64u) {+ bool nxt = k0 + 64u < K; // workgroup-uniform+ if (nxt) fetch_raw(s0, o0, k0 + 64u, tid); // in flight during the multiply below+ [[unroll]] for (uint kt = 0u; kt < 4u; kt++) {+ [[unroll]] for (uint t = 0u; t < TT; t++)+ if (t < ntl) coopMatLoad(A[t], sx, (t * 4u + kt) * 32u, 2u, gl_CooperativeMatrixLayoutRowMajor);+ [[unroll]] for (uint qi = 0u; qi < MAXQ; qi++) {+ uint q = sgi + qi * 4u; // wave64 assumed (requiredSubgroupSize = 64)+ coopMatLoad(B, sg, (q * 4u + kt) * 32u, 2u, gl_CooperativeMatrixLayoutColumnMajor);+ [[unroll]] for (uint t = 0u; t < TT; t++) if (t < ntl) G[qi * TT + t] = coopMatMulAdd(A[t], B, G[qi * TT + t]);+#ifdef GATE_UP+ coopMatLoad(B, su, (q * 4u + kt) * 32u, 2u, gl_CooperativeMatrixLayoutColumnMajor);+ [[unroll]] for (uint t = 0u; t < TT; t++) if (t < ntl) U[qi * TT + t] = coopMatMulAdd(A[t], B, U[qi * TT + t]);+#endif+ }+ }+ barrier();+ if (nxt) { store_raw(tid); barrier(); }+ }++ uint lane = gl_SubgroupInvocationID, sgsz = gl_SubgroupSize;+ [[unroll]] for (uint qi = 0u; qi < MAXQ; qi++) {+ if (qi >= nq) break;+ uint q = sgi + qi * nsg;+ if (q >= 8u) break;+ [[unroll]] for (uint t = 0u; t < TT; t++) {+#ifdef GATE_UP+ for (int e = 0; e < G[qi * TT + t].length(); e++) {+ float gt = G[qi * TT + t][e], ut = U[qi * TT + t][e];+ if (p.limit > 0.0) {+ if (gt > p.limit) gt = p.limit;+ if (ut > p.limit) ut = p.limit;+ if (ut < -p.limit) ut = -p.limit;+ }+ G[qi * TT + t][e] = (gt / (1.0 + exp(-gt))) * ut;+ }+#endif+ uint r0 = s0 + t * 16u;+ if (r0 >= S) continue; // subgroup-uniform+#ifdef GATE_UP+ if (r0 + 16u <= S) {+#else+ if (r0 + 16u <= S && p.kgat == 0) {+#endif+ coopMatStore(G[qi * TT + t], y, (J.xoff + r0) * uint(p.O) + o0 + q * 16u, uint(p.O), gl_CooperativeMatrixLayoutRowMajor);+ } else {+ coopMatStore(G[qi * TT + t], spatch, sgi * 256u, 16u, gl_CooperativeMatrixLayoutRowMajor);+ subgroupBarrier();+ for (uint e = lane; e < 256u; e += sgsz) {+ uint r = e / 16u, n = e % 16u;+ uint yr = J.xoff + r0 + r;+#ifndef GATE_UP+ if (r0 + r < S && p.kgat > 0) yr = uint(amap[yr]);+#endif+ if (r0 + r < S) y[yr * uint(p.O) + o0 + q * 16u + n] = spatch[sgi * 256u + e];+ }+ subgroupBarrier();+ }+ }+ }+}diff --git a/c/shaders/qmatmul_grp.spv b/c/shaders/qmatmul_grp.spvnew file mode 100644index 00000000..3dd17499Binary files /dev/null and b/c/shaders/qmatmul_grp.spv differdiff --git a/c/shaders/qmatmul_grp_gate_up.spv b/c/shaders/qmatmul_grp_gate_up.spvnew file mode 100644index 00000000..6178a8a0Binary files /dev/null and b/c/shaders/qmatmul_grp_gate_up.spv differdiff --git a/c/tests/test_vk_tier.c b/c/tests/test_vk_tier.cindex ee59dbc9..56c1a399 100644--- a/c/tests/test_vk_tier.c+++ b/c/tests/test_vk_tier.c@@ -36,7 +36,8 @@static int fails;#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)-enum { L = 2, E = 10, H = 192, F = 128, K = 3 };+enum { L = 2, E = 10, K = 3, HF_MAX = 512 };+static int H = 192, F = 128; /* hidden and expert width; grouped() widens H */static unsigned rng = 7;static unsigned rnd(void) { rng = rng * 1103515245u + 12345u; return rng >> 8; }static float frnd(void) { return (float)((int)(rnd() % 2001) - 1000) / 1000.0f; }@@ -128,7 +129,7 @@ typedef struct { Mat g, u, d; } Ex;static Ex ex[L][E];static int g_act; static float g_limit;static void expert_ref(const Ex *e, const float *x, float *y) {- float h[F];+ float h[HF_MAX];for (int o = 0; o < F; o++) {double a = 0, b = 0;for (int i = 0; i < H; i++) { a += (double)e->g.w[(size_t)o * H + i] * x[i]; b += (double)e->u.w[(size_t)o * H + i] * x[i]; }@@ -445,7 +446,7 @@ static void qdq128(float *v, int n) {}}static void expert_v4(const Ex *e, const float *xq, float w, float limit, float *y) {- float h[F];+ float h[HF_MAX];for (int o = 0; o < F; o++) {double a = 0, b = 0;for (int i = 0; i < H; i++) { a += (double)e->g.w[(size_t)o * H + i] * xq[i]; b += (double)e->u.w[(size_t)o * H + i] * xq[i]; }@@ -494,10 +495,10 @@ static void v4_act(void) {expert_v4(&ex[l][idx[i]], xq + (size_t)s * H, w[i], limit, y);for (int d = 0; d < H; d++) ref[(size_t)s * H + d] += y[d];if (taken[i] && joined) {- float dr[H];+ float dr[HF_MAX];for (int d = 0; d < H; d++) dr[d] = bf16r(rows[i][d]);double r = rel(dr, y, H); if (r > worst) worst = r;- rows_same += !memcmp(dr, y, sizeof dr);+ rows_same += !memcmp(dr, y, (size_t)H * sizeof(float));for (int d = 0; d < H; d++) out[(size_t)s * H + d] += dr[d];dev++;} else for (int d = 0; d < H; d++) out[(size_t)s * H + d] += cpu[(size_t)i * H + d];@@ -562,7 +563,7 @@ static double stream_step(int layer, int S, const int *idx, const float *w, intint i = s * K + k;if (v4) expert_v4(&ex[layer][idx[i]], xq + (size_t)s * H, w[i], 10.0f, y);else expert_ref(&ex[layer][idx[i]], xq + (size_t)s * H, y);- float dr[H];+ float dr[HF_MAX];const float *c = cpu + (size_t)i * H;if (taken[i] && joined) {for (int d = 0; d < H; d++) dr[d] = v4 ? bf16r(dev[i][d]) : dev[i][d];@@ -697,6 +698,116 @@ static void stream_commit_failure(const char *spv) {unsetenv("COLI_VK_TIER_STREAM_SLOTS"); unsetenv("COLI_VK_TIER_STREAM_ROWS");}+/* The expert batch's grouped GEMM (qmatmul_grp.comp) and a big step's whole-step+ * buffers (coli_vk_xb_step_*): a hidden width of 256 (multiples of 128 take the grouped+ * route), every format it computes (device fmt 1, 2, 4).+ * single steps of 32 rows x top-3 (96 assignments, over its 64) as one batch;+ * whole prompt steps of 40 and 64 rows with every expert resident: sub-batches+ * read their x through the assignment map, write y in place, and+ * vkt_join_sum returns the rows summed on the device, compared with the+ * reference sum and with the engines' own loop over the device rows;+ * mixed the same steps with a budget of four experts: cold ones stream or stay on+ * the CPU, no device sum (*sum NULL), the rows still right.+ * Inputs and weights are rounded to f16 on this route, hence the looser bound. A device+ * without it (no cooperative matrices at subgroup size 64 or bufferDeviceAddress, e.g.+ * Lavapipe) runs the same steps on the per-expert route and vkt_join_sum gives NULL. */+static double sum_step(int layer, int S, const int *idx, const float *w, Books *bk, int *summed) {+ float *x = calloc((size_t)S * H, sizeof(float)), *cpu = calloc((size_t)S * K * H, sizeof(float));+ float *ref = calloc((size_t)S * H, sizeof(float)), *loop = calloc((size_t)S * H, sizeof(float)), *y = malloc(sizeof(float) * H);+ uint8_t *taken = malloc((size_t)S * K); const float **dev = malloc(sizeof(*dev) * S * K);+ for (int i = 0; i < S * H; i++) x[i] = frnd();+ int n = vkt_issue_w(layer, x, S, K, idx, w, taken);+ for (int i = 0; i < S * K; i++) {+ if (taken[i]) continue;+ expert_ref(&ex[layer][idx[i]], x + (size_t)(i / K) * H, cpu + (size_t)i * H);+ VktExpertSrc s = src_of(&ex[layer][idx[i]]); vkt_note(layer, idx[i], &s);+ }+ const float *sum = NULL;+ int joined = n ? vkt_join_sum(dev, w, &sum) : 1;+ CHECK(joined, "grouped: join failed");+ double worst = 0;+ for (int s = 0; s < S; s++)+ for (int k = 0; k < K; k++) {+ int i = s * K + k;+ expert_ref(&ex[layer][idx[i]], x + (size_t)s * H, y);+ const float *c = taken[i] && joined ? dev[i] : cpu + (size_t)i * H;+ if (taken[i] && joined) { double r = rel(c, y, H); if (r > worst) worst = r; }+ for (int d = 0; d < H; d++) { ref[(size_t)s * H + d] += w[i] * y[d]; loop[(size_t)s * H + d] += w[i] * c[d]; }+ bk->routed++; bk->dev += taken[i] != 0;+ }+ double r = rel(loop, ref, S * H);+ if (r > worst) worst = r;+ if (sum) {+ (*summed)++;+ /* the device's sum: the reference's within the bound, the host loop's to rounding */+ double rs = rel(sum, ref, S * H), rl = rel(sum, loop, S * H);+ if (rs > worst) worst = rs;+ CHECK(rl < 1e-6, "grouped: the device sum is off the host loop over its rows (%.3g)", rl);+ }+ free(x); free(cpu); free(ref); free(loop); free(y); free(taken); free(dev);+ return worst;+}+static void grouped(void) {+ struct { VktFmt gu, dn; float limit; const char *name; } cs[] = {+ {{VKT_SRC_I8_ROW, 0}, {VKT_SRC_I8_ROW, 0}, 0, "int8 per row"},+ {{VKT_SRC_I8_AS_I4_ROW, 0}, {VKT_SRC_I4S_PAIRS_ROW, 0}, 0, "int4 per row"},+ {{VKT_SRC_I4U_PAIRS_GS, 64}, {VKT_SRC_I4U_PAIRS_GS, 64}, 2.5f, "int4 v+8 gs64, SwiGLU limit"},+ {{VKT_SRC_I4S_PAIRS_GS, 32}, {VKT_SRC_I4S_PAIRS_GS, 32}, 0, "signed int4 pairs gs32"},+ {{VKT_SRC_I4U_PLANAR64, 64}, {VKT_SRC_I4U_PLANAR64, 64}, 0, "planar int4-g64"},+ };+ int h0 = H; H = 256;+ int any_grouped = 0, any_summed = 0;+ for (size_t c = 0; c < sizeof cs / sizeof *cs; c++) {+ for (int mode = 0; mode < 3; mode++) { /* single, whole, mixed */+ model_make(cs[c].gu, cs[c].dn);+ g_act = VKT_ACT_SWIGLU; g_limit = cs[c].limit;+ set_budget(mode == 2 ? 4 : L * E, cs[c].gu, cs[c].dn);+ setenv("COLI_VK_TIER_RATE", mode ? "0" : "64", 1);+ setenv("COLI_VK_TIER_SYNC", "1", 1);+ if (mode) { setenv("COLI_VK_TIER_STREAM_SLOTS", "4", 1); setenv("COLI_VK_TIER_STREAM_ROWS", "6", 1);+ setenv("COLI_VK_TIER_STREAM_HALF", "70", 1); }+ VktConfig vc = cfg_of(cs[c].gu, cs[c].dn, VKT_ACT_SWIGLU, cs[c].limit);+ if (mode) { vc.load = load_cb; vc.release = release_cb; }+ uint32_t hist[L][E], *hr[L];+ for (int l = 0; l < L; l++) { hr[l] = hist[l]; for (int e = 0; e < E; e++) hist[l][e] = 100 - e; }+ int on = vkt_init(&vc, mode ? hr : NULL);+ unsetenv("COLI_VK_TIER_SYNC"); unsetenv("COLI_VK_TIER_STREAM_HALF");+ CHECK(on, "grouped %s: the tier did not start", cs[c].name);+ if (!on) { model_free(); continue; }+ if (mode) { /* the warm start: whole = every expert, mixed = the four hottest */+ int pl[L * E], pe[L * E], np = vkt_plan(pl, pe, L * E);+ for (int i = 0; i < np; i++) { VktExpertSrc s = src_of(&ex[pl[i]][pe[i]]); vkt_put(pl[i], pe[i], &s); }+ vkt_put_done();+ }+ ColiVkXbStats a, b; coli_vk_xb_stats(&a);+ Books bk = {0, 0}; double worst = 0; int summed = 0;+ static const int Ss[3][4] = {{32, 32, 32, 32}, {40, 64, 40, 64}, {40, 64, 40, 64}};+ for (int t = 0; t < 4; t++)+ for (int l = 0; l < L; l++) {+ int S = Ss[mode][t], idx[64 * K]; float w[64 * K];+ route(S, 0, E, idx, w);+ vkt_begin_forward();+ double r = sum_step(l, S, idx, w, &bk, &summed);+ if (r > worst) worst = r;+ }+ coli_vk_xb_stats(&b);+ unsigned long long gb = b.grouped_batches - a.grouped_batches;+ static const char *mn[3] = {"single", "whole", "mixed"};+ printf(" %-30s %-6s device %3llu of %3llu, %3llu grouped batches, %d summed on the device, worst %.2e\n",+ cs[c].name, mn[mode], bk.dev, bk.routed, gb, summed, worst);+ CHECK(bk.dev > 0, "grouped %s %s: nothing ran on the device", cs[c].name, mn[mode]);+ CHECK(worst < (gb ? 1e-2 : 2e-3), "grouped %s %s: off the reference (%.3g)", cs[c].name, mn[mode], worst);+ any_grouped |= gb > 0; any_summed |= summed > 0;+ vkt_shutdown(); model_free();+ unsetenv("COLI_VK_TIER_STREAM_SLOTS"); unsetenv("COLI_VK_TIER_STREAM_ROWS");+ }+ }+ printf(" grouped route %s, device sums %s\n", any_grouped ? "taken" : "not on this device",+ any_summed ? "taken" : "not on this device");+ CHECK(!any_grouped || any_summed, "grouped: the device took the grouped route but never summed a whole step");+ H = h0;+}+int main(int argc, char **argv) {char buf[1024];const char *spv = argc > 1 ? argv[1] : coli_vk_shader_path(buf, sizeof buf);@@ -704,6 +815,7 @@ int main(int argc, char **argv) {setenv("COLI_VK_TIER_RESERVE_GB", "0", 1);scratch_budget();printf("formats:\n"); formats();+ printf("grouped:\n"); grouped();printf("warm:\n"); warm();printf("adapt:\n"); adapt();printf("partial:\n"); partial();diff --git a/c/vk_tier.c b/c/vk_tier.cindex bb1a835e..8379904c 100644--- a/c/vk_tier.c+++ b/c/vk_tier.c@@ -118,9 +118,11 @@ static struct {unsigned sk; /* sub-batches issued in this step */VSub hf[2]; /* the two in flight */float *sy; size_t csy; /* the step's device rows, S*K*hidden */+ float *sy_step; /* the backend's whole-step rows instead (coli_vk_xb_step_begin), or NULL */+ int big_valid; /* the step's routed assignments (idx >= 0) */int *bcnt, *bofs, *blist, *bcls, cblist; /* per expert of the step: rows, first row, class; the rows grouped */uint32_t *pred; uint8_t *pred_ok; /* per layer: the last big step's rows per expert */- unsigned long long st_steps, st_experts, st_rows, st_bytes, st_subs, st_kept, st_kept_rows, pf_n, pf_used;+ unsigned long long st_steps, st_experts, st_rows, st_bytes, st_subs, st_sums, st_kept, st_kept_rows, pf_n, pf_used;double st_up_ms;} T;@@ -924,6 +926,12 @@ static int issue_big(int layer, const float *x, int S, int K, const int *idx, co}for (int i = 0; i < n; i++) T.map[i] = -1;T.big = 1; T.big_fail = 0; T.big_n = n; T.big_taken = 0; T.sk = 0;+ T.big_valid = 0;+ for (int e = 0; e < E; e++) T.big_valid += cnt[e];+ /* the step's token rows once and its output rows in place, where the backend can+ * (not beside the second device: its rows come back as before) */+ T.sy_step = !nd2 && (ndev || nst) ? coli_vk_xb_step_begin(S, K, x) : NULL;+ float *SY = T.sy_step ? T.sy_step : T.sy;T.can_balance = 0;/* the order: resident experts, then the streamed ones already in a slot, then the rest */int *order = malloc((size_t)E * sizeof(int)), no = 0;@@ -958,7 +966,8 @@ static int issue_big(int layer, const float *x, int S, int K, const int *idx, coint curh = 0;#define SUBMIT_CURRENT() do { \if (cnt_b) { \- int okk = !T.big_fail && coli_vk_xb_sub_issue(curh, bex, brows, cnt_b, bx, bw); \+ int okk = !T.big_fail && (T.sy_step ? coli_vk_xb_sub_issue_step(curh, bex, brows, cnt_b, bidx, bw) \+ : coli_vk_xb_sub_issue(curh, bex, brows, cnt_b, bx, bw)); \if (okk) { \for (int q = 0; q < rows_b; q++) { taken[bidx[q]] = 1; T.map[bidx[q]] = bidx[q]; } \total += rows_b; T.st_subs++; T.sk++; \@@ -1030,7 +1039,7 @@ static int issue_big(int layer, const float *x, int S, int K, const int *idx, cobx[rows_b + q] = x + (size_t)(i / K) * H;bw[rows_b + q] = w ? w[i] : 1.0f;bidx[rows_b + q] = i;- cur->yout[rows_b + q] = T.sy + (size_t)i * H;+ cur->yout[rows_b + q] = SY + (size_t)i * H;}rows_b += take; cur->n = rows_b;if (sl >= 0) {@@ -1049,6 +1058,7 @@ static int issue_big(int layer, const float *x, int S, int K, const int *idx, coT.big_taken = total;if (!total && !T.inflight) { /* nothing went (or everything failed before a submit) */T.big = 0;+ if (T.sy_step) { coli_vk_xb_step_end(); T.sy_step = NULL; }if (T.big_fail) {fprintf(stderr, "[VK] tier %s: a batch failed, its experts are recomputed on the CPU and the tier stops\n", T.engine);T.on = 0;@@ -1071,7 +1081,7 @@ static int join_big(const float **rows) {double dms = 0, t1 = vkt_now_ms();ok1 = coli_vk_xb_join_dev(1, T.d2_y, &dms);T.wait_ms += vkt_now_ms() - t1; T.dev_ms += dms; T.d2_ms += dms;- if (ok1) for (int q = 0; q < T.nd2; q++) memcpy(T.sy + (size_t)T.d2_idx[q] * T.c.hidden, T.d2_y[q], (size_t)T.c.hidden * sizeof(float));+ if (ok1) for (int q = 0; q < T.nd2; q++) memcpy((T.sy_step ? T.sy_step : T.sy) + (size_t)T.d2_idx[q] * T.c.hidden, T.d2_y[q], (size_t)T.c.hidden * sizeof(float));T.d2_inflight = 0;}int cpu_pairs = T.big_n - T.big_taken;@@ -1081,7 +1091,8 @@ static int join_big(const float **rows) {}T.inflight = 0; T.big = 0;int ok = !T.big_fail;- for (int i = 0; i < T.big_n; i++) rows[i] = ok && ok1 && T.map[i] >= 0 ? T.sy + (size_t)i * T.c.hidden : NULL;+ const float *SY = T.sy_step ? T.sy_step : T.sy;+ for (int i = 0; i < T.big_n; i++) rows[i] = ok && ok1 && T.map[i] >= 0 ? SY + (size_t)i * T.c.hidden : NULL;for (int i = 0; T.st && i < T.st_n; i++) if (T.st[i].state == SS_BUSY) { T.st[i].state = SS_FREE; T.st[i].pending = 0; }if (!ok) {fprintf(stderr, "[VK] tier %s: a batch failed, its experts are recomputed on the CPU and the tier stops\n", T.engine);@@ -1289,7 +1300,11 @@ int vkt_issue_w(int layer, const float *x, int S, int K, const int *idx, const fint vkt_join(const float **rows) {if (!T.inflight) return 0;- if (T.big) return join_big(rows);+ if (T.big) {+ int ok = join_big(rows);+ if (T.sy_step) coli_vk_xb_step_end(); /* rows stay readable until the next step */+ return ok;+ }double t0 = vkt_now_ms(), dms = 0, dms1 = 0, tc = t0 - T.t_issued;T.cpu_ms += tc;int ok = !T.d0_inflight || coli_vk_xb_join(T.by, &dms);@@ -1324,6 +1339,30 @@ int vkt_join(const float **rows) {return 1;}+/* vkt_join, and for a big step that ran wholly on the device (every routed assignment+ * taken) on the backend's whole-step buffers, the routed sum per row on the device:+ * *sum = S rows of sum over k in rank order of w[s*K+k] * expert(s, k), as an fma chain+ * from 0 (the engines' own loop); else *sum = NULL and rows as vkt_join gives them. */+int vkt_join_sum(const float **rows, const float *w, const float **sum) {+ *sum = NULL;+ if (!T.inflight) return 0;+ if (!T.big || !T.sy_step) return vkt_join(rows);+ int ok = join_big(rows);+ if (ok && T.big_taken == T.big_valid && T.big_valid > 0) {+ uint8_t *use = malloc((size_t)T.big_n);+ if (use) {+ for (int i = 0; i < T.big_n; i++) use[i] = T.map[i] >= 0;+ double dms = 0, t0 = vkt_now_ms();+ *sum = coli_vk_xb_step_sum(w, use, &dms);+ T.wait_ms += vkt_now_ms() - t0; T.dev_ms += dms;+ if (*sum) T.st_sums++;+ free(use);+ }+ }+ coli_vk_xb_step_end();+ return ok;+}+void vkt_begin_forward(void) { if (T.on) T.begin = 1; }int vkt_resident(int layer, int eid) {return T.on && layer >= 0 && layer < T.c.layers && eid >= 0 && eid < T.c.experts &&@@ -1595,6 +1634,7 @@ void vkt_report(const char *scope, unsigned long long ram_hits, unsigned long loT.st_steps, T.st_experts, human((double)T.st_bytes, hs, sizeof hs),T.st_up_ms > 0 ? (double)T.st_bytes / T.st_up_ms / 1e6 : 0.0, T.st_rows, T.st_subs, T.pf_n, T.pf_used,T.st_kept, T.st_kept_rows);+ if (T.st_sums) fprintf(stderr, ", %llu steps summed on the device", T.st_sums);}if (T.d2 || T.d2_lost) {ColiVkPoolStats p2; coli_vk_pool_stats(3, &p2);diff --git a/c/vk_tier.h b/c/vk_tier.hindex 586240da..0f0a3dcb 100644--- a/c/vk_tier.h+++ b/c/vk_tier.h@@ -222,6 +222,10 @@ int vkt_issue_w(int layer, const float *x, int S, int K, const int *idx, const* the experts the tier will take. No side effects. */int vkt_wants(int layer, int eid);int vkt_join(const float **rows);+/* vkt_join; for a big step that ran wholly on the device, *sum = its S routed rows (the+ * rank-order sum of w[s*K+k] * expert(s, k) as an fma chain from 0, the engines' loop),+ * summed on the device; else *sum = NULL. w as vkt_issue_w's (NULL = 1). */+int vkt_join_sum(const float **rows, const float *w, const float **sum);int vkt_resident(int layer, int eid);/* The next vkt_issue starts a forward (promotion rate, decay, eviction candidates).* Optional: the tier tells forwards apart by the layer index going back, which a model@@ -262,6 +266,7 @@ static inline int vkt_issue(int l,const float *x,int S,int K,const int *i,uint8static inline int vkt_issue_w(int l,const float *x,int S,int K,const int *i,const float *w,uint8_t *t){(void)l;(void)x;(void)S;(void)K;(void)i;(void)w;(void)t;return 0;}static inline int vkt_wants(int l,int e){(void)l;(void)e;return 0;}static inline int vkt_join(const float **r){(void)r;return 0;}+static inline int vkt_join_sum(const float **r, const float *w, const float **s){(void)r;(void)w;*s=0;return 0;}static inline int vkt_resident(int l,int e){(void)l;(void)e;return 0;}static inline void vkt_begin_forward(void){}static inline void vkt_report(const char *s,unsigned long long r,unsigned long long d){(void)s;(void)r;(void)d;}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 6d2d8025..ced0b71e 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -247,6 +247,10 @@ With `COLI_VULKAN=1` every MoE engine (qwen36, qwen38, inkling, olmoe, kimi_k3,| `COLI_VK_TIER_SYNC` | `0` | `1`: every layer step waits for the uploads staged so far before it runs, so which experts are resident depends on the routing alone, not on when the uploader thread ran (the tests set it: a tiny fixture's run can end before that thread is first scheduled). With `COLI_VK_TIER_BALANCE=0` as well, a run with the tier is reproducible. A promotion that displaces a resident while a batch is in flight waits for the batch's join to free it. Costs the overlap of uploads with compute. || `COLI_VK_TIER_BALANCE` | on | `0`: the device takes every resident expert of a step. On, when a join keeps waiting for the device, the step's resident experts the CPU also holds in RAM go back to the CPU beyond the device's share, which moves with every join (down when the CPU waited more than a tenth of its own time, up when the device finished before 80% of it). || `COLI_VK_TIER_GEMM_ROWS` | `16` | Rows from which an expert of a step takes the tiled GEMM (prefill) instead of the per-row GEMV; `0` never. Below it a row's bits do not depend on how many rows share the dispatch. |+| `COLI_VK_XB_GROUPED` | on where supported | A step's experts as one grouped GEMM (`qmatmul_grp.comp`): gate+up with the SwiGLU in one dispatch, down in another, the weights read by device address. Needs cooperative matrices at subgroup size 64, `bufferDeviceAddress`, hidden and expert widths that are multiples of 128, SwiGLU, and int8 or int4 experts (device fmt 1, 2, 4); otherwise the per-expert route. `0` keeps the per-expert route. |+| `COLI_VK_XB_GROUPED_ROWS` | `64` | Assignments (rows x top-k) in a batch from which the grouped GEMM takes it. Below it (decode, an MTP verify) the per-expert route keeps a row's bits independent of the batch. |+| `COLI_VK_XB_STEP` | on where supported | A big (prompt) step on the grouped GEMM: its token rows uploaded once, every expert's output written in place, and when every routed assignment ran on the device the routed sum per row computed there (`expert_sum.comp`, rank order). `0` keeps the packed rows and the host sum. |+| `COLI_VK_BDA` | on where supported | `0`: the device is opened without `bufferDeviceAddress`, so the grouped GEMM is off. || `COLI_VK_TIER_COOP` | on where supported | Cooperative-matrix GEMM for resident and streamed expert batches. The same format and group-size checks as the per-matrix path apply to each gate, up and down projection independently; other projections keep the FP32 GEMM. `0` disables it; `COLI_VK_COOP=0` disables all cooperative pipelines. || `COLI_VK_TIER_COOP_ROWS` | `64` | Minimum rows of one expert for the cooperative path, also subject to `COLI_VK_TIER_GEMM_ROWS`. Small decode and draft-verification batches retain their GEMV arithmetic. || `COLI_VK_TIER_QUEUE` | a second queue | `0`: the tier's batches share the main queue with the dense matmuls (they then serialize). Unset: a second queue of the main family, else a compute-only family's (RADV), else shared (Lavapipe). |diff --git a/docs/vulkan.md b/docs/vulkan.mdindex 3e1245e7..3725ea0e 100644--- a/docs/vulkan.md+++ b/docs/vulkan.md@@ -476,6 +476,9 @@ the device, since no Vulkan tier runs.| `COLI_VK_TIER_WARM` | on | `0`: no warm start; the tier fills as experts pass by. || `COLI_VK_TIER_SYNC` | `0` | `1`: each layer step first waits for the uploads staged so far: residency then follows the routing alone (with `COLI_VK_TIER_BALANCE=0`, the run is reproducible). A promotion that displaces a resident while a batch is in flight waits for the join to free it. For tests and debugging. || `COLI_VK_TIER_GEMM_ROWS` | `16` | Rows from which an expert of a step takes the tiled GEMM instead of the per-row GEMV; `0` never. |+| `COLI_VK_XB_GROUPED` | on where supported | `0`: a step's experts keep their own dispatches where the grouped GEMM would take them (cooperative matrices at subgroup size 64, `bufferDeviceAddress`, widths multiples of 128, int8/int4, SwiGLU). |+| `COLI_VK_XB_GROUPED_ROWS` | `64` | Assignments in a batch from which the grouped GEMM takes it. |+| `COLI_VK_XB_STEP` | on where supported | `0`: a prompt step packs its rows per expert and sums on the host, as before the whole-step buffers. On a Radeon 8060S (Qwen3.6, 1011-token prompt) the grouped GEMM and the step buffers took the experts' device time from 762 to 344 ms and the first token from 2.08 to 1.49 s, perplexity unchanged (7.65). || `COLI_VK_TIER_QUEUE` | a second queue | `0`: the tier shares the main queue (its batches and the dense matmuls then serialize). || `COLI_VK_TIER_STREAM` | on | A prompt step's cold experts streamed to the device ([below](#big-prompt-chunks-and-expert-streaming)); `0` off. `_SLOTS`, `_ROWS`, `_HALF`, `_PAR` there. || `COLI_VK_DENSE` | on, but off on a device sharing the CPU's RAM while the tier is on | `0`: the dense trunk stays on the CPU and the device takes the routed experts only; `1`: the trunk on the device whatever the device. Unset: on a discrete GPU, or with the tier off, on the device; on an integrated GPU or Lavapipe with the tier on, on the CPU. The startup line says which and why. (The GLM engine reads it through the same rule with its own default, off.) |