Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
glm: VK MLA absorb attention core (COLI_VK_ATTN) β the last CPU compute piece on Vulkan-only machines
Steve Markgraf559383daf671
diff --git a/c/Makefile b/c/Makefileindex e39ed07a..0206475f 100644--- a/c/Makefile+++ b/c/Makefile@@ -302,7 +302,7 @@ ifeq ($(VK),1)CFLAGS += -DCOLI_VULKANLDFLAGS += -lvulkanVK_OBJ = backend_vulkan.o-VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv+VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spvendifall: colibri$(EXE)diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex 51bc6596..7934786f 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -29,6 +29,16 @@ typedef struct {VkBuffer buf; VkDeviceMemory mem; void *ptr; size_t cap;} Scratch;+/* Persistent device-side KV latent/rope cache for one layer (MLA attention).+ * Host appends rows as tokens decode (absolute-position indexing); the absorb+ * kernel reads them in place. Allocated once at max_t rows, like the CUDA+ * kv_dev shadow. */+#define VK_KV_LAYERS 160+typedef struct {+ VkBuffer bl, br; VkDeviceMemory ml, mr; void *pl, *pr;+ int rows, K, R;+} VkKvLayer;+static struct {int ready;VkInstance inst;@@ -47,6 +57,9 @@ static struct {/* fused dual gate+up+silu pipeline (6 bindings): x, Wg, gscale, Wu, uscale, hidden */VkShaderModule shader_gu; VkDescriptorSetLayout dsl_gu; VkPipelineLayout plyt_gu;VkPipeline pipe_gu; VkDescriptorPool dpool_gu; VkDescriptorSet dset_gu;+ /* MLA absorb attention core (7 bindings): q, W, scales, Lcache, Rcache, scores, ctx */+ VkShaderModule shader_att; VkDescriptorSetLayout dsl_att; VkPipelineLayout plyt_att;+ VkPipeline pipe_att; VkDescriptorPool dpool_att; VkDescriptorSet dset_att;VkCommandPool cpool;VkCommandBuffer cmd;VkFence fence;@@ -56,6 +69,8 @@ static struct {* one submit with hidden never leaving the GPU. */Scratch eg_x, eg_h, eg_y;VkDescriptorPool eg_pool; VkDescriptorSet eg_gu[64], eg_dn[64]; int eg_nsets;+ Scratch att_sc; /* attention score scratch (GPU-only) */+ VkKvLayer kv[VK_KV_LAYERS]; /* per-layer resident KV latent/rope cache *//* resubmit cache: skip vkUpdateDescriptorSets + command re-record when the bound* tensor / shape / scratch buffers are unchanged from the previous call (the hot-* expert-called-repeatedly pattern). The synchronous fence wait each call means no@@ -67,6 +82,8 @@ static struct {} G;struct PC { int fmt, S, I, O, rowWords; };+/* Push constants of the absorb attention kernel (must match attention_absorb.comp). */+struct PCAttn { int fmt, S, H, Q, R, V, K, st0, T, rowWords, cap; float scale; };static int pick_memtype(void) {VkPhysicalDeviceMemoryProperties m;@@ -148,11 +165,12 @@ static VkShaderModule load_spv(const char *path) {return r == VK_SUCCESS ? m : VK_NULL_HANDLE;}-/* Build a compute pipeline + descriptor pool/set for nbind storage buffers sharing the- * struct PC push constant. Used for both the 4-binding matmul and 6-binding gate_up. */-static int build_pipeline(int nbind, VkShaderModule shader, VkDescriptorSetLayout *dsl,- VkPipelineLayout *plyt, VkPipeline *pipe, VkDescriptorPool *dpool,- VkDescriptorSet *dset) {+/* Build a compute pipeline + descriptor pool/set for nbind storage buffers with a+ * pc_size-byte push constant. Used by the 4-binding matmul, 6-binding gate_up and+ * 7-binding absorb attention pipelines. */+static int build_pipeline(int nbind, size_t pc_size, VkShaderModule shader,+ VkDescriptorSetLayout *dsl, VkPipelineLayout *plyt, VkPipeline *pipe,+ VkDescriptorPool *dpool, VkDescriptorSet *dset) {VkDescriptorSetLayoutBinding b[8];for (int i = 0; i < nbind; i++) b[i] = (VkDescriptorSetLayoutBinding){.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,@@ -160,7 +178,7 @@ static int build_pipeline(int nbind, VkShaderModule shader, VkDescriptorSetLayouVkDescriptorSetLayoutCreateInfo dsli = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,.bindingCount = (uint32_t)nbind, .pBindings = b};VKCHECK(vkCreateDescriptorSetLayout(G.dev, &dsli, NULL, dsl), "descSetLayout");- VkPushConstantRange pcr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, .offset = 0, .size = sizeof(struct PC)};+ VkPushConstantRange pcr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, .offset = 0, .size = (uint32_t)pc_size};VkPipelineLayoutCreateInfo pli = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,.setLayoutCount = 1, .pSetLayouts = dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pcr};VKCHECK(vkCreatePipelineLayout(G.dev, &pli, NULL, plyt), "pipelineLayout");@@ -179,14 +197,21 @@ static int build_pipeline(int nbind, VkShaderModule shader, VkDescriptorSetLayoureturn 1;}-/* "β¦/qmatmul.spv" -> "β¦/qmatmul_gate_up.spv" (sibling of the main shader). */-static void derive_gu_path(const char *spv, char *out, size_t n) {+/* "β¦/qmatmul.spv" -> "β¦/qmatmul<suffix>" (sibling of the main shader). */+static void derive_sibling(const char *spv, const char *suffix, char *out, size_t n) {const char *dot = strstr(spv, ".spv");- if (dot && (size_t)(dot - spv) + 13 < n) {+ if (dot && (size_t)(dot - spv) + strlen(suffix) + 1 < n) {size_t pre = (size_t)(dot - spv);- memcpy(out, spv, pre); strcpy(out + pre, "_gate_up.spv");+ memcpy(out, spv, pre); strcpy(out + pre, suffix);} else snprintf(out, n, "%s", spv);}+/* "β¦/qmatmul.spv" -> "β¦/attention_absorb.spv" (same directory). */+static void derive_dir_file(const char *spv, const char *fname, char *out, size_t n) {+ const char *sl = strrchr(spv, '/');+ size_t pre = sl ? (size_t)(sl - spv) + 1 : 0;+ if (pre + strlen(fname) + 1 < n) { memcpy(out, spv, pre); strcpy(out + pre, fname); }+ else snprintf(out, n, "%s", fname);+}int coli_vk_init(const char *spv_path) {if (G.ready) return 1;@@ -238,13 +263,19 @@ int coli_vk_init(const char *spv_path) {G.shader = load_spv(spv_path);if (!G.shader) return 0;- if (!build_pipeline(4, G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;+ if (!build_pipeline(4, sizeof(struct PC), G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;/* Optional fused gate+up pipeline: skip gracefully if its shader isn't present* (single-matmul path keeps working). */- char gu_path[512]; derive_gu_path(spv_path, gu_path, sizeof(gu_path));+ char gu_path[512]; derive_sibling(spv_path, "_gate_up.spv", gu_path, sizeof(gu_path));G.shader_gu = load_spv(gu_path);- if (G.shader_gu && !build_pipeline(6, G.shader_gu, &G.dsl_gu, &G.plyt_gu, &G.pipe_gu, &G.dpool_gu, &G.dset_gu))+ if (G.shader_gu && !build_pipeline(6, sizeof(struct PC), G.shader_gu, &G.dsl_gu, &G.plyt_gu, &G.pipe_gu, &G.dpool_gu, &G.dset_gu))+ return 0;++ /* Optional MLA absorb attention pipeline (same directory as the main shader). */+ char att_path[512]; derive_dir_file(spv_path, "attention_absorb.spv", att_path, sizeof(att_path));+ G.shader_att = load_spv(att_path);+ if (G.shader_att && !build_pipeline(7, sizeof(struct PCAttn), G.shader_att, &G.dsl_att, &G.plyt_att, &G.pipe_att, &G.dpool_att, &G.dset_att))return 0;VkCommandPoolCreateInfo cpci = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,@@ -258,8 +289,8 @@ int coli_vk_init(const char *spv_path) {G.ready = 1;VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(G.phys, &p);- fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s\n", p.deviceName, G.qfam, G.memtype,- G.shader_gu ? ", fused gate+up" : "");+ fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s%s\n", p.deviceName, G.qfam, G.memtype,+ G.shader_gu ? ", fused gate+up" : "", G.shader_att ? ", absorb attention" : "");return 1;}@@ -507,6 +538,101 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,return 1;}+/* ---- MLA absorb attention core -------------------------------------------------+ * The KV latent (L, [rows,K]) and rope (R, [rows,Rd]) caches live in persistent+ * per-layer device buffers, appended row-by-row as tokens decode (the host stays+ * canonical; glm.c tracks a valid-watermark and re-appends after invalidation).+ * Rows are indexed by ABSOLUTE position, so kv_start windows just skip rows. */++int coli_vk_kv_ensure(int layer, int max_rows, int K, int Rd) {+ if (!G.ready || layer < 0 || layer >= VK_KV_LAYERS || max_rows < 1 || K < 1 || Rd < 1) return 0;+ VkKvLayer *v = &G.kv[layer];+ if (v->bl) return v->rows >= max_rows && v->K == K && v->R == Rd; /* resize goes through coli_vk_kv_reset */+ if (!alloc_hostvis((size_t)max_rows * K * 4, &v->bl, &v->ml, &v->pl)) return 0;+ if (!alloc_hostvis((size_t)max_rows * Rd * 4, &v->br, &v->mr, &v->pr)) {+ vkDestroyBuffer(G.dev, v->bl, NULL); vkFreeMemory(G.dev, v->ml, NULL);+ memset(v, 0, sizeof(*v)); return 0;+ }+ v->rows = max_rows; v->K = K; v->R = Rd;+ return 1;+}++/* Mirror one host cache row into the device copy (write-combined memory: the CPU+ * only ever WRITES these buffers, the GPU reads them). */+int coli_vk_kv_row(int layer, int pos, const float *L, const float *R) {+ if (layer < 0 || layer >= VK_KV_LAYERS) return 0;+ VkKvLayer *v = &G.kv[layer];+ if (!v->pl || pos < 0 || pos >= v->rows) return 0;+ memcpy((float *)v->pl + (size_t)pos * v->K, L, (size_t)v->K * 4);+ memcpy((float *)v->pr + (size_t)pos * v->R, R, (size_t)v->R * 4);+ return 1;+}++/* Drop all per-layer KV device caches (cache resize in kv_alloc). */+void coli_vk_kv_reset(void) {+ for (int i = 0; i < VK_KV_LAYERS; i++) {+ VkKvLayer *v = &G.kv[i];+ if (!v->bl) continue;+ if (G.ready) { /* dead device: leak GPU handles like coli_vk_tensor_free */+ vkDestroyBuffer(G.dev, v->bl, NULL); vkFreeMemory(G.dev, v->ml, NULL);+ vkDestroyBuffer(G.dev, v->br, NULL); vkFreeMemory(G.dev, v->mr, NULL);+ }+ memset(v, 0, sizeof(*v));+ }+}++/* Decode MLA absorption core for S causal query rows of one sequence, one submit:+ * ctx[s,h,:] = softmax((Wnope_h^T q_nope).L_t + q_rope.R_t) weighted latent context+ * projected through the value rows of kv_b. kv_b ([H*(Q+V), K]) uploads on first+ * call and stays resident; L/R rows [st0, T) must already be mirrored via+ * coli_vk_kv_row. Returns 0 -> caller falls back to CPU. */+int coli_vk_attention_absorb(ColiVkTensor **kvb, const void *w, const float *sc, int fmt,+ float *ctx, const float *q, int layer, int S, int H,+ int Q, int R, int V, int K, int st0, int T, float scale) {+ if (!G.ready || !G.pipe_att || S < 1 || H < 1 || layer < 0 || layer >= VK_KV_LAYERS) return 0;+ if (Q > 256 || R > 64 || K > 512 || st0 < 0 || T - S - st0 < 0) return 0; /* shared-array limits */+ VkKvLayer *kv = &G.kv[layer];+ if (!kv->bl || kv->rows < T || kv->K != K || kv->R != R) return 0;+ if (!upload_tensor(kvb, w, sc, fmt, K, H * (Q + V))) return 0;+ ColiVkTensor *t = *kvb;+ int cap = T - st0;+ size_t qb = (size_t)S * H * (Q + R) * 4, cb = (size_t)S * H * V * 4;+ size_t sb = (size_t)S * H * cap * 4;+ if (!scratch_reserve(&G.x, qb) || !scratch_reserve_mt(&G.y, cb, G.memtype_cached) ||+ !scratch_reserve(&G.att_sc, sb)) return 0; /* y (ctx) is read back -> cached */+ memcpy(G.x.ptr, q, qb);++ VkDescriptorBufferInfo bi[7] = {+ {.buffer = G.x.buf, .range = VK_WHOLE_SIZE}, {.buffer = t->wbuf, .range = VK_WHOLE_SIZE},+ {.buffer = t->sbuf, .range = VK_WHOLE_SIZE}, {.buffer = kv->bl, .range = VK_WHOLE_SIZE},+ {.buffer = kv->br, .range = VK_WHOLE_SIZE}, {.buffer = G.att_sc.buf, .range = VK_WHOLE_SIZE},+ {.buffer = G.y.buf, .range = VK_WHOLE_SIZE}};+ VkWriteDescriptorSet wd[7];+ for (int i = 0; i < 7; i++) wd[i] = (VkWriteDescriptorSet){+ .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = G.dset_att,+ .dstBinding = (uint32_t)i, .descriptorCount = 1,+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[i]};+ vkUpdateDescriptorSets(G.dev, 7, wd, 0, NULL);++ VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");+ VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};+ VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");+ vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe_att);+ vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt_att, 0, 1, &G.dset_att, 0, NULL);+ struct PCAttn pc = {fmt, S, H, Q, R, V, K, st0, T, t->rowWords, cap, scale};+ vkCmdPushConstants(G.cmd, G.plyt_att, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ vkCmdDispatch(G.cmd, (uint32_t)H, (uint32_t)S, 1); /* one workgroup per (head, row) */+ VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");++ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};+ VKCHECK(vkResetFences(G.dev, 1, &G.fence), "resetFence");+ VKCHECK(vkQueueSubmit(G.queue, 1, &si, G.fence), "queueSubmit");+ if (vkWaitForFences(G.dev, 1, &G.fence, VK_TRUE, 10000000000ULL) != VK_SUCCESS) { G.ready = 0; return 0; }+ memcpy(ctx, G.y.ptr, cb);+ G.cmd_ready = 0; G.bound_tensor = NULL; /* the shared command buffer/binding was clobbered */+ return 1;+}+void coli_vk_tensor_free(ColiVkTensor *t) {if (!t) return;if (G.bound_tensor == t) { G.bound_tensor = NULL; G.cmd_ready = 0; } /* drop stale cache */@@ -536,6 +662,8 @@ void coli_vk_shutdown(void) {if (G.eg_x.buf) { vkDestroyBuffer(G.dev, G.eg_x.buf, NULL); vkFreeMemory(G.dev, G.eg_x.mem, NULL); }if (G.eg_h.buf) { vkDestroyBuffer(G.dev, G.eg_h.buf, NULL); vkFreeMemory(G.dev, G.eg_h.mem, NULL); }if (G.eg_y.buf) { vkDestroyBuffer(G.dev, G.eg_y.buf, NULL); vkFreeMemory(G.dev, G.eg_y.mem, NULL); }+ if (G.att_sc.buf) { vkDestroyBuffer(G.dev, G.att_sc.buf, NULL); vkFreeMemory(G.dev, G.att_sc.mem, NULL); }+ coli_vk_kv_reset();if (G.eg_pool) vkDestroyDescriptorPool(G.dev, G.eg_pool, NULL);vkDestroyFence(G.dev, G.fence, NULL);vkDestroyCommandPool(G.dev, G.cpool, NULL);@@ -551,6 +679,13 @@ void coli_vk_shutdown(void) {vkDestroyDescriptorSetLayout(G.dev, G.dsl_gu, NULL);vkDestroyShaderModule(G.dev, G.shader_gu, NULL);}+ if (G.shader_att) {+ vkDestroyDescriptorPool(G.dev, G.dpool_att, NULL);+ vkDestroyPipeline(G.dev, G.pipe_att, NULL);+ vkDestroyPipelineLayout(G.dev, G.plyt_att, NULL);+ vkDestroyDescriptorSetLayout(G.dev, G.dsl_att, NULL);+ vkDestroyShaderModule(G.dev, G.shader_att, NULL);+ }vkDestroyDevice(G.dev, NULL);vkDestroyInstance(G.inst, NULL);memset(&G, 0, sizeof(G));@@ -810,6 +945,63 @@ static int run_expert_group(int fmt, int D, int I, int K) {return maxrel > 1e-3 ? 1 : 0;}+/* MLA absorb attention vs a CPU ref that mirrors glm.c's absorb loop exactly:+ * qabs = sum_d q[d]*deq(row rbase+d)*ws, scores over cache rows [st0, T-S+s],+ * softmax, weighted latent, value-row projection. */+static int run_absorb(int fmt, int S, int H, int Q, int R, int V, int K, int st0, int T, int layer) {+ size_t rb = fmt == 1 ? (size_t)K : (size_t)(K + 1) / 2;+ int O = H * (Q + V);+ uint8_t *w = malloc(rb * O); float *ws = malloc((size_t)O * 4);+ float *q = malloc((size_t)S * H * (Q + R) * 4);+ float *L = malloc((size_t)T * K * 4), *Rr = malloc((size_t)T * R * 4);+ float *cg = malloc((size_t)S * H * V * 4), *cc = malloc((size_t)S * H * V * 4);+ for (size_t i = 0; i < rb * (size_t)O; i++) w[i] = rand() & 0xff;+ for (int o = 0; o < O; o++) ws[o] = 0.01f + (rand() % 100) / 10000.0f;+ for (int i = 0; i < S * H * (Q + R); i++) q[i] = (rand() % 200 - 100) / 100.0f;+ for (int i = 0; i < T * K; i++) L[i] = (rand() % 200 - 100) / 100.0f;+ for (int i = 0; i < T * R; i++) Rr[i] = (rand() % 200 - 100) / 100.0f;+ float scale = 0.13f;+ if (!coli_vk_kv_ensure(layer, T, K, R)) { printf("kv_ensure failed\n"); return 1; }+ for (int t = 0; t < T; t++)+ if (!coli_vk_kv_row(layer, t, L + (size_t)t * K, Rr + (size_t)t * R)) { printf("kv_row failed\n"); return 1; }+ ColiVkTensor *kvb = NULL;+ if (!coli_vk_attention_absorb(&kvb, w, ws, fmt, cg, q, layer, S, H, Q, R, V, K, st0, T, scale)) {+ printf("absorb failed\n"); return 1; }+ float *qabs = malloc((size_t)K * 4), *clat = malloc((size_t)K * 4), *sc = malloc((size_t)(T - st0) * 4);+ for (int s = 0; s < S; s++) for (int h = 0; h < H; h++) {+ const float *qp = q + ((size_t)s * H + h) * (Q + R), *qr = qp + Q;+ int rbase = h * (Q + V), nt = (T - S + s + 1) - st0;+ for (int i = 0; i < K; i++) qabs[i] = 0;+ for (int d = 0; d < Q; d++) { const uint8_t *row = w + (size_t)(rbase + d) * rb;+ for (int i = 0; i < K; i++) qabs[i] += qp[d] * deq(row, fmt, i) * ws[rbase + d]; }+ for (int j = 0; j < nt; j++) { int t = st0 + j;+ double a = 0;+ for (int i = 0; i < K; i++) a += qabs[i] * L[(size_t)t * K + i];+ for (int d = 0; d < R; d++) a += qr[d] * Rr[(size_t)t * R + d];+ sc[j] = (float)(a * scale); }+ float mx = sc[0]; for (int j = 1; j < nt; j++) if (sc[j] > mx) mx = sc[j];+ double sum = 0; for (int j = 0; j < nt; j++) { sc[j] = expf(sc[j] - mx); sum += sc[j]; }+ for (int i = 0; i < K; i++) clat[i] = 0;+ for (int j = 0; j < nt; j++) { float a = (float)(sc[j] / sum);+ for (int i = 0; i < K; i++) clat[i] += a * L[(size_t)(st0 + j) * K + i]; }+ for (int v = 0; v < V; v++) { const uint8_t *row = w + (size_t)(rbase + Q + v) * rb;+ double a = 0; for (int i = 0; i < K; i++) a += clat[i] * deq(row, fmt, i);+ cc[((size_t)s * H + h) * V + v] = (float)(a * ws[rbase + Q + v]); }+ }+ double maxrel = 0, maxerr = 0;+ for (int i = 0; i < S * H * V; i++) { double e = fabs(cg[i] - cc[i]); if (e > maxerr) maxerr = e;+ if (fabs(cc[i]) > 1e-2) { double r = e / fabs(cc[i]); if (r > maxrel) maxrel = r; } }+ double t0 = now(); int iters = 20; /* per-call cost, the engine pattern (one submit/layer) */+ for (int k = 0; k < iters; k++)+ coli_vk_attention_absorb(&kvb, w, ws, fmt, cg, q, layer, S, H, Q, R, V, K, st0, T, scale);+ double ms = (now() - t0) * 1000 / iters;+ printf("absorb fmt=%d S=%d H=%d Q=%d R=%d V=%d K=%d st0=%d T=%d | maxerr=%.4g maxrel=%.4g | %.3f ms/call\n",+ fmt, S, H, Q, R, V, K, st0, T, maxerr, maxrel, ms);+ coli_vk_tensor_free(kvb);+ free(w); free(ws); free(q); free(L); free(Rr); free(cg); free(cc); free(qabs); free(clat); free(sc);+ return maxrel > 2e-3 ? 1 : 0;+}+int main(int argc, char **argv) {const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";if (!coli_vk_init(spv)) { printf("vk init failed\n"); return 1; }@@ -866,6 +1058,14 @@ int main(int argc, char **argv) {bad |= run_expert_group(2, 6144, 2048, 1);bad |= run_expert_group(2, 6144, 2048, 8);bad |= run_expert_group(2, 6144, 2048, 32);+ /* MLA absorb attention core (GLM decode shape + window/causal/int8 variants). */+ if (G.pipe_att) {+ bad |= run_absorb(2, 1, 64, 192, 64, 256, 512, 0, 300, 0); // GLM-5.2 decode+ bad |= run_absorb(2, 1, 64, 192, 64, 256, 512, 17, 300, 1); // kv_start window+ bad |= run_absorb(2, 2, 64, 192, 64, 256, 512, 0, 300, 2); // S=2 causal (MTP verify)+ bad |= run_absorb(1, 1, 8, 128, 32, 64, 256, 0, 64, 3); // int8, odd dims+ bad |= run_absorb(2, 1, 64, 192, 64, 256, 512, 0, 2000, 4); // long context+ }printf(bad ? "FAIL\n" : "PASS\n");coli_vk_shutdown();return bad;diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex 0c5c930b..44f08f48 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -52,6 +52,20 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,* then driven by coli_vk_expert_group). Returns 0 on failure/unsupported fmt. */int coli_vk_tensor_ensure(ColiVkTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O);+/* MLA absorb attention core (decode). The KV latent/rope caches live in persistent+ * per-layer device buffers: _ensure allocates a layer's cache at max_rows (once; resize+ * via _reset), _row mirrors one host row (absolute position), _reset drops all layers.+ * The caller keeps a valid-watermark and re-mirrors rows after any invalidation.+ * absorb runs S causal query rows over cache rows [st0, T) in one submit:+ * q [S,H*(Q+R)] roped, kv_b [H*(Q+V), K] uploads once (fmt 1=int8/2=int4),+ * ctx out [S,H*V]. Returns 0 -> caller falls back to CPU. */+int coli_vk_kv_ensure(int layer, int max_rows, int K, int Rd);+int coli_vk_kv_row(int layer, int pos, const float *L, const float *R);+void coli_vk_kv_reset(void);+int coli_vk_attention_absorb(ColiVkTensor **kvb, const void *w, const float *sc, int fmt,+ float *ctx, const float *q, int layer, int S, int H,+ int Q, int R, int V, int K, int st0, int T, float scale);+void coli_vk_tensor_free(ColiVkTensor *t);size_t coli_vk_tensor_bytes(const ColiVkTensor *t);diff --git a/c/colibri.c b/c/colibri.cindex 711783f9..6f1d0c81 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -202,6 +202,9 @@ typedef struct {ESlot **ecache; int *ecn; int ecap; /* LRU expert per-layer */float **kv_dev_L, **kv_dev_R; int *kv_dev_valid; /* ombra KV su device (decode) */float **ln_dev; /* in_ln/post_ln cached on device: [layer*2+{0,1}] (Inc.4) */+#ifdef COLI_VULKAN+ int *vk_kv_valid; /* righe [0,v) specchiate nella cache KV Vulkan */+#endifESlot ws[64]; /* working set del layer corrente (load paralleli) */ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */@@ -295,6 +298,8 @@ static int g_vk_budget; /* COLI_VK_EXPERTS: max experts uploaded to VK VRAstatic int g_vk_resident; /* how many experts currently VK-resident */static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention* projections + shared expert) on Vulkan too */+static int g_vk_attn; /* COLI_VK_ATTN=1: run the MLA absorb attention core on Vulkan+ * (mirrors COLI_CUDA_ATTN; KV latent cache mirrored on-device) */#endif#ifdef COLI_CUDAstatic int g_cuda_enabled;@@ -1205,6 +1210,9 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbitsm->ecap=cap; m->ecache=calloc(NR,sizeof(ESlot*)); m->ecn=calloc(NR,sizeof(int));m->kv_dev_L=calloc(NR,sizeof(float*)); m->kv_dev_R=calloc(NR,sizeof(float*));m->kv_dev_valid=calloc(NR,sizeof(int));+#ifdef COLI_VULKAN+ m->vk_kv_valid=calloc(NR,sizeof(int));+#endifm->eroute=calloc(NR,sizeof(int*)); m->enr=calloc(NR,sizeof(int));m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int));m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*));@@ -2532,6 +2540,10 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p#ifdef COLI_CUDAif(ks==m->kv&&m->kv_dev_valid&&layer<=c->n_layers&&m->kv_dev_valid[layer]>pos)m->kv_dev_valid[layer]=pos; /* riga riscritta: l'ombra si accorcia */+#endif+#ifdef COLI_VULKAN+ if(ks==m->kv&&m->vk_kv_valid&&layer<=c->n_layers&&m->vk_kv_valid[layer]>pos)+ m->vk_kv_valid[layer]=pos; /* riga riscritta: la cache VK si accorcia */#endifmemcpy(Ldst, cs, c->kv_lora*sizeof(float));rmsnorm(Ldst, Ldst, l->kv_a_ln, c->kv_lora, c->eps); /* latente normato */@@ -2733,7 +2745,34 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p}}#endif- if(!cuda_core){+ int vk_core=0;+#ifdef COLI_VULKAN+ /* Vulkan MLA absorb core (COLI_VK_ATTN=1): scores/softmax/latent/value rows for all+ * S x H in ONE submit per layer, reading the persistent on-device KV mirror (rows+ * appended incrementally; vk_kv_valid = watermark, invalidated like the CUDA shadow+ * on rewrite/rebind/resize). Falls back to CPU on DSA top-k selection, ragged KV,+ * the MTP layer, or any backend failure β output identical either way. */+ if(!cuda_core&&g_vk_attn&&!kvs&&!positions&&S<=4&&layer<c->n_layers&&+ (l->kv_b.fmt==1||l->kv_b.fmt==2)&&kvl<=512&&c->qk_nope<=256&&c->qk_rope<=64&&+ m->vk_kv_valid){+ int dsa_on=0; if(dnsel) for(int s=0;s<S;s++) if(dnsel[s]>0) dsa_on=1;+ int st0=m->kv_start[layer], T=pos_base+S;+ if(!dsa_on&&T<=m->max_t&&coli_vk_kv_ensure(layer,m->max_t,kvl,c->qk_rope)){+ int ok=1;+ for(int t=m->vk_kv_valid[layer];t<T&&ok;t++)+ ok=coli_vk_kv_row(layer,t,coli_kv_row(m->Lc[layer],t,kvl),+ coli_kv_row(m->Rc[layer],t,c->qk_rope));+ if(ok){+ m->vk_kv_valid[layer]=T;+ vk_core=coli_vk_attention_absorb(&l->kv_b.vk,+ l->kv_b.fmt==1?(const void*)l->kv_b.q8:(const void*)l->kv_b.q4,+ l->kv_b.s,l->kv_b.fmt,ctx,Q,layer,S,H,c->qk_nope,c->qk_rope,+ vh,kvl,st0,T,c->attn_scale);+ }+ }+ }+#endif+ if(!cuda_core&&!vk_core){/* Causal rows grow with s; round-robin pairs keep that work balanced. */#pragma omp parallel for collapse(2) schedule(static,1)for(int s=0;s<S;s++) for(int h=0;h<H;h++){@@ -4552,6 +4591,12 @@ static void kv_alloc(Model *m, int max_t){if(m->kv_dev_R[i]){ coli_cuda_pipe_free(m->L[i<c->n_layers?i:0].kv_b.cuda_device,m->kv_dev_R[i]); m->kv_dev_R[i]=NULL; }m->kv_dev_valid[i]=0;}+#endif+#ifdef COLI_VULKAN+ if(g_vulkan&&m->vk_kv_valid){ /* dimensioni cambiate: cache VK da rifare */+ coli_vk_kv_reset();+ for(int i=0;i<c->n_layers+1;i++) m->vk_kv_valid[i]=0;+ }#endifif(k->Lc){ for(int i=0;i<c->n_layers+1;i++){#ifdef COLI_METAL@@ -4587,6 +4632,10 @@ static void kv_alloc(Model *m, int max_t){static void kv_bind(Model *m, KVState *k){if(m->kv!=k && m->kv_dev_valid) /* ombra legata al KVState corrente */for(int i=0;i<m->c.n_layers+1;i++) m->kv_dev_valid[i]=0;+#ifdef COLI_VULKAN+ if(m->kv!=k && m->vk_kv_valid) /* la cache VK specchia la KV corrente */+ for(int i=0;i<m->c.n_layers+1;i++) m->vk_kv_valid[i]=0;+#endifm->kv=k; m->Lc=k->Lc; m->Rc=k->Rc; m->Ic=k->Ic;m->max_t=k->max_t; m->kv_start=k->kv_start;}@@ -7189,8 +7238,10 @@ int main(int argc, char **argv){if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (need libvulkan + shaders/qmatmul{,_gate_up}.spv)\n"); return 2; }g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 1024;g_vk_dense = getenv("COLI_VK_DENSE") ? atoi(getenv("COLI_VK_DENSE")) : 0;- fprintf(stderr,"[VK] expert tier active: routed int4 experts on the GPU (budget %d)%s\n",- g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "");+ g_vk_attn = getenv("COLI_VK_ATTN") ? atoi(getenv("COLI_VK_ATTN")) : 0;+ fprintf(stderr,"[VK] expert tier active: routed int4 experts on the GPU (budget %d)%s%s\n",+ g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "",+ g_vk_attn ? " + absorb attention core" : "");}#endif#ifdef COLI_CUDAdiff --git a/c/shaders/attention_absorb.comp b/c/shaders/attention_absorb.compnew file mode 100644index 00000000..f82a2bac--- /dev/null+++ b/c/shaders/attention_absorb.comp@@ -0,0 +1,127 @@+#version 450+// MLA weight-absorption attention core (decode), the VK equivalent of+// coli_cuda_attention_absorb / glm.c's CPU absorb loop. One workgroup per+// (query row s = WorkGroupID.y, head h = WorkGroupID.x); the whole core for+// one head runs in a single dispatch:+// qabs[i] = sum_d q_nope[d] * dequant(Wkv_b[h*(Q+V)+d])[i] (absorbed query)+// score_t = (qabs . L_t + q_rope . R_t) * scale t in [st0, T-S+s] (causal)+// p = softmax(score)+// clat[i] = sum_t p_t * L_t[i] (latent context)+// ctx[v] = clat . dequant(Wkv_b[h*(Q+V)+Q+v]) (value rows)+// L/R are the PERSISTENT device-side latent/rope caches (rows appended by the+// host each token, absolute position indexing). Queries arrive already roped;+// cache rows already normed+roped (host writes them that way).+// fmt: 1=int8, 2=int4 (nibble-8). Limits: Q<=256, R<=64, K<=512 (shared arrays).+#extension GL_KHR_shader_subgroup_arithmetic : require+layout(local_size_x = 256) in;++layout(std430, binding = 0) readonly buffer QB { float q[]; }; // [S,H,Q+R] roped+layout(std430, binding = 1) readonly buffer WB { uint w[]; }; // kv_b [H*(Q+V), K] quant+layout(std430, binding = 2) readonly buffer SB { float wsc[]; }; // kv_b row scales+layout(std430, binding = 3) readonly buffer LB { float lc[]; }; // latent cache [max_t, K]+layout(std430, binding = 4) readonly buffer RB { float rc[]; }; // rope cache [max_t, R]+layout(std430, binding = 5) buffer SC { float sc[]; }; // scores [S,H,cap] scratch+layout(std430, binding = 6) writeonly buffer CB { float ctx[]; }; // [S,H,V] out++layout(push_constant) uniform P {+ int fmt; // 1=int8, 2=int4+ int S, H, Q, R, V, K; // Q=qk_nope, R=qk_rope, V=v_head, K=kv_lora+ int st0, T; // attend rows [st0, T-S+s]; T = absolute rows incl. final query+ int rowWords; // uint32 words per kv_b row (K-dim)+ int cap; // score scratch stride per (s,h) = T-st0+ float scale; // attention scale+} p;++shared float qsc[256]; // q_nope pre-multiplied by its row scale+shared float qr[64]; // q_rope+shared float qabs[512]; // absorbed query in latent space+shared float clat[512]; // latent context+shared float red[256]; // reduction scratch++float i8(uint x, int l) { int b = int((x >> (uint(l) * 8u)) & 0xffu); if (b >= 128) b -= 256; return float(b); }+float i4(uint x, int l) { return float(int((x >> (uint(l) * 4u)) & 0xfu) - 8); }++void main() {+ int h = int(gl_WorkGroupID.x);+ int s = int(gl_WorkGroupID.y);+ int tid = int(gl_LocalInvocationID.x);+ int nth = int(gl_WorkGroupSize.x);+ int lane = int(gl_SubgroupInvocationID);+ int sgsize = int(gl_SubgroupSize);+ int sg = int(gl_SubgroupID);+ int nsg = nth / sgsize;++ int nt = (p.T - p.S + s + 1) - p.st0; // causal window for query s+ int rbase = h * (p.Q + p.V); // head's row block in kv_b+ int qoff = (s * p.H + h) * (p.Q + p.R);+ int scoff = (s * p.H + h) * p.cap;++ for (int d = tid; d < p.Q; d += nth) qsc[d] = q[qoff + d] * wsc[rbase + d];+ for (int d = tid; d < p.R; d += nth) qr[d] = q[qoff + p.Q + d];+ barrier();++ /* 1) absorbed query: qabs = Wnope^T qsc β thread-parallel over latent dim i */+ for (int i = tid; i < p.K; i += nth) {+ float acc = 0.0;+ if (p.fmt == 1) {+ int wi = i >> 2, l = i & 3;+ for (int d = 0; d < p.Q; d++) acc += qsc[d] * i8(w[(rbase + d) * p.rowWords + wi], l);+ } else {+ int wi = i >> 3, l = i & 7;+ for (int d = 0; d < p.Q; d++) acc += qsc[d] * i4(w[(rbase + d) * p.rowWords + wi], l);+ }+ qabs[i] = acc;+ }+ barrier();++ /* 2) scores: one subgroup per cached token, lanes split the K+R dot */+ float lmax = -3.0e38;+ for (int j = sg; j < nt; j += nsg) {+ int t = p.st0 + j;+ float a = 0.0;+ for (int i = lane; i < p.K; i += sgsize) a += qabs[i] * lc[t * p.K + i];+ for (int d = lane; d < p.R; d += sgsize) a += qr[d] * rc[t * p.R + d];+ float tot = subgroupAdd(a) * p.scale;+ if (lane == 0) { sc[scoff + j] = tot; lmax = max(lmax, tot); }+ }+ red[tid] = lmax; barrier();+ for (int o = nth / 2; o > 0; o >>= 1) { if (tid < o) red[tid] = max(red[tid], red[tid + o]); barrier(); }+ float wmax = red[0]; barrier();++ /* 3) softmax: exp in place + workgroup sum */+ float lsum = 0.0;+ for (int j = tid; j < nt; j += nth) { float e = exp(sc[scoff + j] - wmax); sc[scoff + j] = e; lsum += e; }+ red[tid] = lsum; barrier();+ for (int o = nth / 2; o > 0; o >>= 1) { if (tid < o) red[tid] += red[tid + o]; barrier(); }+ float inv = 1.0 / red[0]; barrier();++ /* 4) latent context: clat = sum_t p_t L_t β thread-parallel over i (coalesced rows) */+ for (int i = tid; i < p.K; i += nth) {+ float acc = 0.0;+ for (int j = 0; j < nt; j++) acc += sc[scoff + j] * lc[(p.st0 + j) * p.K + i];+ clat[i] = acc * inv;+ }+ barrier();++ /* 5) value rows: ctx[v] = dequant(row rbase+Q+v) . clat β one subgroup per row */+ for (int v = sg; v < p.V; v += nsg) {+ int row = rbase + p.Q + v;+ uint rb = uint(row) * uint(p.rowWords);+ float acc = 0.0;+ if (p.fmt == 1) {+ int words = (p.K + 3) / 4;+ for (int wi = lane; wi < words; wi += sgsize) {+ uint pk = w[rb + uint(wi)]; int i0 = wi * 4;+ for (int k = 0; k < 4; k++) { int i = i0 + k; if (i < p.K) acc += clat[i] * i8(pk, k); }+ }+ } else {+ int words = (p.K + 7) / 8;+ for (int wi = lane; wi < words; wi += sgsize) {+ uint pk = w[rb + uint(wi)]; int i0 = wi * 8;+ for (int k = 0; k < 8; k++) { int i = i0 + k; if (i < p.K) acc += clat[i] * i4(pk, k); }+ }+ }+ float tot = subgroupAdd(acc);+ if (lane == 0) ctx[(s * p.H + h) * p.V + v] = tot * wsc[row];+ }+}