Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

glm: VK MLA absorb attention core (COLI_VK_ATTN) β€” the last CPU compute piece on Vulkan-only machines

Steve Markgraf559383daf671
diff --git a/c/Makefile b/c/Makefile
index e39ed07a..0206475f 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -302,7 +302,7 @@ ifeq ($(VK),1)
CFLAGS += -DCOLI_VULKAN
LDFLAGS += -lvulkan
VK_OBJ = backend_vulkan.o
-VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv
+VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv
endif
all: colibri$(EXE)
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index 51bc6596..7934786f 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -29,6 +29,16 @@ typedef struct {
VkBuffer buf; VkDeviceMemory mem; void *ptr; size_t cap;
} Scratch;
+/* Persistent device-side KV latent/rope cache for one layer (MLA attention).
+ * Host appends rows as tokens decode (absolute-position indexing); the absorb
+ * kernel reads them in place. Allocated once at max_t rows, like the CUDA
+ * kv_dev shadow. */
+#define VK_KV_LAYERS 160
+typedef struct {
+ VkBuffer bl, br; VkDeviceMemory ml, mr; void *pl, *pr;
+ int rows, K, R;
+} VkKvLayer;
+
static struct {
int ready;
VkInstance inst;
@@ -47,6 +57,9 @@ static struct {
/* fused dual gate+up+silu pipeline (6 bindings): x, Wg, gscale, Wu, uscale, hidden */
VkShaderModule shader_gu; VkDescriptorSetLayout dsl_gu; VkPipelineLayout plyt_gu;
VkPipeline pipe_gu; VkDescriptorPool dpool_gu; VkDescriptorSet dset_gu;
+ /* MLA absorb attention core (7 bindings): q, W, scales, Lcache, Rcache, scores, ctx */
+ VkShaderModule shader_att; VkDescriptorSetLayout dsl_att; VkPipelineLayout plyt_att;
+ VkPipeline pipe_att; VkDescriptorPool dpool_att; VkDescriptorSet dset_att;
VkCommandPool cpool;
VkCommandBuffer cmd;
VkFence fence;
@@ -56,6 +69,8 @@ static struct {
* one submit with hidden never leaving the GPU. */
Scratch eg_x, eg_h, eg_y;
VkDescriptorPool eg_pool; VkDescriptorSet eg_gu[64], eg_dn[64]; int eg_nsets;
+ Scratch att_sc; /* attention score scratch (GPU-only) */
+ VkKvLayer kv[VK_KV_LAYERS]; /* per-layer resident KV latent/rope cache */
/* resubmit cache: skip vkUpdateDescriptorSets + command re-record when the bound
* tensor / shape / scratch buffers are unchanged from the previous call (the hot-
* expert-called-repeatedly pattern). The synchronous fence wait each call means no
@@ -67,6 +82,8 @@ static struct {
} G;
struct PC { int fmt, S, I, O, rowWords; };
+/* Push constants of the absorb attention kernel (must match attention_absorb.comp). */
+struct PCAttn { int fmt, S, H, Q, R, V, K, st0, T, rowWords, cap; float scale; };
static int pick_memtype(void) {
VkPhysicalDeviceMemoryProperties m;
@@ -148,11 +165,12 @@ static VkShaderModule load_spv(const char *path) {
return r == VK_SUCCESS ? m : VK_NULL_HANDLE;
}
-/* Build a compute pipeline + descriptor pool/set for nbind storage buffers sharing the
- * struct PC push constant. Used for both the 4-binding matmul and 6-binding gate_up. */
-static int build_pipeline(int nbind, VkShaderModule shader, VkDescriptorSetLayout *dsl,
- VkPipelineLayout *plyt, VkPipeline *pipe, VkDescriptorPool *dpool,
- VkDescriptorSet *dset) {
+/* Build a compute pipeline + descriptor pool/set for nbind storage buffers with a
+ * pc_size-byte push constant. Used by the 4-binding matmul, 6-binding gate_up and
+ * 7-binding absorb attention pipelines. */
+static int build_pipeline(int nbind, size_t pc_size, VkShaderModule shader,
+ VkDescriptorSetLayout *dsl, VkPipelineLayout *plyt, VkPipeline *pipe,
+ VkDescriptorPool *dpool, VkDescriptorSet *dset) {
VkDescriptorSetLayoutBinding b[8];
for (int i = 0; i < nbind; i++) b[i] = (VkDescriptorSetLayoutBinding){
.binding = (uint32_t)i, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
@@ -160,7 +178,7 @@ static int build_pipeline(int nbind, VkShaderModule shader, VkDescriptorSetLayou
VkDescriptorSetLayoutCreateInfo dsli = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.bindingCount = (uint32_t)nbind, .pBindings = b};
VKCHECK(vkCreateDescriptorSetLayout(G.dev, &dsli, NULL, dsl), "descSetLayout");
- VkPushConstantRange pcr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, .offset = 0, .size = sizeof(struct PC)};
+ VkPushConstantRange pcr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, .offset = 0, .size = (uint32_t)pc_size};
VkPipelineLayoutCreateInfo pli = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
.setLayoutCount = 1, .pSetLayouts = dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pcr};
VKCHECK(vkCreatePipelineLayout(G.dev, &pli, NULL, plyt), "pipelineLayout");
@@ -179,14 +197,21 @@ static int build_pipeline(int nbind, VkShaderModule shader, VkDescriptorSetLayou
return 1;
}
-/* "…/qmatmul.spv" -> "…/qmatmul_gate_up.spv" (sibling of the main shader). */
-static void derive_gu_path(const char *spv, char *out, size_t n) {
+/* "…/qmatmul.spv" -> "…/qmatmul<suffix>" (sibling of the main shader). */
+static void derive_sibling(const char *spv, const char *suffix, char *out, size_t n) {
const char *dot = strstr(spv, ".spv");
- if (dot && (size_t)(dot - spv) + 13 < n) {
+ if (dot && (size_t)(dot - spv) + strlen(suffix) + 1 < n) {
size_t pre = (size_t)(dot - spv);
- memcpy(out, spv, pre); strcpy(out + pre, "_gate_up.spv");
+ memcpy(out, spv, pre); strcpy(out + pre, suffix);
} else snprintf(out, n, "%s", spv);
}
+/* "…/qmatmul.spv" -> "…/attention_absorb.spv" (same directory). */
+static void derive_dir_file(const char *spv, const char *fname, char *out, size_t n) {
+ const char *sl = strrchr(spv, '/');
+ size_t pre = sl ? (size_t)(sl - spv) + 1 : 0;
+ if (pre + strlen(fname) + 1 < n) { memcpy(out, spv, pre); strcpy(out + pre, fname); }
+ else snprintf(out, n, "%s", fname);
+}
int coli_vk_init(const char *spv_path) {
if (G.ready) return 1;
@@ -238,13 +263,19 @@ int coli_vk_init(const char *spv_path) {
G.shader = load_spv(spv_path);
if (!G.shader) return 0;
- if (!build_pipeline(4, G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;
+ if (!build_pipeline(4, sizeof(struct PC), G.shader, &G.dsl, &G.plyt, &G.pipe, &G.dpool, &G.dset)) return 0;
/* Optional fused gate+up pipeline: skip gracefully if its shader isn't present
* (single-matmul path keeps working). */
- char gu_path[512]; derive_gu_path(spv_path, gu_path, sizeof(gu_path));
+ char gu_path[512]; derive_sibling(spv_path, "_gate_up.spv", gu_path, sizeof(gu_path));
G.shader_gu = load_spv(gu_path);
- if (G.shader_gu && !build_pipeline(6, G.shader_gu, &G.dsl_gu, &G.plyt_gu, &G.pipe_gu, &G.dpool_gu, &G.dset_gu))
+ if (G.shader_gu && !build_pipeline(6, sizeof(struct PC), G.shader_gu, &G.dsl_gu, &G.plyt_gu, &G.pipe_gu, &G.dpool_gu, &G.dset_gu))
+ return 0;
+
+ /* Optional MLA absorb attention pipeline (same directory as the main shader). */
+ char att_path[512]; derive_dir_file(spv_path, "attention_absorb.spv", att_path, sizeof(att_path));
+ G.shader_att = load_spv(att_path);
+ if (G.shader_att && !build_pipeline(7, sizeof(struct PCAttn), G.shader_att, &G.dsl_att, &G.plyt_att, &G.pipe_att, &G.dpool_att, &G.dset_att))
return 0;
VkCommandPoolCreateInfo cpci = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
@@ -258,8 +289,8 @@ int coli_vk_init(const char *spv_path) {
G.ready = 1;
VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(G.phys, &p);
- fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s\n", p.deviceName, G.qfam, G.memtype,
- G.shader_gu ? ", fused gate+up" : "");
+ fprintf(stderr, "[VK] ready: %s, compute qfam %u, memtype %u%s%s\n", p.deviceName, G.qfam, G.memtype,
+ G.shader_gu ? ", fused gate+up" : "", G.shader_att ? ", absorb attention" : "");
return 1;
}
@@ -507,6 +538,101 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
return 1;
}
+/* ---- MLA absorb attention core -------------------------------------------------
+ * The KV latent (L, [rows,K]) and rope (R, [rows,Rd]) caches live in persistent
+ * per-layer device buffers, appended row-by-row as tokens decode (the host stays
+ * canonical; glm.c tracks a valid-watermark and re-appends after invalidation).
+ * Rows are indexed by ABSOLUTE position, so kv_start windows just skip rows. */
+
+int coli_vk_kv_ensure(int layer, int max_rows, int K, int Rd) {
+ if (!G.ready || layer < 0 || layer >= VK_KV_LAYERS || max_rows < 1 || K < 1 || Rd < 1) return 0;
+ VkKvLayer *v = &G.kv[layer];
+ if (v->bl) return v->rows >= max_rows && v->K == K && v->R == Rd; /* resize goes through coli_vk_kv_reset */
+ if (!alloc_hostvis((size_t)max_rows * K * 4, &v->bl, &v->ml, &v->pl)) return 0;
+ if (!alloc_hostvis((size_t)max_rows * Rd * 4, &v->br, &v->mr, &v->pr)) {
+ vkDestroyBuffer(G.dev, v->bl, NULL); vkFreeMemory(G.dev, v->ml, NULL);
+ memset(v, 0, sizeof(*v)); return 0;
+ }
+ v->rows = max_rows; v->K = K; v->R = Rd;
+ return 1;
+}
+
+/* Mirror one host cache row into the device copy (write-combined memory: the CPU
+ * only ever WRITES these buffers, the GPU reads them). */
+int coli_vk_kv_row(int layer, int pos, const float *L, const float *R) {
+ if (layer < 0 || layer >= VK_KV_LAYERS) return 0;
+ VkKvLayer *v = &G.kv[layer];
+ if (!v->pl || pos < 0 || pos >= v->rows) return 0;
+ memcpy((float *)v->pl + (size_t)pos * v->K, L, (size_t)v->K * 4);
+ memcpy((float *)v->pr + (size_t)pos * v->R, R, (size_t)v->R * 4);
+ return 1;
+}
+
+/* Drop all per-layer KV device caches (cache resize in kv_alloc). */
+void coli_vk_kv_reset(void) {
+ for (int i = 0; i < VK_KV_LAYERS; i++) {
+ VkKvLayer *v = &G.kv[i];
+ if (!v->bl) continue;
+ if (G.ready) { /* dead device: leak GPU handles like coli_vk_tensor_free */
+ vkDestroyBuffer(G.dev, v->bl, NULL); vkFreeMemory(G.dev, v->ml, NULL);
+ vkDestroyBuffer(G.dev, v->br, NULL); vkFreeMemory(G.dev, v->mr, NULL);
+ }
+ memset(v, 0, sizeof(*v));
+ }
+}
+
+/* Decode MLA absorption core for S causal query rows of one sequence, one submit:
+ * ctx[s,h,:] = softmax((Wnope_h^T q_nope).L_t + q_rope.R_t) weighted latent context
+ * projected through the value rows of kv_b. kv_b ([H*(Q+V), K]) uploads on first
+ * call and stays resident; L/R rows [st0, T) must already be mirrored via
+ * coli_vk_kv_row. Returns 0 -> caller falls back to CPU. */
+int coli_vk_attention_absorb(ColiVkTensor **kvb, const void *w, const float *sc, int fmt,
+ float *ctx, const float *q, int layer, int S, int H,
+ int Q, int R, int V, int K, int st0, int T, float scale) {
+ if (!G.ready || !G.pipe_att || S < 1 || H < 1 || layer < 0 || layer >= VK_KV_LAYERS) return 0;
+ if (Q > 256 || R > 64 || K > 512 || st0 < 0 || T - S - st0 < 0) return 0; /* shared-array limits */
+ VkKvLayer *kv = &G.kv[layer];
+ if (!kv->bl || kv->rows < T || kv->K != K || kv->R != R) return 0;
+ if (!upload_tensor(kvb, w, sc, fmt, K, H * (Q + V))) return 0;
+ ColiVkTensor *t = *kvb;
+ int cap = T - st0;
+ size_t qb = (size_t)S * H * (Q + R) * 4, cb = (size_t)S * H * V * 4;
+ size_t sb = (size_t)S * H * cap * 4;
+ if (!scratch_reserve(&G.x, qb) || !scratch_reserve_mt(&G.y, cb, G.memtype_cached) ||
+ !scratch_reserve(&G.att_sc, sb)) return 0; /* y (ctx) is read back -> cached */
+ memcpy(G.x.ptr, q, qb);
+
+ VkDescriptorBufferInfo bi[7] = {
+ {.buffer = G.x.buf, .range = VK_WHOLE_SIZE}, {.buffer = t->wbuf, .range = VK_WHOLE_SIZE},
+ {.buffer = t->sbuf, .range = VK_WHOLE_SIZE}, {.buffer = kv->bl, .range = VK_WHOLE_SIZE},
+ {.buffer = kv->br, .range = VK_WHOLE_SIZE}, {.buffer = G.att_sc.buf, .range = VK_WHOLE_SIZE},
+ {.buffer = G.y.buf, .range = VK_WHOLE_SIZE}};
+ VkWriteDescriptorSet wd[7];
+ for (int i = 0; i < 7; i++) wd[i] = (VkWriteDescriptorSet){
+ .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = G.dset_att,
+ .dstBinding = (uint32_t)i, .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .pBufferInfo = &bi[i]};
+ vkUpdateDescriptorSets(G.dev, 7, wd, 0, NULL);
+
+ VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");
+ VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
+ VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");
+ vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe_att);
+ vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt_att, 0, 1, &G.dset_att, 0, NULL);
+ struct PCAttn pc = {fmt, S, H, Q, R, V, K, st0, T, t->rowWords, cap, scale};
+ vkCmdPushConstants(G.cmd, G.plyt_att, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+ vkCmdDispatch(G.cmd, (uint32_t)H, (uint32_t)S, 1); /* one workgroup per (head, row) */
+ VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");
+
+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};
+ VKCHECK(vkResetFences(G.dev, 1, &G.fence), "resetFence");
+ VKCHECK(vkQueueSubmit(G.queue, 1, &si, G.fence), "queueSubmit");
+ if (vkWaitForFences(G.dev, 1, &G.fence, VK_TRUE, 10000000000ULL) != VK_SUCCESS) { G.ready = 0; return 0; }
+ memcpy(ctx, G.y.ptr, cb);
+ G.cmd_ready = 0; G.bound_tensor = NULL; /* the shared command buffer/binding was clobbered */
+ return 1;
+}
+
void coli_vk_tensor_free(ColiVkTensor *t) {
if (!t) return;
if (G.bound_tensor == t) { G.bound_tensor = NULL; G.cmd_ready = 0; } /* drop stale cache */
@@ -536,6 +662,8 @@ void coli_vk_shutdown(void) {
if (G.eg_x.buf) { vkDestroyBuffer(G.dev, G.eg_x.buf, NULL); vkFreeMemory(G.dev, G.eg_x.mem, NULL); }
if (G.eg_h.buf) { vkDestroyBuffer(G.dev, G.eg_h.buf, NULL); vkFreeMemory(G.dev, G.eg_h.mem, NULL); }
if (G.eg_y.buf) { vkDestroyBuffer(G.dev, G.eg_y.buf, NULL); vkFreeMemory(G.dev, G.eg_y.mem, NULL); }
+ if (G.att_sc.buf) { vkDestroyBuffer(G.dev, G.att_sc.buf, NULL); vkFreeMemory(G.dev, G.att_sc.mem, NULL); }
+ coli_vk_kv_reset();
if (G.eg_pool) vkDestroyDescriptorPool(G.dev, G.eg_pool, NULL);
vkDestroyFence(G.dev, G.fence, NULL);
vkDestroyCommandPool(G.dev, G.cpool, NULL);
@@ -551,6 +679,13 @@ void coli_vk_shutdown(void) {
vkDestroyDescriptorSetLayout(G.dev, G.dsl_gu, NULL);
vkDestroyShaderModule(G.dev, G.shader_gu, NULL);
}
+ if (G.shader_att) {
+ vkDestroyDescriptorPool(G.dev, G.dpool_att, NULL);
+ vkDestroyPipeline(G.dev, G.pipe_att, NULL);
+ vkDestroyPipelineLayout(G.dev, G.plyt_att, NULL);
+ vkDestroyDescriptorSetLayout(G.dev, G.dsl_att, NULL);
+ vkDestroyShaderModule(G.dev, G.shader_att, NULL);
+ }
vkDestroyDevice(G.dev, NULL);
vkDestroyInstance(G.inst, NULL);
memset(&G, 0, sizeof(G));
@@ -810,6 +945,63 @@ static int run_expert_group(int fmt, int D, int I, int K) {
return maxrel > 1e-3 ? 1 : 0;
}
+/* MLA absorb attention vs a CPU ref that mirrors glm.c's absorb loop exactly:
+ * qabs = sum_d q[d]*deq(row rbase+d)*ws, scores over cache rows [st0, T-S+s],
+ * softmax, weighted latent, value-row projection. */
+static int run_absorb(int fmt, int S, int H, int Q, int R, int V, int K, int st0, int T, int layer) {
+ size_t rb = fmt == 1 ? (size_t)K : (size_t)(K + 1) / 2;
+ int O = H * (Q + V);
+ uint8_t *w = malloc(rb * O); float *ws = malloc((size_t)O * 4);
+ float *q = malloc((size_t)S * H * (Q + R) * 4);
+ float *L = malloc((size_t)T * K * 4), *Rr = malloc((size_t)T * R * 4);
+ float *cg = malloc((size_t)S * H * V * 4), *cc = malloc((size_t)S * H * V * 4);
+ for (size_t i = 0; i < rb * (size_t)O; i++) w[i] = rand() & 0xff;
+ for (int o = 0; o < O; o++) ws[o] = 0.01f + (rand() % 100) / 10000.0f;
+ for (int i = 0; i < S * H * (Q + R); i++) q[i] = (rand() % 200 - 100) / 100.0f;
+ for (int i = 0; i < T * K; i++) L[i] = (rand() % 200 - 100) / 100.0f;
+ for (int i = 0; i < T * R; i++) Rr[i] = (rand() % 200 - 100) / 100.0f;
+ float scale = 0.13f;
+ if (!coli_vk_kv_ensure(layer, T, K, R)) { printf("kv_ensure failed\n"); return 1; }
+ for (int t = 0; t < T; t++)
+ if (!coli_vk_kv_row(layer, t, L + (size_t)t * K, Rr + (size_t)t * R)) { printf("kv_row failed\n"); return 1; }
+ ColiVkTensor *kvb = NULL;
+ if (!coli_vk_attention_absorb(&kvb, w, ws, fmt, cg, q, layer, S, H, Q, R, V, K, st0, T, scale)) {
+ printf("absorb failed\n"); return 1; }
+ float *qabs = malloc((size_t)K * 4), *clat = malloc((size_t)K * 4), *sc = malloc((size_t)(T - st0) * 4);
+ for (int s = 0; s < S; s++) for (int h = 0; h < H; h++) {
+ const float *qp = q + ((size_t)s * H + h) * (Q + R), *qr = qp + Q;
+ int rbase = h * (Q + V), nt = (T - S + s + 1) - st0;
+ for (int i = 0; i < K; i++) qabs[i] = 0;
+ for (int d = 0; d < Q; d++) { const uint8_t *row = w + (size_t)(rbase + d) * rb;
+ for (int i = 0; i < K; i++) qabs[i] += qp[d] * deq(row, fmt, i) * ws[rbase + d]; }
+ for (int j = 0; j < nt; j++) { int t = st0 + j;
+ double a = 0;
+ for (int i = 0; i < K; i++) a += qabs[i] * L[(size_t)t * K + i];
+ for (int d = 0; d < R; d++) a += qr[d] * Rr[(size_t)t * R + d];
+ sc[j] = (float)(a * scale); }
+ float mx = sc[0]; for (int j = 1; j < nt; j++) if (sc[j] > mx) mx = sc[j];
+ double sum = 0; for (int j = 0; j < nt; j++) { sc[j] = expf(sc[j] - mx); sum += sc[j]; }
+ for (int i = 0; i < K; i++) clat[i] = 0;
+ for (int j = 0; j < nt; j++) { float a = (float)(sc[j] / sum);
+ for (int i = 0; i < K; i++) clat[i] += a * L[(size_t)(st0 + j) * K + i]; }
+ for (int v = 0; v < V; v++) { const uint8_t *row = w + (size_t)(rbase + Q + v) * rb;
+ double a = 0; for (int i = 0; i < K; i++) a += clat[i] * deq(row, fmt, i);
+ cc[((size_t)s * H + h) * V + v] = (float)(a * ws[rbase + Q + v]); }
+ }
+ double maxrel = 0, maxerr = 0;
+ for (int i = 0; i < S * H * V; i++) { double e = fabs(cg[i] - cc[i]); if (e > maxerr) maxerr = e;
+ if (fabs(cc[i]) > 1e-2) { double r = e / fabs(cc[i]); if (r > maxrel) maxrel = r; } }
+ double t0 = now(); int iters = 20; /* per-call cost, the engine pattern (one submit/layer) */
+ for (int k = 0; k < iters; k++)
+ coli_vk_attention_absorb(&kvb, w, ws, fmt, cg, q, layer, S, H, Q, R, V, K, st0, T, scale);
+ double ms = (now() - t0) * 1000 / iters;
+ printf("absorb fmt=%d S=%d H=%d Q=%d R=%d V=%d K=%d st0=%d T=%d | maxerr=%.4g maxrel=%.4g | %.3f ms/call\n",
+ fmt, S, H, Q, R, V, K, st0, T, maxerr, maxrel, ms);
+ coli_vk_tensor_free(kvb);
+ free(w); free(ws); free(q); free(L); free(Rr); free(cg); free(cc); free(qabs); free(clat); free(sc);
+ return maxrel > 2e-3 ? 1 : 0;
+}
+
int main(int argc, char **argv) {
const char *spv = argc > 1 ? argv[1] : "shaders/qmatmul.spv";
if (!coli_vk_init(spv)) { printf("vk init failed\n"); return 1; }
@@ -866,6 +1058,14 @@ int main(int argc, char **argv) {
bad |= run_expert_group(2, 6144, 2048, 1);
bad |= run_expert_group(2, 6144, 2048, 8);
bad |= run_expert_group(2, 6144, 2048, 32);
+ /* MLA absorb attention core (GLM decode shape + window/causal/int8 variants). */
+ if (G.pipe_att) {
+ bad |= run_absorb(2, 1, 64, 192, 64, 256, 512, 0, 300, 0); // GLM-5.2 decode
+ bad |= run_absorb(2, 1, 64, 192, 64, 256, 512, 17, 300, 1); // kv_start window
+ bad |= run_absorb(2, 2, 64, 192, 64, 256, 512, 0, 300, 2); // S=2 causal (MTP verify)
+ bad |= run_absorb(1, 1, 8, 128, 32, 64, 256, 0, 64, 3); // int8, odd dims
+ bad |= run_absorb(2, 1, 64, 192, 64, 256, 512, 0, 2000, 4); // long context
+ }
printf(bad ? "FAIL\n" : "PASS\n");
coli_vk_shutdown();
return bad;
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index 0c5c930b..44f08f48 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -52,6 +52,20 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
* then driven by coli_vk_expert_group). Returns 0 on failure/unsupported fmt. */
int coli_vk_tensor_ensure(ColiVkTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O);
+/* MLA absorb attention core (decode). The KV latent/rope caches live in persistent
+ * per-layer device buffers: _ensure allocates a layer's cache at max_rows (once; resize
+ * via _reset), _row mirrors one host row (absolute position), _reset drops all layers.
+ * The caller keeps a valid-watermark and re-mirrors rows after any invalidation.
+ * absorb runs S causal query rows over cache rows [st0, T) in one submit:
+ * q [S,H*(Q+R)] roped, kv_b [H*(Q+V), K] uploads once (fmt 1=int8/2=int4),
+ * ctx out [S,H*V]. Returns 0 -> caller falls back to CPU. */
+int coli_vk_kv_ensure(int layer, int max_rows, int K, int Rd);
+int coli_vk_kv_row(int layer, int pos, const float *L, const float *R);
+void coli_vk_kv_reset(void);
+int coli_vk_attention_absorb(ColiVkTensor **kvb, const void *w, const float *sc, int fmt,
+ float *ctx, const float *q, int layer, int S, int H,
+ int Q, int R, int V, int K, int st0, int T, float scale);
+
void coli_vk_tensor_free(ColiVkTensor *t);
size_t coli_vk_tensor_bytes(const ColiVkTensor *t);
diff --git a/c/colibri.c b/c/colibri.c
index 711783f9..6f1d0c81 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -202,6 +202,9 @@ typedef struct {
ESlot **ecache; int *ecn; int ecap; /* LRU expert per-layer */
float **kv_dev_L, **kv_dev_R; int *kv_dev_valid; /* ombra KV su device (decode) */
float **ln_dev; /* in_ln/post_ln cached on device: [layer*2+{0,1}] (Inc.4) */
+#ifdef COLI_VULKAN
+ int *vk_kv_valid; /* righe [0,v) specchiate nella cache KV Vulkan */
+#endif
ESlot ws[64]; /* working set del layer corrente (load paralleli) */
ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */
uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */
@@ -295,6 +298,8 @@ static int g_vk_budget; /* COLI_VK_EXPERTS: max experts uploaded to VK VRA
static int g_vk_resident; /* how many experts currently VK-resident */
static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention
* projections + shared expert) on Vulkan too */
+static int g_vk_attn; /* COLI_VK_ATTN=1: run the MLA absorb attention core on Vulkan
+ * (mirrors COLI_CUDA_ATTN; KV latent cache mirrored on-device) */
#endif
#ifdef COLI_CUDA
static int g_cuda_enabled;
@@ -1205,6 +1210,9 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
m->ecap=cap; m->ecache=calloc(NR,sizeof(ESlot*)); m->ecn=calloc(NR,sizeof(int));
m->kv_dev_L=calloc(NR,sizeof(float*)); m->kv_dev_R=calloc(NR,sizeof(float*));
m->kv_dev_valid=calloc(NR,sizeof(int));
+#ifdef COLI_VULKAN
+ m->vk_kv_valid=calloc(NR,sizeof(int));
+#endif
m->eroute=calloc(NR,sizeof(int*)); m->enr=calloc(NR,sizeof(int));
m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int));
m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*));
@@ -2532,6 +2540,10 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
#ifdef COLI_CUDA
if(ks==m->kv&&m->kv_dev_valid&&layer<=c->n_layers&&m->kv_dev_valid[layer]>pos)
m->kv_dev_valid[layer]=pos; /* riga riscritta: l'ombra si accorcia */
+#endif
+#ifdef COLI_VULKAN
+ if(ks==m->kv&&m->vk_kv_valid&&layer<=c->n_layers&&m->vk_kv_valid[layer]>pos)
+ m->vk_kv_valid[layer]=pos; /* riga riscritta: la cache VK si accorcia */
#endif
memcpy(Ldst, cs, c->kv_lora*sizeof(float));
rmsnorm(Ldst, Ldst, l->kv_a_ln, c->kv_lora, c->eps); /* latente normato */
@@ -2733,7 +2745,34 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
}
}
#endif
- if(!cuda_core){
+ int vk_core=0;
+#ifdef COLI_VULKAN
+ /* Vulkan MLA absorb core (COLI_VK_ATTN=1): scores/softmax/latent/value rows for all
+ * S x H in ONE submit per layer, reading the persistent on-device KV mirror (rows
+ * appended incrementally; vk_kv_valid = watermark, invalidated like the CUDA shadow
+ * on rewrite/rebind/resize). Falls back to CPU on DSA top-k selection, ragged KV,
+ * the MTP layer, or any backend failure β€” output identical either way. */
+ if(!cuda_core&&g_vk_attn&&!kvs&&!positions&&S<=4&&layer<c->n_layers&&
+ (l->kv_b.fmt==1||l->kv_b.fmt==2)&&kvl<=512&&c->qk_nope<=256&&c->qk_rope<=64&&
+ m->vk_kv_valid){
+ int dsa_on=0; if(dnsel) for(int s=0;s<S;s++) if(dnsel[s]>0) dsa_on=1;
+ int st0=m->kv_start[layer], T=pos_base+S;
+ if(!dsa_on&&T<=m->max_t&&coli_vk_kv_ensure(layer,m->max_t,kvl,c->qk_rope)){
+ int ok=1;
+ for(int t=m->vk_kv_valid[layer];t<T&&ok;t++)
+ ok=coli_vk_kv_row(layer,t,coli_kv_row(m->Lc[layer],t,kvl),
+ coli_kv_row(m->Rc[layer],t,c->qk_rope));
+ if(ok){
+ m->vk_kv_valid[layer]=T;
+ vk_core=coli_vk_attention_absorb(&l->kv_b.vk,
+ l->kv_b.fmt==1?(const void*)l->kv_b.q8:(const void*)l->kv_b.q4,
+ l->kv_b.s,l->kv_b.fmt,ctx,Q,layer,S,H,c->qk_nope,c->qk_rope,
+ vh,kvl,st0,T,c->attn_scale);
+ }
+ }
+ }
+#endif
+ if(!cuda_core&&!vk_core){
/* Causal rows grow with s; round-robin pairs keep that work balanced. */
#pragma omp parallel for collapse(2) schedule(static,1)
for(int s=0;s<S;s++) for(int h=0;h<H;h++){
@@ -4552,6 +4591,12 @@ static void kv_alloc(Model *m, int max_t){
if(m->kv_dev_R[i]){ coli_cuda_pipe_free(m->L[i<c->n_layers?i:0].kv_b.cuda_device,m->kv_dev_R[i]); m->kv_dev_R[i]=NULL; }
m->kv_dev_valid[i]=0;
}
+#endif
+#ifdef COLI_VULKAN
+ if(g_vulkan&&m->vk_kv_valid){ /* dimensioni cambiate: cache VK da rifare */
+ coli_vk_kv_reset();
+ for(int i=0;i<c->n_layers+1;i++) m->vk_kv_valid[i]=0;
+ }
#endif
if(k->Lc){ for(int i=0;i<c->n_layers+1;i++){
#ifdef COLI_METAL
@@ -4587,6 +4632,10 @@ static void kv_alloc(Model *m, int max_t){
static void kv_bind(Model *m, KVState *k){
if(m->kv!=k && m->kv_dev_valid) /* ombra legata al KVState corrente */
for(int i=0;i<m->c.n_layers+1;i++) m->kv_dev_valid[i]=0;
+#ifdef COLI_VULKAN
+ if(m->kv!=k && m->vk_kv_valid) /* la cache VK specchia la KV corrente */
+ for(int i=0;i<m->c.n_layers+1;i++) m->vk_kv_valid[i]=0;
+#endif
m->kv=k; m->Lc=k->Lc; m->Rc=k->Rc; m->Ic=k->Ic;
m->max_t=k->max_t; m->kv_start=k->kv_start;
}
@@ -7189,8 +7238,10 @@ int main(int argc, char **argv){
if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (need libvulkan + shaders/qmatmul{,_gate_up}.spv)\n"); return 2; }
g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 1024;
g_vk_dense = getenv("COLI_VK_DENSE") ? atoi(getenv("COLI_VK_DENSE")) : 0;
- fprintf(stderr,"[VK] expert tier active: routed int4 experts on the GPU (budget %d)%s\n",
- g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "");
+ g_vk_attn = getenv("COLI_VK_ATTN") ? atoi(getenv("COLI_VK_ATTN")) : 0;
+ fprintf(stderr,"[VK] expert tier active: routed int4 experts on the GPU (budget %d)%s%s\n",
+ g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "",
+ g_vk_attn ? " + absorb attention core" : "");
}
#endif
#ifdef COLI_CUDA
diff --git a/c/shaders/attention_absorb.comp b/c/shaders/attention_absorb.comp
new file mode 100644
index 00000000..f82a2bac
--- /dev/null
+++ b/c/shaders/attention_absorb.comp
@@ -0,0 +1,127 @@
+#version 450
+// MLA weight-absorption attention core (decode), the VK equivalent of
+// coli_cuda_attention_absorb / glm.c's CPU absorb loop. One workgroup per
+// (query row s = WorkGroupID.y, head h = WorkGroupID.x); the whole core for
+// one head runs in a single dispatch:
+// qabs[i] = sum_d q_nope[d] * dequant(Wkv_b[h*(Q+V)+d])[i] (absorbed query)
+// score_t = (qabs . L_t + q_rope . R_t) * scale t in [st0, T-S+s] (causal)
+// p = softmax(score)
+// clat[i] = sum_t p_t * L_t[i] (latent context)
+// ctx[v] = clat . dequant(Wkv_b[h*(Q+V)+Q+v]) (value rows)
+// L/R are the PERSISTENT device-side latent/rope caches (rows appended by the
+// host each token, absolute position indexing). Queries arrive already roped;
+// cache rows already normed+roped (host writes them that way).
+// fmt: 1=int8, 2=int4 (nibble-8). Limits: Q<=256, R<=64, K<=512 (shared arrays).
+#extension GL_KHR_shader_subgroup_arithmetic : require
+layout(local_size_x = 256) in;
+
+layout(std430, binding = 0) readonly buffer QB { float q[]; }; // [S,H,Q+R] roped
+layout(std430, binding = 1) readonly buffer WB { uint w[]; }; // kv_b [H*(Q+V), K] quant
+layout(std430, binding = 2) readonly buffer SB { float wsc[]; }; // kv_b row scales
+layout(std430, binding = 3) readonly buffer LB { float lc[]; }; // latent cache [max_t, K]
+layout(std430, binding = 4) readonly buffer RB { float rc[]; }; // rope cache [max_t, R]
+layout(std430, binding = 5) buffer SC { float sc[]; }; // scores [S,H,cap] scratch
+layout(std430, binding = 6) writeonly buffer CB { float ctx[]; }; // [S,H,V] out
+
+layout(push_constant) uniform P {
+ int fmt; // 1=int8, 2=int4
+ int S, H, Q, R, V, K; // Q=qk_nope, R=qk_rope, V=v_head, K=kv_lora
+ int st0, T; // attend rows [st0, T-S+s]; T = absolute rows incl. final query
+ int rowWords; // uint32 words per kv_b row (K-dim)
+ int cap; // score scratch stride per (s,h) = T-st0
+ float scale; // attention scale
+} p;
+
+shared float qsc[256]; // q_nope pre-multiplied by its row scale
+shared float qr[64]; // q_rope
+shared float qabs[512]; // absorbed query in latent space
+shared float clat[512]; // latent context
+shared float red[256]; // reduction scratch
+
+float i8(uint x, int l) { int b = int((x >> (uint(l) * 8u)) & 0xffu); if (b >= 128) b -= 256; return float(b); }
+float i4(uint x, int l) { return float(int((x >> (uint(l) * 4u)) & 0xfu) - 8); }
+
+void main() {
+ int h = int(gl_WorkGroupID.x);
+ int s = int(gl_WorkGroupID.y);
+ int tid = int(gl_LocalInvocationID.x);
+ int nth = int(gl_WorkGroupSize.x);
+ int lane = int(gl_SubgroupInvocationID);
+ int sgsize = int(gl_SubgroupSize);
+ int sg = int(gl_SubgroupID);
+ int nsg = nth / sgsize;
+
+ int nt = (p.T - p.S + s + 1) - p.st0; // causal window for query s
+ int rbase = h * (p.Q + p.V); // head's row block in kv_b
+ int qoff = (s * p.H + h) * (p.Q + p.R);
+ int scoff = (s * p.H + h) * p.cap;
+
+ for (int d = tid; d < p.Q; d += nth) qsc[d] = q[qoff + d] * wsc[rbase + d];
+ for (int d = tid; d < p.R; d += nth) qr[d] = q[qoff + p.Q + d];
+ barrier();
+
+ /* 1) absorbed query: qabs = Wnope^T qsc β€” thread-parallel over latent dim i */
+ for (int i = tid; i < p.K; i += nth) {
+ float acc = 0.0;
+ if (p.fmt == 1) {
+ int wi = i >> 2, l = i & 3;
+ for (int d = 0; d < p.Q; d++) acc += qsc[d] * i8(w[(rbase + d) * p.rowWords + wi], l);
+ } else {
+ int wi = i >> 3, l = i & 7;
+ for (int d = 0; d < p.Q; d++) acc += qsc[d] * i4(w[(rbase + d) * p.rowWords + wi], l);
+ }
+ qabs[i] = acc;
+ }
+ barrier();
+
+ /* 2) scores: one subgroup per cached token, lanes split the K+R dot */
+ float lmax = -3.0e38;
+ for (int j = sg; j < nt; j += nsg) {
+ int t = p.st0 + j;
+ float a = 0.0;
+ for (int i = lane; i < p.K; i += sgsize) a += qabs[i] * lc[t * p.K + i];
+ for (int d = lane; d < p.R; d += sgsize) a += qr[d] * rc[t * p.R + d];
+ float tot = subgroupAdd(a) * p.scale;
+ if (lane == 0) { sc[scoff + j] = tot; lmax = max(lmax, tot); }
+ }
+ red[tid] = lmax; barrier();
+ for (int o = nth / 2; o > 0; o >>= 1) { if (tid < o) red[tid] = max(red[tid], red[tid + o]); barrier(); }
+ float wmax = red[0]; barrier();
+
+ /* 3) softmax: exp in place + workgroup sum */
+ float lsum = 0.0;
+ for (int j = tid; j < nt; j += nth) { float e = exp(sc[scoff + j] - wmax); sc[scoff + j] = e; lsum += e; }
+ red[tid] = lsum; barrier();
+ for (int o = nth / 2; o > 0; o >>= 1) { if (tid < o) red[tid] += red[tid + o]; barrier(); }
+ float inv = 1.0 / red[0]; barrier();
+
+ /* 4) latent context: clat = sum_t p_t L_t β€” thread-parallel over i (coalesced rows) */
+ for (int i = tid; i < p.K; i += nth) {
+ float acc = 0.0;
+ for (int j = 0; j < nt; j++) acc += sc[scoff + j] * lc[(p.st0 + j) * p.K + i];
+ clat[i] = acc * inv;
+ }
+ barrier();
+
+ /* 5) value rows: ctx[v] = dequant(row rbase+Q+v) . clat β€” one subgroup per row */
+ for (int v = sg; v < p.V; v += nsg) {
+ int row = rbase + p.Q + v;
+ uint rb = uint(row) * uint(p.rowWords);
+ float acc = 0.0;
+ if (p.fmt == 1) {
+ int words = (p.K + 3) / 4;
+ for (int wi = lane; wi < words; wi += sgsize) {
+ uint pk = w[rb + uint(wi)]; int i0 = wi * 4;
+ for (int k = 0; k < 4; k++) { int i = i0 + k; if (i < p.K) acc += clat[i] * i8(pk, k); }
+ }
+ } else {
+ int words = (p.K + 7) / 8;
+ for (int wi = lane; wi < words; wi += sgsize) {
+ uint pk = w[rb + uint(wi)]; int i0 = wi * 8;
+ for (int k = 0; k < 8; k++) { int i = i0 + k; if (i < p.K) acc += clat[i] * i4(pk, k); }
+ }
+ }
+ float tot = subgroupAdd(acc);
+ if (lane == 0) ctx[(s * p.H + h) * p.V + v] = tot * wsc[row];
+ }
+}