Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
VK: pinned+async expert tier β heat-ranked startup residency, GPU batch overlapped with CPU rows
Steve Markgrafdd024ecc1ac2
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex ff5190c9..2fce42d0 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -69,6 +69,11 @@ static struct {* one submit with hidden never leaving the GPU. */Scratch eg_x, eg_h, eg_y;VkDescriptorPool eg_pool; VkDescriptorSet eg_gu[64], eg_dn[64]; int eg_nsets;+ /* expert-group ASYNC state: its own command buffer + fence so an in-flight group+ * never collides with the main cmd/fence (dense matmuls, absorb) β issue() returns+ * immediately, the CPU computes its share, take() joins. */+ VkCommandBuffer eg_cmd; VkFence eg_fence; int eg_inflight; size_t eg_pending_yb;+ double eg_t0, eg_t1, eg_t2, eg_t3; int eg_prof;Scratch att_sc; /* attention score scratch (GPU-only) */Scratch att_ctx; /* fused absorb+o: ctx stays on device (GPU-only) */Scratch y2; /* second output of the fused matmul pair (readback) */@@ -287,8 +292,10 @@ int coli_vk_init(const char *spv_path) {VkCommandBufferAllocateInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,.commandPool = G.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};VKCHECK(vkAllocateCommandBuffers(G.dev, &cbi, &G.cmd), "cmdBuf");+ VKCHECK(vkAllocateCommandBuffers(G.dev, &cbi, &G.eg_cmd), "eg cmdBuf");VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};VKCHECK(vkCreateFence(G.dev, &fi, NULL, &G.fence), "fence");+ VKCHECK(vkCreateFence(G.dev, &fi, NULL, &G.eg_fence), "eg fence");G.ready = 1;VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(G.phys, &p);@@ -455,13 +462,16 @@ static void wr_desc(VkDescriptorSet set, int n, const VkDescriptorBufferInfo *bivkUpdateDescriptorSets(G.dev, (uint32_t)n, w, 0, NULL);}-/* Full batched expert MLP for `count` experts in ONE submit, hidden staying on-device:+/* Full batched expert MLP for `count` experts, hidden staying on-device:* for each c, hidden_c = silu(gate_c(x_c))*up_c(x_c) (fused), then y_c = down_c(hidden_c).* x/y are packed [sum(rows)*D]; experts are resident VkTensors (gate/up: D->I, down: I->D).- * Mirrors coli_cuda_expert_group. Returns 0 -> caller falls back to CPU. */-int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,- ColiVkTensor *const *downs, const int *rows, int count,- float *y, const float *x) {+ * Mirrors coli_cuda_expert_group. Split into prepare+submit / take so the caller can+ * overlap the GPU batch with its own CPU share (issue -> CPU rows -> take); the group+ * runs on its OWN command buffer + fence, so in-flight work never collides with the+ * main pipeline (dense matmuls, absorb attention). Returns 0 -> caller falls back. */+static int eg_prepare_submit(ColiVkTensor *const *gates, ColiVkTensor *const *ups,+ ColiVkTensor *const *downs, const int *rows, int count,+ const float *x) {if (!G.ready || !G.shader_gu || count < 1 || count > 64) return 0;ColiVkTensor *g0 = gates[0]; if (!g0) return 0;int D = g0->I, I = g0->O, fmt = g0->fmt, total = 0, off[64];@@ -474,10 +484,10 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,size_t xb = (size_t)total*D*4, hb = (size_t)total*I*4, yb = (size_t)total*D*4;if (!scratch_reserve(&G.eg_x, xb) || !scratch_reserve(&G.eg_h, hb) ||!scratch_reserve_mt(&G.eg_y, yb, G.memtype_cached)) return 0; /* eg_y is read back -> cached */- int prof = getenv("VK_PROF") != NULL; double t0=0,t1=0,t2=0,t3=0,t4=0,t5=0;- if (prof) t0 = vk_now();+ G.eg_prof = getenv("VK_PROF") != NULL;+ if (G.eg_prof) G.eg_t0 = vk_now();memcpy(G.eg_x.ptr, x, xb);- if (prof) t1 = vk_now();+ if (G.eg_prof) G.eg_t1 = vk_now();if (!G.eg_pool) { /* one-time: 64 gate_up (6-binding) + 64 down (4-binding) sets */VkDescriptorPoolSize ps = {.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 64*6 + 64*4};@@ -503,45 +513,75 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,{downs[c]->sbuf, 0, VK_WHOLE_SIZE}, {G.eg_y.buf, yo, (VkDeviceSize)rows[c]*D*4}};wr_desc(G.eg_dn[c], 4, di);}- if (prof) t2 = vk_now();+ if (G.eg_prof) G.eg_t2 = vk_now();- VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");+ VKCHECK(vkResetCommandBuffer(G.eg_cmd, 0), "eg resetCmd");VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};- VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");+ VKCHECK(vkBeginCommandBuffer(G.eg_cmd, &begin), "eg beginCmd");VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT};/* phase 1: fused gate+up+silu -> hidden (per expert, bound to its x/hidden slices) */- vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe_gu);+ vkCmdBindPipeline(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe_gu);for (int c = 0; c < count; c++) {struct PC pc = {fmt, rows[c], D, I, gates[c]->rowWords};- vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt_gu, 0, 1, &G.eg_gu[c], 0, NULL);- vkCmdPushConstants(G.cmd, G.plyt_gu, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- vkCmdDispatch(G.cmd, (uint32_t)((I + 7) / 8), (uint32_t)rows[c], 1);+ vkCmdBindDescriptorSets(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt_gu, 0, 1, &G.eg_gu[c], 0, NULL);+ vkCmdPushConstants(G.eg_cmd, G.plyt_gu, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ vkCmdDispatch(G.eg_cmd, (uint32_t)((I + 7) / 8), (uint32_t)rows[c], 1);}- vkCmdPipelineBarrier(G.cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &mb, 0, NULL, 0, NULL);+ vkCmdPipelineBarrier(G.eg_cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &mb, 0, NULL, 0, NULL);/* phase 2: down projection hidden -> y */- vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);+ vkCmdBindPipeline(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);for (int c = 0; c < count; c++) {struct PC pc = {fmt, rows[c], I, D, downs[c]->rowWords};- vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.eg_dn[c], 0, NULL);- vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);- vkCmdDispatch(G.cmd, (uint32_t)((D + 7) / 8), (uint32_t)rows[c], 1);+ vkCmdBindDescriptorSets(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.eg_dn[c], 0, NULL);+ vkCmdPushConstants(G.eg_cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);+ vkCmdDispatch(G.eg_cmd, (uint32_t)((D + 7) / 8), (uint32_t)rows[c], 1);}- VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");- if (prof) t3 = vk_now();+ VKCHECK(vkEndCommandBuffer(G.eg_cmd), "eg endCmd");+ if (G.eg_prof) G.eg_t3 = vk_now();- VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};- VKCHECK(vkResetFences(G.dev, 1, &G.fence), "resetFence");- VKCHECK(vkQueueSubmit(G.queue, 1, &si, G.fence), "queueSubmit");- if (vkWaitForFences(G.dev, 1, &G.fence, VK_TRUE, 10000000000ULL) != VK_SUCCESS) { G.ready = 0; return 0; }- if (prof) t4 = vk_now();- memcpy(y, G.eg_y.ptr, yb);- if (prof) { t5 = vk_now();- fprintf(stderr, "[VK_PROF] K=%d memcpy_x %.3f | desc %.3f | record %.3f | gpu %.3f | memcpy_y %.3f ms\n",- count, t1-t0, t2-t1, t3-t2, t4-t3, t5-t4); }- G.cmd_ready = 0; G.bound_tensor = NULL;+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.eg_cmd};+ VKCHECK(vkResetFences(G.dev, 1, &G.eg_fence), "eg resetFence");+ VKCHECK(vkQueueSubmit(G.queue, 1, &si, G.eg_fence), "eg queueSubmit");+ G.eg_pending_yb = yb; G.eg_inflight = 1;+ return 1;+}++/* Issue a group asynchronously: submit and return WITHOUT waiting, so the caller+ * computes its CPU share concurrently. Exactly one group may be in flight. */+int coli_vk_expert_group_issue(ColiVkTensor *const *gates, ColiVkTensor *const *ups,+ ColiVkTensor *const *downs, const int *rows, int count,+ const float *x) {+ if (G.eg_inflight) return 0;+ return eg_prepare_submit(gates, ups, downs, rows, count, x);+}++/* Join the in-flight group and read back the packed outputs. */+int coli_vk_expert_group_take(float *y) {+ if (!G.eg_inflight) return 0;+ G.eg_inflight = 0;+ if (vkWaitForFences(G.dev, 1, &G.eg_fence, VK_TRUE, 10000000000ULL) != VK_SUCCESS) {+ fprintf(stderr, "[VK] expert-group fence wait failed β disabling GPU offload\n");+ G.ready = 0; return 0;+ }+ double t4 = G.eg_prof ? vk_now() : 0;+ memcpy(y, G.eg_y.ptr, G.eg_pending_yb);+ if (G.eg_prof) {+ double t5 = vk_now();+ fprintf(stderr, "[VK_PROF] memcpy_x %.3f | desc %.3f | record %.3f | issue->take %.3f | memcpy_y %.3f ms\n",+ G.eg_t1-G.eg_t0, G.eg_t2-G.eg_t1, G.eg_t3-G.eg_t2, t4-G.eg_t3, t5-t4);+ }return 1;}+/* Synchronous form (shared expert, harness): issue + take in one call. */+int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,+ ColiVkTensor *const *downs, const int *rows, int count,+ float *y, const float *x) {+ if (G.eg_inflight) return 0;+ if (!eg_prepare_submit(gates, ups, downs, rows, count, x)) return 0;+ return coli_vk_expert_group_take(y);+}+/* ---- MLA absorb attention core -------------------------------------------------* The KV latent (L, [rows,K]) and rope (R, [rows,Rd]) caches live in persistent* per-layer device buffers, appended row-by-row as tokens decode (the host stays@@ -798,6 +838,7 @@ void coli_vk_shutdown(void) {coli_vk_kv_reset();if (G.eg_pool) vkDestroyDescriptorPool(G.dev, G.eg_pool, NULL);vkDestroyFence(G.dev, G.fence, NULL);+ vkDestroyFence(G.dev, G.eg_fence, NULL);vkDestroyCommandPool(G.dev, G.cpool, NULL);vkDestroyDescriptorPool(G.dev, G.dpool, NULL);vkDestroyPipeline(G.dev, G.pipe, NULL);@@ -1056,16 +1097,28 @@ static int run_expert_group(int fmt, int D, int I, int K) {}double maxrel = 0;for (int i = 0; i < K*D; i++) { double e = fabs(yg[i]-yc[i]); if (fabs(yc[i])>1e-2) { double r = e/fabs(yc[i]); if (r>maxrel) maxrel = r; } }- coli_vk_expert_group(tg, tu, td, rows, K, yg, x); /* warm + leaves G.cmd recorded */+ coli_vk_expert_group(tg, tu, td, rows, K, yg, x); /* warm + leaves G.eg_cmd recorded */+ /* async issue/take must reproduce the sync result exactly (same buffers/records) */+ {+ float *ya = malloc((size_t)K*D*4);+ if (!coli_vk_expert_group_issue(tg, tu, td, rows, K, x) || !coli_vk_expert_group_take(ya)) {+ printf("expert_group issue/take failed\n"); maxrel = 1;+ } else {+ double amr = 0;+ for (int i = 0; i < K*D; i++) { double e = fabs(ya[i]-yg[i]); if (fabs(yg[i])>1e-2) { double r = e/fabs(yg[i]); if (r>amr) amr = r; } }+ if (amr > 1e-6) { printf("issue/take deviates from sync: %.4g\n", amr); maxrel = 1; }+ }+ free(ya);+ }int iters = 20; double t0 = now();for (int k = 0; k < iters; k++) coli_vk_expert_group(tg, tu, td, rows, K, yg, x);double ms = (now()-t0)*1000.0/iters/K;/* GPU-only: re-submit the already-recorded command buffer (skips per-call host setup:* descriptor updates + recording), isolating raw GPU throughput. */- VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};- for (int w = 0; w < 2; w++) { vkResetFences(G.dev,1,&G.fence); vkQueueSubmit(G.queue,1,&si,G.fence); vkWaitForFences(G.dev,1,&G.fence,VK_TRUE,10000000000ULL); }+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.eg_cmd};+ for (int w = 0; w < 2; w++) { vkResetFences(G.dev,1,&G.eg_fence); vkQueueSubmit(G.queue,1,&si,G.eg_fence); vkWaitForFences(G.dev,1,&G.eg_fence,VK_TRUE,10000000000ULL); }double g0 = now();- for (int k = 0; k < iters; k++) { vkResetFences(G.dev,1,&G.fence); vkQueueSubmit(G.queue,1,&si,G.fence); vkWaitForFences(G.dev,1,&G.fence,VK_TRUE,10000000000ULL); }+ for (int k = 0; k < iters; k++) { vkResetFences(G.dev,1,&G.eg_fence); vkQueueSubmit(G.queue,1,&si,G.eg_fence); vkWaitForFences(G.dev,1,&G.eg_fence,VK_TRUE,10000000000ULL); }double gpums = (now()-g0)*1000.0/iters/K;printf("FULL VK expert_group fmt=%d %2d experts | maxrel=%.4g | per-call %.4f GPU-only %.4f ms/expert (ROCm 0.179)\n",fmt, K, maxrel, ms, gpums);diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex ebb44ff4..20fb53a1 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -47,6 +47,13 @@ int coli_vk_gate_up(ColiVkTensor **gate, ColiVkTensor **up,int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,ColiVkTensor *const *downs, const int *rows, int count,float *y, const float *x);+/* Async form: _issue submits the group and returns immediately (one in flight max);+ * the caller computes its CPU share, then _take joins and reads back the packed y.+ * Both return 0 on failure (caller computes those experts on the CPU instead). */+int coli_vk_expert_group_issue(ColiVkTensor *const *gates, ColiVkTensor *const *ups,+ ColiVkTensor *const *downs, const int *rows, int count,+ const float *x);+int coli_vk_expert_group_take(float *y);/* Upload a resident tensor without computing (expert tier: gate/up/down uploaded once,* then driven by coli_vk_expert_group). Returns 0 on failure/unsupported fmt. */diff --git a/c/colibri.c b/c/colibri.cindex 9065573b..1cab6198 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -294,8 +294,16 @@ static int g_repin;static uint64_t g_last_repin;#ifdef COLI_VULKANstatic int g_vulkan; /* COLI_VULKAN=1: compute routed experts on the Vulkan tier */-static int g_vk_budget; /* COLI_VK_EXPERTS: max experts uploaded to VK VRAM (default 1024) */+static int g_vk_budget; /* COLI_VK_EXPERTS: pinned VK expert tier size (0 = tier off) */static int g_vk_resident; /* how many experts currently VK-resident */+/* Pinned VK expert tier: top heat-ranked routed experts uploaded ONCE at startup into a+ * registry keyed by (layer,eid), decoupled from the RAM cache slots β stable residency+ * (no LRU churn, no uploads on the decode critical path), mirroring the HIP VRAM tier. */+static struct ColiVkTensor **g_vk_reg; /* [(layer*E+eid)*3 + {gate,up,down}] */+static int g_vk_reg_n, g_vk_reg_E;+static inline struct ColiVkTensor **vk_reg_at(int layer,int eid){+ return g_vk_reg ? &g_vk_reg[((size_t)layer*g_vk_reg_E+eid)*3] : NULL;+}static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention* projections + shared expert) on Vulkan too */static int g_vk_attn; /* COLI_VK_ATTN=1: run the MLA absorb attention core on Vulkan@@ -3319,8 +3327,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int#endifint vk_active = 0; (void)vk_active;#ifdef COLI_VULKAN- vk_active = g_vulkan && g_vk_budget>0 && !omp_in_parallel() && S<=4; /* budget 0 = tier off,- * experts stay on the normal (parallel) CPU loop */+ vk_active = g_vulkan && g_vk_reg_n>0 && !omp_in_parallel() && S<=4; /* empty registry (tier+ * off / no usage history) = normal CPU expert loop */float *vk_xh = vk_active?falloc((int64_t)S*K*D):NULL;float *vk_yh = vk_active?falloc((int64_t)S*K*D):NULL;#endif@@ -3603,57 +3611,60 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* coli_vk_expert_group (fused gate+up+silu -> down, on-device); the rest + misses* fall back to the CPU below. Decode-only (S<=4). Measured ~35% faster than ROCm. */if(vk_active && !metal_done){- ColiVkTensor *vg[64],*vu[64],*vd[64]; ESlot *ve[64]; int vrows[64], voff[64], vrmap[64*4]; float vwmap[64*4];- ESlot *ce[64]; int cnr[64], crmap[64*4]; float cwmap[64*4];+ /* Pinned+async VK expert tier. Pass 1 partitions by REGISTRY residency (the+ * heat-pinned startup uploads β no uploads, no loads here); pass 2 ISSUES the+ * GPU batch async and computes the CPU share while it runs; pass 3 drains the+ * pipe + RAM-loads the GPU-side slots (cache invariants: every dispatched slot+ * waited, every use[] slot loaded before the end-of-block LRU swap); pass 4+ * takes the GPU results (fallback: recompute those rows on the CPU). */+ ColiVkTensor *vg[64],*vu[64],*vd[64]; ESlot *ve[64]; int vrows[64], voff[64], vrmap[64*4]; float vwmap[64*4]; int vqof[64];+ ESlot *ce[64]; int cnr[64], crmap[64*4]; float cwmap[64*4]; int cqof[64];int nvk=0, vtot=0, ncpu=0;double t0=now_s();for(int j=0;j<nb;j++){ int eid=uniq[base+j]; ESlot *e=use[j];- if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; }int nr=0;for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++)if(idxs[(int64_t)s*K+kk]==eid){ rows[nr]=s; rw[nr]=ws[(int64_t)s*K+kk]; nr++; break; }- if(!nr) continue;- if(!e->slab) expert_load(m,layer,e->eid,e,1);- int elig = e->slab && e->g.fmt==2 && e->u.fmt==2 && e->d.fmt==2 && nvk<64 && (vtot+nr)<=S*K;- if(elig && !e->g.vk_eligible && g_vk_resident < g_vk_budget- && (uint8_t*)e->g.q4>=e->slab && (uint8_t*)e->d.q4>=e->slab){- if(coli_vk_tensor_ensure(&e->g.vk,e->g.q4,e->g.s,2,D,I)- && coli_vk_tensor_ensure(&e->u.vk,e->u.q4,e->u.s,2,D,I)- && coli_vk_tensor_ensure(&e->d.vk,e->d.q4,e->d.s,2,I,D)){- e->g.vk_eligible=e->u.vk_eligible=e->d.vk_eligible=1; g_vk_resident++;- }- }- if(elig && e->g.vk_eligible){+ if(!nr){ if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } continue; }+ ColiVkTensor **reg = layer<c->n_layers ? vk_reg_at(layer,eid) : NULL;+ if(reg && reg[0] && nvk<64 && (vtot+nr)<=S*K){voff[nvk]=vtot;for(int r=0;r<nr;r++){ memcpy(vk_xh+(int64_t)(vtot+r)*D, x+(int64_t)rows[r]*D, D*sizeof(float));vrmap[nvk*S+r]=rows[r]; vwmap[nvk*S+r]=rw[r]; }- vg[nvk]=e->g.vk; vu[nvk]=e->u.vk; vd[nvk]=e->d.vk; ve[nvk]=e; vrows[nvk]=nr; vtot+=nr; nvk++;+ vg[nvk]=reg[0]; vu[nvk]=reg[1]; vd[nvk]=reg[2]; ve[nvk]=e; vrows[nvk]=nr; vqof[nvk]=qof[j]; vtot+=nr; nvk++;} else {- ce[ncpu]=e; cnr[ncpu]=nr;+ ce[ncpu]=e; cnr[ncpu]=nr; cqof[ncpu]=qof[j];for(int r=0;r<nr;r++){ crmap[ncpu*S+r]=rows[r]; cwmap[ncpu*S+r]=rw[r]; }ncpu++;}}- int vk_ok = nvk>0 && coli_vk_expert_group(vg,vu,vd,vrows,nvk,vk_yh,vk_xh);- for(int c=0;c<ncpu;c++){ ESlot *e=ce[c]; int nr=cnr[c];- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)crmap[c*S+r]*D, D*sizeof(float));+ int vk_issued = nvk>0 && coli_vk_expert_group_issue(vg,vu,vd,vrows,nvk,vk_xh);+ for(int c2=0;c2<ncpu;c2++){ ESlot *e=ce[c2]; int nr=cnr[c2];+ if(g_pipe && cqof[c2]>=0){ double tw=now_s(); pipe_wait(cqof[c2]); m->t_ewait += now_s()-tw; }+ if(!e->slab) expert_load(m,layer,e->eid,e,1);+ for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)crmap[c2*S+r]*D, D*sizeof(float));expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];matmul_qt(hh, gg, &e->d, nr);- for(int r=0;r<nr;r++){ float *os=out+(int64_t)crmap[c*S+r]*D, wgt=cwmap[c*S+r], *hr=hh+(int64_t)r*D;+ for(int r=0;r<nr;r++){ float *os=out+(int64_t)crmap[c2*S+r]*D, wgt=cwmap[c2*S+r], *hr=hh+(int64_t)r*D;for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }}- for(int c=0;c<nvk;c++){ int nr=vrows[c];- if(vk_ok){ int o=voff[c];- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c*S+r]*D, wgt=vwmap[c*S+r], *src=vk_yh+(int64_t)(o+r)*D;+ for(int c2=0;c2<nvk;c2++){+ if(g_pipe && vqof[c2]>=0){ double tw=now_s(); pipe_wait(vqof[c2]); m->t_ewait += now_s()-tw; }+ ESlot *e=ve[c2]; if(!e->slab) expert_load(m,layer,e->eid,e,1);+ }+ int vk_ok = vk_issued && coli_vk_expert_group_take(vk_yh);+ for(int c2=0;c2<nvk;c2++){ int nr=vrows[c2];+ if(vk_ok){ int o=voff[c2];+ for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *src=vk_yh+(int64_t)(o+r)*D;for(int d=0;d<D;d++) os[d]+=wgt*src[d]; }- } else { /* group failed: recompute this expert's rows on the CPU via its ESlot */- ESlot *e=ve[c];- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap[c*S+r]*D, D*sizeof(float));+ } else { /* issue/take failed: recompute this expert's rows on the CPU */+ ESlot *e=ve[c2];+ for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap[c2*S+r]*D, D*sizeof(float));expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];matmul_qt(hh, gg, &e->d, nr);- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c*S+r]*D, wgt=vwmap[c*S+r], *hr=hh+(int64_t)r*D;+ for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *hr=hh+(int64_t)r*D;for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }}}@@ -6449,6 +6460,57 @@ static int *read_arr(jval*o,const char*k,int*n){/* telemetry, stats, usage persistence β moved to telemetry.h */+#ifdef COLI_VULKAN+/* Pinned VK expert tier fill: upload the top-COLI_VK_EXPERTS heat-ranked routed experts+ * ONCE at startup (mirrors the HIP VRAM tier: stable residency, zero uploads at decode).+ * Ranking comes from the persistent usage history; already-pinned RAM slots feed the+ * upload directly, the rest stream through one transient slot and are freed after. */+typedef struct { uint32_t u; int layer, eid; } VkCand;+static int vk_cand_cmp(const void *a, const void *b){+ uint32_t ua=((const VkCand*)a)->u, ub=((const VkCand*)b)->u;+ return ua<ub ? 1 : ua>ub ? -1 : 0;+}+static void vk_registry_fill(Model *m){+ Cfg *c=&m->c; int E=c->n_experts, NL=c->n_layers;+ if(!g_vulkan || g_vk_budget<=0) return;+ int64_t nz=0;+ for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++) if(m->eusage[i][e]) nz++;+ if(!nz){ fprintf(stderr,"[VK] expert tier: no usage history yet β tier empty this run "+ "(it seeds from %s as you use the model)\n", g_usage_path); return; }+ VkCand *cand=malloc((size_t)nz*sizeof(VkCand)); if(!cand) return;+ int64_t n=0;+ for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++)+ if(m->eusage[i][e]) cand[n++]=(VkCand){m->eusage[i][e],i,e};+ qsort(cand,(size_t)n,sizeof(VkCand),vk_cand_cmp);+ g_vk_reg=calloc((size_t)NL*E*3,sizeof(*g_vk_reg)); g_vk_reg_E=E;+ if(!g_vk_reg){ free(cand); return; }+ ESlot tmp; memset(&tmp,0,sizeof(tmp)); tmp.eid=-1;+ double t0=now_s(); int64_t bytes=0; int tried=0;+ for(int64_t i2=0;i2<n && g_vk_reg_n<g_vk_budget;i2++){+ int layer=cand[i2].layer, eid=cand[i2].eid; tried++;+ ESlot *src=NULL, *P=m->pin[layer];+ for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid && P[z].slab){ src=&P[z]; break; }+ if(!src){ if(!expert_load(m,layer,eid,&tmp,0)) continue; src=&tmp; }+ if(src->g.fmt!=2||src->u.fmt!=2||src->d.fmt!=2) continue; /* int4 tier only */+ ColiVkTensor **slot=vk_reg_at(layer,eid);+ if(!coli_vk_tensor_ensure(&slot[0],src->g.q4,src->g.s,2,c->hidden,c->moe_inter)||+ !coli_vk_tensor_ensure(&slot[1],src->u.q4,src->u.s,2,c->hidden,c->moe_inter)||+ !coli_vk_tensor_ensure(&slot[2],src->d.q4,src->d.s,2,c->moe_inter,c->hidden)){+ if(slot[0]){coli_vk_tensor_free(slot[0]);slot[0]=NULL;}+ if(slot[1]){coli_vk_tensor_free(slot[1]);slot[1]=NULL;}+ fprintf(stderr,"[VK] expert tier: VRAM full after %d experts\n",g_vk_reg_n);+ break;+ }+ bytes+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);+ g_vk_reg_n++;+ }+ if(tmp.slab){ compat_aligned_free(tmp.slab); free(tmp.fslab); }+ free(cand);+ fprintf(stderr,"[VK] expert tier: %d hot experts resident (%.2f GB VRAM, %.1fs, top-%d of history)\n",+ g_vk_reg_n,bytes/1e9,now_s()-t0,tried);+}+#endif+/* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert* per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB.* Ogni hit evita una lettura dal disco lento. */@@ -7480,6 +7542,9 @@ int main(int argc, char **argv){cap_for_ram(&m, ram_env, ebits, est_ctx);g_prof = getenv("PROF")?atoi(getenv("PROF")):0; /* PROF=1: opt-in performance profile */if(g_prof) prof_config(&m, ram_env, est_ctx); }+#ifdef COLI_VULKAN+ vk_registry_fill(&m); /* pinned VK expert tier: needs the usage history loaded above */+#endifconst char *stats=getenv("STATS"); /* STATS=<file> -> istogramma uso expert a fine run *//* modo scoring per benchmark: SCORE=<requests.txt> -> log-likelihood per riga */