Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

VK: pinned+async expert tier β€” heat-ranked startup residency, GPU batch overlapped with CPU rows

Steve Markgrafdd024ecc1ac2
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index ff5190c9..2fce42d0 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -69,6 +69,11 @@ static struct {
* one submit with hidden never leaving the GPU. */
Scratch eg_x, eg_h, eg_y;
VkDescriptorPool eg_pool; VkDescriptorSet eg_gu[64], eg_dn[64]; int eg_nsets;
+ /* expert-group ASYNC state: its own command buffer + fence so an in-flight group
+ * never collides with the main cmd/fence (dense matmuls, absorb) β€” issue() returns
+ * immediately, the CPU computes its share, take() joins. */
+ VkCommandBuffer eg_cmd; VkFence eg_fence; int eg_inflight; size_t eg_pending_yb;
+ double eg_t0, eg_t1, eg_t2, eg_t3; int eg_prof;
Scratch att_sc; /* attention score scratch (GPU-only) */
Scratch att_ctx; /* fused absorb+o: ctx stays on device (GPU-only) */
Scratch y2; /* second output of the fused matmul pair (readback) */
@@ -287,8 +292,10 @@ int coli_vk_init(const char *spv_path) {
VkCommandBufferAllocateInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
.commandPool = G.cpool, .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1};
VKCHECK(vkAllocateCommandBuffers(G.dev, &cbi, &G.cmd), "cmdBuf");
+ VKCHECK(vkAllocateCommandBuffers(G.dev, &cbi, &G.eg_cmd), "eg cmdBuf");
VkFenceCreateInfo fi = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
VKCHECK(vkCreateFence(G.dev, &fi, NULL, &G.fence), "fence");
+ VKCHECK(vkCreateFence(G.dev, &fi, NULL, &G.eg_fence), "eg fence");
G.ready = 1;
VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(G.phys, &p);
@@ -455,13 +462,16 @@ static void wr_desc(VkDescriptorSet set, int n, const VkDescriptorBufferInfo *bi
vkUpdateDescriptorSets(G.dev, (uint32_t)n, w, 0, NULL);
}
-/* Full batched expert MLP for `count` experts in ONE submit, hidden staying on-device:
+/* Full batched expert MLP for `count` experts, hidden staying on-device:
* for each c, hidden_c = silu(gate_c(x_c))*up_c(x_c) (fused), then y_c = down_c(hidden_c).
* x/y are packed [sum(rows)*D]; experts are resident VkTensors (gate/up: D->I, down: I->D).
- * Mirrors coli_cuda_expert_group. Returns 0 -> caller falls back to CPU. */
-int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
- ColiVkTensor *const *downs, const int *rows, int count,
- float *y, const float *x) {
+ * Mirrors coli_cuda_expert_group. Split into prepare+submit / take so the caller can
+ * overlap the GPU batch with its own CPU share (issue -> CPU rows -> take); the group
+ * runs on its OWN command buffer + fence, so in-flight work never collides with the
+ * main pipeline (dense matmuls, absorb attention). Returns 0 -> caller falls back. */
+static int eg_prepare_submit(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
+ ColiVkTensor *const *downs, const int *rows, int count,
+ const float *x) {
if (!G.ready || !G.shader_gu || count < 1 || count > 64) return 0;
ColiVkTensor *g0 = gates[0]; if (!g0) return 0;
int D = g0->I, I = g0->O, fmt = g0->fmt, total = 0, off[64];
@@ -474,10 +484,10 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
size_t xb = (size_t)total*D*4, hb = (size_t)total*I*4, yb = (size_t)total*D*4;
if (!scratch_reserve(&G.eg_x, xb) || !scratch_reserve(&G.eg_h, hb) ||
!scratch_reserve_mt(&G.eg_y, yb, G.memtype_cached)) return 0; /* eg_y is read back -> cached */
- int prof = getenv("VK_PROF") != NULL; double t0=0,t1=0,t2=0,t3=0,t4=0,t5=0;
- if (prof) t0 = vk_now();
+ G.eg_prof = getenv("VK_PROF") != NULL;
+ if (G.eg_prof) G.eg_t0 = vk_now();
memcpy(G.eg_x.ptr, x, xb);
- if (prof) t1 = vk_now();
+ if (G.eg_prof) G.eg_t1 = vk_now();
if (!G.eg_pool) { /* one-time: 64 gate_up (6-binding) + 64 down (4-binding) sets */
VkDescriptorPoolSize ps = {.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 64*6 + 64*4};
@@ -503,45 +513,75 @@ int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
{downs[c]->sbuf, 0, VK_WHOLE_SIZE}, {G.eg_y.buf, yo, (VkDeviceSize)rows[c]*D*4}};
wr_desc(G.eg_dn[c], 4, di);
}
- if (prof) t2 = vk_now();
+ if (G.eg_prof) G.eg_t2 = vk_now();
- VKCHECK(vkResetCommandBuffer(G.cmd, 0), "resetCmd");
+ VKCHECK(vkResetCommandBuffer(G.eg_cmd, 0), "eg resetCmd");
VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
- VKCHECK(vkBeginCommandBuffer(G.cmd, &begin), "beginCmd");
+ VKCHECK(vkBeginCommandBuffer(G.eg_cmd, &begin), "eg beginCmd");
VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT};
/* phase 1: fused gate+up+silu -> hidden (per expert, bound to its x/hidden slices) */
- vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe_gu);
+ vkCmdBindPipeline(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe_gu);
for (int c = 0; c < count; c++) {
struct PC pc = {fmt, rows[c], D, I, gates[c]->rowWords};
- vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt_gu, 0, 1, &G.eg_gu[c], 0, NULL);
- vkCmdPushConstants(G.cmd, G.plyt_gu, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
- vkCmdDispatch(G.cmd, (uint32_t)((I + 7) / 8), (uint32_t)rows[c], 1);
+ vkCmdBindDescriptorSets(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt_gu, 0, 1, &G.eg_gu[c], 0, NULL);
+ vkCmdPushConstants(G.eg_cmd, G.plyt_gu, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+ vkCmdDispatch(G.eg_cmd, (uint32_t)((I + 7) / 8), (uint32_t)rows[c], 1);
}
- vkCmdPipelineBarrier(G.cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &mb, 0, NULL, 0, NULL);
+ vkCmdPipelineBarrier(G.eg_cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &mb, 0, NULL, 0, NULL);
/* phase 2: down projection hidden -> y */
- vkCmdBindPipeline(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);
+ vkCmdBindPipeline(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.pipe);
for (int c = 0; c < count; c++) {
struct PC pc = {fmt, rows[c], I, D, downs[c]->rowWords};
- vkCmdBindDescriptorSets(G.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.eg_dn[c], 0, NULL);
- vkCmdPushConstants(G.cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
- vkCmdDispatch(G.cmd, (uint32_t)((D + 7) / 8), (uint32_t)rows[c], 1);
+ vkCmdBindDescriptorSets(G.eg_cmd, VK_PIPELINE_BIND_POINT_COMPUTE, G.plyt, 0, 1, &G.eg_dn[c], 0, NULL);
+ vkCmdPushConstants(G.eg_cmd, G.plyt, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+ vkCmdDispatch(G.eg_cmd, (uint32_t)((D + 7) / 8), (uint32_t)rows[c], 1);
}
- VKCHECK(vkEndCommandBuffer(G.cmd), "endCmd");
- if (prof) t3 = vk_now();
+ VKCHECK(vkEndCommandBuffer(G.eg_cmd), "eg endCmd");
+ if (G.eg_prof) G.eg_t3 = vk_now();
- VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};
- VKCHECK(vkResetFences(G.dev, 1, &G.fence), "resetFence");
- VKCHECK(vkQueueSubmit(G.queue, 1, &si, G.fence), "queueSubmit");
- if (vkWaitForFences(G.dev, 1, &G.fence, VK_TRUE, 10000000000ULL) != VK_SUCCESS) { G.ready = 0; return 0; }
- if (prof) t4 = vk_now();
- memcpy(y, G.eg_y.ptr, yb);
- if (prof) { t5 = vk_now();
- fprintf(stderr, "[VK_PROF] K=%d memcpy_x %.3f | desc %.3f | record %.3f | gpu %.3f | memcpy_y %.3f ms\n",
- count, t1-t0, t2-t1, t3-t2, t4-t3, t5-t4); }
- G.cmd_ready = 0; G.bound_tensor = NULL;
+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.eg_cmd};
+ VKCHECK(vkResetFences(G.dev, 1, &G.eg_fence), "eg resetFence");
+ VKCHECK(vkQueueSubmit(G.queue, 1, &si, G.eg_fence), "eg queueSubmit");
+ G.eg_pending_yb = yb; G.eg_inflight = 1;
+ return 1;
+}
+
+/* Issue a group asynchronously: submit and return WITHOUT waiting, so the caller
+ * computes its CPU share concurrently. Exactly one group may be in flight. */
+int coli_vk_expert_group_issue(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
+ ColiVkTensor *const *downs, const int *rows, int count,
+ const float *x) {
+ if (G.eg_inflight) return 0;
+ return eg_prepare_submit(gates, ups, downs, rows, count, x);
+}
+
+/* Join the in-flight group and read back the packed outputs. */
+int coli_vk_expert_group_take(float *y) {
+ if (!G.eg_inflight) return 0;
+ G.eg_inflight = 0;
+ if (vkWaitForFences(G.dev, 1, &G.eg_fence, VK_TRUE, 10000000000ULL) != VK_SUCCESS) {
+ fprintf(stderr, "[VK] expert-group fence wait failed β€” disabling GPU offload\n");
+ G.ready = 0; return 0;
+ }
+ double t4 = G.eg_prof ? vk_now() : 0;
+ memcpy(y, G.eg_y.ptr, G.eg_pending_yb);
+ if (G.eg_prof) {
+ double t5 = vk_now();
+ fprintf(stderr, "[VK_PROF] memcpy_x %.3f | desc %.3f | record %.3f | issue->take %.3f | memcpy_y %.3f ms\n",
+ G.eg_t1-G.eg_t0, G.eg_t2-G.eg_t1, G.eg_t3-G.eg_t2, t4-G.eg_t3, t5-t4);
+ }
return 1;
}
+/* Synchronous form (shared expert, harness): issue + take in one call. */
+int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
+ ColiVkTensor *const *downs, const int *rows, int count,
+ float *y, const float *x) {
+ if (G.eg_inflight) return 0;
+ if (!eg_prepare_submit(gates, ups, downs, rows, count, x)) return 0;
+ return coli_vk_expert_group_take(y);
+}
+
/* ---- MLA absorb attention core -------------------------------------------------
* The KV latent (L, [rows,K]) and rope (R, [rows,Rd]) caches live in persistent
* per-layer device buffers, appended row-by-row as tokens decode (the host stays
@@ -798,6 +838,7 @@ void coli_vk_shutdown(void) {
coli_vk_kv_reset();
if (G.eg_pool) vkDestroyDescriptorPool(G.dev, G.eg_pool, NULL);
vkDestroyFence(G.dev, G.fence, NULL);
+ vkDestroyFence(G.dev, G.eg_fence, NULL);
vkDestroyCommandPool(G.dev, G.cpool, NULL);
vkDestroyDescriptorPool(G.dev, G.dpool, NULL);
vkDestroyPipeline(G.dev, G.pipe, NULL);
@@ -1056,16 +1097,28 @@ static int run_expert_group(int fmt, int D, int I, int K) {
}
double maxrel = 0;
for (int i = 0; i < K*D; i++) { double e = fabs(yg[i]-yc[i]); if (fabs(yc[i])>1e-2) { double r = e/fabs(yc[i]); if (r>maxrel) maxrel = r; } }
- coli_vk_expert_group(tg, tu, td, rows, K, yg, x); /* warm + leaves G.cmd recorded */
+ coli_vk_expert_group(tg, tu, td, rows, K, yg, x); /* warm + leaves G.eg_cmd recorded */
+ /* async issue/take must reproduce the sync result exactly (same buffers/records) */
+ {
+ float *ya = malloc((size_t)K*D*4);
+ if (!coli_vk_expert_group_issue(tg, tu, td, rows, K, x) || !coli_vk_expert_group_take(ya)) {
+ printf("expert_group issue/take failed\n"); maxrel = 1;
+ } else {
+ double amr = 0;
+ for (int i = 0; i < K*D; i++) { double e = fabs(ya[i]-yg[i]); if (fabs(yg[i])>1e-2) { double r = e/fabs(yg[i]); if (r>amr) amr = r; } }
+ if (amr > 1e-6) { printf("issue/take deviates from sync: %.4g\n", amr); maxrel = 1; }
+ }
+ free(ya);
+ }
int iters = 20; double t0 = now();
for (int k = 0; k < iters; k++) coli_vk_expert_group(tg, tu, td, rows, K, yg, x);
double ms = (now()-t0)*1000.0/iters/K;
/* GPU-only: re-submit the already-recorded command buffer (skips per-call host setup:
* descriptor updates + recording), isolating raw GPU throughput. */
- VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.cmd};
- for (int w = 0; w < 2; w++) { vkResetFences(G.dev,1,&G.fence); vkQueueSubmit(G.queue,1,&si,G.fence); vkWaitForFences(G.dev,1,&G.fence,VK_TRUE,10000000000ULL); }
+ VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &G.eg_cmd};
+ for (int w = 0; w < 2; w++) { vkResetFences(G.dev,1,&G.eg_fence); vkQueueSubmit(G.queue,1,&si,G.eg_fence); vkWaitForFences(G.dev,1,&G.eg_fence,VK_TRUE,10000000000ULL); }
double g0 = now();
- for (int k = 0; k < iters; k++) { vkResetFences(G.dev,1,&G.fence); vkQueueSubmit(G.queue,1,&si,G.fence); vkWaitForFences(G.dev,1,&G.fence,VK_TRUE,10000000000ULL); }
+ for (int k = 0; k < iters; k++) { vkResetFences(G.dev,1,&G.eg_fence); vkQueueSubmit(G.queue,1,&si,G.eg_fence); vkWaitForFences(G.dev,1,&G.eg_fence,VK_TRUE,10000000000ULL); }
double gpums = (now()-g0)*1000.0/iters/K;
printf("FULL VK expert_group fmt=%d %2d experts | maxrel=%.4g | per-call %.4f GPU-only %.4f ms/expert (ROCm 0.179)\n",
fmt, K, maxrel, ms, gpums);
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index ebb44ff4..20fb53a1 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -47,6 +47,13 @@ int coli_vk_gate_up(ColiVkTensor **gate, ColiVkTensor **up,
int coli_vk_expert_group(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
ColiVkTensor *const *downs, const int *rows, int count,
float *y, const float *x);
+/* Async form: _issue submits the group and returns immediately (one in flight max);
+ * the caller computes its CPU share, then _take joins and reads back the packed y.
+ * Both return 0 on failure (caller computes those experts on the CPU instead). */
+int coli_vk_expert_group_issue(ColiVkTensor *const *gates, ColiVkTensor *const *ups,
+ ColiVkTensor *const *downs, const int *rows, int count,
+ const float *x);
+int coli_vk_expert_group_take(float *y);
/* Upload a resident tensor without computing (expert tier: gate/up/down uploaded once,
* then driven by coli_vk_expert_group). Returns 0 on failure/unsupported fmt. */
diff --git a/c/colibri.c b/c/colibri.c
index 9065573b..1cab6198 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -294,8 +294,16 @@ static int g_repin;
static uint64_t g_last_repin;
#ifdef COLI_VULKAN
static int g_vulkan; /* COLI_VULKAN=1: compute routed experts on the Vulkan tier */
-static int g_vk_budget; /* COLI_VK_EXPERTS: max experts uploaded to VK VRAM (default 1024) */
+static int g_vk_budget; /* COLI_VK_EXPERTS: pinned VK expert tier size (0 = tier off) */
static int g_vk_resident; /* how many experts currently VK-resident */
+/* Pinned VK expert tier: top heat-ranked routed experts uploaded ONCE at startup into a
+ * registry keyed by (layer,eid), decoupled from the RAM cache slots β€” stable residency
+ * (no LRU churn, no uploads on the decode critical path), mirroring the HIP VRAM tier. */
+static struct ColiVkTensor **g_vk_reg; /* [(layer*E+eid)*3 + {gate,up,down}] */
+static int g_vk_reg_n, g_vk_reg_E;
+static inline struct ColiVkTensor **vk_reg_at(int layer,int eid){
+ return g_vk_reg ? &g_vk_reg[((size_t)layer*g_vk_reg_E+eid)*3] : NULL;
+}
static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention
* projections + shared expert) on Vulkan too */
static int g_vk_attn; /* COLI_VK_ATTN=1: run the MLA absorb attention core on Vulkan
@@ -3319,8 +3327,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
#endif
int vk_active = 0; (void)vk_active;
#ifdef COLI_VULKAN
- vk_active = g_vulkan && g_vk_budget>0 && !omp_in_parallel() && S<=4; /* budget 0 = tier off,
- * experts stay on the normal (parallel) CPU loop */
+ vk_active = g_vulkan && g_vk_reg_n>0 && !omp_in_parallel() && S<=4; /* empty registry (tier
+ * off / no usage history) = normal CPU expert loop */
float *vk_xh = vk_active?falloc((int64_t)S*K*D):NULL;
float *vk_yh = vk_active?falloc((int64_t)S*K*D):NULL;
#endif
@@ -3603,57 +3611,60 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
* coli_vk_expert_group (fused gate+up+silu -> down, on-device); the rest + misses
* fall back to the CPU below. Decode-only (S<=4). Measured ~35% faster than ROCm. */
if(vk_active && !metal_done){
- ColiVkTensor *vg[64],*vu[64],*vd[64]; ESlot *ve[64]; int vrows[64], voff[64], vrmap[64*4]; float vwmap[64*4];
- ESlot *ce[64]; int cnr[64], crmap[64*4]; float cwmap[64*4];
+ /* Pinned+async VK expert tier. Pass 1 partitions by REGISTRY residency (the
+ * heat-pinned startup uploads β€” no uploads, no loads here); pass 2 ISSUES the
+ * GPU batch async and computes the CPU share while it runs; pass 3 drains the
+ * pipe + RAM-loads the GPU-side slots (cache invariants: every dispatched slot
+ * waited, every use[] slot loaded before the end-of-block LRU swap); pass 4
+ * takes the GPU results (fallback: recompute those rows on the CPU). */
+ ColiVkTensor *vg[64],*vu[64],*vd[64]; ESlot *ve[64]; int vrows[64], voff[64], vrmap[64*4]; float vwmap[64*4]; int vqof[64];
+ ESlot *ce[64]; int cnr[64], crmap[64*4]; float cwmap[64*4]; int cqof[64];
int nvk=0, vtot=0, ncpu=0;
double t0=now_s();
for(int j=0;j<nb;j++){ int eid=uniq[base+j]; ESlot *e=use[j];
- if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; }
int nr=0;
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++)
if(idxs[(int64_t)s*K+kk]==eid){ rows[nr]=s; rw[nr]=ws[(int64_t)s*K+kk]; nr++; break; }
- if(!nr) continue;
- if(!e->slab) expert_load(m,layer,e->eid,e,1);
- int elig = e->slab && e->g.fmt==2 && e->u.fmt==2 && e->d.fmt==2 && nvk<64 && (vtot+nr)<=S*K;
- if(elig && !e->g.vk_eligible && g_vk_resident < g_vk_budget
- && (uint8_t*)e->g.q4>=e->slab && (uint8_t*)e->d.q4>=e->slab){
- if(coli_vk_tensor_ensure(&e->g.vk,e->g.q4,e->g.s,2,D,I)
- && coli_vk_tensor_ensure(&e->u.vk,e->u.q4,e->u.s,2,D,I)
- && coli_vk_tensor_ensure(&e->d.vk,e->d.q4,e->d.s,2,I,D)){
- e->g.vk_eligible=e->u.vk_eligible=e->d.vk_eligible=1; g_vk_resident++;
- }
- }
- if(elig && e->g.vk_eligible){
+ if(!nr){ if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } continue; }
+ ColiVkTensor **reg = layer<c->n_layers ? vk_reg_at(layer,eid) : NULL;
+ if(reg && reg[0] && nvk<64 && (vtot+nr)<=S*K){
voff[nvk]=vtot;
for(int r=0;r<nr;r++){ memcpy(vk_xh+(int64_t)(vtot+r)*D, x+(int64_t)rows[r]*D, D*sizeof(float));
vrmap[nvk*S+r]=rows[r]; vwmap[nvk*S+r]=rw[r]; }
- vg[nvk]=e->g.vk; vu[nvk]=e->u.vk; vd[nvk]=e->d.vk; ve[nvk]=e; vrows[nvk]=nr; vtot+=nr; nvk++;
+ vg[nvk]=reg[0]; vu[nvk]=reg[1]; vd[nvk]=reg[2]; ve[nvk]=e; vrows[nvk]=nr; vqof[nvk]=qof[j]; vtot+=nr; nvk++;
} else {
- ce[ncpu]=e; cnr[ncpu]=nr;
+ ce[ncpu]=e; cnr[ncpu]=nr; cqof[ncpu]=qof[j];
for(int r=0;r<nr;r++){ crmap[ncpu*S+r]=rows[r]; cwmap[ncpu*S+r]=rw[r]; }
ncpu++;
}
}
- int vk_ok = nvk>0 && coli_vk_expert_group(vg,vu,vd,vrows,nvk,vk_yh,vk_xh);
- for(int c=0;c<ncpu;c++){ ESlot *e=ce[c]; int nr=cnr[c];
- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)crmap[c*S+r]*D, D*sizeof(float));
+ int vk_issued = nvk>0 && coli_vk_expert_group_issue(vg,vu,vd,vrows,nvk,vk_xh);
+ for(int c2=0;c2<ncpu;c2++){ ESlot *e=ce[c2]; int nr=cnr[c2];
+ if(g_pipe && cqof[c2]>=0){ double tw=now_s(); pipe_wait(cqof[c2]); m->t_ewait += now_s()-tw; }
+ if(!e->slab) expert_load(m,layer,e->eid,e,1);
+ for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)crmap[c2*S+r]*D, D*sizeof(float));
expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);
for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];
matmul_qt(hh, gg, &e->d, nr);
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)crmap[c*S+r]*D, wgt=cwmap[c*S+r], *hr=hh+(int64_t)r*D;
+ for(int r=0;r<nr;r++){ float *os=out+(int64_t)crmap[c2*S+r]*D, wgt=cwmap[c2*S+r], *hr=hh+(int64_t)r*D;
for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
}
- for(int c=0;c<nvk;c++){ int nr=vrows[c];
- if(vk_ok){ int o=voff[c];
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c*S+r]*D, wgt=vwmap[c*S+r], *src=vk_yh+(int64_t)(o+r)*D;
+ for(int c2=0;c2<nvk;c2++){
+ if(g_pipe && vqof[c2]>=0){ double tw=now_s(); pipe_wait(vqof[c2]); m->t_ewait += now_s()-tw; }
+ ESlot *e=ve[c2]; if(!e->slab) expert_load(m,layer,e->eid,e,1);
+ }
+ int vk_ok = vk_issued && coli_vk_expert_group_take(vk_yh);
+ for(int c2=0;c2<nvk;c2++){ int nr=vrows[c2];
+ if(vk_ok){ int o=voff[c2];
+ for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *src=vk_yh+(int64_t)(o+r)*D;
for(int d=0;d<D;d++) os[d]+=wgt*src[d]; }
- } else { /* group failed: recompute this expert's rows on the CPU via its ESlot */
- ESlot *e=ve[c];
- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap[c*S+r]*D, D*sizeof(float));
+ } else { /* issue/take failed: recompute this expert's rows on the CPU */
+ ESlot *e=ve[c2];
+ for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap[c2*S+r]*D, D*sizeof(float));
expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);
for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];
matmul_qt(hh, gg, &e->d, nr);
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c*S+r]*D, wgt=vwmap[c*S+r], *hr=hh+(int64_t)r*D;
+ for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *hr=hh+(int64_t)r*D;
for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
}
}
@@ -6449,6 +6460,57 @@ static int *read_arr(jval*o,const char*k,int*n){
/* telemetry, stats, usage persistence β€” moved to telemetry.h */
+#ifdef COLI_VULKAN
+/* Pinned VK expert tier fill: upload the top-COLI_VK_EXPERTS heat-ranked routed experts
+ * ONCE at startup (mirrors the HIP VRAM tier: stable residency, zero uploads at decode).
+ * Ranking comes from the persistent usage history; already-pinned RAM slots feed the
+ * upload directly, the rest stream through one transient slot and are freed after. */
+typedef struct { uint32_t u; int layer, eid; } VkCand;
+static int vk_cand_cmp(const void *a, const void *b){
+ uint32_t ua=((const VkCand*)a)->u, ub=((const VkCand*)b)->u;
+ return ua<ub ? 1 : ua>ub ? -1 : 0;
+}
+static void vk_registry_fill(Model *m){
+ Cfg *c=&m->c; int E=c->n_experts, NL=c->n_layers;
+ if(!g_vulkan || g_vk_budget<=0) return;
+ int64_t nz=0;
+ for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++) if(m->eusage[i][e]) nz++;
+ if(!nz){ fprintf(stderr,"[VK] expert tier: no usage history yet β€” tier empty this run "
+ "(it seeds from %s as you use the model)\n", g_usage_path); return; }
+ VkCand *cand=malloc((size_t)nz*sizeof(VkCand)); if(!cand) return;
+ int64_t n=0;
+ for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++)
+ if(m->eusage[i][e]) cand[n++]=(VkCand){m->eusage[i][e],i,e};
+ qsort(cand,(size_t)n,sizeof(VkCand),vk_cand_cmp);
+ g_vk_reg=calloc((size_t)NL*E*3,sizeof(*g_vk_reg)); g_vk_reg_E=E;
+ if(!g_vk_reg){ free(cand); return; }
+ ESlot tmp; memset(&tmp,0,sizeof(tmp)); tmp.eid=-1;
+ double t0=now_s(); int64_t bytes=0; int tried=0;
+ for(int64_t i2=0;i2<n && g_vk_reg_n<g_vk_budget;i2++){
+ int layer=cand[i2].layer, eid=cand[i2].eid; tried++;
+ ESlot *src=NULL, *P=m->pin[layer];
+ for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid && P[z].slab){ src=&P[z]; break; }
+ if(!src){ if(!expert_load(m,layer,eid,&tmp,0)) continue; src=&tmp; }
+ if(src->g.fmt!=2||src->u.fmt!=2||src->d.fmt!=2) continue; /* int4 tier only */
+ ColiVkTensor **slot=vk_reg_at(layer,eid);
+ if(!coli_vk_tensor_ensure(&slot[0],src->g.q4,src->g.s,2,c->hidden,c->moe_inter)||
+ !coli_vk_tensor_ensure(&slot[1],src->u.q4,src->u.s,2,c->hidden,c->moe_inter)||
+ !coli_vk_tensor_ensure(&slot[2],src->d.q4,src->d.s,2,c->moe_inter,c->hidden)){
+ if(slot[0]){coli_vk_tensor_free(slot[0]);slot[0]=NULL;}
+ if(slot[1]){coli_vk_tensor_free(slot[1]);slot[1]=NULL;}
+ fprintf(stderr,"[VK] expert tier: VRAM full after %d experts\n",g_vk_reg_n);
+ break;
+ }
+ bytes+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);
+ g_vk_reg_n++;
+ }
+ if(tmp.slab){ compat_aligned_free(tmp.slab); free(tmp.fslab); }
+ free(cand);
+ fprintf(stderr,"[VK] expert tier: %d hot experts resident (%.2f GB VRAM, %.1fs, top-%d of history)\n",
+ g_vk_reg_n,bytes/1e9,now_s()-t0,tried);
+}
+#endif
+
/* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert
* per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB.
* Ogni hit evita una lettura dal disco lento. */
@@ -7480,6 +7542,9 @@ int main(int argc, char **argv){
cap_for_ram(&m, ram_env, ebits, est_ctx);
g_prof = getenv("PROF")?atoi(getenv("PROF")):0; /* PROF=1: opt-in performance profile */
if(g_prof) prof_config(&m, ram_env, est_ctx); }
+#ifdef COLI_VULKAN
+ vk_registry_fill(&m); /* pinned VK expert tier: needs the usage history loaded above */
+#endif
const char *stats=getenv("STATS"); /* STATS=<file> -> istogramma uso expert a fine run */
/* modo scoring per benchmark: SCORE=<requests.txt> -> log-likelihood per riga */