Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: issue dev2 expert group on a worker thread
Steve Markgrafec3cdc7f4054
diff --git a/c/colibri.c b/c/colibri.cindex 58ffded4..9ba34e53 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -3049,6 +3049,20 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba* una volta sola e moltiplicato per tutte le posizioni che lo usano (pesi letti 1 volta);* lo shared expert e' un unico matmul a S righe. Per posizione l'accumulo resta* nell'ordine (routed nel loro ordine di union, poi shared). */+#ifdef COLI_VULKAN+/* dev2 group issue on a worker thread: the Polaris/x4 submit path costs ~0.8ms per+ * dev2-active block serialized on the main thread (measured 3.2s/decode vs 0.15s+ * dev0-only) while the 580's exec itself finishes early (take-wait 0.1s) — off-thread+ * it overlaps dev0's issue + the CPU expert share. All Vulkan state it touches is+ * G2-private, and moe() joins before take2, preserving the one-in-flight invariant. */+typedef struct { ColiVkTensor **g,**u,**d; const int *rows; int n; const float *x; int rc; } Vk2Iss;+static void *vk2_issue_worker(void *p){+ Vk2Iss *j=(Vk2Iss*)p;+ j->rc = coli_vk_expert_group_issue2(j->g,j->u,j->d,j->rows,j->n,j->x);+ return NULL;+}+#endif+/* pin ∪ LRU residency probe (used by CACHE_ROUTE max-rank fill). */static int expert_is_resident(Model *m, int layer, int eid){ESlot *P=m->pin[layer];@@ -3784,8 +3798,15 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}if(g_prof){ g_vkb_cls+=now_s()-t0; g_vkb_blocks++; g_vkb_nvk+=nvk+nvk2; g_vkb_nvk2+=nvk2; g_vkb_ncpu+=ncpu; }double t_iss0=now_s();- /* issue the SLOWER device first so it gets the longest overlap window */- int vk2_issued = nvk2>0 && coli_vk_expert_group_issue2(vg2,vu2,vd2,vrows2,nvk2,vk_xh2);+ /* issue the SLOWER device first so it gets the longest overlap window;+ * dev2's submit runs on a worker thread (joined before take2 below) so its+ * per-block cost overlaps dev0 issue + the CPU share instead of serializing */+ Vk2Iss iss2 = { vg2, vu2, vd2, vrows2, nvk2, vk_xh2, 0 };+ pthread_t iss2_th; int iss2_threaded=0;+ if(nvk2>0){+ if(pthread_create(&iss2_th,NULL,vk2_issue_worker,&iss2)==0) iss2_threaded=1;+ else iss2.rc = coli_vk_expert_group_issue2(vg2,vu2,vd2,vrows2,nvk2,vk_xh2);+ }int vk_issued = nvk>0 && coli_vk_expert_group_issue(vg,vu,vd,vrows,nvk,vk_xh);if(g_prof) g_vkb_issue+=now_s()-t_iss0;/* CPU share stays SERIAL over experts with row-parallel kernels: a decode@@ -3842,7 +3863,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}/* dev2 group: taken AFTER dev0's accumulate so the slower card gets the* extra overlap; same per-expert CPU recompute fallback on failure. */- int vk2_ok = vk2_issued && coli_vk_expert_group_take2(vk_yh2);+ if(iss2_threaded) pthread_join(iss2_th,NULL);+ int vk2_ok = iss2.rc && coli_vk_expert_group_take2(vk_yh2);for(int c2=0;c2<nvk2;c2++){ int nr=vrows2[c2];if(vk2_ok){ int o=voff2[c2];for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap2[c2*S+r]*D, wgt=vwmap2[c2*S+r], *src=vk_yh2+(int64_t)(o+r)*D;