Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

colibri: the routed experts move to the shared Vulkan tier

JustVugg3cc1a21be438
diff --git a/c/colibri.c b/c/colibri.c
index dfd3bed8..9ca04f99 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -107,6 +107,7 @@ static inline void omp_set_num_threads(int n){ (void)n; }
#endif
#ifdef COLI_VULKAN
#include "backend_vulkan.h"
+#include "vk_tier.h" /* the routed-expert tier the MoE engines share */
#endif
#ifdef COLI_XDNA
/* Optional AMD XDNA2 lane. This header pulls in no XRT: the engine owns the
@@ -629,23 +630,33 @@ static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S)
static int g_repin;
static uint64_t g_last_repin;
#ifdef COLI_VULKAN
-static int g_vulkan; /* COLI_VULKAN=1: compute routed experts on the Vulkan tier */
-static int g_vk_budget; /* COLI_VK_EXPERTS: pinned VK expert tier size (0 = tier off) */
+static int g_vulkan; /* COLI_VULKAN=1: the device is open */
+static int g_vk_experts=-1; /* COLI_VK_EXPERTS (deprecated): -1 unset; 0 = no expert tier;
+ * N = the shared tier (vk_tier.c) holds at most N experts */
static int g_vk_resident; /* how many experts currently VK-resident */
-/* Pinned VK expert tier: top heat-ranked routed experts uploaded ONCE at startup into a
- * registry keyed by (layer,eid), decoupled from the RAM cache slots β€” stable residency
- * (no LRU churn, no uploads on the decode critical path), mirroring the HIP VRAM tier. */
+/* The routed experts on the device live in the shared tier (vk_tier.c: adaptive, with
+ * eviction and a budget). The registry below is the second device's only
+ * (COLI_VK_DEV2): its top heat-ranked experts, uploaded ONCE at startup, keyed by
+ * (layer,eid) -- stable residency, no uploads on the decode path. */
static struct ColiVkTensor **g_vk_reg; /* [(layer*E+eid)*3 + {gate,up,down}] */
-static int g_vk_reg_n, g_vk_reg_E;
+static int g_vk_reg_E;
static inline struct ColiVkTensor **vk_reg_at(int layer,int eid){
return g_vk_reg ? &g_vk_reg[((size_t)layer*g_vk_reg_E+eid)*3] : NULL;
}
static int g_vk_reg_NL; /* registry layer bound (n_layers, MTP excluded) */
-/* Will the VK tier serve this expert at decode? (prefetch/pilot can skip its I/O.) */
+/* Will the second device serve this expert? (prefetch can skip its I/O.) */
static inline int vk_reg_served(int layer,int eid){
if(!g_vk_reg || layer<0 || layer>=g_vk_reg_NL || eid<0 || eid>=g_vk_reg_E) return 0;
return g_vk_reg[((size_t)layer*g_vk_reg_E+eid)*3] != NULL;
}
+/* The shared tier's view of this model's experts (vk_tier_start). */
+static struct {
+ int on; /* vkt_init accepted the experts */
+ int gu_fmt, gu_gs, dn_fmt, dn_gs; /* the QT formats it was given (gate = up) */
+ size_t exp_bytes; /* one expert on the device */
+} g_vkt;
+static int vk_tier_resident_count(Model *m);
+static void vk_tier_turn(Model *m); /* "[VK] tier colibri turn: ..." after a serve turn */
static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention
* projections + shared expert) on Vulkan too */
static int g_vk_budget2; /* COLI_VK_EXPERTS2: dev2 expert-tier cap (with COLI_VK_DEV2) */
@@ -1612,8 +1623,8 @@ static inline float siluf(float x){ return x/(1.f+expf(-x)); }
* (Q^T x) dal chiamante via E8_XE (#452: una copia ruotata per chiamata, MAI per expert).
* La rotazione dell'input del down e' per-expert e vive qui: `d->fmt==6` la applica
* SEMPRE β€” e' la forma CANONICA, pura funzione del fmt del down-tensor. Perche' resta
- * byte-identica al vecchio inline: (a) i fallback device-lost Vulkan ricaricano solo
- * expert che erano in registry (vk_reg_at), e vk_registry_fill ammette solo fmt 2/4/5 β€”
+ * byte-identica al vecchio inline: (a) i fallback device-lost Vulkan ricalcolano solo
+ * expert che il tier Vulkan teneva, e il tier ammette solo fmt 0/1/2/4/5 β€”
* li' d->fmt!=6 e la rotazione e' un no-op; (b) l'oracolo e' single-format, e una
* conversione normale single-pass produce un modello single-format: le due copie che
* POSSONO vedere fmt=6 lo fanno solo su un container mixed-format, che il tooling
@@ -5399,8 +5410,32 @@ static void ecache_hide(Model *m,int layer,ESlot *s){
s->eid=-1;
}
-/* pin βˆͺ LRU residency probe (used by CACHE_ROUTE max-rank fill). */
+/* The LRU promotion at the end of a block of moe(): the block's nmiss loaded working-set
+ * slots m->ws[0..nmiss) swap into the layer's cache, the last loaded first, displacing
+ * the least recently used (swap buffer: the victim's slab becomes the ws slot's).
+ * Every ws slot must have been waited for (PIPE) before this runs. */
+static void ecache_promote_ws(Model *m,int layer,int nmiss){
+ ESlot *Sl=m->ecache[layer]; int *nn=&m->ecn[layer];
+ int promo = nmiss<m->ecap ? nmiss : m->ecap;
+ for(int a=0;a<promo;a++){ int q=nmiss-1-a; ESlot *dst;
+ if(*nn<m->ecap) dst=&Sl[(*nn)++];
+ else { int lru=eslot_lru_victim(Sl,*nn,m->ecap);
+ if(lru<0){ static int warned;
+ if(!warned){ warned=1; fprintf(stderr,"[CUDA] no reusable LRU expert slot (in flight or cap reached); skipping cache promotion\n"); }
+ continue; }
+ dst=&Sl[lru]; }
+ ecache_unindex(m,layer,dst);
+ ESlot tmp=*dst; *dst=m->ws[q]; m->ws[q]=tmp;
+ ecache_publish(m,layer,dst,dst->eid);
+ dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); }
+}
+
+/* pin βˆͺ LRU residency probe (used by CACHE_ROUTE max-rank fill), plus the Vulkan expert
+ * tier's residents: no disk read either (vkt_resident is 0 without COLI_VULKAN=1). */
static int expert_is_resident(Model *m, int layer, int eid){
+#ifdef COLI_VULKAN
+ if(vkt_resident(layer,eid)) return 1;
+#endif
return pin_indexed(m,layer,eid)!=NULL || ecache_indexed(m,layer,eid,0)!=NULL;
}
@@ -5458,6 +5493,182 @@ static int mb_gs_compat(int est_fmt, int est_gs, int fmt, int gs){
return !(est_fmt==4 && fmt==4 && gs!=est_gs);
}
+#ifdef COLI_VULKAN
+/* ---- the routed experts on the Vulkan device (COLI_VULKAN=1) -----------------------
+ * The shared tier (vk_tier.c) holds a cache of routed experts on the device: warm from
+ * .coli_usage at startup, then adapting while you chat (an expert the CPU computes is
+ * offered to it and promoted when there is room or when it is hotter than the coldest
+ * resident, which is evicted), under a budget. COLI_VK_DEV2's registry holds the next
+ * hottest on a second device, fixed at startup.
+ *
+ * moe_vk() takes a MoE layer's routed experts, by blocks of GLM_VK_ROWS rows: the tier's
+ * resident (row, rank) pairs go to the device as one batch (vkt_issue), the dev2 ones
+ * to the second device on a worker thread, and the CPU resolves, loads and computes the
+ * rest (vk_cpu_pairs: the pin / LRU / disk path of moe()'s loop) into output rows of
+ * their own while the batch runs. Then every rank of every row joins `out` in routing
+ * (rank) order, the device's and the CPU's alike, so the order of the sum never
+ * depends on which experts happened to be resident. moe()'s CPU-only loop adds them
+ * in union order and is untouched: without COLI_VULKAN nothing here runs. */
+#define GLM_VK_ROWS 64
+static const void *vk_qt_codes(const QT *t){
+ return t->fmt==0 ? (const void*)t->qf : (t->fmt==1||t->fmt==8) ? (const void*)t->q8 : (const void*)t->q4;
+}
+/* An expert slot as the tier reads it, or 0 when its tensors are not in the format the
+ * tier was configured with (a mixed container): such an expert stays on the CPU. */
+static int vk_slot_src(const ESlot *e, VktExpertSrc *src){
+ if(!g_vkt.on || e->g.fmt!=g_vkt.gu_fmt || e->u.fmt!=g_vkt.gu_fmt || e->g.gs!=g_vkt.gu_gs ||
+ e->u.gs!=g_vkt.gu_gs || e->d.fmt!=g_vkt.dn_fmt || e->d.gs!=g_vkt.dn_gs ||
+ e->g.planar || e->u.planar || e->d.planar) return 0;
+ *src=(VktExpertSrc){vk_qt_codes(&e->g),vk_qt_codes(&e->u),vk_qt_codes(&e->d),e->g.s,e->u.s,e->d.s};
+ return src->g && src->u && src->d;
+}
+/* Does this MoE call go through moe_vk? Layers 0..n_layers-1 only: the MTP head's
+ * experts are int8 and stay on the CPU. CUDA, Metal, the cluster workers and the
+ * ablation harness keep moe()'s own loop. */
+static int moe_vk_on(Model *m,int layer){
+ if(!g_vulkan || layer<0 || layer>=m->c.n_layers || omp_in_parallel()) return 0;
+ if(!vkt_ready() && g_vk_reg_n2<=0) return 0;
+ if(g_abl.mode || g_metal_enabled || g_pre_idx) return 0;
+#if !defined(_WIN32)
+ if(g_cluster_n) return 0;
+#endif
+#ifdef COLI_CUDA
+ if(g_cuda_enabled) return 0;
+#endif
+ return 1;
+}
+/* The pairs i of one block of rows that want[i] marks, on the CPU, as moe()'s loop does
+ * them: the union of their experts in first-seen order, 64 at a time; each resolved
+ * from the pin set, the LRU or a working-set slot loaded from disk (PIPE or the blocking
+ * parallel load), computed once for all its rows into ctb[i]; the block's misses then
+ * promoted into the LRU. note: offer each computed expert to the tier. */
+static void vk_cpu_pairs(Model *m,int layer,const float *x,int rows,int K,const int *ib,const uint8_t *want,
+ float *ctb,float *xg,float *gg,float *uu,float *hh,int note){
+ Cfg *c=&m->c; int D=c->hidden, I=c->moe_inter, E=c->n_experts, n=rows*K;
+ int *uq=xalloc((size_t)n*sizeof(int),"vk uq"), *rws=xalloc((size_t)rows*sizeof(int),"vk rws"), nu=0;
+ unsigned char *seen=xzalloc((size_t)E,"vk seen");
+ for(int i=0;i<n;i++) if(want[i] && ib[i]>=0 && !seen[ib[i]]){ seen[ib[i]]=1; uq[nu++]=ib[i]; }
+ for(int base=0;base<nu;base+=64){
+ int nb = nu-base<64 ? nu-base : 64;
+ ESlot *use[64]; int missk[64], qof[64], nmiss=0;
+ for(int j=0;j<nb;j++){ int eid=uq[base+j]; qof[j]=-1;
+ use[j]=pin_indexed(m,layer,eid);
+ if(use[j]){ m->hits++; m->hit_pin++; continue; }
+ use[j]=ecache_indexed(m,layer,eid,0);
+ if(use[j]){ m->hits++; m->hit_ecache++; use[j]->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); continue; }
+ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++;
+ if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; }
+ }
+ if(nmiss){
+ if(g_pipe){
+ if(!g_pp.started) pipe_init(m);
+ double t0=now_s(); int eids[64]; for(int q=0;q<nmiss;q++) eids[q]=uq[base+missk[q]];
+ pipe_dispatch(m,layer,eids,nmiss);
+ m->t_ewait += now_s()-t0;
+ } else { double t0=now_s();
+ #pragma omp parallel for schedule(dynamic,1)
+ for(int q=0;q<nmiss;q++) expert_load(m,layer,uq[base+missk[q]],&m->ws[q],1,1); /* demand=1: the miss path */
+ m->t_ewait += now_s()-t0; }
+ }
+ if(base+64<nu){ /* readahead of the next 64 while these compute */
+ int nb2 = nu-(base+64)<64 ? nu-(base+64) : 64;
+ for(int j=0;j<nb2;j++){ int eid=uq[base+64+j];
+ if(!expert_is_resident(m,layer,eid)) expert_prefetch(m,layer,eid); }
+ }
+ for(int j=0;j<nb;j++){ int eid=uq[base+j]; ESlot *e=use[j];
+ if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; }
+ int nr=0;
+ for(int i=0;i<n;i++) if(want[i] && ib[i]==eid){
+ memcpy(xg+(int64_t)nr*D, x+(int64_t)(i/K)*D, (size_t)D*sizeof(float)); rws[nr++]=i; }
+ if(e->g.fmt==6) e8_rot_rows(xg,nr,D); /* fmt=6 reads Q^T x, row by row as E8_XE does */
+ double t0=now_s();
+ expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);
+ for(int r=0;r<nr;r++) memcpy(ctb+(int64_t)rws[r]*D, hh+(int64_t)r*D, (size_t)D*sizeof(float));
+ double dt=now_s()-t0; m->t_emm+=dt;
+ if(g_prof){ m->t_ecpu+=dt; m->cpu_expert_rows+=(uint64_t)nr;
+ m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); }
+ VktExpertSrc vs;
+ if(note && vk_slot_src(e,&vs)) vkt_note(layer,eid,&vs); /* may promote it to the device */
+ }
+ ecache_promote_ws(m,layer,nmiss);
+ }
+ free(uq); free(rws); free(seen);
+}
+static void moe_vk(Model *m,int layer,const float *x,int S,float *out,const int *idxs,const float *ws,
+ const int *keff,float *xg,float *gg,float *uu,float *hh){
+ Cfg *c=&m->c; int D=c->hidden, K=c->topk, E=c->n_experts;
+ int B = S<GLM_VK_ROWS ? S : GLM_VK_ROWS, nmax=B*K, dev2=g_vk_reg_n2>0;
+ int *ib=xalloc((size_t)nmax*sizeof(int),"vk ib"), *r2=xalloc((size_t)nmax*sizeof(int),"vk r2");
+ uint8_t *taken=xalloc((size_t)nmax,"vk taken"), *want=xalloc((size_t)nmax,"vk want"), *on2=xzalloc((size_t)nmax,"vk on2");
+ const float **dev=xalloc((size_t)nmax*sizeof(*dev),"vk dev");
+ float *ctb=falloc((int64_t)nmax*D);
+ float *x2=dev2?falloc((int64_t)nmax*D):NULL, *y2=dev2?falloc((int64_t)nmax*D):NULL;
+ unsigned char *seen=xzalloc((size_t)E,"vk seen");
+ for(int s0=0;s0<S;s0+=B){
+ int rows = S-s0<B ? S-s0 : B, n=rows*K;
+ const float *xb=x+(int64_t)s0*D;
+ for(int s=0;s<rows;s++) for(int k=0;k<K;k++)
+ ib[s*K+k] = k<keff[s0+s] ? idxs[(int64_t)(s0+s)*K+k] : -1;
+ double t0=now_s();
+ int ndev=vkt_issue(layer,xb,rows,K,ib,taken); /* returns at once; 0 = nothing on the device */
+ if(g_prof){ g_vkb_issue+=now_s()-t0; g_vkb_blocks++; }
+ memset(seen,0,(size_t)E); memset(on2,0,(size_t)n);
+ for(int i=0;i<n;i++) if(taken[i] && !seen[ib[i]]){ seen[ib[i]]=1; m->hits++; m->hit_vk++; if(g_prof) g_vkb_nvk++; }
+ /* the second device: the pairs the tier did not take whose expert is in its
+ * registry, one group (64 experts at most), issued on a worker thread */
+ ColiVkTensor *g2[64],*u2[64],*d2[64]; int n2rows[64], ne2=0, tot2=0;
+ Vk2Iss iss2={0}; pthread_t iss2_th; int iss2_threaded=0, iss2_on=0;
+ if(dev2){
+ int eo[64];
+ for(int i=0;i<n && ne2<64;i++){ int e=ib[i];
+ if(e<0 || taken[i] || seen[e] || !vk_reg_served(layer,e)) continue;
+ seen[e]=1; eo[ne2++]=e; }
+ for(int q=0;q<ne2;q++){ ColiVkTensor **rg=vk_reg_at(layer,eo[q]); int nr=0;
+ for(int i=0;i<n;i++) if(ib[i]==eo[q] && !taken[i]){
+ memcpy(x2+(int64_t)tot2*D, xb+(int64_t)(i/K)*D, (size_t)D*sizeof(float));
+ r2[i]=tot2++; on2[i]=1; nr++; }
+ g2[q]=rg[0]; u2[q]=rg[1]; d2[q]=rg[2]; n2rows[q]=nr;
+ m->hits++; m->hit_vk++; }
+ if(ne2){
+ iss2=(Vk2Iss){g2,u2,d2,n2rows,ne2,x2,0,0}; iss2_on=1;
+ if(g_prof) g_vkb_nvk2+=ne2;
+ if(pthread_create(&iss2_th,NULL,vk2_issue_worker,&iss2)==0) iss2_threaded=1;
+ else iss2.rc=coli_vk_expert_group_issue2(g2,u2,d2,n2rows,ne2,x2);
+ }
+ }
+ for(int i=0;i<n;i++) want[i] = ib[i]>=0 && !taken[i] && !on2[i];
+ if(g_prof) for(int i=0;i<n;i++) if(want[i]) g_vkb_ncpu++;
+ vk_cpu_pairs(m,layer,xb,rows,K,ib,want,ctb,xg,gg,uu,hh,1); /* while the batches run */
+ double tj=now_s();
+ if(ndev && !vkt_join(dev)){ /* the batch failed (the tier stops): those pairs here */
+ vk_cpu_pairs(m,layer,xb,rows,K,ib,taken,ctb,xg,gg,uu,hh,0);
+ memset(taken,0,(size_t)n);
+ }
+ if(g_prof) m->t_egpu+=now_s()-tj;
+ if(iss2_on){
+ if(iss2_threaded){ double t_j0=now_s(); pthread_join(iss2_th,NULL);
+ if(g_prof){ g_vkb_join+=now_s()-t_j0; g_vkb_wrk+=iss2.dt; } }
+ if(!(iss2.rc && coli_vk_expert_group_take2(y2))){ /* device lost: those pairs here */
+ vk_cpu_pairs(m,layer,xb,rows,K,ib,on2,ctb,xg,gg,uu,hh,0);
+ memset(on2,0,(size_t)n);
+ }
+ }
+ /* every rank of every row in routing order, wherever it was computed */
+ for(int s=0;s<rows;s++){
+ float *os=out+(int64_t)(s0+s)*D;
+ for(int k=0;k<keff[s0+s];k++){ int i=s*K+k;
+ if(ib[i]<0) continue;
+ const float *src = taken[i] ? dev[i] : on2[i] ? y2+(int64_t)r2[i]*D : ctb+(int64_t)i*D;
+ float w=ws[(int64_t)(s0+s)*K+k];
+ for(int d=0;d<D;d++) os[d]+=w*src[d];
+ }
+ }
+ double ta=now_s()-tj; m->t_emm+=ta; if(g_prof) g_vkb_acc+=ta;
+ }
+ free(ib); free(r2); free(taken); free(want); free(on2); free(dev); free(ctb); free(x2); free(y2); free(seen);
+}
+#endif
+
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int with_shared){
if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta
* l'eventuale load-pilota in volo sullo stesso layer (dopodiche' il
@@ -5909,15 +6120,14 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
#endif
int vk_active = 0; (void)vk_active;
#ifdef COLI_VULKAN
- vk_active = g_vulkan && (g_vk_reg_n+g_vk_reg_n2)>0 && !omp_in_parallel() && S<=4; /* empty
- * registry (tier off / no usage history) = normal CPU expert loop */
- float *vk_xh = vk_active?falloc((int64_t)S*K*D):NULL;
- float *vk_yh = vk_active?falloc((int64_t)S*K*D):NULL;
- int vk2_on = vk_active && g_vk_reg_n2>0; /* dev2 tier live: second async group */
- float *vk_xh2 = vk2_on?falloc((int64_t)S*K*D):NULL;
- float *vk_yh2 = vk2_on?falloc((int64_t)S*K*D):NULL;
+ /* COLI_VULKAN=1 with an expert tier (vk_tier.c, or COLI_VK_DEV2's registry):
+ * moe_vk() takes this call's routed experts, by blocks of rows, in place of the
+ * loop below; the shared expert stays in FASE E. Off, nothing here runs. */
+ vk_active = moe_vk_on(m,layer);
+ if(vk_active) moe_vk(m,layer,x,S,out,idxs,ws,keff,xg,gg,uu,hh);
#endif
int shared_on_gpu=0; (void)shared_on_gpu; /* set by the Metal path when Phase E was fused */
+ if(!vk_active)
for(int base=0;base<nu;base+=64){
int nb = nu-base<64 ? nu-base : 64;
#if !defined(_WIN32)
@@ -5927,19 +6137,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
}
#endif
ESlot *use[64]; int missk[64]; int qof[64]; int nmiss=0;
-#ifdef COLI_VULKAN
- int vk_hit[64]={0};
-#endif
for(int j=0;j<nb;j++){ int eid=uniq[base+j]; use[j]=NULL; qof[j]=-1;
-#ifdef COLI_VULKAN
- /* VK VRAM tier first: registry-served experts need NO RAM slot and NO disk
- * load (the whole point) β€” and skipping the LRU recency bump lets them age
- * out of the RAM cache, freeing capacity for the CPU-served experts. */
- if(vk_active && layer<c->n_layers){
- ColiVkTensor **rg=vk_reg_at(layer,eid);
- if(rg && rg[0]){ vk_hit[j]=1; m->hits++; m->hit_vk++; continue; }
- }
-#endif
use[j]=pin_indexed(m,layer,eid);
if(use[j]){ m->hits++; m->hit_pin++; }
if(!use[j]){
@@ -6054,9 +6252,6 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(base+64<nu){
int nb2 = nu-(base+64)<64 ? nu-(base+64) : 64;
for(int j=0;j<nb2;j++){ int eid=uniq[base+64+j]; int found=0;
-#ifdef COLI_VULKAN
- if(vk_active && vk_reg_served(layer,eid)) found=1; /* VK-tier-served at decode */
-#endif
if(!found) found=expert_is_resident(m,layer,eid);
if(!found) expert_prefetch(m,layer,eid);
}
@@ -6244,131 +6439,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
}
}
}
-#ifdef COLI_VULKAN
- /* Vulkan expert tier (COLI_VULKAN=1): upload routed int4/int3-g64 experts to the GPU once
- * (capped by COLI_VK_EXPERTS), then compute the resident ones as one batched
- * coli_vk_expert_group (fused gate+up+silu -> down, on-device); the rest + misses
- * fall back to the CPU below. Decode-only (S<=4). Measured ~35% faster than ROCm. */
- if(vk_active && !metal_done){
- /* Pinned+async VK expert tier. Pass 1 partitions by REGISTRY residency (the
- * heat-pinned startup uploads β€” no uploads, no loads here); pass 2 ISSUES the
- * GPU batch async and computes the CPU share while it runs; pass 3 drains the
- * pipe + RAM-loads the GPU-side slots (cache invariants: every dispatched slot
- * waited, every use[] slot loaded before the end-of-block LRU swap); pass 4
- * takes the GPU results (fallback: recompute those rows on the CPU). */
- ColiVkTensor *vg[64],*vu[64],*vd[64]; int veid[64]; int vrows[64], voff[64], vrmap[64*4]; float vwmap[64*4];
- ColiVkTensor *vg2[64],*vu2[64],*vd2[64]; int veid2[64]; int vrows2[64], voff2[64], vrmap2[64*4]; float vwmap2[64*4];
- ESlot *ce[64]; int cnr[64], crmap[64*4]; float cwmap[64*4]; int cqof[64];
- int nvk=0, vtot=0, nvk2=0, vtot2=0, ncpu=0;
- double t0=now_s();
- for(int j=0;j<nb;j++){ int eid=uniq[base+j];
- int nr=0;
- for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++)
- if(idxs[(int64_t)s*K+kk]==eid){ rows[nr]=s; rw[nr]=ws[(int64_t)s*K+kk]; nr++; break; }
- if(!nr){ if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } continue; }
- if(vk_hit[j]){ /* registry-served: no RAM slot, no disk load */
- ColiVkTensor **reg=vk_reg_at(layer,eid);
- if(vk2_on && coli_vk_tensor_dev(reg[0])==1){ /* dev2 tier expert */
- voff2[nvk2]=vtot2;
- for(int r=0;r<nr;r++){ memcpy(vk_xh2+(int64_t)(vtot2+r)*D, x+(int64_t)rows[r]*D, D*sizeof(float));
- vrmap2[nvk2*S+r]=rows[r]; vwmap2[nvk2*S+r]=rw[r]; }
- vg2[nvk2]=reg[0]; vu2[nvk2]=reg[1]; vd2[nvk2]=reg[2]; veid2[nvk2]=eid; vrows2[nvk2]=nr; vtot2+=nr; nvk2++;
- } else {
- voff[nvk]=vtot;
- for(int r=0;r<nr;r++){ memcpy(vk_xh+(int64_t)(vtot+r)*D, x+(int64_t)rows[r]*D, D*sizeof(float));
- vrmap[nvk*S+r]=rows[r]; vwmap[nvk*S+r]=rw[r]; }
- vg[nvk]=reg[0]; vu[nvk]=reg[1]; vd[nvk]=reg[2]; veid[nvk]=eid; vrows[nvk]=nr; vtot+=nr; nvk++;
- }
- } else {
- ce[ncpu]=use[j]; cnr[ncpu]=nr; cqof[ncpu]=qof[j];
- for(int r=0;r<nr;r++){ crmap[ncpu*S+r]=rows[r]; cwmap[ncpu*S+r]=rw[r]; }
- ncpu++;
- }
- }
- if(g_prof){ g_vkb_cls+=now_s()-t0; g_vkb_blocks++; g_vkb_nvk+=nvk+nvk2; g_vkb_nvk2+=nvk2; g_vkb_ncpu+=ncpu; }
- double t_iss0=now_s();
- /* issue the SLOWER device first so it gets the longest overlap window;
- * dev2's submit runs on a worker thread (joined before take2 below) so its
- * per-block cost overlaps dev0 issue + the CPU share instead of serializing */
- Vk2Iss iss2 = { vg2, vu2, vd2, vrows2, nvk2, vk_xh2, 0, 0 };
- pthread_t iss2_th; int iss2_threaded=0;
- if(nvk2>0){
- if(pthread_create(&iss2_th,NULL,vk2_issue_worker,&iss2)==0) iss2_threaded=1;
- else iss2.rc = coli_vk_expert_group_issue2(vg2,vu2,vd2,vrows2,nvk2,vk_xh2);
- }
- int vk_issued = nvk>0 && coli_vk_expert_group_issue(vg,vu,vd,vrows,nvk,vk_xh);
- if(g_prof) g_vkb_issue+=now_s()-t_iss0;
- /* CPU share stays SERIAL over experts with row-parallel kernels: a decode
- * block leaves only ~5-6 CPU experts (the tier absorbs the hot head), fewer
- * than the OMP pool, so one-task-per-expert ran each expert single-threaded
- * at ~2.5 GB/s while cores idled β€” measured -23% vs this structure (A/B
- * 2026-07-20); the kernels' internal parallel-for already uses every core.
- * What IS reordered: pipe-ready and cache-resident experts run FIRST, so a
- * still-loading expert gets its I/O hidden behind their matmuls instead of
- * head-of-line blocking the block. The class is an ordering HINT only β€”
- * the body still pipe_waits/loads every entry, so a stale probe costs
- * nothing but order. t_ecpu counts kernel time only (the default path's
- * meaning); the waits land in t_ewait. */
- {
- uint8_t vcls[64];
- for(int c2=0;c2<ncpu;c2++){
- if(g_pipe && cqof[c2]>=0) vcls[c2] = pipe_ready(cqof[c2]) ? 0 : 2; /* loaded : in flight */
- else vcls[c2] = ce[c2]->slab ? 0 : 1; /* resident : sync miss */
- }
- int vord[64], no=0;
- for(uint8_t k2=0;k2<3;k2++) for(int c2=0;c2<ncpu;c2++) if(vcls[c2]==k2) vord[no++]=c2;
- for(int oi=0;oi<no;oi++){ int c2=vord[oi]; ESlot *e=ce[c2]; int nr=cnr[c2];
- if(g_pipe && cqof[c2]>=0){ double tw=now_s(); pipe_wait(cqof[c2]); m->t_ewait += now_s()-tw; }
- if(!e->slab) expert_load(m,layer,e->eid,e,1,1); /* demand=1: moe miss path (FASE A snapshot valid) */
- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)crmap[c2*S+r]*D, D*sizeof(float));
- double te0=now_s();
- expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)crmap[c2*S+r]*D, wgt=cwmap[c2*S+r], *hr=hh+(int64_t)r*D;
- for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
- if(g_prof){ m->t_ecpu+=now_s()-te0;
- m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d);
- m->cpu_expert_rows+=(uint64_t)nr; }
- }
- }
- double t_take0=now_s();
- int vk_ok = vk_issued && coli_vk_expert_group_take(vk_yh);
- if(g_prof) m->t_egpu+=now_s()-t_take0;
- for(int c2=0;c2<nvk;c2++){ int nr=vrows[c2];
- if(vk_ok){ int o=voff[c2];
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *src=vk_yh+(int64_t)(o+r)*D;
- for(int d=0;d<D;d++) os[d]+=wgt*src[d]; }
- } else { /* issue/take failed (device lost): load + recompute on the CPU */
- ESlot *e=&m->ws[nmiss<63?nmiss:63];
- if(e->eid!=veid[c2] || !e->slab) expert_load(m,layer,veid[c2],e,1,0); /* device-lost recovery: DISK-CLASS leaves it unclassified */
- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap[c2*S+r]*D, D*sizeof(float));
- expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *hr=hh+(int64_t)r*D;
- for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
- }
- }
- /* dev2 group: taken AFTER dev0's accumulate so the slower card gets the
- * extra overlap; same per-expert CPU recompute fallback on failure. */
- if(iss2_threaded){ double t_j0=now_s(); pthread_join(iss2_th,NULL);
- if(g_prof){ g_vkb_join+=now_s()-t_j0; g_vkb_wrk+=iss2.dt; } }
- int vk2_ok = iss2.rc && coli_vk_expert_group_take2(vk_yh2);
- for(int c2=0;c2<nvk2;c2++){ int nr=vrows2[c2];
- if(vk2_ok){ int o=voff2[c2];
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap2[c2*S+r]*D, wgt=vwmap2[c2*S+r], *src=vk_yh2+(int64_t)(o+r)*D;
- for(int d=0;d<D;d++) os[d]+=wgt*src[d]; }
- } else {
- ESlot *e=&m->ws[nmiss<63?nmiss:63];
- if(e->eid!=veid2[c2] || !e->slab) expert_load(m,layer,veid2[c2],e,1,0);
- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap2[c2*S+r]*D, D*sizeof(float));
- expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);
- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap2[c2*S+r]*D, wgt=vwmap2[c2*S+r], *hr=hh+(int64_t)r*D;
- for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
- }
- }
- double dt=now_s()-t0; m->t_emm+=dt;
- if(g_prof) g_vkb_acc+=now_s()-t_take0; /* take-wait (t_egpu) + result accumulate */
- }
-#endif
- if(!metal_done && !xexp_done && !vk_active)
+ if(!metal_done && !xexp_done)
for(int j=0;j<nb;j++){ int eid=uniq[base+j]; ESlot *e=use[j];
#ifdef COLI_CUDA
if(early_issued && done_j[j]) continue; /* computing on the GPU right now */
@@ -6594,20 +6665,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
* dispatched miss slot, before the nr==0 skip) already waited on all ws[] loads
* for this block, so they are complete before the LRU swap β€” and the gen-tagged
* cursor keeps any still-spinning worker off a wrong-generation slot. */
- { ESlot *Sl=m->ecache[layer]; int *nn=&m->ecn[layer]; /* promozione LRU (swap buffer) */
- int promo = nmiss<m->ecap ? nmiss : m->ecap;
- for(int a=0;a<promo;a++){ int q=nmiss-1-a; ESlot *dst;
- if(*nn<m->ecap) dst=&Sl[(*nn)++];
- else { int lru=eslot_lru_victim(Sl,*nn,m->ecap);
- if(lru<0){ static int warned;
- if(!warned){ warned=1; fprintf(stderr,"[CUDA] no reusable LRU expert slot (in flight or cap reached); skipping cache promotion\n"); }
- continue; }
- dst=&Sl[lru]; }
- ecache_unindex(m,layer,dst);
- ESlot tmp=*dst; *dst=m->ws[q]; m->ws[q]=tmp;
- ecache_publish(m,layer,dst,dst->eid);
- dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); }
- }
+ ecache_promote_ws(m,layer,nmiss); /* promozione LRU (swap buffer) */
}
/* ---- FASE E: shared expert (PIPE2: gia' sul device; Metal CB: gia' sommata) ---- */
if(!with_shared) goto shared_done;
@@ -6709,9 +6767,6 @@ shared_done:
free(group_x);free(group_y);
free(group_row); free(group_weight);
#endif
-#ifdef COLI_VULKAN
- free(vk_xh); free(vk_yh);
-#endif
}
static void dense_mlp(Layer *l, float *x, int S, int D, int I, float *out){
@@ -9960,7 +10015,11 @@ static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){
printf("DONE %llu STAT %d %.2f %.1f %.2f %d %d\n",r->id,r->emitted,
r->emitted/dt,(dh+dm)>0?100.0*dh/(dh+dm):0.0,rss_gb(),
r->prompt_tokens,r->length_limited);
- fflush(stdout); kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len);
+ fflush(stdout);
+#ifdef COLI_VULKAN
+ vk_tier_turn(m); /* the expert tier's line, on stderr */
+#endif
+ kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len);
/* PROF window = this request's lifetime; with KV_SLOTS>1 concurrent slots
* share the batched forwards, so the shares describe the engine, not the
* single request (same convention as the STAT hit%% above). */
@@ -10453,6 +10512,9 @@ static void run_serve(Model *m, const char *snap){
printf("\n\x01\x01" "END" "\x01\x01\n");
printf("STAT %d %.2f %.1f %.2f\n", prod, prod/tdt, (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb());
fflush(stdout);
+#ifdef COLI_VULKAN
+ vk_tier_turn(m); /* the expert tier's line, on stderr */
+#endif
if(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */
kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */
if(nr<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; }
@@ -10549,6 +10611,9 @@ static void run_serve(Model *m, const char *snap){
agree_pct,kl_mean);
printf("\n");
fflush(stdout);
+#ifdef COLI_VULKAN
+ vk_tier_turn(m); /* the expert tier's line, on stderr */
+#endif
if(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */
free(raw); g_temp=base_temp; g_nuc=base_nuc;
usage_save(m); /* la cache che impara: storia aggiornata a ogni turno */
@@ -10573,15 +10638,6 @@ static void run_serve(Model *m, const char *snap){
/* telemetry, stats, usage persistence β€” moved to telemetry.h */
#ifdef COLI_VULKAN
-/* Pinned VK expert tier fill: upload the top-COLI_VK_EXPERTS heat-ranked routed experts
- * ONCE at startup (mirrors the HIP VRAM tier: stable residency, zero uploads at decode).
- * Ranking comes from the persistent usage history; already-pinned RAM slots feed the
- * upload directly, the rest stream through one transient slot and are freed after. */
-typedef struct { uint32_t u; int layer, eid; } VkCand;
-static int vk_cand_cmp(const void *a, const void *b){
- uint32_t ua=((const VkCand*)a)->u, ub=((const VkCand*)b)->u;
- return ua<ub ? 1 : ua>ub ? -1 : 0;
-}
/* Upload the VK dense working set (attention projections, absorb kv_b, o-proj,
* shared expert) at startup, BEFORE the tier fill. These ~8 GB otherwise
* allocate lazily on the first forwards β€” AFTER the tier has filled to its
@@ -10613,107 +10669,194 @@ static void vk_dense_preload(Model *m){
nt,bytes/1e9,now_s()-t0);
}
-static void vk_registry_fill(Model *m){
+/* ---- COLI_VULKAN=1: the routed experts' places on the device, at startup ----------
+ * vk_tier_start describes the routed experts to the shared tier (vk_tier.c) and warms
+ * it from the history; vk_dev2_fill then gives COLI_VK_DEV2's registry the hottest
+ * experts the tier did not take. */
+
+/* The format one routed expert tensor will have in RAM, from the container's header
+ * alone (no read): qt_resolve_fmt for a quantized container (the .qs companion), else
+ * the format qt_alloc gives the bits the experts are quantized to at load. -1: no such
+ * tensor. */
+static int vk_expert_fmt(Model *m,int layer,int eid,int k,int *gs){
+ Cfg *c=&m->c; int O = k<2 ? c->moe_inter : c->hidden, I = k<2 ? c->hidden : c->moe_inter;
+ const char suf[3][16]={"gate_proj","up_proj","down_proj"};
+ char nm[288], qn[320], q0[320];
+ snprintf(nm,sizeof nm,"model.layers.%d.mlp.experts.%d.%s.weight",layer,eid,suf[k]);
+ snprintf(qn,sizeof qn,"%s.qs",nm);
+ snprintf(q0,sizeof q0,"model.layers.%d.mlp.experts.%d.gate_proj.weight.qs",layer,eid);
+ *gs=0;
+ st_tensor *tw=st_find(&m->S,nm);
+ if(!tw) return -1;
+ if(st_has(&m->S,q0)){ /* expert_load_impl's own test */
+ st_tensor *tq=st_find(&m->S,qn);
+ if(!tq) return -1;
+ return qt_resolve_fmt(nm,O,I,tw->nbytes,tq->nbytes,gs,st_fmt_stamp(&m->S,nm));
+ }
+ int b=m->ebits; /* qt_alloc's rule */
+ return b>=16 ? 0 : (b>=5||g_nopack) ? 1 : b>=4 ? 2 : b==3 ? 5 : 3;
+}
+/* The tier's source kind for a colibri QT format; 0 when the device has no form of it
+ * (int2, the E8/IQ3 lattice of fmt=6 with its rotated input, fp8): those stay on the CPU. */
+static int vk_src_kind(int fmt,int gs,VktFmt *f){
+ switch(fmt){
+ case 0: *f=(VktFmt){VKT_SRC_F32,0}; return 1;
+ case 1: *f=(VktFmt){VKT_SRC_I8_ROW,0}; return 1;
+ case 2: *f=(VktFmt){VKT_SRC_I4U_PAIRS_ROW,0}; return 1;
+ case 4: *f=(VktFmt){VKT_SRC_I4U_PAIRS_GS,gs}; return gs>=8 && gs%8==0;
+ case 5: *f=(VktFmt){VKT_SRC_I3_G64,64}; return 1;
+ default: return 0;
+ }
+}
+static int vk_in_ram(void *ctx,int layer,int eid){
+ Model *m=ctx;
+ return pin_indexed(m,layer,eid)!=NULL || ecache_indexed(m,layer,eid,0)!=NULL;
+}
+/* Is this model's expert set quantized at load (no .qs companions: qt_from_disk's
+ * qt_alloc buffers, one per tensor) rather than read from a container (a slab, or
+ * views of a COLI_MMAP mapping that own nothing)? */
+static int vk_experts_rtq(Model *m,int layer){
+ char q0[320]; snprintf(q0,sizeof q0,"model.layers.%d.mlp.experts.0.gate_proj.weight.qs",layer);
+ return !st_has(&m->S,q0);
+}
+/* A startup slot's memory, then the slot cleared. */
+static void vk_tmp_slot_free(ESlot *t,int rtq){
+ if(t->slab){ compat_aligned_free(t->slab); free(t->fslab); }
+ else if(rtq){ QT *q[3]={&t->g,&t->u,&t->d};
+ for(int k=0;k<3;k++){ free(q[k]->qf); free(q[k]->q8); free(q[k]->q4); free(q[k]->s); } }
+ memset(t,0,sizeof(*t)); t->eid=-1;
+}
+static Model *g_vk_model; /* the run's report at exit */
+static void vk_tier_report_run(void){
+ if(g_vk_model) vkt_report("run",g_vk_model->hit_pin+g_vk_model->hit_ecache,g_vk_model->miss);
+}
+static void vk_tier_turn(Model *m){ vkt_report("turn",m->hit_pin+m->hit_ecache,m->miss); }
+static int vk_tier_resident_count(Model *m){
+ int n=0; if(!vkt_ready()) return 0;
+ for(int i=0;i<m->c.n_layers;i++) for(int e=0;e<m->c.n_experts;e++) n+=vkt_resident(i,e);
+ return n;
+}
+static void vk_tier_start(Model *m){
+ Cfg *c=&m->c; int E=c->n_experts, NL=c->n_layers;
+ if(!g_vulkan || E<=0 || g_vk_experts==0 || !vkt_wanted()) return;
+#ifdef COLI_CUDA
+ if(g_cuda_enabled){ fprintf(stderr,"[VK] tier colibri: the CUDA expert tier is on and wins; the Vulkan tier stays off\n"); return; }
+#endif
+ int pl=-1; for(int i=0;i<NL;i++) if(m->L[i].sparse){ pl=i; break; }
+ if(pl<0) return;
+ int f[3], gs[3]; for(int k=0;k<3;k++) f[k]=vk_expert_fmt(m,pl,0,k,&gs[k]);
+ VktFmt gu, dn;
+ if(f[0]!=f[1] || gs[0]!=gs[1] || !vk_src_kind(f[0],gs[0],&gu) || !vk_src_kind(f[2],gs[2],&dn)){
+ fprintf(stderr,"[VK] tier colibri: experts in fmt %d/%d/%d (gate/up/down) have no device form "
+ "(int2, E8/IQ3 fmt=6 and fp8 stay on the CPU, gate and up must match), the routed experts stay on the CPU\n",f[0],f[1],f[2]);
+ return;
+ }
+ /* What the device still has to hold besides the experts: the absorb core's KV
+ * mirror (COLI_VK_ATTN, allocated at the first decode, CTX rows a layer), and
+ * COLI_VK_RESERVE_GB, the old name of this reserve, beyond the tier's own. The dense
+ * set is already on the device (vk_dense_preload ran first) and counted as used. */
+ double dense=0;
+ if(g_vk_attn){ int ctx=getenv("CTX")?atoi(getenv("CTX")):4096; if(ctx<1) ctx=4096;
+ dense+=(double)NL*ctx*(c->kv_lora+c->qk_rope)*4.0; }
+ { const char *r=getenv("COLI_VK_RESERVE_GB"), *tr=getenv("COLI_VK_TIER_RESERVE_GB");
+ if(r && *r){ double extra=atof(r)-(tr&&*tr?atof(tr):1.0); if(extra>0) dense+=extra*1073741824.0; } }
+ VktConfig vc={.engine="colibri", .layers=NL, .experts=E, .hidden=c->hidden, .inter=c->moe_inter, .topk=c->topk,
+ .gate_up=gu, .down=dn, .act=VKT_ACT_SWIGLU, .act_limit=0.f,
+ .max_rows=GLM_VK_ROWS*c->topk,
+ .ram_reserve=(size_t)((double)m->ecap*expert_cache_row_bytes(m,m->ebits)),
+ .dense_bytes=(size_t)dense,
+ .in_ram=vk_in_ram, .ram_ctx=m,
+ .max_experts=g_vk_experts>0 ? g_vk_experts : 0};
+ if(!vkt_init(&vc,m->eusage)) return;
+ g_vkt.on=1; g_vkt.gu_fmt=f[0]; g_vkt.gu_gs=gs[0]; g_vkt.dn_fmt=f[2]; g_vkt.dn_gs=gs[2];
+ g_vkt.exp_bytes=vkt_expert_bytes(c->hidden,c->moe_inter,gu,dn);
+ atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
+ atexit(vkt_shutdown);
+ g_vk_model=m; atexit(vk_tier_report_run); /* runs first: the tier is still up */
+ /* warm start: the history's hottest experts, read in parallel (pinned ones from RAM) */
+ int all=NL*E;
+ int *ql=malloc((size_t)all*sizeof(int)), *qe=malloc((size_t)all*sizeof(int));
+ const char *warm=getenv("COLI_VK_TIER_WARM"); /* 0: no warm start, the tier fills as experts pass by */
+ int n = ql&&qe&&!(warm&&*warm=='0') ? vkt_plan(ql,qe,all) : 0;
+ if(n>0){
+ double t0=now_s(); int nt=omp_get_max_threads();
+ ESlot *tmp=xzalloc((size_t)nt*sizeof(ESlot),"vk warm slots");
+ for(int t=0;t<nt;t++) tmp[t].eid=-1;
+ #pragma omp parallel for schedule(dynamic,4)
+ for(int i=0;i<n;i++){
+ VktExpertSrc vs; ESlot *p=pin_indexed(m,ql[i],qe[i]);
+ if(p && vk_slot_src(p,&vs)){ vkt_put(ql[i],qe[i],&vs); continue; }
+ ESlot *t=&tmp[omp_get_thread_num()];
+ int ok = expert_load(m,ql[i],qe[i],t,0,0)==0 && vk_slot_src(t,&vs); /* demand=0: startup */
+ vkt_put(ql[i],qe[i],ok?&vs:NULL); /* NULL: planned but not placed (a mixed container) */
+ }
+ vkt_put_done();
+ for(int t=0;t<nt;t++) vk_tmp_slot_free(&tmp[t],vk_experts_rtq(m,pl));
+ free(tmp);
+ fprintf(stderr,"[VK] tier colibri: warm start, %d experts from the history in %.1fs\n",n,now_s()-t0);
+ }
+ free(ql); free(qe);
+}
+
+/* COLI_VK_DEV2: the second device's registry takes the hottest experts of the history
+ * that the tier does not hold, up to COLI_VK_EXPERTS2 and COLI_VK_RESERVE2_GB, uploaded
+ * once (int4 per row, int4-gs and int3-g64 experts). */
+typedef struct { uint32_t u; int layer, eid; } VkCand;
+static int vk_cand_cmp(const void *a, const void *b){
+ uint32_t ua=((const VkCand*)a)->u, ub=((const VkCand*)b)->u;
+ return ua<ub ? 1 : ua>ub ? -1 : 0;
+}
+static void vk_dev2_fill(Model *m){
Cfg *c=&m->c; int E=c->n_experts, NL=c->n_layers;
- if(!g_vulkan || g_vk_budget<=0) return;
+ if(!g_vulkan || g_vk_budget2<=0 || !coli_vk_dev2_available()) return;
int64_t nz=0;
for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++) if(m->eusage[i][e]) nz++;
- if(!nz){ fprintf(stderr,"[VK] expert tier: no usage history yet β€” tier empty this run "
+ if(!nz){ fprintf(stderr,"[VK] dev2 tier: no usage history yet β€” tier empty this run "
"(it seeds from %s as you use the model)\n", g_usage_path); return; }
VkCand *cand=malloc((size_t)nz*sizeof(VkCand)); if(!cand) return;
int64_t n=0;
for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++)
- if(m->eusage[i][e]) cand[n++]=(VkCand){m->eusage[i][e],i,e};
+ if(m->eusage[i][e] && !vkt_resident(i,e)) cand[n++]=(VkCand){m->eusage[i][e],i,e};
qsort(cand,(size_t)n,sizeof(VkCand),vk_cand_cmp);
g_vk_reg=calloc((size_t)NL*E*3,sizeof(*g_vk_reg)); g_vk_reg_E=E; g_vk_reg_NL=NL;
if(!g_vk_reg){ free(cand); return; }
ESlot tmp; memset(&tmp,0,sizeof(tmp)); tmp.eid=-1;
- double t0=now_s(); int64_t bytes=0; int tried=0, loadfail=0;
- /* Pressure-proofing: tier weights are the EVICTABLE class (0.4 vs scratch/KV 1.0,
- * dense 0.75) so an oversubscribed heap sheds cold experts instead of thrashing
- * the per-token attention submits; and the fill STOPS while the device-local
- * budget still holds COLI_VK_RESERVE_GB (default 3) for the lazily-allocated
- * dense weights + KV mirror + staging (measured ~1.7 GB at 4k ctx, growing with
- * max_t). Without the budget extension the count cap alone applies, as before. */
- double vkr_reserve = getenv("COLI_VK_RESERVE_GB")?atof(getenv("COLI_VK_RESERVE_GB")):3.0;
- int vkr_stopped=0; double vkr_used=0, vkr_budget=0;
- int64_t i2=0;
- coli_vk_alloc_priority(0.4f);
- for(;i2<n && g_vk_reg_n<g_vk_budget;i2++){
- if(vkr_reserve>0 && (g_vk_reg_n&7)==0 && coli_vk_mem_budget(&vkr_used,&vkr_budget)
- && vkr_budget-vkr_used < vkr_reserve){ vkr_stopped=1; break; }
- int layer=cand[i2].layer, eid=cand[i2].eid; tried++;
+ double t20=now_s(); int64_t bytes2=0; int tried2=0, loadfail=0;
+ double vkr2_reserve = getenv("COLI_VK_RESERVE2_GB")?atof(getenv("COLI_VK_RESERVE2_GB")):0.5;
+ int vkr2_stopped=0; double u2=0,b2=0;
+ for(int64_t i2=0;i2<n && g_vk_reg_n2<g_vk_budget2;i2++){
+ if(vkr2_reserve>0 && (g_vk_reg_n2&7)==0 && coli_vk_mem_budget2(&u2,&b2)
+ && b2-u2 < vkr2_reserve){ vkr2_stopped=1; break; }
+ int layer=cand[i2].layer, eid=cand[i2].eid; tried2++;
ESlot *src=pin_indexed(m,layer,eid);
if(src&&!src->slab) src=NULL;
- if(!src){ if(expert_load(m,layer,eid,&tmp,0,0)!=0){ /* 0 = success (impl convention); demand=0: startup tier fill */
- if(++loadfail<4) fprintf(stderr,"[VK] tier fill: expert_load(%d,%d) failed\n",layer,eid);
- if(loadfail>=64) break; /* disk trouble: stop burning time */
+ if(!src){ if(expert_load(m,layer,eid,&tmp,0,0)!=0){ /* 0 = success; demand=0: startup fill */
+ if(++loadfail>=64) break; /* disk trouble: stop burning time */
continue; } src=&tmp; }
- int xf=src->g.fmt; /* int4 (2), grouped int4 (4), int3-g64 (5) tiers; gate/up must
- * share fmt for the fused gate_up shader, down may differ */
+ int xf=src->g.fmt; /* int4 (2), grouped int4 (4), int3-g64 (5); gate/up share fmt
+ * for the fused gate_up shader, down may differ */
if((xf!=2&&xf!=4&&xf!=5)||src->u.fmt!=xf||src->u.gs!=src->g.gs
||(src->d.fmt!=2&&src->d.fmt!=4&&src->d.fmt!=5)
- ||(xf==4&&(src->g.gs<8||src->g.gs%8))||(src->d.fmt==4&&(src->d.gs<8||src->d.gs%8))){
- if(tried<4) fprintf(stderr,"[VK] tier fill: (%d,%d) fmt %d/%d/%d not int4/int3-g64 (src=%s)\n",
- layer,eid,src->g.fmt,src->u.fmt,src->d.fmt,src==&tmp?"load":"pin");
- continue; }
+ ||(xf==4&&(src->g.gs<8||src->g.gs%8))||(src->d.fmt==4&&(src->d.gs<8||src->d.gs%8)))
+ continue;
ColiVkTensor **slot=vk_reg_at(layer,eid);
- if(!coli_vk_tensor_ensure(&slot[0],src->g.q4,src->g.s,xf,c->hidden,c->moe_inter,src->g.gs)||
- !coli_vk_tensor_ensure(&slot[1],src->u.q4,src->u.s,xf,c->hidden,c->moe_inter,src->u.gs)||
- !coli_vk_tensor_ensure(&slot[2],src->d.q4,src->d.s,src->d.fmt,c->moe_inter,c->hidden,src->d.gs)){
+ if(!coli_vk_tensor_ensure2(&slot[0],src->g.q4,src->g.s,xf,c->hidden,c->moe_inter,src->g.gs)||
+ !coli_vk_tensor_ensure2(&slot[1],src->u.q4,src->u.s,xf,c->hidden,c->moe_inter,src->u.gs)||
+ !coli_vk_tensor_ensure2(&slot[2],src->d.q4,src->d.s,src->d.fmt,c->moe_inter,c->hidden,src->d.gs)){
if(slot[0]){coli_vk_tensor_free(slot[0]);slot[0]=NULL;}
if(slot[1]){coli_vk_tensor_free(slot[1]);slot[1]=NULL;}
- fprintf(stderr,"[VK] expert tier: VRAM full after %d experts\n",g_vk_reg_n);
+ fprintf(stderr,"[VK] dev2 tier: VRAM full after %d experts\n",g_vk_reg_n2);
break;
}
- bytes+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);
- g_vk_reg_n++;
- }
- coli_vk_alloc_priority(0.75f); /* back to the dense/default class */
- fprintf(stderr,"[VK] expert tier: %d hot experts resident (%.2f GB VRAM, %.1fs, top-%d of history)\n",
- g_vk_reg_n,bytes/1e9,now_s()-t0,tried);
- if(vkr_stopped)
- fprintf(stderr,"[VK] expert tier: budget stop at %d experts β€” %.1f of %.1f GB device-local used, %.1f GB reserved (COLI_VK_RESERVE_GB)\n",
- g_vk_reg_n,vkr_used,vkr_budget,vkr_reserve);
- /* DEV2 tier: continue down the heat ranking onto the second GPU (COLI_VK_DEV2),
- * starting at the candidate dev0 stopped on. Same fmt gates, its own budget. */
- if(g_vk_budget2>0 && coli_vk_dev2_available()){
- double t20=now_s(); int64_t bytes2=0; int tried2=0;
- double vkr2_reserve = getenv("COLI_VK_RESERVE2_GB")?atof(getenv("COLI_VK_RESERVE2_GB")):0.5;
- int vkr2_stopped=0; double u2=0,b2=0;
- for(;i2<n && g_vk_reg_n2<g_vk_budget2;i2++){
- if(vkr2_reserve>0 && (g_vk_reg_n2&7)==0 && coli_vk_mem_budget2(&u2,&b2)
- && b2-u2 < vkr2_reserve){ vkr2_stopped=1; break; }
- int layer=cand[i2].layer, eid=cand[i2].eid; tried2++;
- ESlot *src=pin_indexed(m,layer,eid);
- if(src&&!src->slab) src=NULL;
- if(!src){ if(expert_load(m,layer,eid,&tmp,0,0)!=0){
- if(++loadfail>=64) break;
- continue; } src=&tmp; }
- int xf=src->g.fmt;
- if((xf!=2&&xf!=4&&xf!=5)||src->u.fmt!=xf||src->u.gs!=src->g.gs
- ||(src->d.fmt!=2&&src->d.fmt!=4&&src->d.fmt!=5)
- ||(xf==4&&(src->g.gs<8||src->g.gs%8))||(src->d.fmt==4&&(src->d.gs<8||src->d.gs%8)))
- continue;
- ColiVkTensor **slot=vk_reg_at(layer,eid);
- if(!coli_vk_tensor_ensure2(&slot[0],src->g.q4,src->g.s,xf,c->hidden,c->moe_inter,src->g.gs)||
- !coli_vk_tensor_ensure2(&slot[1],src->u.q4,src->u.s,xf,c->hidden,c->moe_inter,src->u.gs)||
- !coli_vk_tensor_ensure2(&slot[2],src->d.q4,src->d.s,src->d.fmt,c->moe_inter,c->hidden,src->d.gs)){
- if(slot[0]){coli_vk_tensor_free(slot[0]);slot[0]=NULL;}
- if(slot[1]){coli_vk_tensor_free(slot[1]);slot[1]=NULL;}
- fprintf(stderr,"[VK] dev2 tier: VRAM full after %d experts\n",g_vk_reg_n2);
- break;
- }
- bytes2+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);
- g_vk_reg_n2++;
- }
- fprintf(stderr,"[VK] dev2 tier: %d experts resident (%.2f GB VRAM, %.1fs, next-%d of history)\n",
- g_vk_reg_n2,bytes2/1e9,now_s()-t20,tried2);
- if(vkr2_stopped)
- fprintf(stderr,"[VK] dev2 tier: budget stop at %d experts β€” %.1f of %.1f GB device-local used, %.1f GB reserved (COLI_VK_RESERVE2_GB)\n",
- g_vk_reg_n2,u2,b2,vkr2_reserve);
+ bytes2+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);
+ g_vk_reg_n2++;
}
- if(tmp.slab){ compat_aligned_free(tmp.slab); free(tmp.fslab); }
+ fprintf(stderr,"[VK] dev2 tier: %d experts resident (%.2f GB VRAM, %.1fs, next-%d of history)\n",
+ g_vk_reg_n2,bytes2/1e9,now_s()-t20,tried2);
+ if(vkr2_stopped)
+ fprintf(stderr,"[VK] dev2 tier: budget stop at %d experts β€” %.1f of %.1f GB device-local used, %.1f GB reserved (COLI_VK_RESERVE2_GB)\n",
+ g_vk_reg_n2,u2,b2,vkr2_reserve);
+ for(int i=0;i<NL;i++) if(m->L[i].sparse){ vk_tmp_slot_free(&tmp,vk_experts_rtq(m,i)); break; }
free(cand);
}
#endif
@@ -12321,14 +12464,26 @@ int main(int argc, char **argv){
if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (tried %s; need libvulkan + "
"the compiled shaders β€” point COLI_VK_SHADERS at the shader directory "
"or the qmatmul.spv file, or run `make VK=1` to build them)\n", spv); return 2; }
- /* 320 = sweep optimum on a 16 GB card (256-384 measured flat, 320 best median;
- * ~6 GB tier + ~8 GB dense leaves headroom for the long-context KV mirror). */
- g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 320;
- g_vk_dense = coli_vk_dense_decide(NULL, g_vk_budget > 0, 0); /* COLI_VK_DENSE; this engine's default is off */
+ /* The routed experts go to the shared tier (vk_tier.c), sized by its budget
+ * (COLI_VK_TIER_GB). COLI_VK_EXPERTS, the count of the fixed set this engine
+ * uploaded before (320 by default), is kept as a deprecated alias: N caps the
+ * tier at N experts, 0 turns it off as COLI_VK_TIER=0 does. */
+ { const char *ve=getenv("COLI_VK_EXPERTS"); g_vk_experts = ve&&*ve ? atoi(ve) : -1; }
+ int vk_tier_on = vkt_wanted() && g_vk_experts!=0;
+ g_vk_dense = coli_vk_dense_decide(NULL, vk_tier_on, 0); /* COLI_VK_DENSE; this engine's default is off */
g_vk_attn = getenv("COLI_VK_ATTN") ? atoi(getenv("COLI_VK_ATTN")) : 0;
- fprintf(stderr,"[VK] expert tier active: routed quantized experts on the GPU (budget %d)%s%s\n",
- g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "",
- g_vk_attn ? " + absorb attention core" : "");
+ if(vk_tier_on)
+ fprintf(stderr,"[VK] expert tier active: routed experts on the shared adaptive tier (vk_tier.c)%s%s\n",
+ g_vk_dense ? " + dense projections + shared expert" : "",
+ g_vk_attn ? " + absorb attention core" : "");
+ else
+ fprintf(stderr,"[VK] expert tier off (%s): routed experts on the CPU%s%s\n",
+ g_vk_experts==0 ? "COLI_VK_EXPERTS=0" : "COLI_VK_TIER=0",
+ g_vk_dense ? "; dense projections + shared expert on the GPU" : "",
+ g_vk_attn ? "; absorb attention core on the GPU" : "");
+ if(g_vk_experts>0)
+ fprintf(stderr,"[VK] COLI_VK_EXPERTS=%d is deprecated: it caps the shared tier at %d experts; "
+ "COLI_VK_TIER_GB sizes it in GiB\n", g_vk_experts, g_vk_experts);
/* COLI_VK_DEV2=auto|<index>: bring up a SECOND GPU for tier experts only
* (e.g. an RX 580 beside the primary card). The dev2 tier fills with the
* next heat-ranked experts after dev0's budget stop, capped by
@@ -12724,43 +12879,9 @@ int main(int argc, char **argv){
{ double ram_env = getenv("RAM_GB")?atof(getenv("RAM_GB")):0.0;
int est_ctx = getenv("CTX")?atoi(getenv("CTX")):4096; /* stesso default di run_serve */
snprintf(g_usage_path,sizeof(g_usage_path),"%s/.coli_usage",snap);
-#ifdef COLI_VULKAN
- /* #653's correction, for the Vulkan tier. On an integrated GPU the tier's
- * HOST_VISIBLE|DEVICE_LOCAL allocation is the SAME physical RAM that
- * expert_avail()/cap_for_ram() below hand to the pin set and the LRU.
- * Unlike the CUDA tier this one cannot be subtracted after the fact:
- * vk_registry_fill() runs at the END of init, long after both decisions
- * are made, so the planned size has to be reserved here instead. Sized
- * from a routed layer's row width x the configured expert count.
- * Discrete GPUs have their own pool -> deviceType is not INTEGRATED and
- * this is a no-op, as with #653. */
- if(g_vulkan && g_vk_budget>0 && g_mem_avail_boot>0 && coli_vk_device_integrated()){
- int probe_l = m.c.n_layers>1 ? m.c.n_layers/2 : 0;
- double per = (double)expert_bytes_row(&m,probe_l,m.ebits);
- double tier_gb = per>0 ? (double)g_vk_budget*per/1e9 : 0.0;
- /* COLI_VK_EXPERTS is a REQUEST, not a placement: vk_registry_fill() stops
- * early when the device-local budget runs out (COLI_VK_RESERVE_GB), so
- * pricing the request would over-reserve badly -- measured 95.6 GB reserved
- * against 66.0 GB actually placed at 4500, and at 6000 the unclamped
- * reservation starved MemAvailable to the 1 GB floor and killed the run.
- * Clamp to what the device can actually take, and never take so much that
- * the host side has nothing left to plan with. */
- double vk_used=0, vk_bud=0;
- if(tier_gb>0 && coli_vk_mem_budget(&vk_used,&vk_bud) && vk_bud>vk_used){
- double reserve = getenv("COLI_VK_RESERVE_GB")?atof(getenv("COLI_VK_RESERVE_GB")):3.0;
- double placeable = vk_bud - vk_used - reserve;
- if(placeable>0 && tier_gb>placeable) tier_gb = placeable;
- }
- double host_floor = g_mem_avail_boot*0.35; /* the planner keeps at least this */
- if(tier_gb > g_mem_avail_boot - host_floor) tier_gb = g_mem_avail_boot - host_floor;
- if(tier_gb>0){
- g_mem_avail_boot -= tier_gb;
- fprintf(stderr,"[VK] integrated/unified memory: expert tier will share physical RAM; "
- "RAM budget snapshot reduced by %.2f GB (%d experts requested) -> MemAvailable=%.1f GB\n",
- tier_gb, g_vk_budget, g_mem_avail_boot);
- }
- }
-#endif
+ /* An integrated GPU's expert tier is RAM too. It is not reserved here: the shared
+ * tier (vk_tier_start, at the end of init) sizes itself from what MemAvailable has
+ * left once this LRU has grown to its cap (VktConfig.ram_reserve). */
int64_t hist = usage_load(&m,g_usage_path);
if(hist>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",(long long)hist,g_usage_path);
int autopin = getenv("AUTOPIN")?atoi(getenv("AUTOPIN")):1;
@@ -12818,8 +12939,9 @@ int main(int argc, char **argv){
g_prof = getenv("PROF")?atoi(getenv("PROF")):0; /* PROF=1: opt-in performance profile */
if(g_prof) prof_config(&m, ram_env, est_ctx); }
#ifdef COLI_VULKAN
- vk_dense_preload(&m); /* dense claims VRAM first β€” the tier fill sizes to the remainder */
- vk_registry_fill(&m); /* pinned VK expert tier: needs the usage history loaded above */
+ vk_dense_preload(&m); /* dense claims VRAM first β€” the tier sizes to the remainder */
+ vk_tier_start(&m); /* the shared expert tier: needs the usage history and the cap above */
+ vk_dev2_fill(&m); /* COLI_VK_DEV2: the hottest experts the tier does not hold */
#endif
const char *stats=getenv("STATS"); /* STATS=<file> -> istogramma uso expert a fine run */
@@ -12945,9 +13067,11 @@ int main(int argc, char **argv){
double tot=m.hits+m.miss;
printf("N-gram speculation (DRAFT=%d): %.2f tokens/forward (%llu forwards per %llu tokens)\n",
g_draft, m.n_fw?(double)m.n_emit/m.n_fw:1.0, (unsigned long long)m.n_fw, (unsigned long long)m.n_emit);
- printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru / %llu miss) | RSS: %.2f GB | %.1f tok/s\n",
+ char vkhits[48]=""; /* experts a Vulkan device served (#336's split) */
+ if(m.hit_vk) snprintf(vkhits,sizeof vkhits," + %llu vk",(unsigned long long)m.hit_vk);
+ printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru%s / %llu miss) | RSS: %.2f GB | %.1f tok/s\n",
tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hit_pin, (unsigned long long)m.hit_ecache,
- (unsigned long long)m.miss, rss_gb(), n_new/dt);
+ vkhits, (unsigned long long)m.miss, rss_gb(), n_new/dt);
profile_print(&m,dt);
if(g_prof) prof_report(&m,&pb,dt,n_new,stdout);
#ifdef COLI_CUDA
diff --git a/c/telemetry.h b/c/telemetry.h
index 52ab105f..94f13f51 100644
--- a/c/telemetry.h
+++ b/c/telemetry.h
@@ -193,8 +193,9 @@ static void tiers_emit(Model *m){
#ifdef COLI_CUDA
vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9;
#elif defined(COLI_VULKAN)
- vram=g_vk_reg_n+g_vk_reg_n2;
- int64_t bytes=0;
+ int vk_tier_n=vk_tier_resident_count(m); /* the shared tier's residents, then dev2's */
+ vram=vk_tier_n+g_vk_reg_n2;
+ int64_t bytes=(int64_t)vk_tier_n*(int64_t)g_vkt.exp_bytes;
if(g_vk_reg) for(int i=0;i<g_vk_reg_NL;i++) for(int e=0;e<g_vk_reg_E;e++){
ColiVkTensor **slot=vk_reg_at(i,e);
for(int j=0;j<3;j++) if(slot[j]) bytes+=(int64_t)coli_vk_tensor_bytes(slot[j]);
@@ -234,7 +235,7 @@ static void emap_emit(Model *m){
for(int e=0;e<cols;e++){
int tier=0;
#ifdef COLI_VULKAN
- if(vk_reg_served(i,e)) tier=2;
+ if(vkt_resident(i,e) || vk_reg_served(i,e)) tier=2; /* on a Vulkan device */
#endif
ESlot *P=m->pin[i];
for(int z=0;!tier&&z<m->npin[i];z++) if(P[z].eid==e){