Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
colibri: the routed experts move to the shared Vulkan tier
JustVugg3cc1a21be438
diff --git a/c/colibri.c b/c/colibri.cindex dfd3bed8..9ca04f99 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -107,6 +107,7 @@ static inline void omp_set_num_threads(int n){ (void)n; }#endif#ifdef COLI_VULKAN#include "backend_vulkan.h"+#include "vk_tier.h" /* the routed-expert tier the MoE engines share */#endif#ifdef COLI_XDNA/* Optional AMD XDNA2 lane. This header pulls in no XRT: the engine owns the@@ -629,23 +630,33 @@ static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S)static int g_repin;static uint64_t g_last_repin;#ifdef COLI_VULKAN-static int g_vulkan; /* COLI_VULKAN=1: compute routed experts on the Vulkan tier */-static int g_vk_budget; /* COLI_VK_EXPERTS: pinned VK expert tier size (0 = tier off) */+static int g_vulkan; /* COLI_VULKAN=1: the device is open */+static int g_vk_experts=-1; /* COLI_VK_EXPERTS (deprecated): -1 unset; 0 = no expert tier;+ * N = the shared tier (vk_tier.c) holds at most N experts */static int g_vk_resident; /* how many experts currently VK-resident */-/* Pinned VK expert tier: top heat-ranked routed experts uploaded ONCE at startup into a- * registry keyed by (layer,eid), decoupled from the RAM cache slots β stable residency- * (no LRU churn, no uploads on the decode critical path), mirroring the HIP VRAM tier. */+/* The routed experts on the device live in the shared tier (vk_tier.c: adaptive, with+ * eviction and a budget). The registry below is the second device's only+ * (COLI_VK_DEV2): its top heat-ranked experts, uploaded ONCE at startup, keyed by+ * (layer,eid) -- stable residency, no uploads on the decode path. */static struct ColiVkTensor **g_vk_reg; /* [(layer*E+eid)*3 + {gate,up,down}] */-static int g_vk_reg_n, g_vk_reg_E;+static int g_vk_reg_E;static inline struct ColiVkTensor **vk_reg_at(int layer,int eid){return g_vk_reg ? &g_vk_reg[((size_t)layer*g_vk_reg_E+eid)*3] : NULL;}static int g_vk_reg_NL; /* registry layer bound (n_layers, MTP excluded) */-/* Will the VK tier serve this expert at decode? (prefetch/pilot can skip its I/O.) */+/* Will the second device serve this expert? (prefetch can skip its I/O.) */static inline int vk_reg_served(int layer,int eid){if(!g_vk_reg || layer<0 || layer>=g_vk_reg_NL || eid<0 || eid>=g_vk_reg_E) return 0;return g_vk_reg[((size_t)layer*g_vk_reg_E+eid)*3] != NULL;}+/* The shared tier's view of this model's experts (vk_tier_start). */+static struct {+ int on; /* vkt_init accepted the experts */+ int gu_fmt, gu_gs, dn_fmt, dn_gs; /* the QT formats it was given (gate = up) */+ size_t exp_bytes; /* one expert on the device */+} g_vkt;+static int vk_tier_resident_count(Model *m);+static void vk_tier_turn(Model *m); /* "[VK] tier colibri turn: ..." after a serve turn */static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention* projections + shared expert) on Vulkan too */static int g_vk_budget2; /* COLI_VK_EXPERTS2: dev2 expert-tier cap (with COLI_VK_DEV2) */@@ -1612,8 +1623,8 @@ static inline float siluf(float x){ return x/(1.f+expf(-x)); }* (Q^T x) dal chiamante via E8_XE (#452: una copia ruotata per chiamata, MAI per expert).* La rotazione dell'input del down e' per-expert e vive qui: `d->fmt==6` la applica* SEMPRE β e' la forma CANONICA, pura funzione del fmt del down-tensor. Perche' resta- * byte-identica al vecchio inline: (a) i fallback device-lost Vulkan ricaricano solo- * expert che erano in registry (vk_reg_at), e vk_registry_fill ammette solo fmt 2/4/5 β+ * byte-identica al vecchio inline: (a) i fallback device-lost Vulkan ricalcolano solo+ * expert che il tier Vulkan teneva, e il tier ammette solo fmt 0/1/2/4/5 β* li' d->fmt!=6 e la rotazione e' un no-op; (b) l'oracolo e' single-format, e una* conversione normale single-pass produce un modello single-format: le due copie che* POSSONO vedere fmt=6 lo fanno solo su un container mixed-format, che il tooling@@ -5399,8 +5410,32 @@ static void ecache_hide(Model *m,int layer,ESlot *s){s->eid=-1;}-/* pin βͺ LRU residency probe (used by CACHE_ROUTE max-rank fill). */+/* The LRU promotion at the end of a block of moe(): the block's nmiss loaded working-set+ * slots m->ws[0..nmiss) swap into the layer's cache, the last loaded first, displacing+ * the least recently used (swap buffer: the victim's slab becomes the ws slot's).+ * Every ws slot must have been waited for (PIPE) before this runs. */+static void ecache_promote_ws(Model *m,int layer,int nmiss){+ ESlot *Sl=m->ecache[layer]; int *nn=&m->ecn[layer];+ int promo = nmiss<m->ecap ? nmiss : m->ecap;+ for(int a=0;a<promo;a++){ int q=nmiss-1-a; ESlot *dst;+ if(*nn<m->ecap) dst=&Sl[(*nn)++];+ else { int lru=eslot_lru_victim(Sl,*nn,m->ecap);+ if(lru<0){ static int warned;+ if(!warned){ warned=1; fprintf(stderr,"[CUDA] no reusable LRU expert slot (in flight or cap reached); skipping cache promotion\n"); }+ continue; }+ dst=&Sl[lru]; }+ ecache_unindex(m,layer,dst);+ ESlot tmp=*dst; *dst=m->ws[q]; m->ws[q]=tmp;+ ecache_publish(m,layer,dst,dst->eid);+ dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); }+}++/* pin βͺ LRU residency probe (used by CACHE_ROUTE max-rank fill), plus the Vulkan expert+ * tier's residents: no disk read either (vkt_resident is 0 without COLI_VULKAN=1). */static int expert_is_resident(Model *m, int layer, int eid){+#ifdef COLI_VULKAN+ if(vkt_resident(layer,eid)) return 1;+#endifreturn pin_indexed(m,layer,eid)!=NULL || ecache_indexed(m,layer,eid,0)!=NULL;}@@ -5458,6 +5493,182 @@ static int mb_gs_compat(int est_fmt, int est_gs, int fmt, int gs){return !(est_fmt==4 && fmt==4 && gs!=est_gs);}+#ifdef COLI_VULKAN+/* ---- the routed experts on the Vulkan device (COLI_VULKAN=1) -----------------------+ * The shared tier (vk_tier.c) holds a cache of routed experts on the device: warm from+ * .coli_usage at startup, then adapting while you chat (an expert the CPU computes is+ * offered to it and promoted when there is room or when it is hotter than the coldest+ * resident, which is evicted), under a budget. COLI_VK_DEV2's registry holds the next+ * hottest on a second device, fixed at startup.+ *+ * moe_vk() takes a MoE layer's routed experts, by blocks of GLM_VK_ROWS rows: the tier's+ * resident (row, rank) pairs go to the device as one batch (vkt_issue), the dev2 ones+ * to the second device on a worker thread, and the CPU resolves, loads and computes the+ * rest (vk_cpu_pairs: the pin / LRU / disk path of moe()'s loop) into output rows of+ * their own while the batch runs. Then every rank of every row joins `out` in routing+ * (rank) order, the device's and the CPU's alike, so the order of the sum never+ * depends on which experts happened to be resident. moe()'s CPU-only loop adds them+ * in union order and is untouched: without COLI_VULKAN nothing here runs. */+#define GLM_VK_ROWS 64+static const void *vk_qt_codes(const QT *t){+ return t->fmt==0 ? (const void*)t->qf : (t->fmt==1||t->fmt==8) ? (const void*)t->q8 : (const void*)t->q4;+}+/* An expert slot as the tier reads it, or 0 when its tensors are not in the format the+ * tier was configured with (a mixed container): such an expert stays on the CPU. */+static int vk_slot_src(const ESlot *e, VktExpertSrc *src){+ if(!g_vkt.on || e->g.fmt!=g_vkt.gu_fmt || e->u.fmt!=g_vkt.gu_fmt || e->g.gs!=g_vkt.gu_gs ||+ e->u.gs!=g_vkt.gu_gs || e->d.fmt!=g_vkt.dn_fmt || e->d.gs!=g_vkt.dn_gs ||+ e->g.planar || e->u.planar || e->d.planar) return 0;+ *src=(VktExpertSrc){vk_qt_codes(&e->g),vk_qt_codes(&e->u),vk_qt_codes(&e->d),e->g.s,e->u.s,e->d.s};+ return src->g && src->u && src->d;+}+/* Does this MoE call go through moe_vk? Layers 0..n_layers-1 only: the MTP head's+ * experts are int8 and stay on the CPU. CUDA, Metal, the cluster workers and the+ * ablation harness keep moe()'s own loop. */+static int moe_vk_on(Model *m,int layer){+ if(!g_vulkan || layer<0 || layer>=m->c.n_layers || omp_in_parallel()) return 0;+ if(!vkt_ready() && g_vk_reg_n2<=0) return 0;+ if(g_abl.mode || g_metal_enabled || g_pre_idx) return 0;+#if !defined(_WIN32)+ if(g_cluster_n) return 0;+#endif+#ifdef COLI_CUDA+ if(g_cuda_enabled) return 0;+#endif+ return 1;+}+/* The pairs i of one block of rows that want[i] marks, on the CPU, as moe()'s loop does+ * them: the union of their experts in first-seen order, 64 at a time; each resolved+ * from the pin set, the LRU or a working-set slot loaded from disk (PIPE or the blocking+ * parallel load), computed once for all its rows into ctb[i]; the block's misses then+ * promoted into the LRU. note: offer each computed expert to the tier. */+static void vk_cpu_pairs(Model *m,int layer,const float *x,int rows,int K,const int *ib,const uint8_t *want,+ float *ctb,float *xg,float *gg,float *uu,float *hh,int note){+ Cfg *c=&m->c; int D=c->hidden, I=c->moe_inter, E=c->n_experts, n=rows*K;+ int *uq=xalloc((size_t)n*sizeof(int),"vk uq"), *rws=xalloc((size_t)rows*sizeof(int),"vk rws"), nu=0;+ unsigned char *seen=xzalloc((size_t)E,"vk seen");+ for(int i=0;i<n;i++) if(want[i] && ib[i]>=0 && !seen[ib[i]]){ seen[ib[i]]=1; uq[nu++]=ib[i]; }+ for(int base=0;base<nu;base+=64){+ int nb = nu-base<64 ? nu-base : 64;+ ESlot *use[64]; int missk[64], qof[64], nmiss=0;+ for(int j=0;j<nb;j++){ int eid=uq[base+j]; qof[j]=-1;+ use[j]=pin_indexed(m,layer,eid);+ if(use[j]){ m->hits++; m->hit_pin++; continue; }+ use[j]=ecache_indexed(m,layer,eid,0);+ if(use[j]){ m->hits++; m->hit_ecache++; use[j]->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); continue; }+ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++;+ if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; }+ }+ if(nmiss){+ if(g_pipe){+ if(!g_pp.started) pipe_init(m);+ double t0=now_s(); int eids[64]; for(int q=0;q<nmiss;q++) eids[q]=uq[base+missk[q]];+ pipe_dispatch(m,layer,eids,nmiss);+ m->t_ewait += now_s()-t0;+ } else { double t0=now_s();+ #pragma omp parallel for schedule(dynamic,1)+ for(int q=0;q<nmiss;q++) expert_load(m,layer,uq[base+missk[q]],&m->ws[q],1,1); /* demand=1: the miss path */+ m->t_ewait += now_s()-t0; }+ }+ if(base+64<nu){ /* readahead of the next 64 while these compute */+ int nb2 = nu-(base+64)<64 ? nu-(base+64) : 64;+ for(int j=0;j<nb2;j++){ int eid=uq[base+64+j];+ if(!expert_is_resident(m,layer,eid)) expert_prefetch(m,layer,eid); }+ }+ for(int j=0;j<nb;j++){ int eid=uq[base+j]; ESlot *e=use[j];+ if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; }+ int nr=0;+ for(int i=0;i<n;i++) if(want[i] && ib[i]==eid){+ memcpy(xg+(int64_t)nr*D, x+(int64_t)(i/K)*D, (size_t)D*sizeof(float)); rws[nr++]=i; }+ if(e->g.fmt==6) e8_rot_rows(xg,nr,D); /* fmt=6 reads Q^T x, row by row as E8_XE does */+ double t0=now_s();+ expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);+ for(int r=0;r<nr;r++) memcpy(ctb+(int64_t)rws[r]*D, hh+(int64_t)r*D, (size_t)D*sizeof(float));+ double dt=now_s()-t0; m->t_emm+=dt;+ if(g_prof){ m->t_ecpu+=dt; m->cpu_expert_rows+=(uint64_t)nr;+ m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); }+ VktExpertSrc vs;+ if(note && vk_slot_src(e,&vs)) vkt_note(layer,eid,&vs); /* may promote it to the device */+ }+ ecache_promote_ws(m,layer,nmiss);+ }+ free(uq); free(rws); free(seen);+}+static void moe_vk(Model *m,int layer,const float *x,int S,float *out,const int *idxs,const float *ws,+ const int *keff,float *xg,float *gg,float *uu,float *hh){+ Cfg *c=&m->c; int D=c->hidden, K=c->topk, E=c->n_experts;+ int B = S<GLM_VK_ROWS ? S : GLM_VK_ROWS, nmax=B*K, dev2=g_vk_reg_n2>0;+ int *ib=xalloc((size_t)nmax*sizeof(int),"vk ib"), *r2=xalloc((size_t)nmax*sizeof(int),"vk r2");+ uint8_t *taken=xalloc((size_t)nmax,"vk taken"), *want=xalloc((size_t)nmax,"vk want"), *on2=xzalloc((size_t)nmax,"vk on2");+ const float **dev=xalloc((size_t)nmax*sizeof(*dev),"vk dev");+ float *ctb=falloc((int64_t)nmax*D);+ float *x2=dev2?falloc((int64_t)nmax*D):NULL, *y2=dev2?falloc((int64_t)nmax*D):NULL;+ unsigned char *seen=xzalloc((size_t)E,"vk seen");+ for(int s0=0;s0<S;s0+=B){+ int rows = S-s0<B ? S-s0 : B, n=rows*K;+ const float *xb=x+(int64_t)s0*D;+ for(int s=0;s<rows;s++) for(int k=0;k<K;k++)+ ib[s*K+k] = k<keff[s0+s] ? idxs[(int64_t)(s0+s)*K+k] : -1;+ double t0=now_s();+ int ndev=vkt_issue(layer,xb,rows,K,ib,taken); /* returns at once; 0 = nothing on the device */+ if(g_prof){ g_vkb_issue+=now_s()-t0; g_vkb_blocks++; }+ memset(seen,0,(size_t)E); memset(on2,0,(size_t)n);+ for(int i=0;i<n;i++) if(taken[i] && !seen[ib[i]]){ seen[ib[i]]=1; m->hits++; m->hit_vk++; if(g_prof) g_vkb_nvk++; }+ /* the second device: the pairs the tier did not take whose expert is in its+ * registry, one group (64 experts at most), issued on a worker thread */+ ColiVkTensor *g2[64],*u2[64],*d2[64]; int n2rows[64], ne2=0, tot2=0;+ Vk2Iss iss2={0}; pthread_t iss2_th; int iss2_threaded=0, iss2_on=0;+ if(dev2){+ int eo[64];+ for(int i=0;i<n && ne2<64;i++){ int e=ib[i];+ if(e<0 || taken[i] || seen[e] || !vk_reg_served(layer,e)) continue;+ seen[e]=1; eo[ne2++]=e; }+ for(int q=0;q<ne2;q++){ ColiVkTensor **rg=vk_reg_at(layer,eo[q]); int nr=0;+ for(int i=0;i<n;i++) if(ib[i]==eo[q] && !taken[i]){+ memcpy(x2+(int64_t)tot2*D, xb+(int64_t)(i/K)*D, (size_t)D*sizeof(float));+ r2[i]=tot2++; on2[i]=1; nr++; }+ g2[q]=rg[0]; u2[q]=rg[1]; d2[q]=rg[2]; n2rows[q]=nr;+ m->hits++; m->hit_vk++; }+ if(ne2){+ iss2=(Vk2Iss){g2,u2,d2,n2rows,ne2,x2,0,0}; iss2_on=1;+ if(g_prof) g_vkb_nvk2+=ne2;+ if(pthread_create(&iss2_th,NULL,vk2_issue_worker,&iss2)==0) iss2_threaded=1;+ else iss2.rc=coli_vk_expert_group_issue2(g2,u2,d2,n2rows,ne2,x2);+ }+ }+ for(int i=0;i<n;i++) want[i] = ib[i]>=0 && !taken[i] && !on2[i];+ if(g_prof) for(int i=0;i<n;i++) if(want[i]) g_vkb_ncpu++;+ vk_cpu_pairs(m,layer,xb,rows,K,ib,want,ctb,xg,gg,uu,hh,1); /* while the batches run */+ double tj=now_s();+ if(ndev && !vkt_join(dev)){ /* the batch failed (the tier stops): those pairs here */+ vk_cpu_pairs(m,layer,xb,rows,K,ib,taken,ctb,xg,gg,uu,hh,0);+ memset(taken,0,(size_t)n);+ }+ if(g_prof) m->t_egpu+=now_s()-tj;+ if(iss2_on){+ if(iss2_threaded){ double t_j0=now_s(); pthread_join(iss2_th,NULL);+ if(g_prof){ g_vkb_join+=now_s()-t_j0; g_vkb_wrk+=iss2.dt; } }+ if(!(iss2.rc && coli_vk_expert_group_take2(y2))){ /* device lost: those pairs here */+ vk_cpu_pairs(m,layer,xb,rows,K,ib,on2,ctb,xg,gg,uu,hh,0);+ memset(on2,0,(size_t)n);+ }+ }+ /* every rank of every row in routing order, wherever it was computed */+ for(int s=0;s<rows;s++){+ float *os=out+(int64_t)(s0+s)*D;+ for(int k=0;k<keff[s0+s];k++){ int i=s*K+k;+ if(ib[i]<0) continue;+ const float *src = taken[i] ? dev[i] : on2[i] ? y2+(int64_t)r2[i]*D : ctb+(int64_t)i*D;+ float w=ws[(int64_t)(s0+s)*K+k];+ for(int d=0;d<D;d++) os[d]+=w*src[d];+ }+ }+ double ta=now_s()-tj; m->t_emm+=ta; if(g_prof) g_vkb_acc+=ta;+ }+ free(ib); free(r2); free(taken); free(want); free(on2); free(dev); free(ctb); free(x2); free(y2); free(seen);+}+#endif+static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int with_shared){if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta* l'eventuale load-pilota in volo sullo stesso layer (dopodiche' il@@ -5909,15 +6120,14 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int#endifint vk_active = 0; (void)vk_active;#ifdef COLI_VULKAN- vk_active = g_vulkan && (g_vk_reg_n+g_vk_reg_n2)>0 && !omp_in_parallel() && S<=4; /* empty- * registry (tier off / no usage history) = normal CPU expert loop */- float *vk_xh = vk_active?falloc((int64_t)S*K*D):NULL;- float *vk_yh = vk_active?falloc((int64_t)S*K*D):NULL;- int vk2_on = vk_active && g_vk_reg_n2>0; /* dev2 tier live: second async group */- float *vk_xh2 = vk2_on?falloc((int64_t)S*K*D):NULL;- float *vk_yh2 = vk2_on?falloc((int64_t)S*K*D):NULL;+ /* COLI_VULKAN=1 with an expert tier (vk_tier.c, or COLI_VK_DEV2's registry):+ * moe_vk() takes this call's routed experts, by blocks of rows, in place of the+ * loop below; the shared expert stays in FASE E. Off, nothing here runs. */+ vk_active = moe_vk_on(m,layer);+ if(vk_active) moe_vk(m,layer,x,S,out,idxs,ws,keff,xg,gg,uu,hh);#endifint shared_on_gpu=0; (void)shared_on_gpu; /* set by the Metal path when Phase E was fused */+ if(!vk_active)for(int base=0;base<nu;base+=64){int nb = nu-base<64 ? nu-base : 64;#if !defined(_WIN32)@@ -5927,19 +6137,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}#endifESlot *use[64]; int missk[64]; int qof[64]; int nmiss=0;-#ifdef COLI_VULKAN- int vk_hit[64]={0};-#endiffor(int j=0;j<nb;j++){ int eid=uniq[base+j]; use[j]=NULL; qof[j]=-1;-#ifdef COLI_VULKAN- /* VK VRAM tier first: registry-served experts need NO RAM slot and NO disk- * load (the whole point) β and skipping the LRU recency bump lets them age- * out of the RAM cache, freeing capacity for the CPU-served experts. */- if(vk_active && layer<c->n_layers){- ColiVkTensor **rg=vk_reg_at(layer,eid);- if(rg && rg[0]){ vk_hit[j]=1; m->hits++; m->hit_vk++; continue; }- }-#endifuse[j]=pin_indexed(m,layer,eid);if(use[j]){ m->hits++; m->hit_pin++; }if(!use[j]){@@ -6054,9 +6252,6 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(base+64<nu){int nb2 = nu-(base+64)<64 ? nu-(base+64) : 64;for(int j=0;j<nb2;j++){ int eid=uniq[base+64+j]; int found=0;-#ifdef COLI_VULKAN- if(vk_active && vk_reg_served(layer,eid)) found=1; /* VK-tier-served at decode */-#endifif(!found) found=expert_is_resident(m,layer,eid);if(!found) expert_prefetch(m,layer,eid);}@@ -6244,131 +6439,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}}}-#ifdef COLI_VULKAN- /* Vulkan expert tier (COLI_VULKAN=1): upload routed int4/int3-g64 experts to the GPU once- * (capped by COLI_VK_EXPERTS), then compute the resident ones as one batched- * coli_vk_expert_group (fused gate+up+silu -> down, on-device); the rest + misses- * fall back to the CPU below. Decode-only (S<=4). Measured ~35% faster than ROCm. */- if(vk_active && !metal_done){- /* Pinned+async VK expert tier. Pass 1 partitions by REGISTRY residency (the- * heat-pinned startup uploads β no uploads, no loads here); pass 2 ISSUES the- * GPU batch async and computes the CPU share while it runs; pass 3 drains the- * pipe + RAM-loads the GPU-side slots (cache invariants: every dispatched slot- * waited, every use[] slot loaded before the end-of-block LRU swap); pass 4- * takes the GPU results (fallback: recompute those rows on the CPU). */- ColiVkTensor *vg[64],*vu[64],*vd[64]; int veid[64]; int vrows[64], voff[64], vrmap[64*4]; float vwmap[64*4];- ColiVkTensor *vg2[64],*vu2[64],*vd2[64]; int veid2[64]; int vrows2[64], voff2[64], vrmap2[64*4]; float vwmap2[64*4];- ESlot *ce[64]; int cnr[64], crmap[64*4]; float cwmap[64*4]; int cqof[64];- int nvk=0, vtot=0, nvk2=0, vtot2=0, ncpu=0;- double t0=now_s();- for(int j=0;j<nb;j++){ int eid=uniq[base+j];- int nr=0;- for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++)- if(idxs[(int64_t)s*K+kk]==eid){ rows[nr]=s; rw[nr]=ws[(int64_t)s*K+kk]; nr++; break; }- if(!nr){ if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } continue; }- if(vk_hit[j]){ /* registry-served: no RAM slot, no disk load */- ColiVkTensor **reg=vk_reg_at(layer,eid);- if(vk2_on && coli_vk_tensor_dev(reg[0])==1){ /* dev2 tier expert */- voff2[nvk2]=vtot2;- for(int r=0;r<nr;r++){ memcpy(vk_xh2+(int64_t)(vtot2+r)*D, x+(int64_t)rows[r]*D, D*sizeof(float));- vrmap2[nvk2*S+r]=rows[r]; vwmap2[nvk2*S+r]=rw[r]; }- vg2[nvk2]=reg[0]; vu2[nvk2]=reg[1]; vd2[nvk2]=reg[2]; veid2[nvk2]=eid; vrows2[nvk2]=nr; vtot2+=nr; nvk2++;- } else {- voff[nvk]=vtot;- for(int r=0;r<nr;r++){ memcpy(vk_xh+(int64_t)(vtot+r)*D, x+(int64_t)rows[r]*D, D*sizeof(float));- vrmap[nvk*S+r]=rows[r]; vwmap[nvk*S+r]=rw[r]; }- vg[nvk]=reg[0]; vu[nvk]=reg[1]; vd[nvk]=reg[2]; veid[nvk]=eid; vrows[nvk]=nr; vtot+=nr; nvk++;- }- } else {- ce[ncpu]=use[j]; cnr[ncpu]=nr; cqof[ncpu]=qof[j];- for(int r=0;r<nr;r++){ crmap[ncpu*S+r]=rows[r]; cwmap[ncpu*S+r]=rw[r]; }- ncpu++;- }- }- if(g_prof){ g_vkb_cls+=now_s()-t0; g_vkb_blocks++; g_vkb_nvk+=nvk+nvk2; g_vkb_nvk2+=nvk2; g_vkb_ncpu+=ncpu; }- double t_iss0=now_s();- /* issue the SLOWER device first so it gets the longest overlap window;- * dev2's submit runs on a worker thread (joined before take2 below) so its- * per-block cost overlaps dev0 issue + the CPU share instead of serializing */- Vk2Iss iss2 = { vg2, vu2, vd2, vrows2, nvk2, vk_xh2, 0, 0 };- pthread_t iss2_th; int iss2_threaded=0;- if(nvk2>0){- if(pthread_create(&iss2_th,NULL,vk2_issue_worker,&iss2)==0) iss2_threaded=1;- else iss2.rc = coli_vk_expert_group_issue2(vg2,vu2,vd2,vrows2,nvk2,vk_xh2);- }- int vk_issued = nvk>0 && coli_vk_expert_group_issue(vg,vu,vd,vrows,nvk,vk_xh);- if(g_prof) g_vkb_issue+=now_s()-t_iss0;- /* CPU share stays SERIAL over experts with row-parallel kernels: a decode- * block leaves only ~5-6 CPU experts (the tier absorbs the hot head), fewer- * than the OMP pool, so one-task-per-expert ran each expert single-threaded- * at ~2.5 GB/s while cores idled β measured -23% vs this structure (A/B- * 2026-07-20); the kernels' internal parallel-for already uses every core.- * What IS reordered: pipe-ready and cache-resident experts run FIRST, so a- * still-loading expert gets its I/O hidden behind their matmuls instead of- * head-of-line blocking the block. The class is an ordering HINT only β- * the body still pipe_waits/loads every entry, so a stale probe costs- * nothing but order. t_ecpu counts kernel time only (the default path's- * meaning); the waits land in t_ewait. */- {- uint8_t vcls[64];- for(int c2=0;c2<ncpu;c2++){- if(g_pipe && cqof[c2]>=0) vcls[c2] = pipe_ready(cqof[c2]) ? 0 : 2; /* loaded : in flight */- else vcls[c2] = ce[c2]->slab ? 0 : 1; /* resident : sync miss */- }- int vord[64], no=0;- for(uint8_t k2=0;k2<3;k2++) for(int c2=0;c2<ncpu;c2++) if(vcls[c2]==k2) vord[no++]=c2;- for(int oi=0;oi<no;oi++){ int c2=vord[oi]; ESlot *e=ce[c2]; int nr=cnr[c2];- if(g_pipe && cqof[c2]>=0){ double tw=now_s(); pipe_wait(cqof[c2]); m->t_ewait += now_s()-tw; }- if(!e->slab) expert_load(m,layer,e->eid,e,1,1); /* demand=1: moe miss path (FASE A snapshot valid) */- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)crmap[c2*S+r]*D, D*sizeof(float));- double te0=now_s();- expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);- for(int r=0;r<nr;r++){ float *os=out+(int64_t)crmap[c2*S+r]*D, wgt=cwmap[c2*S+r], *hr=hh+(int64_t)r*D;- for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }- if(g_prof){ m->t_ecpu+=now_s()-te0;- m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d);- m->cpu_expert_rows+=(uint64_t)nr; }- }- }- double t_take0=now_s();- int vk_ok = vk_issued && coli_vk_expert_group_take(vk_yh);- if(g_prof) m->t_egpu+=now_s()-t_take0;- for(int c2=0;c2<nvk;c2++){ int nr=vrows[c2];- if(vk_ok){ int o=voff[c2];- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *src=vk_yh+(int64_t)(o+r)*D;- for(int d=0;d<D;d++) os[d]+=wgt*src[d]; }- } else { /* issue/take failed (device lost): load + recompute on the CPU */- ESlot *e=&m->ws[nmiss<63?nmiss:63];- if(e->eid!=veid[c2] || !e->slab) expert_load(m,layer,veid[c2],e,1,0); /* device-lost recovery: DISK-CLASS leaves it unclassified */- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap[c2*S+r]*D, D*sizeof(float));- expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap[c2*S+r]*D, wgt=vwmap[c2*S+r], *hr=hh+(int64_t)r*D;- for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }- }- }- /* dev2 group: taken AFTER dev0's accumulate so the slower card gets the- * extra overlap; same per-expert CPU recompute fallback on failure. */- if(iss2_threaded){ double t_j0=now_s(); pthread_join(iss2_th,NULL);- if(g_prof){ g_vkb_join+=now_s()-t_j0; g_vkb_wrk+=iss2.dt; } }- int vk2_ok = iss2.rc && coli_vk_expert_group_take2(vk_yh2);- for(int c2=0;c2<nvk2;c2++){ int nr=vrows2[c2];- if(vk2_ok){ int o=voff2[c2];- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap2[c2*S+r]*D, wgt=vwmap2[c2*S+r], *src=vk_yh2+(int64_t)(o+r)*D;- for(int d=0;d<D;d++) os[d]+=wgt*src[d]; }- } else {- ESlot *e=&m->ws[nmiss<63?nmiss:63];- if(e->eid!=veid2[c2] || !e->slab) expert_load(m,layer,veid2[c2],e,1,0);- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)vrmap2[c2*S+r]*D, D*sizeof(float));- expert_ffn(hh,gg,uu,xg,&e->g,&e->u,&e->d,nr,I);- for(int r=0;r<nr;r++){ float *os=out+(int64_t)vrmap2[c2*S+r]*D, wgt=vwmap2[c2*S+r], *hr=hh+(int64_t)r*D;- for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }- }- }- double dt=now_s()-t0; m->t_emm+=dt;- if(g_prof) g_vkb_acc+=now_s()-t_take0; /* take-wait (t_egpu) + result accumulate */- }-#endif- if(!metal_done && !xexp_done && !vk_active)+ if(!metal_done && !xexp_done)for(int j=0;j<nb;j++){ int eid=uniq[base+j]; ESlot *e=use[j];#ifdef COLI_CUDAif(early_issued && done_j[j]) continue; /* computing on the GPU right now */@@ -6594,20 +6665,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* dispatched miss slot, before the nr==0 skip) already waited on all ws[] loads* for this block, so they are complete before the LRU swap β and the gen-tagged* cursor keeps any still-spinning worker off a wrong-generation slot. */- { ESlot *Sl=m->ecache[layer]; int *nn=&m->ecn[layer]; /* promozione LRU (swap buffer) */- int promo = nmiss<m->ecap ? nmiss : m->ecap;- for(int a=0;a<promo;a++){ int q=nmiss-1-a; ESlot *dst;- if(*nn<m->ecap) dst=&Sl[(*nn)++];- else { int lru=eslot_lru_victim(Sl,*nn,m->ecap);- if(lru<0){ static int warned;- if(!warned){ warned=1; fprintf(stderr,"[CUDA] no reusable LRU expert slot (in flight or cap reached); skipping cache promotion\n"); }- continue; }- dst=&Sl[lru]; }- ecache_unindex(m,layer,dst);- ESlot tmp=*dst; *dst=m->ws[q]; m->ws[q]=tmp;- ecache_publish(m,layer,dst,dst->eid);- dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); }- }+ ecache_promote_ws(m,layer,nmiss); /* promozione LRU (swap buffer) */}/* ---- FASE E: shared expert (PIPE2: gia' sul device; Metal CB: gia' sommata) ---- */if(!with_shared) goto shared_done;@@ -6709,9 +6767,6 @@ shared_done:free(group_x);free(group_y);free(group_row); free(group_weight);#endif-#ifdef COLI_VULKAN- free(vk_xh); free(vk_yh);-#endif}static void dense_mlp(Layer *l, float *x, int S, int D, int I, float *out){@@ -9960,7 +10015,11 @@ static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){printf("DONE %llu STAT %d %.2f %.1f %.2f %d %d\n",r->id,r->emitted,r->emitted/dt,(dh+dm)>0?100.0*dh/(dh+dm):0.0,rss_gb(),r->prompt_tokens,r->length_limited);- fflush(stdout); kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len);+ fflush(stdout);+#ifdef COLI_VULKAN+ vk_tier_turn(m); /* the expert tier's line, on stderr */+#endif+ kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len);/* PROF window = this request's lifetime; with KV_SLOTS>1 concurrent slots* share the batched forwards, so the shares describe the engine, not the* single request (same convention as the STAT hit%% above). */@@ -10453,6 +10512,9 @@ static void run_serve(Model *m, const char *snap){printf("\n\x01\x01" "END" "\x01\x01\n");printf("STAT %d %.2f %.1f %.2f\n", prod, prod/tdt, (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb());fflush(stdout);+#ifdef COLI_VULKAN+ vk_tier_turn(m); /* the expert tier's line, on stderr */+#endifif(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */if(nr<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; }@@ -10549,6 +10611,9 @@ static void run_serve(Model *m, const char *snap){agree_pct,kl_mean);printf("\n");fflush(stdout);+#ifdef COLI_VULKAN+ vk_tier_turn(m); /* the expert tier's line, on stderr */+#endifif(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */free(raw); g_temp=base_temp; g_nuc=base_nuc;usage_save(m); /* la cache che impara: storia aggiornata a ogni turno */@@ -10573,15 +10638,6 @@ static void run_serve(Model *m, const char *snap){/* telemetry, stats, usage persistence β moved to telemetry.h */#ifdef COLI_VULKAN-/* Pinned VK expert tier fill: upload the top-COLI_VK_EXPERTS heat-ranked routed experts- * ONCE at startup (mirrors the HIP VRAM tier: stable residency, zero uploads at decode).- * Ranking comes from the persistent usage history; already-pinned RAM slots feed the- * upload directly, the rest stream through one transient slot and are freed after. */-typedef struct { uint32_t u; int layer, eid; } VkCand;-static int vk_cand_cmp(const void *a, const void *b){- uint32_t ua=((const VkCand*)a)->u, ub=((const VkCand*)b)->u;- return ua<ub ? 1 : ua>ub ? -1 : 0;-}/* Upload the VK dense working set (attention projections, absorb kv_b, o-proj,* shared expert) at startup, BEFORE the tier fill. These ~8 GB otherwise* allocate lazily on the first forwards β AFTER the tier has filled to its@@ -10613,107 +10669,194 @@ static void vk_dense_preload(Model *m){nt,bytes/1e9,now_s()-t0);}-static void vk_registry_fill(Model *m){+/* ---- COLI_VULKAN=1: the routed experts' places on the device, at startup ----------+ * vk_tier_start describes the routed experts to the shared tier (vk_tier.c) and warms+ * it from the history; vk_dev2_fill then gives COLI_VK_DEV2's registry the hottest+ * experts the tier did not take. */++/* The format one routed expert tensor will have in RAM, from the container's header+ * alone (no read): qt_resolve_fmt for a quantized container (the .qs companion), else+ * the format qt_alloc gives the bits the experts are quantized to at load. -1: no such+ * tensor. */+static int vk_expert_fmt(Model *m,int layer,int eid,int k,int *gs){+ Cfg *c=&m->c; int O = k<2 ? c->moe_inter : c->hidden, I = k<2 ? c->hidden : c->moe_inter;+ const char suf[3][16]={"gate_proj","up_proj","down_proj"};+ char nm[288], qn[320], q0[320];+ snprintf(nm,sizeof nm,"model.layers.%d.mlp.experts.%d.%s.weight",layer,eid,suf[k]);+ snprintf(qn,sizeof qn,"%s.qs",nm);+ snprintf(q0,sizeof q0,"model.layers.%d.mlp.experts.%d.gate_proj.weight.qs",layer,eid);+ *gs=0;+ st_tensor *tw=st_find(&m->S,nm);+ if(!tw) return -1;+ if(st_has(&m->S,q0)){ /* expert_load_impl's own test */+ st_tensor *tq=st_find(&m->S,qn);+ if(!tq) return -1;+ return qt_resolve_fmt(nm,O,I,tw->nbytes,tq->nbytes,gs,st_fmt_stamp(&m->S,nm));+ }+ int b=m->ebits; /* qt_alloc's rule */+ return b>=16 ? 0 : (b>=5||g_nopack) ? 1 : b>=4 ? 2 : b==3 ? 5 : 3;+}+/* The tier's source kind for a colibri QT format; 0 when the device has no form of it+ * (int2, the E8/IQ3 lattice of fmt=6 with its rotated input, fp8): those stay on the CPU. */+static int vk_src_kind(int fmt,int gs,VktFmt *f){+ switch(fmt){+ case 0: *f=(VktFmt){VKT_SRC_F32,0}; return 1;+ case 1: *f=(VktFmt){VKT_SRC_I8_ROW,0}; return 1;+ case 2: *f=(VktFmt){VKT_SRC_I4U_PAIRS_ROW,0}; return 1;+ case 4: *f=(VktFmt){VKT_SRC_I4U_PAIRS_GS,gs}; return gs>=8 && gs%8==0;+ case 5: *f=(VktFmt){VKT_SRC_I3_G64,64}; return 1;+ default: return 0;+ }+}+static int vk_in_ram(void *ctx,int layer,int eid){+ Model *m=ctx;+ return pin_indexed(m,layer,eid)!=NULL || ecache_indexed(m,layer,eid,0)!=NULL;+}+/* Is this model's expert set quantized at load (no .qs companions: qt_from_disk's+ * qt_alloc buffers, one per tensor) rather than read from a container (a slab, or+ * views of a COLI_MMAP mapping that own nothing)? */+static int vk_experts_rtq(Model *m,int layer){+ char q0[320]; snprintf(q0,sizeof q0,"model.layers.%d.mlp.experts.0.gate_proj.weight.qs",layer);+ return !st_has(&m->S,q0);+}+/* A startup slot's memory, then the slot cleared. */+static void vk_tmp_slot_free(ESlot *t,int rtq){+ if(t->slab){ compat_aligned_free(t->slab); free(t->fslab); }+ else if(rtq){ QT *q[3]={&t->g,&t->u,&t->d};+ for(int k=0;k<3;k++){ free(q[k]->qf); free(q[k]->q8); free(q[k]->q4); free(q[k]->s); } }+ memset(t,0,sizeof(*t)); t->eid=-1;+}+static Model *g_vk_model; /* the run's report at exit */+static void vk_tier_report_run(void){+ if(g_vk_model) vkt_report("run",g_vk_model->hit_pin+g_vk_model->hit_ecache,g_vk_model->miss);+}+static void vk_tier_turn(Model *m){ vkt_report("turn",m->hit_pin+m->hit_ecache,m->miss); }+static int vk_tier_resident_count(Model *m){+ int n=0; if(!vkt_ready()) return 0;+ for(int i=0;i<m->c.n_layers;i++) for(int e=0;e<m->c.n_experts;e++) n+=vkt_resident(i,e);+ return n;+}+static void vk_tier_start(Model *m){+ Cfg *c=&m->c; int E=c->n_experts, NL=c->n_layers;+ if(!g_vulkan || E<=0 || g_vk_experts==0 || !vkt_wanted()) return;+#ifdef COLI_CUDA+ if(g_cuda_enabled){ fprintf(stderr,"[VK] tier colibri: the CUDA expert tier is on and wins; the Vulkan tier stays off\n"); return; }+#endif+ int pl=-1; for(int i=0;i<NL;i++) if(m->L[i].sparse){ pl=i; break; }+ if(pl<0) return;+ int f[3], gs[3]; for(int k=0;k<3;k++) f[k]=vk_expert_fmt(m,pl,0,k,&gs[k]);+ VktFmt gu, dn;+ if(f[0]!=f[1] || gs[0]!=gs[1] || !vk_src_kind(f[0],gs[0],&gu) || !vk_src_kind(f[2],gs[2],&dn)){+ fprintf(stderr,"[VK] tier colibri: experts in fmt %d/%d/%d (gate/up/down) have no device form "+ "(int2, E8/IQ3 fmt=6 and fp8 stay on the CPU, gate and up must match), the routed experts stay on the CPU\n",f[0],f[1],f[2]);+ return;+ }+ /* What the device still has to hold besides the experts: the absorb core's KV+ * mirror (COLI_VK_ATTN, allocated at the first decode, CTX rows a layer), and+ * COLI_VK_RESERVE_GB, the old name of this reserve, beyond the tier's own. The dense+ * set is already on the device (vk_dense_preload ran first) and counted as used. */+ double dense=0;+ if(g_vk_attn){ int ctx=getenv("CTX")?atoi(getenv("CTX")):4096; if(ctx<1) ctx=4096;+ dense+=(double)NL*ctx*(c->kv_lora+c->qk_rope)*4.0; }+ { const char *r=getenv("COLI_VK_RESERVE_GB"), *tr=getenv("COLI_VK_TIER_RESERVE_GB");+ if(r && *r){ double extra=atof(r)-(tr&&*tr?atof(tr):1.0); if(extra>0) dense+=extra*1073741824.0; } }+ VktConfig vc={.engine="colibri", .layers=NL, .experts=E, .hidden=c->hidden, .inter=c->moe_inter, .topk=c->topk,+ .gate_up=gu, .down=dn, .act=VKT_ACT_SWIGLU, .act_limit=0.f,+ .max_rows=GLM_VK_ROWS*c->topk,+ .ram_reserve=(size_t)((double)m->ecap*expert_cache_row_bytes(m,m->ebits)),+ .dense_bytes=(size_t)dense,+ .in_ram=vk_in_ram, .ram_ctx=m,+ .max_experts=g_vk_experts>0 ? g_vk_experts : 0};+ if(!vkt_init(&vc,m->eusage)) return;+ g_vkt.on=1; g_vkt.gu_fmt=f[0]; g_vkt.gu_gs=gs[0]; g_vkt.dn_fmt=f[2]; g_vkt.dn_gs=gs[2];+ g_vkt.exp_bytes=vkt_expert_bytes(c->hidden,c->moe_inter,gu,dn);+ atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */+ atexit(vkt_shutdown);+ g_vk_model=m; atexit(vk_tier_report_run); /* runs first: the tier is still up */+ /* warm start: the history's hottest experts, read in parallel (pinned ones from RAM) */+ int all=NL*E;+ int *ql=malloc((size_t)all*sizeof(int)), *qe=malloc((size_t)all*sizeof(int));+ const char *warm=getenv("COLI_VK_TIER_WARM"); /* 0: no warm start, the tier fills as experts pass by */+ int n = ql&&qe&&!(warm&&*warm=='0') ? vkt_plan(ql,qe,all) : 0;+ if(n>0){+ double t0=now_s(); int nt=omp_get_max_threads();+ ESlot *tmp=xzalloc((size_t)nt*sizeof(ESlot),"vk warm slots");+ for(int t=0;t<nt;t++) tmp[t].eid=-1;+ #pragma omp parallel for schedule(dynamic,4)+ for(int i=0;i<n;i++){+ VktExpertSrc vs; ESlot *p=pin_indexed(m,ql[i],qe[i]);+ if(p && vk_slot_src(p,&vs)){ vkt_put(ql[i],qe[i],&vs); continue; }+ ESlot *t=&tmp[omp_get_thread_num()];+ int ok = expert_load(m,ql[i],qe[i],t,0,0)==0 && vk_slot_src(t,&vs); /* demand=0: startup */+ vkt_put(ql[i],qe[i],ok?&vs:NULL); /* NULL: planned but not placed (a mixed container) */+ }+ vkt_put_done();+ for(int t=0;t<nt;t++) vk_tmp_slot_free(&tmp[t],vk_experts_rtq(m,pl));+ free(tmp);+ fprintf(stderr,"[VK] tier colibri: warm start, %d experts from the history in %.1fs\n",n,now_s()-t0);+ }+ free(ql); free(qe);+}++/* COLI_VK_DEV2: the second device's registry takes the hottest experts of the history+ * that the tier does not hold, up to COLI_VK_EXPERTS2 and COLI_VK_RESERVE2_GB, uploaded+ * once (int4 per row, int4-gs and int3-g64 experts). */+typedef struct { uint32_t u; int layer, eid; } VkCand;+static int vk_cand_cmp(const void *a, const void *b){+ uint32_t ua=((const VkCand*)a)->u, ub=((const VkCand*)b)->u;+ return ua<ub ? 1 : ua>ub ? -1 : 0;+}+static void vk_dev2_fill(Model *m){Cfg *c=&m->c; int E=c->n_experts, NL=c->n_layers;- if(!g_vulkan || g_vk_budget<=0) return;+ if(!g_vulkan || g_vk_budget2<=0 || !coli_vk_dev2_available()) return;int64_t nz=0;for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++) if(m->eusage[i][e]) nz++;- if(!nz){ fprintf(stderr,"[VK] expert tier: no usage history yet β tier empty this run "+ if(!nz){ fprintf(stderr,"[VK] dev2 tier: no usage history yet β tier empty this run ""(it seeds from %s as you use the model)\n", g_usage_path); return; }VkCand *cand=malloc((size_t)nz*sizeof(VkCand)); if(!cand) return;int64_t n=0;for(int i=0;i<NL;i++) if(m->eusage[i]) for(int e=0;e<E;e++)- if(m->eusage[i][e]) cand[n++]=(VkCand){m->eusage[i][e],i,e};+ if(m->eusage[i][e] && !vkt_resident(i,e)) cand[n++]=(VkCand){m->eusage[i][e],i,e};qsort(cand,(size_t)n,sizeof(VkCand),vk_cand_cmp);g_vk_reg=calloc((size_t)NL*E*3,sizeof(*g_vk_reg)); g_vk_reg_E=E; g_vk_reg_NL=NL;if(!g_vk_reg){ free(cand); return; }ESlot tmp; memset(&tmp,0,sizeof(tmp)); tmp.eid=-1;- double t0=now_s(); int64_t bytes=0; int tried=0, loadfail=0;- /* Pressure-proofing: tier weights are the EVICTABLE class (0.4 vs scratch/KV 1.0,- * dense 0.75) so an oversubscribed heap sheds cold experts instead of thrashing- * the per-token attention submits; and the fill STOPS while the device-local- * budget still holds COLI_VK_RESERVE_GB (default 3) for the lazily-allocated- * dense weights + KV mirror + staging (measured ~1.7 GB at 4k ctx, growing with- * max_t). Without the budget extension the count cap alone applies, as before. */- double vkr_reserve = getenv("COLI_VK_RESERVE_GB")?atof(getenv("COLI_VK_RESERVE_GB")):3.0;- int vkr_stopped=0; double vkr_used=0, vkr_budget=0;- int64_t i2=0;- coli_vk_alloc_priority(0.4f);- for(;i2<n && g_vk_reg_n<g_vk_budget;i2++){- if(vkr_reserve>0 && (g_vk_reg_n&7)==0 && coli_vk_mem_budget(&vkr_used,&vkr_budget)- && vkr_budget-vkr_used < vkr_reserve){ vkr_stopped=1; break; }- int layer=cand[i2].layer, eid=cand[i2].eid; tried++;+ double t20=now_s(); int64_t bytes2=0; int tried2=0, loadfail=0;+ double vkr2_reserve = getenv("COLI_VK_RESERVE2_GB")?atof(getenv("COLI_VK_RESERVE2_GB")):0.5;+ int vkr2_stopped=0; double u2=0,b2=0;+ for(int64_t i2=0;i2<n && g_vk_reg_n2<g_vk_budget2;i2++){+ if(vkr2_reserve>0 && (g_vk_reg_n2&7)==0 && coli_vk_mem_budget2(&u2,&b2)+ && b2-u2 < vkr2_reserve){ vkr2_stopped=1; break; }+ int layer=cand[i2].layer, eid=cand[i2].eid; tried2++;ESlot *src=pin_indexed(m,layer,eid);if(src&&!src->slab) src=NULL;- if(!src){ if(expert_load(m,layer,eid,&tmp,0,0)!=0){ /* 0 = success (impl convention); demand=0: startup tier fill */- if(++loadfail<4) fprintf(stderr,"[VK] tier fill: expert_load(%d,%d) failed\n",layer,eid);- if(loadfail>=64) break; /* disk trouble: stop burning time */+ if(!src){ if(expert_load(m,layer,eid,&tmp,0,0)!=0){ /* 0 = success; demand=0: startup fill */+ if(++loadfail>=64) break; /* disk trouble: stop burning time */continue; } src=&tmp; }- int xf=src->g.fmt; /* int4 (2), grouped int4 (4), int3-g64 (5) tiers; gate/up must- * share fmt for the fused gate_up shader, down may differ */+ int xf=src->g.fmt; /* int4 (2), grouped int4 (4), int3-g64 (5); gate/up share fmt+ * for the fused gate_up shader, down may differ */if((xf!=2&&xf!=4&&xf!=5)||src->u.fmt!=xf||src->u.gs!=src->g.gs||(src->d.fmt!=2&&src->d.fmt!=4&&src->d.fmt!=5)- ||(xf==4&&(src->g.gs<8||src->g.gs%8))||(src->d.fmt==4&&(src->d.gs<8||src->d.gs%8))){- if(tried<4) fprintf(stderr,"[VK] tier fill: (%d,%d) fmt %d/%d/%d not int4/int3-g64 (src=%s)\n",- layer,eid,src->g.fmt,src->u.fmt,src->d.fmt,src==&tmp?"load":"pin");- continue; }+ ||(xf==4&&(src->g.gs<8||src->g.gs%8))||(src->d.fmt==4&&(src->d.gs<8||src->d.gs%8)))+ continue;ColiVkTensor **slot=vk_reg_at(layer,eid);- if(!coli_vk_tensor_ensure(&slot[0],src->g.q4,src->g.s,xf,c->hidden,c->moe_inter,src->g.gs)||- !coli_vk_tensor_ensure(&slot[1],src->u.q4,src->u.s,xf,c->hidden,c->moe_inter,src->u.gs)||- !coli_vk_tensor_ensure(&slot[2],src->d.q4,src->d.s,src->d.fmt,c->moe_inter,c->hidden,src->d.gs)){+ if(!coli_vk_tensor_ensure2(&slot[0],src->g.q4,src->g.s,xf,c->hidden,c->moe_inter,src->g.gs)||+ !coli_vk_tensor_ensure2(&slot[1],src->u.q4,src->u.s,xf,c->hidden,c->moe_inter,src->u.gs)||+ !coli_vk_tensor_ensure2(&slot[2],src->d.q4,src->d.s,src->d.fmt,c->moe_inter,c->hidden,src->d.gs)){if(slot[0]){coli_vk_tensor_free(slot[0]);slot[0]=NULL;}if(slot[1]){coli_vk_tensor_free(slot[1]);slot[1]=NULL;}- fprintf(stderr,"[VK] expert tier: VRAM full after %d experts\n",g_vk_reg_n);+ fprintf(stderr,"[VK] dev2 tier: VRAM full after %d experts\n",g_vk_reg_n2);break;}- bytes+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);- g_vk_reg_n++;- }- coli_vk_alloc_priority(0.75f); /* back to the dense/default class */- fprintf(stderr,"[VK] expert tier: %d hot experts resident (%.2f GB VRAM, %.1fs, top-%d of history)\n",- g_vk_reg_n,bytes/1e9,now_s()-t0,tried);- if(vkr_stopped)- fprintf(stderr,"[VK] expert tier: budget stop at %d experts β %.1f of %.1f GB device-local used, %.1f GB reserved (COLI_VK_RESERVE_GB)\n",- g_vk_reg_n,vkr_used,vkr_budget,vkr_reserve);- /* DEV2 tier: continue down the heat ranking onto the second GPU (COLI_VK_DEV2),- * starting at the candidate dev0 stopped on. Same fmt gates, its own budget. */- if(g_vk_budget2>0 && coli_vk_dev2_available()){- double t20=now_s(); int64_t bytes2=0; int tried2=0;- double vkr2_reserve = getenv("COLI_VK_RESERVE2_GB")?atof(getenv("COLI_VK_RESERVE2_GB")):0.5;- int vkr2_stopped=0; double u2=0,b2=0;- for(;i2<n && g_vk_reg_n2<g_vk_budget2;i2++){- if(vkr2_reserve>0 && (g_vk_reg_n2&7)==0 && coli_vk_mem_budget2(&u2,&b2)- && b2-u2 < vkr2_reserve){ vkr2_stopped=1; break; }- int layer=cand[i2].layer, eid=cand[i2].eid; tried2++;- ESlot *src=pin_indexed(m,layer,eid);- if(src&&!src->slab) src=NULL;- if(!src){ if(expert_load(m,layer,eid,&tmp,0,0)!=0){- if(++loadfail>=64) break;- continue; } src=&tmp; }- int xf=src->g.fmt;- if((xf!=2&&xf!=4&&xf!=5)||src->u.fmt!=xf||src->u.gs!=src->g.gs- ||(src->d.fmt!=2&&src->d.fmt!=4&&src->d.fmt!=5)- ||(xf==4&&(src->g.gs<8||src->g.gs%8))||(src->d.fmt==4&&(src->d.gs<8||src->d.gs%8)))- continue;- ColiVkTensor **slot=vk_reg_at(layer,eid);- if(!coli_vk_tensor_ensure2(&slot[0],src->g.q4,src->g.s,xf,c->hidden,c->moe_inter,src->g.gs)||- !coli_vk_tensor_ensure2(&slot[1],src->u.q4,src->u.s,xf,c->hidden,c->moe_inter,src->u.gs)||- !coli_vk_tensor_ensure2(&slot[2],src->d.q4,src->d.s,src->d.fmt,c->moe_inter,c->hidden,src->d.gs)){- if(slot[0]){coli_vk_tensor_free(slot[0]);slot[0]=NULL;}- if(slot[1]){coli_vk_tensor_free(slot[1]);slot[1]=NULL;}- fprintf(stderr,"[VK] dev2 tier: VRAM full after %d experts\n",g_vk_reg_n2);- break;- }- bytes2+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);- g_vk_reg_n2++;- }- fprintf(stderr,"[VK] dev2 tier: %d experts resident (%.2f GB VRAM, %.1fs, next-%d of history)\n",- g_vk_reg_n2,bytes2/1e9,now_s()-t20,tried2);- if(vkr2_stopped)- fprintf(stderr,"[VK] dev2 tier: budget stop at %d experts β %.1f of %.1f GB device-local used, %.1f GB reserved (COLI_VK_RESERVE2_GB)\n",- g_vk_reg_n2,u2,b2,vkr2_reserve);+ bytes2+=coli_vk_tensor_bytes(slot[0])+coli_vk_tensor_bytes(slot[1])+coli_vk_tensor_bytes(slot[2]);+ g_vk_reg_n2++;}- if(tmp.slab){ compat_aligned_free(tmp.slab); free(tmp.fslab); }+ fprintf(stderr,"[VK] dev2 tier: %d experts resident (%.2f GB VRAM, %.1fs, next-%d of history)\n",+ g_vk_reg_n2,bytes2/1e9,now_s()-t20,tried2);+ if(vkr2_stopped)+ fprintf(stderr,"[VK] dev2 tier: budget stop at %d experts β %.1f of %.1f GB device-local used, %.1f GB reserved (COLI_VK_RESERVE2_GB)\n",+ g_vk_reg_n2,u2,b2,vkr2_reserve);+ for(int i=0;i<NL;i++) if(m->L[i].sparse){ vk_tmp_slot_free(&tmp,vk_experts_rtq(m,i)); break; }free(cand);}#endif@@ -12321,14 +12464,26 @@ int main(int argc, char **argv){if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (tried %s; need libvulkan + ""the compiled shaders β point COLI_VK_SHADERS at the shader directory ""or the qmatmul.spv file, or run `make VK=1` to build them)\n", spv); return 2; }- /* 320 = sweep optimum on a 16 GB card (256-384 measured flat, 320 best median;- * ~6 GB tier + ~8 GB dense leaves headroom for the long-context KV mirror). */- g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 320;- g_vk_dense = coli_vk_dense_decide(NULL, g_vk_budget > 0, 0); /* COLI_VK_DENSE; this engine's default is off */+ /* The routed experts go to the shared tier (vk_tier.c), sized by its budget+ * (COLI_VK_TIER_GB). COLI_VK_EXPERTS, the count of the fixed set this engine+ * uploaded before (320 by default), is kept as a deprecated alias: N caps the+ * tier at N experts, 0 turns it off as COLI_VK_TIER=0 does. */+ { const char *ve=getenv("COLI_VK_EXPERTS"); g_vk_experts = ve&&*ve ? atoi(ve) : -1; }+ int vk_tier_on = vkt_wanted() && g_vk_experts!=0;+ g_vk_dense = coli_vk_dense_decide(NULL, vk_tier_on, 0); /* COLI_VK_DENSE; this engine's default is off */g_vk_attn = getenv("COLI_VK_ATTN") ? atoi(getenv("COLI_VK_ATTN")) : 0;- fprintf(stderr,"[VK] expert tier active: routed quantized experts on the GPU (budget %d)%s%s\n",- g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "",- g_vk_attn ? " + absorb attention core" : "");+ if(vk_tier_on)+ fprintf(stderr,"[VK] expert tier active: routed experts on the shared adaptive tier (vk_tier.c)%s%s\n",+ g_vk_dense ? " + dense projections + shared expert" : "",+ g_vk_attn ? " + absorb attention core" : "");+ else+ fprintf(stderr,"[VK] expert tier off (%s): routed experts on the CPU%s%s\n",+ g_vk_experts==0 ? "COLI_VK_EXPERTS=0" : "COLI_VK_TIER=0",+ g_vk_dense ? "; dense projections + shared expert on the GPU" : "",+ g_vk_attn ? "; absorb attention core on the GPU" : "");+ if(g_vk_experts>0)+ fprintf(stderr,"[VK] COLI_VK_EXPERTS=%d is deprecated: it caps the shared tier at %d experts; "+ "COLI_VK_TIER_GB sizes it in GiB\n", g_vk_experts, g_vk_experts);/* COLI_VK_DEV2=auto|<index>: bring up a SECOND GPU for tier experts only* (e.g. an RX 580 beside the primary card). The dev2 tier fills with the* next heat-ranked experts after dev0's budget stop, capped by@@ -12724,43 +12879,9 @@ int main(int argc, char **argv){{ double ram_env = getenv("RAM_GB")?atof(getenv("RAM_GB")):0.0;int est_ctx = getenv("CTX")?atoi(getenv("CTX")):4096; /* stesso default di run_serve */snprintf(g_usage_path,sizeof(g_usage_path),"%s/.coli_usage",snap);-#ifdef COLI_VULKAN- /* #653's correction, for the Vulkan tier. On an integrated GPU the tier's- * HOST_VISIBLE|DEVICE_LOCAL allocation is the SAME physical RAM that- * expert_avail()/cap_for_ram() below hand to the pin set and the LRU.- * Unlike the CUDA tier this one cannot be subtracted after the fact:- * vk_registry_fill() runs at the END of init, long after both decisions- * are made, so the planned size has to be reserved here instead. Sized- * from a routed layer's row width x the configured expert count.- * Discrete GPUs have their own pool -> deviceType is not INTEGRATED and- * this is a no-op, as with #653. */- if(g_vulkan && g_vk_budget>0 && g_mem_avail_boot>0 && coli_vk_device_integrated()){- int probe_l = m.c.n_layers>1 ? m.c.n_layers/2 : 0;- double per = (double)expert_bytes_row(&m,probe_l,m.ebits);- double tier_gb = per>0 ? (double)g_vk_budget*per/1e9 : 0.0;- /* COLI_VK_EXPERTS is a REQUEST, not a placement: vk_registry_fill() stops- * early when the device-local budget runs out (COLI_VK_RESERVE_GB), so- * pricing the request would over-reserve badly -- measured 95.6 GB reserved- * against 66.0 GB actually placed at 4500, and at 6000 the unclamped- * reservation starved MemAvailable to the 1 GB floor and killed the run.- * Clamp to what the device can actually take, and never take so much that- * the host side has nothing left to plan with. */- double vk_used=0, vk_bud=0;- if(tier_gb>0 && coli_vk_mem_budget(&vk_used,&vk_bud) && vk_bud>vk_used){- double reserve = getenv("COLI_VK_RESERVE_GB")?atof(getenv("COLI_VK_RESERVE_GB")):3.0;- double placeable = vk_bud - vk_used - reserve;- if(placeable>0 && tier_gb>placeable) tier_gb = placeable;- }- double host_floor = g_mem_avail_boot*0.35; /* the planner keeps at least this */- if(tier_gb > g_mem_avail_boot - host_floor) tier_gb = g_mem_avail_boot - host_floor;- if(tier_gb>0){- g_mem_avail_boot -= tier_gb;- fprintf(stderr,"[VK] integrated/unified memory: expert tier will share physical RAM; "- "RAM budget snapshot reduced by %.2f GB (%d experts requested) -> MemAvailable=%.1f GB\n",- tier_gb, g_vk_budget, g_mem_avail_boot);- }- }-#endif+ /* An integrated GPU's expert tier is RAM too. It is not reserved here: the shared+ * tier (vk_tier_start, at the end of init) sizes itself from what MemAvailable has+ * left once this LRU has grown to its cap (VktConfig.ram_reserve). */int64_t hist = usage_load(&m,g_usage_path);if(hist>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",(long long)hist,g_usage_path);int autopin = getenv("AUTOPIN")?atoi(getenv("AUTOPIN")):1;@@ -12818,8 +12939,9 @@ int main(int argc, char **argv){g_prof = getenv("PROF")?atoi(getenv("PROF")):0; /* PROF=1: opt-in performance profile */if(g_prof) prof_config(&m, ram_env, est_ctx); }#ifdef COLI_VULKAN- vk_dense_preload(&m); /* dense claims VRAM first β the tier fill sizes to the remainder */- vk_registry_fill(&m); /* pinned VK expert tier: needs the usage history loaded above */+ vk_dense_preload(&m); /* dense claims VRAM first β the tier sizes to the remainder */+ vk_tier_start(&m); /* the shared expert tier: needs the usage history and the cap above */+ vk_dev2_fill(&m); /* COLI_VK_DEV2: the hottest experts the tier does not hold */#endifconst char *stats=getenv("STATS"); /* STATS=<file> -> istogramma uso expert a fine run */@@ -12945,9 +13067,11 @@ int main(int argc, char **argv){double tot=m.hits+m.miss;printf("N-gram speculation (DRAFT=%d): %.2f tokens/forward (%llu forwards per %llu tokens)\n",g_draft, m.n_fw?(double)m.n_emit/m.n_fw:1.0, (unsigned long long)m.n_fw, (unsigned long long)m.n_emit);- printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru / %llu miss) | RSS: %.2f GB | %.1f tok/s\n",+ char vkhits[48]=""; /* experts a Vulkan device served (#336's split) */+ if(m.hit_vk) snprintf(vkhits,sizeof vkhits," + %llu vk",(unsigned long long)m.hit_vk);+ printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru%s / %llu miss) | RSS: %.2f GB | %.1f tok/s\n",tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hit_pin, (unsigned long long)m.hit_ecache,- (unsigned long long)m.miss, rss_gb(), n_new/dt);+ vkhits, (unsigned long long)m.miss, rss_gb(), n_new/dt);profile_print(&m,dt);if(g_prof) prof_report(&m,&pb,dt,n_new,stdout);#ifdef COLI_CUDAdiff --git a/c/telemetry.h b/c/telemetry.hindex 52ab105f..94f13f51 100644--- a/c/telemetry.h+++ b/c/telemetry.h@@ -193,8 +193,9 @@ static void tiers_emit(Model *m){#ifdef COLI_CUDAvram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9;#elif defined(COLI_VULKAN)- vram=g_vk_reg_n+g_vk_reg_n2;- int64_t bytes=0;+ int vk_tier_n=vk_tier_resident_count(m); /* the shared tier's residents, then dev2's */+ vram=vk_tier_n+g_vk_reg_n2;+ int64_t bytes=(int64_t)vk_tier_n*(int64_t)g_vkt.exp_bytes;if(g_vk_reg) for(int i=0;i<g_vk_reg_NL;i++) for(int e=0;e<g_vk_reg_E;e++){ColiVkTensor **slot=vk_reg_at(i,e);for(int j=0;j<3;j++) if(slot[j]) bytes+=(int64_t)coli_vk_tensor_bytes(slot[j]);@@ -234,7 +235,7 @@ static void emap_emit(Model *m){for(int e=0;e<cols;e++){int tier=0;#ifdef COLI_VULKAN- if(vk_reg_served(i,e)) tier=2;+ if(vkt_resident(i,e) || vk_reg_served(i,e)) tier=2; /* on a Vulkan device */#endifESlot *P=m->pin[i];for(int z=0;!tier&&z<m->npin[i];z++) if(P[z].eid==e){