Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(vulkan): colibri's partial dense chain
JustVuggd1460173f962
diff --git a/c/colibri.c b/c/colibri.cindex d6620c5d..471838f4 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -665,6 +665,11 @@ static int vk_tier_resident_count(Model *m);static void vk_tier_turn(Model *m); /* "[VK] tier colibri turn: ..." after a serve turn */static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention* projections + shared expert) on Vulkan too */+/* A partial chain (glm_chain.h, vkc_fit): the device holds the first N layers only, or not+ * the head and the MTP layer. The per-matrix path then multiplies on the device only what+ * is there already: a matrix without a device copy stays on the CPU (VK_MAY). */+static int g_glmc_partial;+#define VK_MAY(t) (!g_glmc_partial || (t)->vk != NULL)static int g_vk_budget2; /* COLI_VK_EXPERTS2: dev2 expert-tier cap (with COLI_VK_DEV2) */static int g_vk_reg_n2; /* experts resident on the dev2 tier *//* Resolve the main shader path (#523): COLI_VK_SHADERS may be the qmatmul.spv file itself@@ -772,13 +777,13 @@ static int qt_dho_matmul(QT *t, float *y, const float *x, int S){return coli_vk_matmul(&t->vk, y, x, NULL, NULL, qt_vk_fmt(t), S, t->I, t->O, t->fmt==4 ? t->gs : 0);}static int vk_matmul_qt(QT *t, float *y, const float *x, int S){- if(!g_vk_dense || !VK_FMT_OK(t)) return 0;+ if(!g_vk_dense || !VK_FMT_OK(t) || !VK_MAY(t)) return 0;const void *w = t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;return coli_vk_matmul(&t->vk, y, x, w, t->s, t->fmt, S, t->I, t->O, t->gs);}/* Two same-input resident matmuls in one submit (q_a + kv_a read the same x). */static int vk_matmul_pair_qt(QT *a, float *ya, QT *b, float *yb, const float *x, int S){- if(!g_vk_dense || a->fmt!=b->fmt || !VK_FMT_OK(a) || a->gs!=b->gs || a->I!=b->I) return 0;+ if(!g_vk_dense || a->fmt!=b->fmt || !VK_FMT_OK(a) || a->gs!=b->gs || a->I!=b->I || !VK_MAY(a) || !VK_MAY(b)) return 0;const void *wa = a->fmt==1 ? (const void*)a->q8 : (const void*)a->q4;const void *wb = b->fmt==1 ? (const void*)b->q8 : (const void*)b->q4;return coli_vk_matmul_pair(&a->vk, ya, wa, a->s, a->O,@@ -4599,7 +4604,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pfloat *dbgQ=NULL,*dbgC=NULL;if(g_vk_qprep==2){ dbgQ=falloc((int64_t)S*l->q_b.O); dbgC=falloc((int64_t)S*l->kv_a.O); }if(g_vk_qprep && g_vk_dense && l->q_a.fmt==l->kv_a.fmt && l->q_a.fmt==l->q_b.fmt && VK_FMT_OK(&l->q_a)- && l->q_a.gs==l->kv_a.gs && l->q_a.gs==l->q_b.gs)+ && l->q_a.gs==l->kv_a.gs && l->q_a.gs==l->q_b.gs && VK_MAY(&l->q_a) && VK_MAY(&l->kv_a) && VK_MAY(&l->q_b))vk_qp=coli_vk_attn_qprep(layer,&l->q_a.vk, l->q_a.fmt==1?(const void*)l->q_a.q8:(const void*)l->q_a.q4, l->q_a.s, l->q_a.O,&l->kv_a.vk, l->kv_a.fmt==1?(const void*)l->kv_a.q8:(const void*)l->kv_a.q4, l->kv_a.s, l->kv_a.O,@@ -4919,7 +4924,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p* on rewrite/rebind/resize). Falls back to CPU on DSA top-k selection, ragged KV,* the MTP layer, or any backend failure β output identical either way. */if(!cuda_core&&g_vk_attn&&!kvs&&!positions&&S<=4&&layer<c->n_layers&&- VK_FMT_OK(&l->kv_b)&&kvl<=512&&c->qk_nope<=256&&c->qk_rope<=64&&+ VK_FMT_OK(&l->kv_b)&&VK_MAY(&l->kv_b)&&kvl<=512&&c->qk_nope<=256&&c->qk_rope<=64&&m->vk_kv_valid&&m->Lc[layer]&&m->Rc[layer]){ /* f32 KV only: a quantized-KV cache* (upstream #399 KV8/TQ leaves Lc/Rc NULL)* falls back to the CPU path until the VK@@ -4935,7 +4940,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pm->vk_kv_valid[layer]=T;const void *kw=l->kv_b.fmt==1?(const void*)l->kv_b.q8:(const void*)l->kv_b.q4;/* fused absorb + o-projection: ctx never leaves the device */- if(VK_FMT_OK(&l->o)&&+ if(VK_FMT_OK(&l->o)&&VK_MAY(&l->o)&&coli_vk_attention_absorb_project(&l->kv_b.vk,kw,l->kv_b.s,l->kv_b.fmt,l->kv_b.gs,&l->o.vk,l->o.fmt==1?(const void*)l->o.q8:(const void*)l->o.q4,l->o.s,l->o.fmt,l->o.gs,out,Q,layer,S,H,c->qk_nope,c->qk_rope,@@ -6743,7 +6748,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* (x read once, 1 fence instead of 3). Falls through to the per-matmul chain. */int fsh=l->sh_gate.fmt;if(g_vk_dense && !omp_in_parallel() && (fsh==1||fsh==2||fsh==5) &&- l->sh_up.fmt==fsh && l->sh_down.fmt==fsh){+ l->sh_up.fmt==fsh && l->sh_down.fmt==fsh && VK_MAY(&l->sh_gate) && VK_MAY(&l->sh_up) && VK_MAY(&l->sh_down)){#define SW_(t) ((t).fmt==1?(const void*)(t).q8:(const void*)(t).q4)if(coli_vk_tensor_ensure(&l->sh_gate.vk,SW_(l->sh_gate),l->sh_gate.s,fsh,D,sI,l->sh_gate.gs)&&coli_vk_tensor_ensure(&l->sh_up.vk, SW_(l->sh_up), l->sh_up.s, fsh,D,sI,l->sh_up.gs)&&@@ -7500,6 +7505,11 @@ static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_blayer_forward_rows(m,l,li,x,S,pos_base,NULL,NULL,nrm,tmp);}#ifdef COLI_VULKAN+/* COLI_VK_DENSE_HOST around the chain's setup (below glm_chain.h): the decision, a placed+ * layer's host copies, the rest once the chain is set up */+static void glm_dho_start(Model *m);+static void glm_dho_drop_layer(Model *m, int i);+static void glm_dho_finish(Model *m);#include "glm_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */#endifstatic void layers_forward_rows_range(Model *m, float *x, int S, int pos_base,@@ -7510,10 +7520,15 @@ static void layers_forward_rows_range(Model *m, float *x, int S, int pos_base,#ifdef COLI_VULKAN/* the whole forward on the device (glm_chain.h); 0: the CPU path below, as before.* A decode batch of one row of the bound KV state (a single-slot serve) is a step- * like any other; rows of several states stay on the CPU. */+ * like any other; rows of several states stay on the CPU. A partial chain (the+ * device holds the first N layers) hands x back after layer N - 1: the CPU runs the+ * rest from there. */if(g_vk_chain && layer_begin==0 && layer_end==c->n_layers){- if(!kvs && !positions && glmc_forward(m,x,S,pos_base)) return;- if(kvs && S==1 && kvs[0]==m->kv && glmc_forward(m,x,1,positions?positions[0]:pos_base)) return;+ int done=0;+ if(!kvs && !positions) done=glmc_forward(m,x,S,pos_base);+ else if(kvs && S==1 && kvs[0]==m->kv) done=glmc_forward(m,x,1,positions?positions[0]:pos_base);+ if(done>=c->n_layers) return;+ layer_begin=done;}#endifif(g_pilot_real){ /* nuovo forward: il possesso-layer riparte da -1 (i layer si rifanno da 0) */@@ -10736,7 +10751,7 @@ static void run_serve(Model *m, const char *snap){* and self-size to what actually fits in device memory. */static void vk_dense_preload(Model *m){Cfg *c=&m->c;- if(!g_vulkan || !g_vk_dense) return;+ if(!g_vulkan || !g_vk_dense || g_glmc_partial) return; /* a partial chain: nothing more goes up */double t0=now_s(); int64_t bytes=0; int nt=0, full=0;for(int i=0;i<=c->n_layers && !full;i++){Layer *l = i<c->n_layers ? &m->L[i] : &m->mtpL;@@ -10882,7 +10897,10 @@ static void vk_tier_start(Model *m){* set is already on the device (vk_dense_preload ran first) and counted as used. */double dense=0;if(g_vk_attn){ int ctx=getenv("CTX")?atoi(getenv("CTX")):4096; if(ctx<1) ctx=4096;- dense+=(double)NL*ctx*(c->kv_lora+c->qk_rope)*4.0; }+ dense+=(double)(g_glmc_partial ? g_glmc_fit.n : NL)*ctx*(c->kv_lora+c->qk_rope)*4.0; }+ /* a partial chain: nothing more of the trunk goes up, but the chain's first forward+ * allocates its scratch and the N layers' KV mirrors */+ if(g_glmc_partial) dense+=(double)g_glmc_lazy;{ const char *r=getenv("COLI_VK_RESERVE_GB"), *tr=getenv("COLI_VK_TIER_RESERVE_GB");if(r && *r){ double extra=atof(r)-(tr&&*tr?atof(tr):1.0); if(extra>0) dense+=extra*1073741824.0; } }VktConfig vc={.engine="colibri", .layers=NL, .experts=E, .hidden=c->hidden, .inter=c->moe_inter, .topk=c->topk,@@ -12269,27 +12287,36 @@ static void qt_dho_visit(QT *t, int keep, int drop, DhoPass *p){coli_vk_dense_host_dropped((size_t)b);p->n++; p->bytes+=b;}-static void glm_dho_pass(Model *m, int drop, DhoPass *p){+/* Layer i's matrices (i == n_layers: the MTP layer's) */+static void glm_dho_layer(Model *m, int i, int drop, DhoPass *p){Cfg *c=&m->c;- for(int i=0;i<=c->n_layers;i++){- if(i==c->n_layers&&!m->has_mtp) break;- Layer *l = i<c->n_layers ? &m->L[i] : &m->mtpL;- int mtp = i==c->n_layers;- qt_dho_visit(&l->q_a,0,drop,p); qt_dho_visit(&l->q_b,0,drop,p); qt_dho_visit(&l->kv_a,0,drop,p);- qt_dho_visit(&l->kv_b,mtp||p->keep_kvb,drop,p); qt_dho_visit(&l->o,0,drop,p);- if(l->sparse){ qt_dho_visit(&l->sh_gate,p->keep_shared,drop,p); qt_dho_visit(&l->sh_up,p->keep_shared,drop,p);- qt_dho_visit(&l->sh_down,p->keep_shared,drop,p); }- else { qt_dho_visit(&l->gate_proj,0,drop,p); qt_dho_visit(&l->up_proj,0,drop,p); qt_dho_visit(&l->down_proj,0,drop,p); }- if(!mtp&&m->ix_wq&&m->has_dsa&&c->idx_type[i]){- qt_dho_visit(&m->ix_wq[i],p->keep_kvb,drop,p); qt_dho_visit(&m->ix_wk[i],p->keep_kvb,drop,p);- qt_dho_visit(&m->ix_wp[i],p->keep_kvb,drop,p);- }- }+ Layer *l = i<c->n_layers ? &m->L[i] : &m->mtpL;+ int mtp = i==c->n_layers;+ qt_dho_visit(&l->q_a,0,drop,p); qt_dho_visit(&l->q_b,0,drop,p); qt_dho_visit(&l->kv_a,0,drop,p);+ qt_dho_visit(&l->kv_b,mtp||p->keep_kvb,drop,p); qt_dho_visit(&l->o,0,drop,p);+ if(l->sparse){ qt_dho_visit(&l->sh_gate,p->keep_shared,drop,p); qt_dho_visit(&l->sh_up,p->keep_shared,drop,p);+ qt_dho_visit(&l->sh_down,p->keep_shared,drop,p); }+ else { qt_dho_visit(&l->gate_proj,0,drop,p); qt_dho_visit(&l->up_proj,0,drop,p); qt_dho_visit(&l->down_proj,0,drop,p); }+ if(!mtp&&m->ix_wq&&m->has_dsa&&c->idx_type[i]){+ qt_dho_visit(&m->ix_wq[i],p->keep_kvb,drop,p); qt_dho_visit(&m->ix_wk[i],p->keep_kvb,drop,p);+ qt_dho_visit(&m->ix_wp[i],p->keep_kvb,drop,p);+ }+}+/* Layers [0, n) and, with tail, the MTP layer, eh_proj and lm_head */+static void glm_dho_pass(Model *m, int drop, DhoPass *p, int n, int tail){+ for(int i=0;i<n;i++) glm_dho_layer(m,i,drop,p);+ if(!tail) return;+ if(m->has_mtp) glm_dho_layer(m,m->c.n_layers,drop,p);if(m->has_mtp) qt_dho_visit(&m->eh_proj,0,drop,p);qt_dho_visit(&m->lm_head,0,drop,p);}-/* After the load, before the pins and cap_for_ram: the decision (the chain's own, made- * silently here and printed by glmc_start), the uploads and the RAM given back. */+static DhoPass g_glm_dho; /* the drop's keep rules and what it gave back */+static int g_glm_dho_on;+/* After the load, before the pins and cap_for_ram (glmc_start calls it once the chain's+ * fit is known): the decision (the chain's own, made silently here and printed by+ * glmc_start). With a fit only the N layers on the device drop their copies, each as+ * glmc_setup places it, and the MTP layer, eh_proj and lm_head only when the whole chain+ * and that tail fit (glm_dho_finish); without one, every matrix here, as before. */static void glm_dho_start(Model *m){if(!g_vulkan) return;int tier_on = vkt_wanted() && g_vk_experts!=0 && m->c.n_experts>0;@@ -12303,17 +12330,37 @@ static void glm_dho_start(Model *m){const char *xv = getenv("COLI_EXACT_VERIFY"); /* exact_verify_on() prints its line later, where it always did */p.keep_kvb = chain!=COLI_VK_CHAIN_ON || g_looka || g_pilot || (xv && atoi(xv)) || slots>1;p.keep_shared = g_pilot || g_looka;- glm_dho_pass(m, 0, &p);- if(!coli_vk_dense_host_decide("colibri", !cuda && (chain!=COLI_VK_CHAIN_OFF || g_vk_dense), (size_t)p.bytes)) return;+ int L = m->c.n_layers, n = g_glmc_fitted ? g_glmc_fit.n : L, tail = g_glmc_fitted ? g_glmc_fit.tail : 1;+ glm_dho_pass(m, 0, &p, n, tail);+ int on_device = !cuda && (g_glmc_fitted ? n>0 : (chain!=COLI_VK_CHAIN_OFF || g_vk_dense));+ if(!coli_vk_dense_host_decide("colibri", on_device, (size_t)p.bytes)) return;g_dho_model = m; g_dho_thread = pthread_self();g_vk_dense = 1; /* the steps the chain does not run take the device too: the CPU has no copy */p.n = 0; p.bytes = 0;- glm_dho_pass(m, 1, &p);- m->resident_bytes -= p.bytes; /* what cap_for_ram and autopin count in RAM */- char kept[256];- snprintf(kept, sizeof kept, "the embedding, routers and norms, the MTP layer's kv_b%s%s",- p.keep_kvb ? ", every kv_b and DSA indexer (the chain will not run every forward: the CPU's attention reads them)" : "",- p.keep_shared ? ", the shared experts (PILOT/LOOKA multiply them on the host)" : "");+ g_glm_dho = p; g_glm_dho_on = 1;+ if(g_glmc_fitted){ coli_vk_dense_host_layers(n, L); return; } /* glmc_setup drops each layer it places */+ glm_dho_pass(m, 1, &g_glm_dho, L, 1);+ glm_dho_finish(m);+}+/* glmc_setup placed all of layer i: its host copies go */+static void glm_dho_drop_layer(Model *m, int i){+ if(g_glm_dho_on) glm_dho_layer(m, i, 1, &g_glm_dho);+}+/* The tail (a full chain with room for it), the RAM given back, the line. */+static void glm_dho_finish(Model *m){+ if(!g_glm_dho_on) return;+ int part = 0;+ if(g_glmc_fitted){+ coli_vk_dense_host_layers(g_glmc_fit.n, m->c.n_layers);+ if(g_glmc_fit.tail) glm_dho_pass(m, 1, &g_glm_dho, 0, 1);+ part = vkc_fit_partial(&g_glmc_fit);+ }+ m->resident_bytes -= g_glm_dho.bytes; /* what cap_for_ram and autopin count in RAM */+ char kept[384];+ snprintf(kept, sizeof kept, "the embedding, routers and norms, the MTP layer's kv_b%s%s%s",+ g_glm_dho.keep_kvb ? ", every kv_b and DSA indexer (the chain will not run every forward: the CPU's attention reads them)" : "",+ g_glm_dho.keep_shared ? ", the shared experts (PILOT/LOOKA multiply them on the host)" : "",+ part ? ", lm_head, eh_proj and the MTP layer (the chain's tail runs on the CPU)" : "");coli_vk_dense_host_placed("colibri", kept);}#endif@@ -13045,7 +13092,10 @@ int main(int argc, char **argv){}if(getenv("DSA_TOPK")) m.c.index_topk=atoi(getenv("DSA_TOPK")); /* override per test */#ifdef COLI_VULKAN- glm_dho_start(&m); /* COLI_VK_DENSE_HOST: the trunk on the device only, before the pins and the RAM cap */+ /* COLI_VK_CHAIN: decided, fitted (how many layers the device holds) and placed, with+ * COLI_VK_DENSE_HOST's host copies of those layers given back, before the pins and the+ * RAM cap (they count only what was dropped) and before the tier sizes itself */+ glmc_start(&m);#endif/* Il path MUX (SERVE_BATCH=1, cioe' `coli serve`) forza g_draft=0 sotto β* la speculazione non e' ragged-safe nel batch multi-slot. Segnalarlo QUI,@@ -13187,7 +13237,6 @@ int main(int argc, char **argv){if(g_prof) prof_config(&m, ram_env, est_ctx); }#ifdef COLI_VULKANvk_dense_preload(&m); /* dense claims VRAM first β the tier sizes to the remainder */- glmc_start(&m); /* COLI_VK_CHAIN: the chain's trunk on the device, before the tier sizes itself */vk_tier_start(&m); /* the shared expert tier: needs the usage history and the cap above */vk_dev2_fill(&m); /* COLI_VK_DEV2: the hottest experts the tier does not hold */glmc_atexit(&m); /* after the tier's: the chain goes before the device */diff --git a/c/glm_chain.h b/c/glm_chain.hindex 57ae7349..9ee5b186 100644--- a/c/glm_chain.h+++ b/c/glm_chain.h@@ -53,12 +53,26 @@* of COLI_EXACT_VERIFY, the CUDA backend, and a model whose matrices or geometry its* shaders do not take (int2, E8/IQ3, fp8 dense matrices; kv_lora above 1024, qk_rope* above 128, qk_nope above 1024; an indexer above 64 heads or 4096 query floats). The- * MTP head runs on the CPU, as before: its own KV row is not mirrored. */+ * MTP head runs on the CPU, as before: its own KV row is not mirrored.+ *+ * A partial chain (vk_chain.h, vkc_fit; docs/vulkan.md, "A partial chain"): when the+ * device's free memory does not hold every layer, the chain takes layers 0..N-1 and the+ * CPU runs N..L-1, the final norm and lm_head (and the MTP head, eh_proj and lm_head stay+ * on the host unless the whole chain and that tail fit). N is decided at startup, before+ * any upload and before the pins and the RAM cap (glmc_start, from colibri.c's main right+ * after the load); with COLI_VK_DENSE_HOST only the N layers give their host copies back,+ * each once all of it reached the device. A forward runs every chunk of its rows through+ * the N device layers, then hands the rows to the CPU's layer loop from layer N. The+ * crossing: the residual rows (D floats a row) and, when layer N reuses a DSA selection+ * (a shared indexer layer after a full one on the device), that selection (1 + index_topk+ * ints a row), once per chunk. The device's KV mirror, its watermarks and the KV split+ * cover the N layers; the CPU layers' rows are the host's alone. */#include "vk_chain.h"#include "vk_kvsplit.h"typedef struct {int ok, failed;+ int n; /* layers on the device: 0..n-1 (the rest run on the CPU) */int rows; /* scratch capacity in rows */int cap; /* host positions the mirror covers */int dev_rows; /* device latent/rope rows a layer: cap, or the split's */@@ -72,6 +86,7 @@ typedef struct {VkcMlaScratch sc;VkcBuf *x, *nrm, *tmp, *h2, *gs, *us, *hs, *ds, *ikd, *iq, *ihw, *isc, *sel;VkcBuf *h2d, *kvd, *xd, *routed, *cs;+ VkcBuf *seld; /* a partial chain: the DSA selection layer n reuses, read back */size_t kvd_layer; /* floats per layer in kvd */float *host_routed;unsigned long long forwards;@@ -80,6 +95,11 @@ typedef struct {static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipelines are up */static int g_glmc_inited = 0; /* vkc_init ran: vkc_shutdown at exit, whatever came after */+static VkcFit g_glmc_fit; /* how many layers the device holds (vkc_fit), when g_glmc_fitted */+static int g_glmc_fitted;+static size_t g_glmc_lazy; /* what the chain allocates at its first forward: one chunk's+ * scratch and the N layers' KV mirrors at their first size */+static size_t *g_glmc_mir; /* each layer's KV mirror at its first size *//* The device's copy of a resident QT, uploaded once (the per-matrix path's own copy* when COLI_VK_DENSE put it there); f32 goes up as the backend's fmt 10. NULL: no@@ -98,12 +118,36 @@ static ColiVkTensor *glmc_tensor(QT *t) {if (!w || t->O <= 0 || t->I <= 0) return NULL;return coli_vk_tensor_ensure(&t->vk, w, t->fmt == 0 ? NULL : t->s, fmt, t->I, t->O, t->fmt == 4 ? t->gs : 0) ? t->vk : NULL;}+/* glmc_tensor's backend format for t without uploading it (-1: no device form), and its+ * group size; the fit counts the bytes from it. */+static int glmc_form(const QT *t, int *gs) {+ *gs = 0;+ if (t->O <= 0 || t->I <= 0) return -1;+ switch (t->fmt) {+ case 0: return t->qf ? 10 : -1;+ case 1: return t->q8 ? 1 : -1;+ case 2: return t->planar || !t->q4 ? -1 : 2;+ case 4: if (t->planar || t->gs < 8 || t->gs % 8 || !t->q4) return -1; *gs = t->gs; return 4;+ case 5: return t->q4 ? 5 : -1;+ default: return -1;+ }+}+static int glmc_full(const Model *m, int i);+/* Layer i's matrices on the device (the chain's set; up to 11) */+static int glmc_layer_qts(Model *m, int i, QT **q) {+ Layer *l = &m->L[i]; int n = 0;+ q[n++] = &l->q_a; q[n++] = &l->q_b; q[n++] = &l->kv_a; q[n++] = &l->kv_b; q[n++] = &l->o;+ if (l->sparse) { q[n++] = &l->sh_gate; q[n++] = &l->sh_up; q[n++] = &l->sh_down; }+ else { q[n++] = &l->gate_proj; q[n++] = &l->up_proj; q[n++] = &l->down_proj; }+ if (glmc_full(m, i)) { q[n++] = &m->ix_wq[i]; q[n++] = &m->ix_wk[i]; q[n++] = &m->ix_wp[i]; }+ return n;+}/* ---- the host's writes to its KV cache: the mirror's watermark follows ------------ *//* The CPU's attention writes rows [pos..] of each row's KV state for this layer. */static void glmc_cpu_rows(Model *m, int layer, KVState *const *kvs, const int *positions, int pos_base, int S) {GlmChain *ch = (GlmChain *)m->vkchain;- if (!ch || !ch->kv_valid || layer < 0 || layer >= m->c.n_layers) return;+ if (!ch || !ch->kv_valid || layer < 0 || layer >= ch->n) return; /* a CPU layer of a partial chain: no mirror */for (int s = 0; s < S; s++) {KVState *ks = kvs ? kvs[s] : m->kv;int pos = positions ? positions[s] : pos_base + s;@@ -115,7 +159,7 @@ static void glmc_cpu_rows(Model *m, int layer, KVState *const *kvs, const int *pstatic void glmc_host_rows(Model *m, KVState *k, int from) {GlmChain *ch = (GlmChain *)m->vkchain;if (!ch || !ch->kv_valid || k != ch->owner) return;- for (int i = 0; i < m->c.n_layers; i++) {+ for (int i = 0; i < ch->n; i++) {if (ch->kv_valid[i] > from) ch->kv_valid[i] = from < 0 ? 0 : from;vkc_kv_lower(&ch->ks, i, from < 0 ? 0 : from);}@@ -124,7 +168,7 @@ static void glmc_host_rows(Model *m, KVState *k, int from) {static void glmc_kv_reset(Model *m) {GlmChain *ch = (GlmChain *)m->vkchain;if (!ch || !ch->kv_valid) return;- for (int i = 0; i < m->c.n_layers; i++) ch->kv_valid[i] = 0;+ for (int i = 0; i < ch->n; i++) ch->kv_valid[i] = 0;vkc_kv_reset(&ch->ks);ch->owner = NULL;}@@ -132,15 +176,103 @@ static void glmc_kv_reset(Model *m) {/* ---- setup ---------------------------------------------------------------------------- */static int glmc_full(const Model *m, int i) { return m->has_dsa && m->c.idx_type[i]; }-/* The parameters and the tensors on the device; 0 = the chain cannot run this model. */-static int glmc_setup(Model *m) {- Cfg *c = &m->c; int L = c->n_layers, D = c->hidden;- const char *why = NULL;+/* What the chain's shaders do not take (NULL: nothing), from the shapes alone: no upload. */+static const char *glmc_check(Model *m) {+ Cfg *c = &m->c;if (c->kv_lora > 1024 || c->qk_rope > 128 || (c->qk_rope & 1) || c->qk_nope > 1024 || c->q_lora < 1)- why = "an attention geometry its shaders do not take";- else if (m->has_dsa && (c->index_nh > 64 || c->index_nh * c->index_hd > 4096 || c->index_hd < c->qk_rope || !c->idx_type[0]))- why = "an indexer its shaders do not take";- if (why) { fprintf(stderr, "[VK] colibri chain: %s; per-matrix path\n", why); return 0; }+ return "an attention geometry its shaders do not take";+ if (m->has_dsa && (c->index_nh > 64 || c->index_nh * c->index_hd > 4096 || c->index_hd < c->qk_rope || !c->idx_type[0]))+ return "an indexer its shaders do not take";+ QT *q[11]; int gs;+ for (int i = 0; i < c->n_layers; i++) {+ int n = glmc_layer_qts(m, i, q);+ for (int k = 0; k < n; k++)+ if (glmc_form(q[k], &gs) < 0) return "a matrix with no device form (int2, E8/IQ3, fp8, planar int4)";+ }+ return NULL;+}++/* glmc_res counts instead of reserving while g_glmc_count >= 0 (the chunk's sizing); with+ * g_glmc_count_fit, each buffer at the bytes the device gives it (the fit's scratch) */+static long long g_glmc_count = -1;+static int g_glmc_count_fit;+static int glmc_res(VkcBuf **b, size_t floats, int kind) {+ size_t bytes = (floats ? floats : 1) * sizeof(float);+ if (g_glmc_count >= 0) { g_glmc_count += (long long)(g_glmc_count_fit ? vkc_fit_buf(bytes) : bytes); return 1; }+ return vkc_reserve(b, bytes, kind);+}+static int glmc_scratch(GlmChain *ch, Model *m, int rows, int ctx);++/* The fit (vkc_fit), before any upload: each layer's device bytes (its matrices as+ * glmc_tensor uploads them, its share of the parameter arena, its KV mirror at the first+ * size glmc_mirror gives it), the chain's fixed bytes (the scratch of one prompt chunk of+ * vkc_fit_rows(256) rows at the context CTX, the arena's alignment) and the tail: what+ * goes up only with every layer there (the MTP layer, eh_proj and lm_head, which+ * COLI_VK_DENSE_HOST and the per-matrix path place). */+static void glmc_fit_plan(Model *m) {+ Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, ID = c->index_hd;+ size_t *lb = calloc(L ? L : 1, sizeof(size_t)), *mb = calloc(L ? L : 1, sizeof(size_t));+ size_t *mir = calloc(L ? L : 1, sizeof(size_t));+ if (!lb || !mb || !mir) { free(lb); free(mb); free(mir); return; }+ const int cap0 = 256; /* glmc_mirror's first size */+ for (int i = 0; i < L; i++) {+ QT *q[11]; int gs, n = glmc_layer_qts(m, i, q);+ for (int k = 0; k < n; k++) {+ int f = glmc_form(q[k], &gs);+ lb[i] += vkc_fit_tensor(f, q[k]->I, q[k]->O, gs);+ mb[i] += coli_vk_tensor_payload(f, q[k]->I, q[k]->O, gs);+ }+ lb[i] += (size_t)(2 * D + c->q_lora + c->kv_lora + (glmc_full(m, i) ? 2 * ID : 0)) * sizeof(float);+ mir[i] = vkc_fit_buf((size_t)cap0 * c->kv_lora * sizeof(float)) ++ (c->qk_rope > 0 ? vkc_fit_buf((size_t)cap0 * c->qk_rope * sizeof(float)) : 0) ++ (glmc_full(m, i) ? vkc_fit_buf((size_t)cap0 * ID * sizeof(float)) : 0);+ lb[i] += mir[i];+ }+ /* one chunk's scratch: glmc_scratch's buffers (every layer's KV rows down) and the MLA+ * scratch, at the context the run plans for (the DSA scores: rows x context, at most+ * the 32M floats the forward allows a selecting chunk) */+ int R = vkc_fit_rows(256), ctx = getenv("CTX") ? atoi(getenv("CTX")) : 4096;+ if (ctx < 1) ctx = 4096;+ if ((long long)R * ctx > (32LL << 20)) ctx = (int)((32LL << 20) / R);+ if (ctx < 1) ctx = 1;+ GlmChain tmp; memset(&tmp, 0, sizeof tmp); tmp.n = L;+ g_glmc_count = 0; g_glmc_count_fit = 1; glmc_scratch(&tmp, m, R, ctx);+ size_t scratch = (size_t)g_glmc_count;+ g_glmc_count = -1; g_glmc_count_fit = 0;+ size_t r = (size_t)R, H = (size_t)c->n_heads;+ scratch += vkc_fit_buf(r * c->q_lora * 4) + vkc_fit_buf(r * H * (c->qk_nope + c->qk_rope) * 4) ++ vkc_fit_buf(r * (c->kv_lora + c->qk_rope) * 4) + 2 * vkc_fit_buf(r * H * c->kv_lora * 4) ++ vkc_fit_buf(r * H * c->v_head * 4);+ size_t fixed = scratch + vkc_fit_buf(4); /* + the arena's alignment */+ size_t tail = 0;+ {+ QT *t[10]; int nt = 0, gs;+ if (m->has_mtp) {+ Layer *l = &m->mtpL;+ t[nt++] = &l->q_a; t[nt++] = &l->q_b; t[nt++] = &l->kv_a; t[nt++] = &l->kv_b; t[nt++] = &l->o;+ t[nt++] = &l->sh_gate; t[nt++] = &l->sh_up; t[nt++] = &l->sh_down; t[nt++] = &m->eh_proj;+ }+ t[nt++] = &m->lm_head;+ for (int k = 0; k < nt; k++) {+ int f = glmc_form(t[k], &gs);+ if (f >= 0) tail += vkc_fit_tensor(f, t[k]->I, t[k]->O, gs);+ }+ }+ vkc_fit("colibri", L, lb, mb, fixed, tail, &g_glmc_fit);+ g_glmc_fitted = 1;+ g_glmc_lazy = scratch;+ g_glmc_mir = mir;+ free(lb); free(mb);+}++/* The parameters and layers 0..N-1 on the device (N = the fit's), layer by layer: each+ * layer's matrices, then (COLI_VK_DENSE_HOST) its host copies given back once all of it is+ * there. A layer that does not reach the device (a driver refusing what the budget+ * promised, a failed upload) is freed whole and the chain keeps the layers before it+ * (vkc_fit_shrink); nothing of it stays on the device and its host copies were never+ * dropped. 0 = no layer on the device: the chain stays off. */+static int glmc_setup(Model *m) {+ Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, N = g_glmc_fit.n;GlmChain *ch = (GlmChain *)calloc(1, sizeof *ch);if (!ch) return 0;ch->o_in = calloc(L, sizeof(size_t)); ch->o_post = calloc(L, sizeof(size_t));@@ -149,17 +281,18 @@ static int glmc_setup(Model *m) {ch->ik = calloc(L, sizeof(VkcBuf *)); ch->kv_valid = calloc(L, sizeof(int));if (!ch->o_in || !ch->o_post || !ch->o_ixw || !ch->o_ixb || !ch->mla || !ch->kv || !ch->ik || !ch->kv_valid) return 0;m->vkchain = ch;- /* the parameter arena: in_ln, post_ln, the q and kv latent norms, the index key norm */+ /* the parameter arena of the N layers: in_ln, post_ln, the q and kv latent norms, the+ * index key norm */size_t n = 0, *o_qn = calloc(L, sizeof(size_t)), *o_kn = calloc(L, sizeof(size_t));if (!o_qn || !o_kn) { free(o_qn); free(o_kn); return 0; }- for (int i = 0; i < L; i++) {+ for (int i = 0; i < N; i++) {ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D;o_qn[i] = n; n += c->q_lora; o_kn[i] = n; n += c->kv_lora;if (glmc_full(m, i)) { ch->o_ixw[i] = n; n += c->index_hd; ch->o_ixb[i] = n; n += c->index_hd; }}- float *arena = calloc(n, sizeof(float));+ float *arena = calloc(n ? n : 1, sizeof(float));if (!arena) { free(o_qn); free(o_kn); return 0; }- for (int i = 0; i < L; i++) {+ for (int i = 0; i < N; i++) {Layer *l = &m->L[i];memcpy(arena + ch->o_in[i], l->in_ln, D * sizeof(float));memcpy(arena + ch->o_post[i], l->post_ln, D * sizeof(float));@@ -173,34 +306,41 @@ static int glmc_setup(Model *m) {ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, arena, n * sizeof(float)) && vkc_submit(1);free(arena);+ if (!ok) {+ vkc_free(ch->prm); ch->prm = NULL;+ free(o_qn); free(o_kn);+ vkc_fit_shrink("colibri", &g_glmc_fit, 0, "the parameters' buffer was refused");+ return 0;+ }/* the tensors: the same device copies the per-matrix path uses */- for (int i = 0; i < L && ok; i++) {+ for (int i = 0; i < N; i++) {Layer *l = &m->L[i];- VkcMla *a = &ch->mla[i];- *a = (VkcMla){c->n_heads, c->qk_nope, c->qk_rope, c->v_head, c->kv_lora, D, c->q_lora, c->eps, c->attn_scale,- VKC_ROPE_INTERLEAVED, glmc_tensor(&l->q_a), glmc_tensor(&l->q_b), glmc_tensor(&l->kv_a),- glmc_tensor(&l->kv_b), NULL, NULL, glmc_tensor(&l->o), ch->prm, o_qn[i], o_kn[i]};- ok = a->q_a && a->q_b && a->kv_a && a->kv_b && a->o;- if (ok && l->sparse) ok = glmc_tensor(&l->sh_gate) && glmc_tensor(&l->sh_up) && glmc_tensor(&l->sh_down);- if (ok && !l->sparse) ok = glmc_tensor(&l->gate_proj) && glmc_tensor(&l->up_proj) && glmc_tensor(&l->down_proj);- if (ok && glmc_full(m, i)) ok = glmc_tensor(&m->ix_wq[i]) && glmc_tensor(&m->ix_wk[i]) && glmc_tensor(&m->ix_wp[i]);+ QT *q[11]; int nq = glmc_layer_qts(m, i, q), up = 1;+ for (int k = 0; k < nq && up; k++) up = glmc_tensor(q[k]) != NULL;+ if (!up) { /* free what it placed: the layer runs on the CPU, from its host copies */+ ColiVkTensor **t[11];+ for (int k = 0; k < nq; k++) t[k] = &q[k]->vk;+ vkc_layer_free(t, nq, NULL, 0);+ vkc_fit_shrink("colibri", &g_glmc_fit, i, vkc_lost() ? "the device was lost" : "a matrix was refused");+ break;+ }+ ch->mla[i] = (VkcMla){c->n_heads, c->qk_nope, c->qk_rope, c->v_head, c->kv_lora, D, c->q_lora, c->eps, c->attn_scale,+ VKC_ROPE_INTERLEAVED, l->q_a.vk, l->q_b.vk, l->kv_a.vk, l->kv_b.vk, NULL, NULL, l->o.vk,+ ch->prm, o_qn[i], o_kn[i]};+ glm_dho_drop_layer(m, i); /* COLI_VK_DENSE_HOST: all of the layer is there, its host copies go */+ vkc_fit_mark(&g_glmc_fit, i);}free(o_qn); free(o_kn);- if (!ok) { fprintf(stderr, "[VK] colibri chain: a matrix did not reach the device (or has no device form); per-matrix path\n"); return 0; }+ ch->n = g_glmc_fit.n;+ if (!ch->n) { vkc_free(ch->prm); ch->prm = NULL; return 0; }ch->ok = 1;- int nfull = 0; for (int i = 0; i < L; i++) nfull += glmc_full(m, i);+ int nfull = 0, ndense = 0;+ for (int i = 0; i < ch->n; i++) { nfull += glmc_full(m, i); ndense += !m->L[i].sparse; }fprintf(stderr, "[VK] colibri chain: %d layers on the device (%d dense, %d with the DSA indexer), %.1f MiB of parameters\n",- L, c->first_dense, nfull, n * 4 / 1048576.0);+ ch->n, ndense, nfull, n * 4 / 1048576.0);return 1;}-/* glmc_res counts instead of reserving while g_glmc_count >= 0 (the chunk's sizing) */-static long long g_glmc_count = -1;-static int glmc_res(VkcBuf **b, size_t floats, int kind) {- if (g_glmc_count >= 0) { g_glmc_count += (long long)(floats ? floats : 1) * (long long)sizeof(float); return 1; }- return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind);-}-static int glmc_scratch(GlmChain *ch, Model *m, int rows, int ctx);/* Prompt rows per chunk (vkc_chunk_rows): the chain's scratch a row, counted from the* reservations (the MLA scratch as vkc_mla_scratch sizes it; the DSA scores at the* model's context), and the routed experts' outputs (the tier's rows, the CPU's@@ -232,7 +372,7 @@ static int glmc_scratch(GlmChain *ch, Model *m, int rows, int ctx) {glmc_res(&ch->x, r * D, VKC_DEV) && glmc_res(&ch->nrm, r * D, VKC_DEV) && glmc_res(&ch->tmp, r * D, VKC_DEV) &&glmc_res(&ch->h2, r * D, VKC_DEV) && glmc_res(&ch->gs, r * MI, VKC_DEV) && glmc_res(&ch->us, r * MI, VKC_DEV) &&glmc_res(&ch->hs, r * MI, VKC_DEV) && glmc_res(&ch->ds, r * D, VKC_DEV) &&- glmc_res(&ch->h2d, r * D, VKC_DOWN) && glmc_res(&ch->kvd, (size_t)L * ch->kvd_layer, VKC_DOWN) &&+ glmc_res(&ch->h2d, r * D, VKC_DOWN) && glmc_res(&ch->kvd, (size_t)ch->n * ch->kvd_layer, VKC_DOWN) &&glmc_res(&ch->xd, r * D, VKC_DOWN) && glmc_res(&ch->routed, r * D, VKC_UP) &&glmc_res(&ch->cs, r * (c->qk_rope > 0 ? c->qk_rope : 2), VKC_UP);if (ok && ch->ks.on && g_glmc_count >= 0)@@ -248,6 +388,10 @@ static int glmc_scratch(GlmChain *ch, Model *m, int rows, int ctx) {ok = glmc_res(&ch->ikd, r * ID, VKC_DEV) && glmc_res(&ch->iq, r * IH * ID, VKC_DEV) &&glmc_res(&ch->ihw, r * IH, VKC_DEV) && glmc_res(&ch->isc, r * (size_t)ctx, VKC_DEV) &&glmc_res(&ch->sel, r * (1 + (size_t)c->index_topk), VKC_DEV);+ /* a partial chain whose CPU layer n reuses the selection of a device layer: it comes+ * back with the rows (the fit counts it whenever there is an indexer) */+ if (ok && m->has_dsa && (g_glmc_count_fit || (ch->n < L && !glmc_full(m, ch->n))))+ ok = glmc_res(&ch->seld, r * (1 + (size_t)c->index_topk), VKC_DOWN);if (!ok || g_glmc_count >= 0) return ok; /* counting: the buffers only */if (ch->rows < rows) {float *hr = realloc(ch->host_routed, r * D * sizeof(float));@@ -262,7 +406,7 @@ static int glmc_scratch(GlmChain *ch, Model *m, int rows, int ctx) {* planned again (vkc_kv_plan): whole when it fits the device's budget, else the split* (latent and rope rows; the index keys stay whole), for steps of `rows` rows. */static int glmc_mirror(GlmChain *ch, Model *m, int need, int rows) {- Cfg *c = &m->c; int L = c->n_layers;+ Cfg *c = &m->c; int L = ch->n; /* the device's layers (a partial chain: the first n) */if (ch->owner != m->kv) { for (int i = 0; i < L; i++) ch->kv_valid[i] = 0; ch->owner = m->kv; vkc_kv_reset(&ch->ks); }if (ch->cap >= need) return 1;int cap = 256; while (cap < need) cap *= 2;@@ -291,7 +435,7 @@ static int glmc_mirror(GlmChain *ch, Model *m, int need, int rows) {* n_rows rows (the split: its window placed first). */static int glmc_push_kv(GlmChain *ch, Model *m, int pos_base, int n_rows) {Cfg *c = &m->c; int ok = 1, K = c->kv_lora, R = c->qk_rope, ID = c->index_hd;- for (int i = 0; i < c->n_layers && ok; i++) {+ for (int i = 0; i < ch->n && ok; i++) {if (ch->ks.on) { /* the split: the window placed, its rows below pos_base uploaded */VkcKvPart pt[2] = {{1, K, m->Lc[i], 0, ch->kv[i].lat, 0}, {1, R, m->Rc[i], 0, ch->kv[i].rope, 0}};vkc_kv_place(&ch->ks, i, pos_base, n_rows);@@ -366,17 +510,28 @@ static int glmc_combine(GlmChain *ch, Model *m, int n) {return vkc_ew(ch->x, ch->x, ch->routed, ch->ds, NULL, &p);}-/* Every layer for S rows of xh (positions pos_base..), the final rows back into xh.- * 0 = not taken: the CPU runs the forward (from xh, untouched). */+/* The device's layers (every layer, or the first n of a partial chain) for S rows of xh+ * (positions pos_base..), the rows after the last of them back into xh. Returns how many+ * layers ran: L, or n (the caller's CPU loop runs layers n..L-1 from xh); 0 = not taken:+ * the CPU runs the forward (from xh, untouched). */static int glmc_forward(Model *m, float *xh, int S, int pos_base) {if (!g_vk_chain) return 0;if (g_vk_chain == COLI_VK_CHAIN_PREFILL && S <= 2) return 0; /* prompts only: decode on the CPU */if (g_pilot || g_pilot_real || g_looka || g_kv8 || g_tq || (exact_verify_on() && g_spec_live)) return 0;GlmChain *ch = (GlmChain *)m->vkchain;if (!ch || !ch->ok || ch->failed || vkc_lost()) return 0;- Cfg *c = &m->c; int D = c->hidden, L = c->n_layers, R = c->qk_rope;+ Cfg *c = &m->c; int D = c->hidden, L = c->n_layers, R = c->qk_rope, N = ch->n;if (!m->Lc || !m->Rc || (m->has_dsa && !m->Ic)) return 0;for (int i = 0; i < L; i++) if (m->kv_start[i] != 0 || !m->Lc[i]) return 0;+ /* layer N on the CPU reuses the DSA selection of the device's last full layer (a+ * shared indexer layer): the selection comes back with the rows, into the CPU's */+ int hand_sel = N < L && m->has_dsa && !glmc_full(m, N), TK = c->index_topk;+ if (hand_sel && (int64_t)S * TK > m->dsa_scap) {+ int *sl = malloc((size_t)S * TK * sizeof(int)), *ns = malloc((size_t)S * sizeof(int));+ if (!sl || !ns) { free(sl); free(ns); return 0; }+ free(m->dsa_sel); free(m->dsa_nsel);+ m->dsa_sel = sl; m->dsa_nsel = ns; m->dsa_scap = (int)((int64_t)S * TK);+ }int mirror_ok = glmc_mirror(ch, m, pos_base + S, 1);int CH = mirror_ok ? glmc_chunk_rows(ch, m) : 1;if (m->has_dsa && (pos_base + S > c->index_topk || g_dsa_force)) { /* the scores' scratch: rows x context */@@ -409,7 +564,7 @@ static int glmc_forward(Model *m, float *xh, int S, int pos_base) {if (!vkc_begin() || !vkc_write(ch->x, 0, xh + (size_t)c0 * D, (size_t)n * D * sizeof(float)) ||!glmc_push_kv(ch, m, pb, n)) goto lost;int ok = 1, pending = 0, pulled = 0, sel_on = 0;- for (int i = 0; i < L && ok; i++) {+ for (int i = 0; i < N && ok; i++) {Layer *l = &m->L[i];if (g_spec && g_prefetch && l->sparse && m->enr[i] > 0) /* the CPU path's I/O hint */for (int z = 0; z < m->enr[i]; z++) if (!(n <= 4 && vk_reg_served(i, m->eroute[i][z]))) expert_prefetch(m, i, m->eroute[i][z]);@@ -451,16 +606,23 @@ static int glmc_forward(Model *m, float *xh, int S, int pos_base) {pending = 1;}if (ok && pending) ok = glmc_combine(ch, m, n);- ok = ok && vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D) && vkc_submit(1);+ ok = ok && vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D) &&+ (!hand_sel || !sel_on || vkc_copy(ch->seld, 0, ch->sel, 0, (size_t)n * (1 + TK))) && vkc_submit(1);if (!ok) goto lost;- glmc_pull_kv(ch, m, pulled, L, pb, n);+ glmc_pull_kv(ch, m, pulled, N, pb, n);memcpy(outs + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));- for (int i = 0; i < L; i++) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, i, pb + n); }+ for (int s = 0; hand_sel && s < n; s++) { /* in the CPU's form: a count (0 = every position), the positions */+ const int *sr = (const int *)vkc_ptr(ch->seld) + (size_t)s * (1 + TK);+ int k = sel_on && sr[0] > 0 ? (sr[0] < TK ? sr[0] : TK) : 0;+ if (k) memcpy(m->dsa_sel + (size_t)(c0 + s) * TK, sr + 1, (size_t)k * sizeof(int));+ m->dsa_nsel[c0 + s] = k;+ }+ for (int i = 0; i < N; i++) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, i, pb + n); }}memcpy(xh, outs, (size_t)S * D * sizeof(float));free(inv); free(outs);ch->forwards++;- return 1;+ return N;lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */free(inv); free(outs);if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }@@ -483,8 +645,11 @@ static void glmc_report(Model *m) {static Model *g_glmc_model;static void glmc_report_atexit(void) { glmc_report(g_glmc_model); }-/* COLI_VK_CHAIN at startup, before the tier sizes itself (the trunk's device copies- * count as used): the decision, the pipelines, the tensors. */+/* COLI_VK_CHAIN at startup, right after the load: before the pins, the RAM cap and the+ * tier (the trunk's device copies count as used): the decision, the pipelines, the fit+ * (how many layers the device holds), COLI_VK_DENSE_HOST's decision for those layers+ * (glm_dho_start), the tensors layer by layer, then the tail's host copies and the RAM+ * given back (glm_dho_finish). Without a fit (the chain off) glm_dho_start does as before. */static void glmc_start(Model *m) {if (!g_vulkan) return;int tier_on = vkt_wanted() && g_vk_experts != 0 && m->c.n_experts > 0;@@ -498,9 +663,16 @@ static void glmc_start(Model *m) {if (on && !no && !(g_glmc_inited = vkc_init())) no = "the chain's pipelines did not come up";if (on && !no && !vkc_mla_ready()) no = "the MLA shaders are missing (chain_mla, chain_hgemv, chain_dsa)";if (no) fprintf(stderr, "[VK] colibri: %s: the dense chain stays off\n", no);- if (!on || no) return;- if (!glmc_setup(m)) return;- g_vk_chain = on;+ const char *why = on && !no ? glmc_check(m) : NULL;+ if (why) fprintf(stderr, "[VK] colibri chain: %s; per-matrix path\n", why);+ if (on && !no && !why) glmc_fit_plan(m); /* before any upload */+ glm_dho_start(m);+ if (!g_glmc_fitted) return;+ if (g_glmc_fit.n > 0 && glmc_setup(m)) g_vk_chain = on;+ g_glmc_partial = vkc_fit_partial(&g_glmc_fit);+ for (int i = 0; i < g_glmc_fit.n; i++) g_glmc_lazy += g_glmc_mir[i]; /* for the tier (vk_tier_start) */+ vkc_fit_placed("colibri", &g_glmc_fit);+ glm_dho_finish(m);}/* After the tier's: at exit the report runs first, then the chain goes, then the device. */static void glmc_atexit(Model *m) {