Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(vulkan): glm53's partial dense chain
JustVugg4471874ac80f
diff --git a/c/glm53.c b/c/glm53.cindex 8a8a23c1..1b692319 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -89,6 +89,11 @@ static uint64_t g_metal_moe_rows = 0;#include "vk_tier.h" /* the routed-expert tier the MoE engines share */static int g_vk_ready = 0; /* COLI_VULKAN=1 and the device opened */static int g_vk_dense = 0; /* the resident matrices run there (coli_vk_dense_decide) */+/* A partial chain (glm53_chain.h, vkc_fit): the device holds the first N layers only, or+ * not the head. mv and mm then multiply on the device only what is there already: a+ * matrix without a device copy stays on the CPU (G53_VK_MAY). */+static int g_g53_partial = 0;+#define G53_VK_MAY(w) (!g_g53_partial || (w)->vk != NULL)#endif#include "compat.h"#include "serve_poll.h" /* CANCEL a meta' turno (#1332) */@@ -744,6 +749,7 @@ static void glm53_vk_report(const GModel *m, const char *scope); /* the [VK] lstatic void g53c_sync_host(const GModel *m, const GSession *s);static void g53c_host_wrote(const GSession *s);static void g53c_session_gone(const GSession *s);+static void g53c_start(GModel *m); /* COLI_VK_CHAIN at load: the decision, the fit, the placement */#endif/* `want` e' quanti valori legge chi usa il tensore, contati dalla config. Il@@ -1091,7 +1097,7 @@ static void mv(float *out, const Mat *w, const float *x) {if (g53_dho_matmul(w, out, x, 1)) return;g53_dho_reload((Mat *)w);}- if (g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4)) {+ if (g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4) && G53_VK_MAY(w)) {Mat *mutable_w = (Mat *)w;if (coli_vk_matmul((ColiVkTensor **)&mutable_w->vk, out, x,w->fmt == 4 ? (const void *)w->q4 : (const void *)w->q8,@@ -1124,14 +1130,14 @@ static void mm(float *out, const Mat *w, const float *x, int S) {for (int t = 0; t < S; t++) mv(out + (size_t)t * w->rows, w, x + (size_t)t * w->columns);return;}- if (S > 1 && g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4)) {+ if (S > 1 && g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4) && G53_VK_MAY(w)) {Mat *mutable_w = (Mat *)w;if (coli_vk_matmul((ColiVkTensor **)&mutable_w->vk, out, x,w->fmt == 4 ? (const void *)w->q4 : (const void *)w->q8,w->s, w->fmt, S, w->columns, w->rows, w->gs))return;}- gpu |= g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4);+ gpu |= g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4) && G53_VK_MAY(w);#endifif (S == 1 || gpu) {for (int t = 0; t < S; t++)@@ -2411,41 +2417,68 @@ static int g53_dho_drop(Mat *w, int keep, int drop, size_t *bytes) {*bytes += b;return 1;}-/* Every resident matrix but the routed experts: the head (unless tied to the embedding),- * each layer's attention (KDA or MLA with its indexer), dense MLP and shared expert. The- * absorbed kv_b halves stay when the chain will not run every forward: outside it the- * CPU's attention reads them by row (mm_rows, mv_rows). */-static int g53_dho_pass(GModel *m, int keep_kvb, int drop, size_t *bytes) {- int n = g53_dho_drop(&m->head, 0, drop, bytes);- for (int i = m->layer_begin; i < m->layer_end; i++) {- GLayer *l = &m->layer[i];- Mat *all[] = {&l->kq, &l->kk, &l->kv, &l->ko, &l->kga, &l->kgb, &l->kfa, &l->kfb, &l->kb,- &l->qa, &l->qb, &l->kva, &l->o, &l->iwq, &l->iwk, &l->iwp, &l->ikpg,- &l->dg, &l->du, &l->dd, &l->rg, &l->ru, &l->rd};- for (size_t k = 0; k < sizeof(all) / sizeof(all[0]); k++) n += g53_dho_drop(all[k], 0, drop, bytes);- n += g53_dho_drop(&l->kvb_kt, keep_kvb, drop, bytes);- n += g53_dho_drop(&l->kvb_v, keep_kvb, drop, bytes);- }+/* Layer i's resident matrices: its attention (KDA or MLA with its indexer), dense MLP and+ * shared expert. The absorbed kv_b halves stay when the chain will not run every forward:+ * outside it the CPU's attention reads them by row (mm_rows, mv_rows). */+static int g53_dho_layer(GModel *m, int i, int keep_kvb, int drop, size_t *bytes) {+ GLayer *l = &m->layer[i];+ Mat *all[] = {&l->kq, &l->kk, &l->kv, &l->ko, &l->kga, &l->kgb, &l->kfa, &l->kfb, &l->kb,+ &l->qa, &l->qb, &l->kva, &l->o, &l->iwq, &l->iwk, &l->iwp, &l->ikpg,+ &l->dg, &l->du, &l->dd, &l->rg, &l->ru, &l->rd};+ int n = 0;+ for (size_t k = 0; k < sizeof(all) / sizeof(all[0]); k++) n += g53_dho_drop(all[k], 0, drop, bytes);+ n += g53_dho_drop(&l->kvb_kt, keep_kvb, drop, bytes);+ n += g53_dho_drop(&l->kvb_v, keep_kvb, drop, bytes);return n;}-/* Once the device is open, before expert_cache_init reads the free memory: the decision- * (the chain's own, made silently here and printed by g53c_start), the uploads and the- * RAM given back. */-static void g53_dho_start(GModel *m, int tier) {+/* Every resident matrix but the routed experts: the head (unless tied to the embedding;+ * with tail) and layers [layer_begin, layer_begin + layers). */+static int g53_dho_pass(GModel *m, int keep_kvb, int drop, size_t *bytes, int layers, int tail) {+ int n = tail ? g53_dho_drop(&m->head, 0, drop, bytes) : 0;+ for (int i = m->layer_begin; i < m->layer_begin + layers && i < m->layer_end; i++) n += g53_dho_layer(m, i, keep_kvb, drop, bytes);+ return n;+}+static int g_g53_dho_on, g_g53_dho_keep;+static void g53_dho_finish(GModel *m, int fit_n, int fit_tail, int partial);+/* Once the device is open, before expert_cache_init reads the free memory (g53c_start+ * calls it once the chain's fit is known): the decision (the chain's own, made silently+ * here and printed by g53c_start). With a fit (fit_n >= 0) only the N layers on the device+ * drop their copies, each as g53c_setup places it, and the head only when the whole+ * chain and the head fit (g53_dho_finish); without one, every matrix here, as before. */+static void g53_dho_start(GModel *m, int tier, int fit_n, int fit_tail) {if (!g_vk_ready || !m->has_io || m->layer_begin != 0 || m->layer_end != m->c.n_layers) return;const int chain = coli_vk_chain_decide(NULL, tier, COLI_VK_CHAIN_UNMEASURED);const int keep_kvb = chain != COLI_VK_CHAIN_ON;+ const int L = m->c.n_layers, fitted = fit_n >= 0, n = fitted ? fit_n : L, tail = fitted ? fit_tail : 1;size_t bytes = 0;- g53_dho_pass(m, keep_kvb, 0, &bytes);- if (!coli_vk_dense_host_decide("glm53", chain != COLI_VK_CHAIN_OFF || g_vk_dense, bytes)) return;+ g53_dho_pass(m, keep_kvb, 0, &bytes, n, tail);+ if (!coli_vk_dense_host_decide("glm53", fitted ? n > 0 : (chain != COLI_VK_CHAIN_OFF || g_vk_dense), bytes)) return;g_g53_dho_model = m;g_vk_dense = 1; /* the steps the chain does not run take the device too: the CPU has no copy */+ g_g53_dho_on = 1; g_g53_dho_keep = keep_kvb;+ if (fitted) { coli_vk_dense_host_layers(n, L); return; } /* g53c_setup drops each layer it places */bytes = 0;- g53_dho_pass(m, keep_kvb, 1, &bytes);- coli_vk_dense_host_placed("glm53", keep_kvb- ? "the embedding (and a head tied to it), the routers, the mHC mixes and norms, the vision tower, "- "the absorbed kv_b (the chain will not run every forward: the CPU's attention reads it)"- : "the embedding (and a head tied to it), the routers, the mHC mixes and norms, the vision tower");+ g53_dho_pass(m, keep_kvb, 1, &bytes, L, 1);+ g53_dho_finish(m, -1, 1, 0);+}+/* g53c_setup placed all of layer i: its host copies go */+static void g53_dho_drop_layer(GModel *m, int i) {+ size_t b = 0;+ if (g_g53_dho_on) g53_dho_layer(m, i, g_g53_dho_keep, 1, &b);+}+/* The head (a full chain with room for it), the line. */+static void g53_dho_finish(GModel *m, int fit_n, int fit_tail, int partial) {+ if (!g_g53_dho_on) return;+ if (fit_n >= 0) {+ size_t b = 0;+ coli_vk_dense_host_layers(fit_n, m->c.n_layers);+ if (fit_tail) g53_dho_drop(&m->head, 0, 1, &b);+ }+ char kept[384];+ snprintf(kept, sizeof kept, "the embedding (and a head tied to it), the routers, the mHC mixes and norms, the vision tower%s%s",+ g_g53_dho_keep ? ", the absorbed kv_b (the chain will not run every forward: the CPU's attention reads it)" : "",+ partial ? ", the head (the chain's tail runs on the CPU)" : "");+ coli_vk_dense_host_placed("glm53", kept);}#endif@@ -2630,7 +2663,10 @@ static void model_load_range(GModel *m, const char *dir, int layer_begin,if (g_vk_ready) coli_vk_set_swiglu_limit(m->c.swiglu_limit);if (!g_vk_ready) fprintf(stderr, "Vulkan: no usable device, falling back to CPU\n");else if (g_vk_dense) fprintf(stderr, "Vulkan: active for resident matrices\n");- g53_dho_start(m, tier); /* COLI_VK_DENSE_HOST: the trunk on the device only, before the expert cache sizes itself */+ /* COLI_VK_CHAIN: decided, fitted (how many layers the device holds) and placed, with+ * COLI_VK_DENSE_HOST's host copies of those layers given back, before the expert cache+ * sizes itself from the free memory and before the tier */+ g53c_start(m);}#endif/* La cache si dimensiona qui, non prima: quanto si puo' spendere dipende@@ -2815,10 +2851,14 @@ static float *run_layers(GModel *m, GSession *s, float *streams, float *next,const Cfg *c = &m->c;const int H = c->hc_mult, D = c->hidden;#ifdef COLI_VULKAN- /* every layer on the device (glm53_chain.h); 0: the CPU below, its state current first */+ /* every layer on the device (glm53_chain.h); 0: the CPU below, its state current+ * first; a partial chain (the first N layers on the device) hands the streams back+ * after layer N - 1 and the CPU runs the rest from there */if (g_vk_chain && begin == 0 && end == c->n_layers) {- if (g53c_forward(m, s, streams, n, start)) return streams;- g53c_cpu_step(m, s, start);+ const int done = g53c_forward(m, s, streams, n, start);+ if (done >= c->n_layers) return streams;+ if (!done) g53c_cpu_step(m, s, start);+ begin = done;}#endiffloat *collapsed = malloc((size_t)n * D * sizeof(float));@@ -4264,7 +4304,10 @@ static void glm53_vk_tier_start(GModel *m) {.act = VKT_ACT_SWIGLU, .act_limit = c->swiglu_limit,.max_rows = GLM53_VK_ROWS * c->topk,.ram_reserve = (size_t)cap * (size_t)sparse * (size_t)m->e_slot,- .dense_bytes = g_vk_dense && !coli_vk_dense_device_only() ? glm53_dense_dev_bytes(m) : 0,+ /* a partial chain: nothing more of the trunk goes up, but the chain's first+ * forward allocates its scratch and the N layers' MLA caches */+ .dense_bytes = g_g53_partial ? g_g53c_lazy+ : g_vk_dense && !coli_vk_dense_device_only() ? glm53_dense_dev_bytes(m) : 0,.in_ram = glm53_in_ram, .ram_ctx = m,.load = glm53_vk_load, .release = glm53_vk_unhold, .load_ctx = m,.load_batch = glm53_vk_load_batch};@@ -4401,7 +4444,6 @@ int main(int argc, char **argv) {model_load(&served, snap);glm53_telemetry_init(snap, &served.c);#ifdef COLI_VULKAN- g53c_start(&served); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */glm53_vk_tier_start(&served); /* after the history: the warm start reads it */g53c_atexit(); /* after the tier's: the chain goes before the device */#endif@@ -4464,7 +4506,6 @@ int main(int argc, char **argv) {model_load(&model, dir);glm53_telemetry_init(dir, &model.c);#ifdef COLI_VULKAN- g53c_start(&model); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */glm53_vk_tier_start(&model); /* after the history: the warm start reads it */g53c_atexit(); /* after the tier's: the chain goes before the device */#endifdiff --git a/c/glm53_chain.h b/c/glm53_chain.hindex c6162bce..9cd95b1c 100644--- a/c/glm53_chain.h+++ b/c/glm53_chain.h@@ -58,6 +58,7 @@typedef struct {int ok, failed, rows, cap, pcap;+ int n; /* layers on the device: 0..n-1 (the rest run on the CPU) */const GSession *owner;int where; /* G53C_HOST / DEV / BOTH for owner's KDA state */VkcBuf *prm;@@ -84,6 +85,11 @@ typedef struct {static G53Chain *g_g53c;static int g_vk_chain = 0;static int g_g53c_inited = 0;+static VkcFit g_g53c_fit; /* how many layers the device holds (vkc_fit), when g_g53c_fitted */+static int g_g53c_fitted;+static size_t g_g53c_lazy; /* what the chain allocates at its first forward: one chunk's+ * scratch and the N layers' MLA caches at their first size */+static size_t *g_g53c_mir; /* each layer's MLA caches at their first size *//* A Mat's device copy (the per-matrix path's own, where it made one); f32 as fmt 10. */@@ -110,7 +116,7 @@ static void g53c_sync_host(const GModel *m, const GSession *s) {if (!ch || !ch->ok || ch->owner != s || ch->where != G53C_DEV || !s) return;const Cfg *c = &m->c;size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)3 * c->kda_proj * c->conv_k;- for (int i = 0; i < c->n_layers; i++) {+ for (int i = 0; i < ch->n; i++) { /* the device's layers: a CPU layer's state is the host's already */if (c->is_full[i]) continue;if (!vkc_read(ch->st[i], 0, s->layer[i].kda_state, ns * sizeof(float)) ||!vkc_read(ch->win[i], 0, s->layer[i].kda_window, nw * sizeof(float))) {@@ -138,7 +144,7 @@ static void g53c_cpu_step(const GModel *m, const GSession *s, int start) {if (!ch || !ch->ok || ch->owner != s) return;g53c_sync_host(m, s);ch->where = G53C_HOST;- for (int i = 0; i < m->c.n_layers; i++) {+ for (int i = 0; i < ch->n; i++) {if (ch->kv_valid[i] > start) ch->kv_valid[i] = start;if (ch->pool_valid[i] > start / m->c.index_kpool) ch->pool_valid[i] = start / m->c.index_kpool;if (m->c.is_full[i]) vkc_kv_lower(&ch->ks, ch->mla_ord[i], start);@@ -146,17 +152,142 @@ static void g53c_cpu_step(const GModel *m, const GSession *s, int start) {}/* ---- setup ----------------------------------------------------------------------------- */+/* g53c_tensor's backend format for w without uploading it (-1: no device form) and its+ * group size; the fit counts the bytes from it. */+static int g53c_form(const Mat *w, int *gs) {+ *gs = 0;+ if (w->rows < 1 || w->columns < 1) return -1;+ switch (w->fmt) {+ case 0: return w->f ? 10 : -1;+ case 1: return w->q8 ? 1 : -1;+ case 4: if (w->gs < 8 || w->gs % 8 || !w->q4) return -1; *gs = w->gs; return 4;+ default: return -1;+ }+}+/* Layer i's matrices on the device (the chain's set; up to 13) */+static int g53c_layer_mats(const GModel *m, int i, Mat **w) {+ const Cfg *c = &m->c; GLayer *l = &m->layer[i]; int n = 0;+ if (c->is_full[i]) {+ Mat *a[] = {&l->qa, &l->qb, &l->kva, &l->kvb_kt, &l->kvb_v, &l->o, &l->iwq, &l->iwk, &l->iwp, &l->ikpg};+ for (size_t k = 0; k < sizeof a / sizeof *a; k++) w[n++] = a[k];+ } else {+ Mat *a[] = {&l->kq, &l->kk, &l->kv, &l->ko, &l->kga, &l->kgb, &l->kfa, &l->kfb, &l->kb};+ for (size_t k = 0; k < sizeof a / sizeof *a; k++) w[n++] = a[k];+ }+ if (i < c->first_dense) { w[n++] = &l->dg; w[n++] = &l->du; w[n++] = &l->dd; }+ else { w[n++] = &l->rg; w[n++] = &l->ru; w[n++] = &l->rd; }+ return n;+}+/* What the chain's shaders do not take (NULL: nothing), from the shapes alone: no upload. */+static const char *g53c_check(const GModel *m) {+ const Cfg *c = &m->c; int L = c->n_layers, H = c->hc_mult, P = c->kda_proj;+ if (m->layer_begin != 0 || m->layer_end != L || !m->has_io) return "a layer range";+ if (c->kv_lora > 1024 || c->qk_rope != 0 || c->qk_nope > 1024 || c->q_lora < 1) return "an attention geometry its shaders do not take";+ if (c->index_nh > 64 || c->index_nh * c->index_hd > 4096 || c->index_kpool < 2 || c->index_topk % c->index_kpool ||+ c->index_topk / c->index_kpool > 1024) return "an indexer its shaders do not take";+ if (P > 0 && (c->kda_hd > 128 || c->conv_k > 8)) return "a KDA head its shaders do not take";+ if (H < 1 || H > 8 || c->hc_iters < 1) return "hyper-connections its shaders do not take";+ Mat *w[16]; int gs;+ for (int i = 0; i < L; i++) {+ int n = g53c_layer_mats(m, i, w);+ for (int k = 0; k < n; k++)+ if (g53c_form(w[k], &gs) < 0) return "a matrix with no device form";+ }+ return NULL;+}+/* Layer i's share of the parameter arena, in floats */+static size_t g53c_arena_floats(const Cfg *c, int i) {+ size_t nm = (size_t)(2 + c->hc_mult) * c->hc_mult, n = 2 * (size_t)c->hidden + 2 * (3 + nm);+ if (c->is_full[i]) n += (size_t)c->q_lora + c->kv_lora + 2 * (size_t)c->index_hd + (size_t)c->index_kpool * c->index_hd;+ else n += (size_t)3 * c->kda_proj * c->conv_k + (size_t)c->kda_heads + c->kda_proj + c->kda_hd;+ return n;+}++/* g53c_res counts instead of reserving while g_g53c_count >= 0 (the chunk's sizing); with+ * g_g53c_count_fit, each buffer at the bytes the device gives it (the fit's scratch) */+static long long g_g53c_count = -1;+static int g_g53c_count_fit;+static int g53c_res(VkcBuf **b, size_t floats, int kind) {+ size_t bytes = (floats ? floats : 1) * sizeof(float);+ if (g_g53c_count >= 0) { g_g53c_count += (long long)(g_g53c_count_fit ? vkc_fit_buf(bytes) : bytes); return 1; }+ return vkc_reserve(b, bytes, kind);+}+static int g53c_scratch(G53Chain *ch, const GModel *m, int rows, int ctx);++/* The fit (vkc_fit), before any upload: each layer's device bytes (the mHC mixes and its+ * matrices as g53c_setup uploads them, its share of the parameter arena, a KDA layer's+ * state and window, an MLA layer's caches at the first size g53c_mirror gives them), the+ * chain's fixed bytes (the scratch of one prompt chunk of vkc_fit_rows(128) rows at the+ * context GLM53_MAXT, the arena's alignment) and the tail: the head, which goes up only+ * with every layer there (COLI_VK_DENSE_HOST and the per-matrix path place it). */+static void g53c_fit_plan(GModel *m) {+ const Cfg *c = &m->c;+ int L = c->n_layers, D = c->hidden, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, P = c->kda_proj, ID = c->index_hd;+ size_t *lb = calloc(L ? L : 1, sizeof(size_t)), *mb = calloc(L ? L : 1, sizeof(size_t));+ size_t *mir = calloc(L ? L : 1, sizeof(size_t));+ if (!lb || !mb || !mir) { free(lb); free(mb); free(mir); return; }+ const int cap0 = 256; /* g53c_mirror's first size */+ for (int i = 0; i < L; i++) {+ lb[i] = 2 * vkc_fit_tensor(10, HD, nm, 0);+ mb[i] = 2 * coli_vk_tensor_payload(10, HD, nm, 0);+ Mat *w[16]; int gs, n = g53c_layer_mats(m, i, w);+ for (int k = 0; k < n; k++) {+ int f = g53c_form(w[k], &gs);+ lb[i] += vkc_fit_tensor(f, w[k]->columns, w[k]->rows, gs);+ mb[i] += coli_vk_tensor_payload(f, w[k]->columns, w[k]->rows, gs);+ }+ lb[i] += g53c_arena_floats(c, i) * sizeof(float);+ if (c->is_full[i])+ mir[i] = vkc_fit_buf((size_t)cap0 * c->kv_lora * sizeof(float)) + 2 * vkc_fit_buf((size_t)cap0 * ID * sizeof(float)) ++ vkc_fit_buf((size_t)(cap0 / c->index_kpool + 1) * ID * sizeof(float));+ else+ lb[i] += vkc_fit_buf((size_t)3 * P * c->conv_k * sizeof(float)) ++ vkc_fit_buf((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));+ lb[i] += mir[i];+ }+ /* one chunk's scratch: g53c_scratch's buffers (every layer's rows down) and the MLA+ * scratch at the context of a serve slot */+ int R = vkc_fit_rows(128), ctx = getenv("GLM53_MAXT") ? atoi(getenv("GLM53_MAXT")) : 8192;+ if (ctx < 64) ctx = 64;+ G53Chain tmp; memset(&tmp, 0, sizeof tmp); tmp.n = L;+ g_g53c_count = 0; g_g53c_count_fit = 1; g53c_scratch(&tmp, m, R, ctx);+ size_t scratch = (size_t)g_g53c_count;+ g_g53c_count = -1; g_g53c_count_fit = 0;+ int any_mla = 0; for (int i = 0; i < L; i++) any_mla |= c->is_full[i];+ if (any_mla) {+ size_t r = (size_t)R, Hh = (size_t)c->n_heads;+ scratch += vkc_fit_buf(r * c->q_lora * 4) + vkc_fit_buf(r * Hh * c->qk_nope * 4) + vkc_fit_buf(r * c->kv_lora * 4) ++ 2 * vkc_fit_buf(r * Hh * c->kv_lora * 4) + vkc_fit_buf(r * Hh * c->v_head * 4);+ }+ size_t fixed = scratch + vkc_fit_buf(4), tail = 0; /* + the arena's alignment */+ { int gs, f = m->head.resident ? g53c_form(&m->head, &gs) : -1;+ if (f >= 0) tail = vkc_fit_tensor(f, m->head.columns, m->head.rows, gs); }+ vkc_fit("glm53", L, lb, mb, fixed, tail, &g_g53c_fit);+ g_g53c_fitted = 1;+ g_g53c_lazy = scratch;+ g_g53c_mir = mir;+ free(lb); free(mb);+}++/* Layer i's device objects (its mHC mixes, matrices, KDA state and window) freed: a layer+ * that did not fully reach the device leaves nothing there. */+static void g53c_layer_free(G53Chain *ch, GModel *m, int i) {+ Mat *w[16]; int n = g53c_layer_mats(m, i, w);+ ColiVkTensor **t[18]; int nt = 0;+ t[nt++] = &ch->fna[i]; t[nt++] = &ch->fnf[i];+ for (int k = 0; k < n; k++) t[nt++] = (ColiVkTensor **)&w[k]->vk;+ VkcBuf **b[2] = {&ch->win[i], &ch->st[i]};+ vkc_layer_free(t, nt, b, 2);+}++/* The parameters and layers 0..N-1 on the device (N = the fit's), layer by layer: its mHC+ * mixes, its matrices and a KDA layer's state and window, then (COLI_VK_DENSE_HOST) its+ * host copies given back once all of it is there. A layer that does not reach the device+ * is freed whole and the chain keeps the layers before it (vkc_fit_shrink). 0 = no layer+ * on the device: the chain stays off. */static int g53c_setup(GModel *m) {const Cfg *c = &m->c;- int L = c->n_layers, D = c->hidden, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, P = c->kda_proj;- const char *why = NULL;- if (m->layer_begin != 0 || m->layer_end != L || !m->has_io) why = "a layer range";- else if (c->kv_lora > 1024 || c->qk_rope != 0 || c->qk_nope > 1024 || c->q_lora < 1) why = "an attention geometry its shaders do not take";- else if (c->index_nh > 64 || c->index_nh * c->index_hd > 4096 || c->index_kpool < 2 || c->index_topk % c->index_kpool ||- c->index_topk / c->index_kpool > 1024) why = "an indexer its shaders do not take";- else if (P > 0 && (c->kda_hd > 128 || c->conv_k > 8)) why = "a KDA head its shaders do not take";- else if (H < 1 || H > 8 || c->hc_iters < 1) why = "hyper-connections its shaders do not take";- if (why) { fprintf(stderr, "[VK] glm53 chain: %s; per-matrix path\n", why); return 0; }+ int L = c->n_layers, D = c->hidden, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, P = c->kda_proj, N = g_g53c_fit.n;G53Chain *ch = calloc(1, sizeof *ch);if (!ch) return 0;size_t **offs[] = {&ch->o_in, &ch->o_post, &ch->o_hca, &ch->o_hcf, &ch->o_qn, &ch->o_kn, &ch->o_ikw, &ch->o_ikb,@@ -169,11 +300,10 @@ static int g53c_setup(GModel *m) {ch->kv_valid = calloc(L, sizeof(int)); ch->pool_valid = calloc(L, sizeof(int)); ch->mla_ord = calloc(L, sizeof(int));if (!ch->fna || !ch->fnf || !ch->mla || !ch->kv || !ch->ik || !ch->ig || !ch->pk || !ch->win || !ch->st ||!ch->kv_valid || !ch->pool_valid || !ch->mla_ord) return 0;- for (int i = 0; i < L; i++) if (c->is_full[i]) ch->mla_ord[i] = ch->n_mla++;g_g53c = ch;- /* the parameter arena */+ /* the parameter arena of the N layers */size_t n = 0;- for (int i = 0; i < L; i++) {+ for (int i = 0; i < N; i++) {ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D;ch->o_hca[i] = n; n += 3 + nm; ch->o_hcf[i] = n; n += 3 + nm;if (c->is_full[i]) {@@ -185,9 +315,9 @@ static int g53c_setup(GModel *m) {ch->o_kda[i] = n; n += (size_t)c->kda_heads + P + c->kda_hd;}}- float *a = calloc(n, sizeof(float));+ float *a = calloc(n ? n : 1, sizeof(float));if (!a) return 0;- for (int i = 0; i < L; i++) {+ for (int i = 0; i < N; i++) {const GLayer *l = &m->layer[i];memcpy(a + ch->o_in[i], l->in_ln, D * sizeof(float));memcpy(a + ch->o_post[i], l->post_ln, D * sizeof(float));@@ -211,44 +341,45 @@ static int g53c_setup(GModel *m) {ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);free(a);- for (int i = 0; i < L && ok; i++) {- const GLayer *l = &m->layer[i];- ok = coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn, NULL, 10, HD, nm, 0) &&- coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn, NULL, 10, HD, nm, 0);- if (ok && c->is_full[i]) {- ch->mla[i] = (VkcMla){c->n_heads, c->qk_nope, 0, c->v_head, c->kv_lora, D, c->q_lora, c->eps,- 1.0f / sqrtf((float)c->qk_nope), VKC_ROPE_HALF, g53c_tensor(&l->qa), g53c_tensor(&l->qb),- g53c_tensor(&l->kva), NULL, g53c_tensor(&l->kvb_kt), g53c_tensor(&l->kvb_v),- g53c_tensor(&l->o), ch->prm, ch->o_qn[i], ch->o_kn[i]};- ok = ch->mla[i].q_a && ch->mla[i].q_b && ch->mla[i].kv_a && ch->mla[i].k_abs && ch->mla[i].v_abs && ch->mla[i].o &&- g53c_tensor(&l->iwq) && g53c_tensor(&l->iwk) && g53c_tensor(&l->iwp) && g53c_tensor(&l->ikpg);- } else if (ok)- ok = g53c_tensor(&l->kq) && g53c_tensor(&l->kk) && g53c_tensor(&l->kv) && g53c_tensor(&l->ko) &&- g53c_tensor(&l->kga) && g53c_tensor(&l->kgb) && g53c_tensor(&l->kfa) && g53c_tensor(&l->kfb) &&- g53c_tensor(&l->kb);- if (ok && i < c->first_dense) ok = g53c_tensor(&l->dg) && g53c_tensor(&l->du) && g53c_tensor(&l->dd);- if (ok && i >= c->first_dense) ok = g53c_tensor(&l->rg) && g53c_tensor(&l->ru) && g53c_tensor(&l->rd);- if (ok && !c->is_full[i]) {+ if (!ok) {+ vkc_free(ch->prm); ch->prm = NULL;+ vkc_fit_shrink("glm53", &g_g53c_fit, 0, "the parameters' buffer was refused");+ return 0;+ }+ for (int i = 0; i < N; i++) {+ GLayer *l = &m->layer[i];+ Mat *w[16]; int nw = g53c_layer_mats(m, i, w);+ int up = coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn, NULL, 10, HD, nm, 0) &&+ coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn, NULL, 10, HD, nm, 0);+ for (int k = 0; k < nw && up; k++) up = g53c_tensor(w[k]) != NULL;+ if (up && !c->is_full[i]) {ch->win[i] = vkc_buf((size_t)3 * P * c->conv_k * sizeof(float), VKC_DEV);ch->st[i] = vkc_buf((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float), VKC_DEV);- ok = ch->win[i] && ch->st[i];+ up = ch->win[i] && ch->st[i];+ }+ if (!up) { /* free what it placed: the layer runs on the CPU, from its host copies */+ g53c_layer_free(ch, m, i);+ vkc_fit_shrink("glm53", &g_g53c_fit, i, vkc_lost() ? "the device was lost" : "a matrix or buffer was refused");+ break;}+ if (c->is_full[i])+ ch->mla[i] = (VkcMla){c->n_heads, c->qk_nope, 0, c->v_head, c->kv_lora, D, c->q_lora, c->eps,+ 1.0f / sqrtf((float)c->qk_nope), VKC_ROPE_HALF, l->qa.vk, l->qb.vk, l->kva.vk, NULL,+ l->kvb_kt.vk, l->kvb_v.vk, l->o.vk, ch->prm, ch->o_qn[i], ch->o_kn[i]};+ g53_dho_drop_layer(m, i); /* COLI_VK_DENSE_HOST: all of the layer is there, its host copies go */+ vkc_fit_mark(&g_g53c_fit, i);}- if (!ok) { fprintf(stderr, "[VK] glm53 chain: a matrix did not reach the device (or has no device form); per-matrix path\n"); return 0; }+ ch->n = g_g53c_fit.n;+ if (!ch->n) { vkc_free(ch->prm); ch->prm = NULL; return 0; }+ for (int i = 0; i < ch->n; i++) if (c->is_full[i]) ch->mla_ord[i] = ch->n_mla++;ch->ok = 1;- int full = 0; for (int i = 0; i < L; i++) full += c->is_full[i];+ int full = 0, dense = 0;+ for (int i = 0; i < ch->n; i++) { full += c->is_full[i]; dense += i < c->first_dense; }fprintf(stderr, "[VK] glm53 chain: %d layers on the device (%d KDA, %d MLA with the k-pooled indexer, %d dense), "- "%d streams, %.1f MiB of parameters\n", L, L - full, full, c->first_dense, H, n * 4 / 1048576.0);+ "%d streams, %.1f MiB of parameters\n", ch->n, ch->n - full, full, dense, H, n * 4 / 1048576.0);return 1;}-/* g53c_res counts instead of reserving while g_g53c_count >= 0 (the chunk's sizing) */-static long long g_g53c_count = -1;-static int g53c_res(VkcBuf **b, size_t floats, int kind) {- if (g_g53c_count >= 0) { g_g53c_count += (long long)(floats ? floats : 1) * (long long)sizeof(float); return 1; }- return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind);-}-static int g53c_scratch(G53Chain *ch, const GModel *m, int rows, int ctx);/* Prompt rows per chunk (vkc_chunk_rows): the chain's scratch a row, counted from the* reservations (the MLA scratch as vkc_mla_scratch sizes it, the indexer's scores at* the session's capacity), and the routed experts' outputs (the tier's rows, the@@ -257,7 +388,7 @@ static int g53c_rows(G53Chain *ch, const GModel *m, int ctx) {if (!vkc_chunk_auto()) return vkc_chunk_rows("glm53", 0);const Cfg *c = &m->c;size_t mla = 0;- for (int i = 0; i < c->n_layers; i++) if (c->is_full[i]) {+ for (int i = 0; i < ch->n; i++) if (c->is_full[i]) {const VkcMla *a = &ch->mla[i];mla = (size_t)(a->q_lora > 0 ? a->q_lora : 1) + (size_t)a->H * (a->Q + a->R) + (size_t)(a->K + a->R) +2 * (size_t)a->H * a->K + (size_t)a->H * a->V;@@ -277,10 +408,10 @@ static int g53c_rows(G53Chain *ch, const GModel *m, int ctx) {static int g53c_scratch(G53Chain *ch, const GModel *m, int rows, int ctx) {const Cfg *c = &m->c;- int D = c->hidden, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, P = c->kda_proj, L = c->n_layers;+ int D = c->hidden, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, P = c->kda_proj;int IH = c->index_nh, ID = c->index_hd, MI = c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter;int width = coli_sparse_index_width(c->index_topk, c->index_kpool, c->index_kpool_tail);- int mfull = -1; for (int i = 0; i < L; i++) if (c->is_full[i]) { mfull = i; break; }+ int mfull = -1; for (int i = 0; i < ch->n; i++) if (c->is_full[i]) { mfull = i; break; } /* the device's MLA layers */size_t r = (size_t)rows;ch->kvd_layer = r * (c->kv_lora + 2 * ID);int ok = (mfull < 0 || g_g53c_count >= 0 || vkc_mla_scratch(&ch->sc, &ch->mla[mfull], rows)) &&@@ -288,7 +419,7 @@ static int g53c_scratch(G53Chain *ch, const GModel *m, int rows, int ctx) {g53c_res(&ch->nrm, r * D, VKC_DEV) && g53c_res(&ch->br, r * D, VKC_DEV) && g53c_res(&ch->mix, r * nm, VKC_DEV) &&g53c_res(&ch->hp, r * (2 * H + H * H), VKC_DEV) && g53c_res(&ch->gs, r * MI, VKC_DEV) &&g53c_res(&ch->us, r * MI, VKC_DEV) && g53c_res(&ch->hs, r * MI, VKC_DEV) && g53c_res(&ch->ds, r * D, VKC_DEV) &&- g53c_res(&ch->h2d, r * D, VKC_DOWN) && g53c_res(&ch->kvd, (size_t)L * ch->kvd_layer, VKC_DOWN) &&+ g53c_res(&ch->h2d, r * D, VKC_DOWN) && g53c_res(&ch->kvd, (size_t)ch->n * ch->kvd_layer, VKC_DOWN) &&g53c_res(&ch->xd, r * HD, VKC_DOWN) && g53c_res(&ch->routed, r * D, VKC_UP);if (ok && P > 0)ok = g53c_res(&ch->qkv3, 3 * r * P, VKC_DEV) && g53c_res(&ch->cm, r * 3 * P, VKC_DEV) && g53c_res(&ch->kf, r * P, VKC_DEV) &&@@ -322,7 +453,7 @@ static int g53c_scratch(G53Chain *ch, const GModel *m, int rows, int ctx) {* split (the latent rows; the index keys, pool gates and pooled keys stay whole), for* steps of `rows` rows. */static int g53c_mirror(G53Chain *ch, const GModel *m, int need, int limit, int rows) {- const Cfg *c = &m->c; int L = c->n_layers, ID = c->index_hd, pool = c->index_kpool;+ const Cfg *c = &m->c; int L = ch->n, ID = c->index_hd, pool = c->index_kpool; /* the device's layers */if (ch->cap >= need) return 1;int cap = 256; while (cap < need) cap *= 2;if (cap > limit) cap = limit;@@ -355,13 +486,13 @@ static int g53c_push(G53Chain *ch, const GModel *m, const GSession *s, int startconst Cfg *c = &m->c; int ok = 1, K = c->kv_lora, ID = c->index_hd;if (ch->where == G53C_HOST) {size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)3 * c->kda_proj * c->conv_k;- for (int i = 0; i < c->n_layers && ok; i++)+ for (int i = 0; i < ch->n && ok; i++)if (!c->is_full[i])ok = vkc_write(ch->st[i], 0, s->layer[i].kda_state, ns * sizeof(float)) &&vkc_write(ch->win[i], 0, s->layer[i].kda_window, nw * sizeof(float));ch->where = G53C_BOTH;}- for (int i = 0; i < c->n_layers && ok; i++) {+ for (int i = 0; i < ch->n && ok; i++) {if (!c->is_full[i]) continue;const GLayerState *ls = &s->layer[i];int t0 = ch->kv_valid[i], n = start - t0;@@ -482,12 +613,13 @@ static void g53c_recover(GModel *m, GSession *s, int start, int dev) {int n = start - ch->rec_base;fprintf(stderr, "[VK] glm53 chain: the device was lost; rebuilding the state of %d positions on the CPU, ""which runs from here on\n", n);+ /* the device's layers only: a partial chain's CPU layers ran every position already */if (n <= 0) return;float *xs = malloc((size_t)n * H * D * sizeof(float)), *xn = malloc((size_t)n * H * D * sizeof(float));if (!xs || !xn) { fprintf(stderr, "OOM rebuilding the state\n"); exit(1); }for (int t = 0; t < n; t++) for (int h = 0; h < H; h++)memcpy(xs + ((size_t)t * H + h) * D, ch->rec + (size_t)t * D, (size_t)D * sizeof(float));- run_layers(m, s, xs, xn, n, ch->rec_base, 0, c->n_layers);+ run_layers(m, s, xs, xn, n, ch->rec_base, 0, ch->n);free(xs); free(xn);}@@ -500,13 +632,15 @@ static int g53c_prefill_rows(GModel *m, GSession *s, int n) {int rows = g53c_rows(ch, m, s->cap);return ch->ks.on && rows > ch->ks.chunk ? ch->ks.chunk : rows;}-/* Every layer for n rows of `streams` (the session's positions start..), the final- * streams back into it. 0 = not taken: the CPU runs the layers (streams untouched). */+/* The device's layers (every layer, or the first N of a partial chain) for n rows of+ * `streams` (the session's positions start..), the streams after the last of them back+ * into it. Returns how many layers ran: L, or N (run_layers goes on from layer N on the+ * CPU); 0 = not taken: the CPU runs the layers (streams untouched). */static int g53c_forward(GModel *m, GSession *s, float *streams, int n, int start) {G53Chain *ch = g_g53c;if (!g_vk_chain || !ch || !ch->ok || ch->failed) return 0;if (g_vk_chain == COLI_VK_CHAIN_PREFILL && n <= 2) return 0;- const Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, H = c->hc_mult, HD = H * D;+ const Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, H = c->hc_mult, HD = H * D, N = ch->n;if (ch->owner != s) { /* another session takes the device */if (ch->owner) g53c_sync_host(m, ch->owner);ch->owner = s; ch->where = G53C_HOST;@@ -552,7 +686,7 @@ static int g53c_forward(GModel *m, GSession *s, float *streams, int n, int startif (!vkc_begin() || !g53c_push(ch, m, s, pb, nr) || !vkc_write(ch->xs, 0, streams + (size_t)c0 * HD, (size_t)nr * HD * sizeof(float)))goto lost;int ok = 1, pending = 0, pulled = 0;- for (int i = 0; i < L && ok; i++) {+ for (int i = 0; i < N && ok; i++) {const GLayer *l = &m->layer[i];if (pending) { /* the FFN branch of the layer before: shared + routed, written back */VkcEw add = {VKC_EW_ADD, nr * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};@@ -593,15 +727,15 @@ static int g53c_forward(GModel *m, GSession *s, float *streams, int n, int start}ok = ok && vkc_copy(ch->xd, 0, ch->xs, 0, (size_t)nr * HD) && vkc_submit(1);if (!ok) goto lost;- g53c_pull(ch, m, s, pulled, L, pb, nr);+ g53c_pull(ch, m, s, pulled, N, pb, nr);memcpy(outs + (size_t)c0 * HD, vkc_ptr(ch->xd), (size_t)nr * HD * sizeof(float));- for (int i = 0; i < L; i++) if (c->is_full[i]) { ch->kv_valid[i] = pb + nr; vkc_kv_done(&ch->ks, ch->mla_ord[i], pb + nr); }+ for (int i = 0; i < N; i++) if (c->is_full[i]) { ch->kv_valid[i] = pb + nr; vkc_kv_done(&ch->ks, ch->mla_ord[i], pb + nr); }ch->where = G53C_DEV;}memcpy(streams, outs, (size_t)n * HD * sizeof(float));free(outs);ch->forwards++;- return 1;+ return N;lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */free(outs);if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }@@ -620,20 +754,33 @@ static void g53c_report(void) {vkc_prof_print();}-/* COLI_VK_CHAIN at startup, before the tier sizes itself: the decision, the pipelines, the tensors. */+/* COLI_VK_CHAIN at load, once the device is open (model_load_range): before the expert+ * cache sizes itself from the free memory and before the tier (the trunk's device copies+ * count as used): the decision, the pipelines, the fit (how many layers the device holds),+ * COLI_VK_DENSE_HOST's decision for those layers (g53_dho_start), the tensors layer by+ * layer, then the head's host copy (g53_dho_finish). Without a fit (the chain off)+ * g53_dho_start does as before. */static void g53c_start(GModel *m) {if (!g_vk_ready) return;const Cfg *c = &m->c;int tier_on = vkt_wanted() && m->streaming && c->n_experts > 0 && c->swiglu_limit > 0.f;+ if (!m->has_io || m->layer_begin != 0 || m->layer_end != c->n_layers) return; /* a segment: no chain, no drop */int on = coli_vk_chain_decide("glm53", tier_on, COLI_VK_CHAIN_UNMEASURED);const char *no = NULL;if (on && !(g_g53c_inited = vkc_init())) no = "the chain's pipelines did not come up";if (on && !no && !(vkc_mla_ready() && vkc_kda_ready() && vkc_mhc_ready()))no = "the MLA, KDA or mHC shaders are missing";if (no) fprintf(stderr, "[VK] glm53: %s: the dense chain stays off\n", no);- if (!on || no) return;- if (!g53c_setup(m)) return;- g_vk_chain = on;+ const char *why = on && !no ? g53c_check(m) : NULL;+ if (why) fprintf(stderr, "[VK] glm53 chain: %s; per-matrix path\n", why);+ if (on && !no && !why) g53c_fit_plan(m); /* before any upload */+ g53_dho_start(m, tier_on, g_g53c_fitted ? g_g53c_fit.n : -1, g_g53c_fitted ? g_g53c_fit.tail : 1);+ if (!g_g53c_fitted) return;+ if (g_g53c_fit.n > 0 && g53c_setup(m)) g_vk_chain = on;+ g_g53_partial = vkc_fit_partial(&g_g53c_fit);+ for (int i = 0; i < g_g53c_fit.n; i++) g_g53c_lazy += g_g53c_mir[i]; /* for the tier (glm53_vk_tier_start) */+ vkc_fit_placed("glm53", &g_g53c_fit);+ g53_dho_finish(m, g_g53c_fit.n, g_g53c_fit.tail, g_g53_partial);}/* After the tier's: at exit the chain goes before the device. */static void g53c_atexit(void) {