Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

glm53: the dense chain on the device (COLI_VK_CHAIN=1)

JustVugga547ac73cbb3
diff --git a/c/glm53.c b/c/glm53.c
index 06a482b0..d539a069 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -727,6 +727,10 @@ typedef struct {
} GModel;
#ifdef COLI_VULKAN
static void glm53_vk_report(const GModel *m, const char *scope); /* the [VK] lines of a run or turn */
+/* the dense chain (glm53_chain.h): the session's KDA state between the host and the device */
+static void g53c_sync_host(const GModel *m, const GSession *s);
+static void g53c_host_wrote(const GSession *s);
+static void g53c_session_gone(const GSession *s);
#endif
/* `want` e' quanti valori legge chi usa il tensore, contati dalla config. Il
@@ -2026,8 +2030,10 @@ static void ffn_layer_vk(GModel *m, int index, const float *x, int tokens, const
*
* Fare l'unione paga due volte: le letture vanno insieme, e un esperto che
* serve a piu' token del blocco si legge una volta sola. */
-static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
- int tokens, float *out) {
+/* with_shared 0 (the dense chain, glm53_chain.h): the routed experts only, from out = 0;
+ * the device runs the shared expert. */
+static void ffn_layer_ex(GModel *m, const GLayer *l, int index, const float *x,
+ int tokens, float *out, int with_shared) {
const Cfg *c = &m->c;
const int wide = c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter;
@@ -2105,7 +2111,8 @@ static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
if (!sg || !su || !tmp || !xg || !row_t || !row_w) { fprintf(stderr, "OOM in MoE\n"); exit(1); }
/* l'esperto condiviso e' sempre attivo e non passa dalla cache */
- mlp3_rows(out, x, tokens, &l->rg, &l->ru, &l->rd, c->swiglu_limit, sg, su);
+ if (with_shared) mlp3_rows(out, x, tokens, &l->rg, &l->ru, &l->rd, c->swiglu_limit, sg, su);
+ else memset(out, 0, (size_t)tokens * c->hidden * sizeof(float));
if (!m->streaming) {
for (int t = 0; t < tokens; t++)
@@ -2243,6 +2250,11 @@ static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
free(row_w); free(row_t); free(xg); free(tmp); free(su); free(sg); free(weight); free(chosen);
}
+static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
+ int tokens, float *out) {
+ ffn_layer_ex(m, l, index, x, tokens, out, 1);
+}
+
/* ---------- caricamento ---------- */
static void vision_load(GModel *m);
static void model_load_range(GModel *m, const char *dir, int b, int e, int io);
@@ -2583,6 +2595,9 @@ static GSession *session_open(const GModel *m, int cap) {
static void session_close(const GModel *m, GSession *s) {
if (!s) return;
+#ifdef COLI_VULKAN
+ g53c_session_gone(s); /* the dense chain's copy of its state goes with it */
+#endif
for (int i = 0; i < m->c.n_layers; i++) {
GLayerState *st = &s->layer[i];
free(st->latent); free(st->ikeys); free(st->igates);
@@ -2600,10 +2615,22 @@ static void session_close(const GModel *m, GSession *s) {
* `next` e' il secondo banco, della stessa misura: il passaggio li scambia a
* ogni sito, quindi alla fine il risultato puo' essere in uno o nell'altro, e
* la funzione restituisce quale. */
+#ifdef COLI_VULKAN
+static int g53c_forward(GModel *m, GSession *s, float *streams, int n, int start);
+static void g53c_cpu_step(const GModel *m, const GSession *s, int start);
+static int g_vk_chain;
+#endif
static float *run_layers(GModel *m, GSession *s, float *streams, float *next,
int n, int start, int begin, int end) {
const Cfg *c = &m->c;
const int H = c->hc_mult, D = c->hidden;
+#ifdef COLI_VULKAN
+ /* every layer on the device (glm53_chain.h); 0: the CPU below, its state current first */
+ if (g_vk_chain && begin == 0 && end == c->n_layers) {
+ if (g53c_forward(m, s, streams, n, start)) return streams;
+ g53c_cpu_step(m, s, start);
+ }
+#endif
float *collapsed = malloc((size_t)n * D * sizeof(float));
float *normed = malloc((size_t)n * D * sizeof(float));
float *branch = malloc((size_t)n * D * sizeof(float));
@@ -2651,6 +2678,9 @@ static float *run_layers(GModel *m, GSession *s, float *streams, float *next,
free(comb); free(post); free(branch); free(normed); free(collapsed);
return streams;
}
+#ifdef COLI_VULKAN
+#include "glm53_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */
+#endif
/* Rilascio completo del modello.
*
@@ -2852,6 +2882,13 @@ static float *forward_span_rows(GModel *m, GSession *s, const int *tokens, int n
if (need < 1 || need > n) need = n;
mm(logits + (size_t)(n - need) * c->vocab, &m->head, normed + (size_t)(n - need) * D, need);
m->t_head += now_s() - t_head0;
+#ifdef COLI_VULKAN
+ { /* DUMP=<path> (VK=1 builds): every logits row computed, raw f32, for the Vulkan gates */
+ static FILE *dump; static int said;
+ if (!said) { said = 1; const char *p = getenv("DUMP"); if (p && *p) dump = fopen(p, "wb"); }
+ if (dump) { fwrite(logits + (size_t)(n - need) * c->vocab, sizeof(float), (size_t)need * c->vocab, dump); fflush(dump); }
+ }
+#endif
m->forwards++;
free(normed); free(collapsed);
@@ -3174,6 +3211,9 @@ static int slot_pin_save(const GModel *m, KVSlot *slot, const int *tokens, int n
/* Sessione nuova: gli scatti vecchi parlano di righe DSA che non esistono
* piu, e rimetterli risponderebbe da posizioni inventate, in silenzio. */
if (slot->pin_session != slot->session) slot_pin_drop(m, slot);
+#ifdef COLI_VULKAN
+ g53c_sync_host(m, slot->session); /* the dense chain may hold the newest state */
+#endif
coli_pin_pool_init(&slot->pins, c->vocab);
ColiPin *k = coli_pin_store(&slot->pins, tokens, n, logit);
if (!k) return 0;
@@ -3232,6 +3272,9 @@ static int slot_pin_restore(const GModel *m, KVSlot *slot, const int *tokens, in
memcpy(ls->kda_state, st->state[i], ns * sizeof(float));
memcpy(ls->kda_window, st->window[i], nw * sizeof(float));
}
+#ifdef COLI_VULKAN
+ g53c_host_wrote(slot->session); /* the dense chain's copy goes up again */
+#endif
slot->session->filled = k->len;
coli_pin_touch(&slot->pins, s);
return k->len;
@@ -3247,6 +3290,9 @@ static int slot_pin_restore(const GModel *m, KVSlot *slot, const int *tokens, in
* 64x128x128 piu' la finestra della convoluzione. */
static int glm53_state_capture(const GModel *m, Glm53PinState **into, const GSession *s) {
const Cfg *c = &m->c;
+#ifdef COLI_VULKAN
+ g53c_sync_host(m, s); /* the dense chain may hold the newest state */
+#endif
const size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd;
const size_t nw = (size_t)3 * c->kda_proj * c->conv_k;
Glm53PinState *st = *into;
@@ -3275,6 +3321,9 @@ static int glm53_state_capture(const GModel *m, Glm53PinState **into, const GSes
static void glm53_state_restore(const GModel *m, const Glm53PinState *st, GSession *s) {
const Cfg *c = &m->c;
+#ifdef COLI_VULKAN
+ g53c_host_wrote(s); /* the dense chain's copy goes up again */
+#endif
const size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd;
const size_t nw = (size_t)3 * c->kda_proj * c->conv_k;
for (int i = 0; i < c->n_layers; i++) {
@@ -3886,6 +3935,7 @@ static int serve_one(GModel *m, Tok *tokenizer, ServeReq *q) {
rss_gb(), prompt_tokens, limited);
#ifdef COLI_VULKAN
glm53_vk_report(m, "turn");
+ g53c_report(); /* the dense chain's line, when it ran */
#endif
free(sequence);
return input_eof ? -1 : 0;
@@ -4153,7 +4203,9 @@ int main(int argc, char **argv) {
model_load(&served, snap);
glm53_telemetry_init(snap, &served.c);
#ifdef COLI_VULKAN
+ g53c_start(&served); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */
glm53_vk_tier_start(&served); /* after the history: the warm start reads it */
+ g53c_atexit(); /* after the tier's: the chain goes before the device */
#endif
Tok serve_tok;
char tokenizer_path[1024];
@@ -4214,7 +4266,9 @@ int main(int argc, char **argv) {
model_load(&model, dir);
glm53_telemetry_init(dir, &model.c);
#ifdef COLI_VULKAN
+ g53c_start(&model); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */
glm53_vk_tier_start(&model); /* after the history: the warm start reads it */
+ g53c_atexit(); /* after the tier's: the chain goes before the device */
#endif
const double load_seconds = now_s() - load_start;
if (getenv("GLM53_VERBOSE")) cfg_report(&model.c);
@@ -4317,6 +4371,7 @@ int main(int argc, char **argv) {
model.hits, model.miss, (unsigned long long)model.ebytes);
#ifdef COLI_VULKAN
glm53_vk_report(&model, "run");
+ g53c_report();
#endif
#ifdef COLI_METAL
if (g_metal_ready && getenv("GLM53_VERBOSE") && atoi(getenv("GLM53_VERBOSE")))
diff --git a/c/glm53_chain.h b/c/glm53_chain.h
new file mode 100644
index 00000000..a15135ea
--- /dev/null
+++ b/c/glm53_chain.h
@@ -0,0 +1,566 @@
+/* glm53_chain.h -- GLM-5.3 Flash's layers as a dense chain on the Vulkan device
+ * (vk_chain.h). Included once by glm53.c in a COLI_VULKAN build, after run_layers, the
+ * CPU forward it stands in for; COLI_VK_CHAIN decides (coli_vk_chain_decide, this
+ * engine's integrated GPU default off: not measured on a GLM-5.3 checkpoint).
+ *
+ * The residual is hc_mult streams per position (mHC). What runs where, per layer, for
+ * one block of rows (decode: one row):
+ * device, frame A1: the previous layer's FFN branch joins the streams (the routed sum
+ * from the host plus the shared expert, then mHC's write back); the attention site:
+ * mHC's mix (hc_fn, the split with Sinkhorn, the collapse), the input RMSNorm, then
+ * the KDA layer (the q/k/v, decay, beta and gate projections, the short convolution
+ * with its window, the delta rule with its state, the output norm and gate, o) or the
+ * MLA layer (vkc_mla_qkv, the k-pooled indexer: index keys with their LayerNorm, the
+ * pool gates, each completed pool's key, every row's pools; vkc_mla_attn over the
+ * selection), mHC's write back; the FFN site's mix, collapse and post-attention
+ * RMSNorm, and on a dense layer its MLP (clamped SwiGLU) and write back with no host
+ * step. Then the frame is waited for.
+ * host: the router and the routed experts (ffn_layer without the shared expert: the
+ * expert tier's batch and the CPU's share), the new MLA rows into the host's cache.
+ * device, frame A2 (not waited for): the shared expert (clamped SwiGLU).
+ * The final streams come back to the host, which collapses them and runs the final norm
+ * and the head as before.
+ *
+ * State and who owns it (one session at a time, `owner`):
+ * - the MLA caches (latent, index keys, pool gates): the host's stay canonical (every
+ * step copies its new rows back); the device mirrors them behind a watermark that
+ * every host write lowers (a CPU forward of the session, a pin restore rewinding
+ * `filled`); the pools' keys have a watermark of their own.
+ * - the KDA state and convolution windows: on the device while the chain runs (too
+ * large to copy per token). `where` says which side holds the newest copy; the host's
+ * is brought back before anything reads it there (a pin, a state capture, a CPU
+ * forward of the session, another session taking the device) and pushed up after
+ * anything writes it there (a restored pin or state).
+ * A device lost while the device holds the newest KDA state: the state is rebuilt on the
+ * CPU from the input rows of the positions since the host's copy was last current (the
+ * chain keeps them: the embedding rows, or the vision tower's), the forward runs again on
+ * the CPU and the CPU runs from there. COLI_VK_CHAIN_FAULT=n fakes the loss at the n-th
+ * frame.
+ *
+ * The chain declines (the CPU path runs, the state synced first) for a layer range (a
+ * segment), a session above the device's limits, and matrices or a geometry its shaders
+ * do not take (kv_lora above 1024, qk_rope above 0 is not GLM-5.3's and is declined too,
+ * a KDA head above 256 key or 128 value floats, an indexer above 64 heads, 4096 query
+ * floats or k-pooling below 2). */
+#include "vk_chain.h"
+
+#define G53C_HOST 0
+#define G53C_DEV 1
+#define G53C_BOTH 2
+
+typedef struct {
+ int ok, failed, rows, cap, pcap;
+ const GSession *owner;
+ int where; /* G53C_HOST / DEV / BOTH for owner's KDA state */
+ VkcBuf *prm;
+ size_t *o_in, *o_post, *o_hca, *o_hcf, *o_qn, *o_kn, *o_ikw, *o_ikb, *o_ape, *o_conv, *o_kda;
+ ColiVkTensor **fna, **fnf; /* the mHC mix matrices, f32 */
+ VkcMla *mla;
+ VkcMlaCache *kv; VkcBuf **ik, **ig, **pk, **win, **st;
+ int *kv_valid, *pool_valid;
+ VkcMlaScratch sc;
+ VkcBuf *xs, *xn, *col, *nrm, *br, *mix, *hp, *gs, *us, *hs, *ds, *qkv3, *cm, *kf, *kb, *kg, *klow, *ky;
+ VkcBuf *ikd, *iq, *ihw, *igd, *isc, *sel;
+ VkcBuf *h2d, *kvd, *xd, *routed;
+ size_t kvd_layer;
+ float *host_routed;
+ /* the input rows since the host's KDA state was last current: [rec_base, rec_base + rec_len) */
+ float *rec; int rec_base, rec_len, rec_cap, rec_ok;
+ unsigned long long forwards;
+ double host_ms;
+} G53Chain;
+
+static G53Chain *g_g53c;
+static int g_vk_chain = 0;
+static int g_g53c_inited = 0;
+
+static int g53c_rows(void) {
+ const char *e = getenv("COLI_VK_CHAIN_ROWS");
+ int v = e && *e ? atoi(e) : 512;
+ return v < 1 ? 1 : v > 65535 ? 65535 : v;
+}
+
+/* A Mat's device copy (the per-matrix path's own, where it made one); f32 as fmt 10. */
+static ColiVkTensor *g53c_tensor(const Mat *wc) {
+ Mat *w = (Mat *)wc;
+ if (w->vk) return (ColiVkTensor *)w->vk;
+ int fmt; const void *p;
+ switch (w->fmt) {
+ case 0: fmt = 10; p = w->f; break;
+ case 1: fmt = 1; p = w->q8; break;
+ case 4: if (w->gs < 8 || w->gs % 8) return NULL; fmt = 4; p = w->q4; break;
+ default: return NULL;
+ }
+ if (!p || w->rows < 1 || w->columns < 1) return NULL;
+ return coli_vk_tensor_ensure((ColiVkTensor **)&w->vk, p, w->fmt ? w->s : NULL, fmt, w->columns, w->rows,
+ w->fmt == 4 ? w->gs : 0) ? (ColiVkTensor *)w->vk : NULL;
+}
+
+/* ---- the KDA state between the host and the device --------------------------------- */
+static void g53c_rec_reset(G53Chain *ch, int base) { ch->rec_base = base; ch->rec_len = 0; ch->rec_ok = 1; }
+/* The host's copy of s's KDA state made current (before anything reads it there). */
+static void g53c_sync_host(const GModel *m, const GSession *s) {
+ G53Chain *ch = g_g53c;
+ if (!ch || !ch->ok || ch->owner != s || ch->where != G53C_DEV || !s) return;
+ const Cfg *c = &m->c;
+ size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)3 * c->kda_proj * c->conv_k;
+ for (int i = 0; i < c->n_layers; i++) {
+ if (c->is_full[i]) continue;
+ if (!vkc_read(ch->st[i], 0, s->layer[i].kda_state, ns * sizeof(float)) ||
+ !vkc_read(ch->win[i], 0, s->layer[i].kda_window, nw * sizeof(float))) {
+ fprintf(stderr, "[VK] glm53 chain: the device was lost holding the KDA state; it is rebuilt at the next step\n");
+ return; /* where stays DEV: the next forward rebuilds from the record */
+ }
+ }
+ ch->where = G53C_BOTH;
+ g53c_rec_reset(ch, s->filled);
+}
+/* The host wrote s's KDA state (a restored pin or state). */
+static void g53c_host_wrote(const GSession *s) {
+ G53Chain *ch = g_g53c;
+ if (ch && ch->owner == s) ch->where = G53C_HOST;
+}
+/* s goes away: whatever the device holds for it goes with it. */
+static void g53c_session_gone(const GSession *s) {
+ G53Chain *ch = g_g53c;
+ if (ch && ch->owner == s) { ch->owner = NULL; ch->where = G53C_HOST; }
+}
+/* A CPU forward of s from `start`: its state current on the host first, the device's
+ * stale after (and the MLA rows from start on). */
+static void g53c_cpu_step(const GModel *m, const GSession *s, int start) {
+ G53Chain *ch = g_g53c;
+ if (!ch || !ch->ok || ch->owner != s) return;
+ g53c_sync_host(m, s);
+ ch->where = G53C_HOST;
+ for (int i = 0; i < m->c.n_layers; i++) {
+ if (ch->kv_valid[i] > start) ch->kv_valid[i] = start;
+ if (ch->pool_valid[i] > start / m->c.index_kpool) ch->pool_valid[i] = start / m->c.index_kpool;
+ }
+}
+
+/* ---- setup ----------------------------------------------------------------------------- */
+static int g53c_setup(GModel *m) {
+ const Cfg *c = &m->c;
+ int L = c->n_layers, D = c->hidden, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, P = c->kda_proj;
+ const char *why = NULL;
+ if (m->layer_begin != 0 || m->layer_end != L || !m->has_io) why = "a layer range";
+ else if (c->kv_lora > 1024 || c->qk_rope != 0 || c->qk_nope > 1024 || c->q_lora < 1) why = "an attention geometry its shaders do not take";
+ else if (c->index_nh > 64 || c->index_nh * c->index_hd > 4096 || c->index_kpool < 2 || c->index_topk % c->index_kpool ||
+ c->index_topk / c->index_kpool > 1024) why = "an indexer its shaders do not take";
+ else if (P > 0 && (c->kda_hd > 128 || c->conv_k > 8)) why = "a KDA head its shaders do not take";
+ else if (H < 1 || H > 8 || c->hc_iters < 1) why = "hyper-connections its shaders do not take";
+ if (why) { fprintf(stderr, "[VK] glm53 chain: %s; per-matrix path\n", why); return 0; }
+ G53Chain *ch = calloc(1, sizeof *ch);
+ if (!ch) return 0;
+ size_t **offs[] = {&ch->o_in, &ch->o_post, &ch->o_hca, &ch->o_hcf, &ch->o_qn, &ch->o_kn, &ch->o_ikw, &ch->o_ikb,
+ &ch->o_ape, &ch->o_conv, &ch->o_kda};
+ for (size_t k = 0; k < sizeof offs / sizeof *offs; k++) if (!(*offs[k] = calloc(L, sizeof(size_t)))) return 0;
+ ch->fna = calloc(L, sizeof(void *)); ch->fnf = calloc(L, sizeof(void *));
+ ch->mla = calloc(L, sizeof(VkcMla)); ch->kv = calloc(L, sizeof(VkcMlaCache));
+ ch->ik = calloc(L, sizeof(void *)); ch->ig = calloc(L, sizeof(void *)); ch->pk = calloc(L, sizeof(void *));
+ ch->win = calloc(L, sizeof(void *)); ch->st = calloc(L, sizeof(void *));
+ ch->kv_valid = calloc(L, sizeof(int)); ch->pool_valid = calloc(L, sizeof(int));
+ if (!ch->fna || !ch->fnf || !ch->mla || !ch->kv || !ch->ik || !ch->ig || !ch->pk || !ch->win || !ch->st ||
+ !ch->kv_valid || !ch->pool_valid) return 0;
+ g_g53c = ch;
+ /* the parameter arena */
+ size_t n = 0;
+ for (int i = 0; i < L; i++) {
+ ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D;
+ ch->o_hca[i] = n; n += 3 + nm; ch->o_hcf[i] = n; n += 3 + nm;
+ if (c->is_full[i]) {
+ ch->o_qn[i] = n; n += c->q_lora; ch->o_kn[i] = n; n += c->kv_lora;
+ ch->o_ikw[i] = n; n += c->index_hd; ch->o_ikb[i] = n; n += c->index_hd;
+ ch->o_ape[i] = n; n += (size_t)c->index_kpool * c->index_hd;
+ } else {
+ ch->o_conv[i] = n; n += (size_t)3 * P * c->conv_k;
+ ch->o_kda[i] = n; n += (size_t)c->kda_heads + P + c->kda_hd;
+ }
+ }
+ float *a = calloc(n, sizeof(float));
+ if (!a) return 0;
+ for (int i = 0; i < L; i++) {
+ const GLayer *l = &m->layer[i];
+ memcpy(a + ch->o_in[i], l->in_ln, D * sizeof(float));
+ memcpy(a + ch->o_post[i], l->post_ln, D * sizeof(float));
+ memcpy(a + ch->o_hca[i], l->hc_attn_scale, 3 * sizeof(float));
+ memcpy(a + ch->o_hca[i] + 3, l->hc_attn_base, nm * sizeof(float));
+ memcpy(a + ch->o_hcf[i], l->hc_ffn_scale, 3 * sizeof(float));
+ memcpy(a + ch->o_hcf[i] + 3, l->hc_ffn_base, nm * sizeof(float));
+ if (c->is_full[i]) {
+ memcpy(a + ch->o_qn[i], l->qa_ln, c->q_lora * sizeof(float));
+ memcpy(a + ch->o_kn[i], l->kva_ln, c->kv_lora * sizeof(float));
+ memcpy(a + ch->o_ikw[i], l->ik_nw, c->index_hd * sizeof(float));
+ if (l->ik_nb) memcpy(a + ch->o_ikb[i], l->ik_nb, c->index_hd * sizeof(float));
+ memcpy(a + ch->o_ape[i], l->ikpa, (size_t)c->index_kpool * c->index_hd * sizeof(float));
+ } else {
+ memcpy(a + ch->o_conv[i], l->conv, (size_t)3 * P * c->conv_k * sizeof(float));
+ memcpy(a + ch->o_kda[i], l->alog, c->kda_heads * sizeof(float));
+ memcpy(a + ch->o_kda[i] + c->kda_heads, l->dt, P * sizeof(float));
+ memcpy(a + ch->o_kda[i] + c->kda_heads + P, l->onorm, c->kda_hd * sizeof(float));
+ }
+ }
+ ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);
+ int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);
+ free(a);
+ for (int i = 0; i < L && ok; i++) {
+ const GLayer *l = &m->layer[i];
+ ok = coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn, NULL, 10, HD, nm, 0) &&
+ coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn, NULL, 10, HD, nm, 0);
+ if (ok && c->is_full[i]) {
+ ch->mla[i] = (VkcMla){c->n_heads, c->qk_nope, 0, c->v_head, c->kv_lora, D, c->q_lora, c->eps,
+ 1.0f / sqrtf((float)c->qk_nope), VKC_ROPE_HALF, g53c_tensor(&l->qa), g53c_tensor(&l->qb),
+ g53c_tensor(&l->kva), NULL, g53c_tensor(&l->kvb_kt), g53c_tensor(&l->kvb_v),
+ g53c_tensor(&l->o), ch->prm, ch->o_qn[i], ch->o_kn[i]};
+ ok = ch->mla[i].q_a && ch->mla[i].q_b && ch->mla[i].kv_a && ch->mla[i].k_abs && ch->mla[i].v_abs && ch->mla[i].o &&
+ g53c_tensor(&l->iwq) && g53c_tensor(&l->iwk) && g53c_tensor(&l->iwp) && g53c_tensor(&l->ikpg);
+ } else if (ok)
+ ok = g53c_tensor(&l->kq) && g53c_tensor(&l->kk) && g53c_tensor(&l->kv) && g53c_tensor(&l->ko) &&
+ g53c_tensor(&l->kga) && g53c_tensor(&l->kgb) && g53c_tensor(&l->kfa) && g53c_tensor(&l->kfb) &&
+ g53c_tensor(&l->kb);
+ if (ok && i < c->first_dense) ok = g53c_tensor(&l->dg) && g53c_tensor(&l->du) && g53c_tensor(&l->dd);
+ if (ok && i >= c->first_dense) ok = g53c_tensor(&l->rg) && g53c_tensor(&l->ru) && g53c_tensor(&l->rd);
+ if (ok && !c->is_full[i]) {
+ ch->win[i] = vkc_buf((size_t)3 * P * c->conv_k * sizeof(float), VKC_DEV);
+ ch->st[i] = vkc_buf((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float), VKC_DEV);
+ ok = ch->win[i] && ch->st[i];
+ }
+ }
+ if (!ok) { fprintf(stderr, "[VK] glm53 chain: a matrix did not reach the device (or has no device form); per-matrix path\n"); return 0; }
+ ch->ok = 1;
+ int full = 0; for (int i = 0; i < L; i++) full += c->is_full[i];
+ fprintf(stderr, "[VK] glm53 chain: %d layers on the device (%d KDA, %d MLA with the k-pooled indexer, %d dense), "
+ "%d streams, %.1f MiB of parameters\n", L, L - full, full, c->first_dense, H, n * 4 / 1048576.0);
+ return 1;
+}
+
+static int g53c_res(VkcBuf **b, size_t floats, int kind) { return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind); }
+
+static int g53c_scratch(G53Chain *ch, const GModel *m, int rows, int ctx) {
+ const Cfg *c = &m->c;
+ int D = c->hidden, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, P = c->kda_proj, L = c->n_layers;
+ int IH = c->index_nh, ID = c->index_hd, MI = c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter;
+ int width = coli_sparse_index_width(c->index_topk, c->index_kpool, c->index_kpool_tail);
+ int mfull = -1; for (int i = 0; i < L; i++) if (c->is_full[i]) { mfull = i; break; }
+ size_t r = (size_t)rows;
+ ch->kvd_layer = r * (c->kv_lora + 2 * ID);
+ int ok = (mfull < 0 || vkc_mla_scratch(&ch->sc, &ch->mla[mfull], rows)) &&
+ g53c_res(&ch->xs, r * HD, VKC_DEV) && g53c_res(&ch->xn, r * HD, VKC_DEV) && g53c_res(&ch->col, r * D, VKC_DEV) &&
+ g53c_res(&ch->nrm, r * D, VKC_DEV) && g53c_res(&ch->br, r * D, VKC_DEV) && g53c_res(&ch->mix, r * nm, VKC_DEV) &&
+ g53c_res(&ch->hp, r * (2 * H + H * H), VKC_DEV) && g53c_res(&ch->gs, r * MI, VKC_DEV) &&
+ g53c_res(&ch->us, r * MI, VKC_DEV) && g53c_res(&ch->hs, r * MI, VKC_DEV) && g53c_res(&ch->ds, r * D, VKC_DEV) &&
+ g53c_res(&ch->h2d, r * D, VKC_DOWN) && g53c_res(&ch->kvd, (size_t)L * ch->kvd_layer, VKC_DOWN) &&
+ g53c_res(&ch->xd, r * HD, VKC_DOWN) && g53c_res(&ch->routed, r * D, VKC_UP);
+ if (ok && P > 0)
+ ok = g53c_res(&ch->qkv3, 3 * r * P, VKC_DEV) && g53c_res(&ch->cm, r * 3 * P, VKC_DEV) && g53c_res(&ch->kf, r * P, VKC_DEV) &&
+ g53c_res(&ch->kb, r * c->kda_heads, VKC_DEV) && g53c_res(&ch->kg, r * P, VKC_DEV) &&
+ g53c_res(&ch->klow, r * c->kda_hd, VKC_DEV) && g53c_res(&ch->ky, r * P, VKC_DEV);
+ if (ok && mfull >= 0)
+ ok = g53c_res(&ch->ikd, r * ID, VKC_DEV) && g53c_res(&ch->iq, r * IH * ID, VKC_DEV) && g53c_res(&ch->ihw, r * IH, VKC_DEV) &&
+ g53c_res(&ch->igd, r * ID, VKC_DEV) && g53c_res(&ch->isc, r * (size_t)(ctx / c->index_kpool + 1), VKC_DEV) &&
+ g53c_res(&ch->sel, r * (size_t)(1 + width), VKC_DEV);
+ if (!ok) return 0;
+ if (ch->rows < rows) {
+ float *hr = realloc(ch->host_routed, r * D * sizeof(float));
+ if (!hr) return 0;
+ ch->host_routed = hr; ch->rows = rows;
+ }
+ return 1;
+}
+
+/* The MLA mirror: room for `need` positions (the owner's), filled again when it grows. */
+static int g53c_mirror(G53Chain *ch, const GModel *m, int need, int limit) {
+ const Cfg *c = &m->c; int L = c->n_layers, ID = c->index_hd, pool = c->index_kpool;
+ if (ch->cap >= need) return 1;
+ int cap = 256; while (cap < need) cap *= 2;
+ if (cap > limit) cap = limit;
+ if (cap < need) return 0;
+ for (int i = 0; i < L; i++) {
+ if (!c->is_full[i]) continue;
+ vkc_free(ch->kv[i].lat); vkc_free(ch->ik[i]); vkc_free(ch->ig[i]); vkc_free(ch->pk[i]);
+ ch->kv[i] = (VkcMlaCache){NULL, NULL, 0}; ch->ik[i] = ch->ig[i] = ch->pk[i] = NULL;
+ ch->kv_valid[i] = ch->pool_valid[i] = 0;
+ }
+ ch->cap = 0;
+ for (int i = 0; i < L; i++) {
+ if (!c->is_full[i]) continue;
+ ch->kv[i].lat = vkc_buf((size_t)cap * c->kv_lora * sizeof(float), VKC_DEV); ch->kv[i].cap = cap;
+ ch->ik[i] = vkc_buf((size_t)cap * ID * sizeof(float), VKC_DEV);
+ ch->ig[i] = vkc_buf((size_t)cap * ID * sizeof(float), VKC_DEV);
+ ch->pk[i] = vkc_buf((size_t)(cap / pool + 1) * ID * sizeof(float), VKC_DEV);
+ if (!ch->kv[i].lat || !ch->ik[i] || !ch->ig[i] || !ch->pk[i]) return 0;
+ }
+ ch->cap = cap;
+ return 1;
+}
+
+/* Record the uploads that make the device the owner's below `start`. */
+static int g53c_push(G53Chain *ch, const GModel *m, const GSession *s, int start) {
+ const Cfg *c = &m->c; int ok = 1, K = c->kv_lora, ID = c->index_hd;
+ if (ch->where == G53C_HOST) {
+ size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)3 * c->kda_proj * c->conv_k;
+ for (int i = 0; i < c->n_layers && ok; i++)
+ if (!c->is_full[i])
+ ok = vkc_write(ch->st[i], 0, s->layer[i].kda_state, ns * sizeof(float)) &&
+ vkc_write(ch->win[i], 0, s->layer[i].kda_window, nw * sizeof(float));
+ ch->where = G53C_BOTH;
+ }
+ for (int i = 0; i < c->n_layers && ok; i++) {
+ if (!c->is_full[i]) continue;
+ const GLayerState *ls = &s->layer[i];
+ int t0 = ch->kv_valid[i], n = start - t0;
+ if (ch->pool_valid[i] > start / c->index_kpool) ch->pool_valid[i] = start / c->index_kpool; /* rows from start change */
+ if (n <= 0) continue;
+ ok = vkc_write(ch->kv[i].lat, (size_t)t0 * K, ls->latent + (size_t)t0 * K, (size_t)n * K * sizeof(float)) &&
+ vkc_write(ch->ik[i], (size_t)t0 * ID, ls->ikeys + (size_t)t0 * ID, (size_t)n * ID * sizeof(float)) &&
+ vkc_write(ch->ig[i], (size_t)t0 * ID, ls->igates + (size_t)t0 * ID, (size_t)n * ID * sizeof(float));
+ ch->kv_valid[i] = start;
+ }
+ return ok;
+}
+static void g53c_pull(G53Chain *ch, const GModel *m, GSession *s, int from, int to, int pb, int n) {
+ const Cfg *c = &m->c; int K = c->kv_lora, ID = c->index_hd;
+ for (int i = from; i < to; i++) {
+ if (!c->is_full[i]) continue;
+ const float *kv = (const float *)vkc_ptr(ch->kvd) + (size_t)i * ch->kvd_layer;
+ GLayerState *ls = &s->layer[i];
+ memcpy(ls->latent + (size_t)pb * K, kv, (size_t)n * K * sizeof(float));
+ memcpy(ls->ikeys + (size_t)pb * ID, kv + (size_t)n * K, (size_t)n * ID * sizeof(float));
+ memcpy(ls->igates + (size_t)pb * ID, kv + (size_t)n * (K + ID), (size_t)n * ID * sizeof(float));
+ }
+}
+
+/* ---- one layer's pieces ---------------------------------------------------------------- */
+/* mHC's entry for a site: the mix, the split, the collapse, the RMSNorm into nrm */
+static int g53c_pre(G53Chain *ch, const GModel *m, ColiVkTensor *fn, size_t hco, size_t lno, int n) {
+ const Cfg *c = &m->c; int H = c->hc_mult, D = c->hidden, HD = H * D, nm = (2 + H) * H, hr = 2 * H + H * H;
+ VkcMhc sp = {n, H, D, c->hc_iters, 0, HD, 0, nm, 0, hr, 0, D, (int)hco, 0, c->eps, c->hc_eps, 0.f};
+ VkcNorm nr = {n, D, 1, 0, D, D, 0, D, D, (int)lno, 0, 0, c->eps, 1.f};
+ return vkc_matmul(fn, ch->xs, 0, ch->mix, 0, n) && vkc_mhc(VKC_MHC_SPLIT, ch->xs, ch->mix, ch->hp, ch->prm, NULL, &sp) &&
+ vkc_mhc(VKC_MHC_COLLAPSE, ch->xs, NULL, ch->hp, NULL, ch->col, &sp) && vkc_norm(ch->col, ch->prm, ch->nrm, &nr);
+}
+/* mHC's exit: xn = mix(xs) + post * br, then the two swap */
+static int g53c_post(G53Chain *ch, const GModel *m, int n) {
+ const Cfg *c = &m->c; int H = c->hc_mult, D = c->hidden, HD = H * D, hr = 2 * H + H * H;
+ VkcMhc po = {n, H, D, c->hc_iters, 0, HD, 0, D, 0, hr, 0, HD, 0, 0, c->eps, c->hc_eps, 0.f};
+ if (!vkc_mhc(VKC_MHC_POST, ch->xs, ch->br, ch->hp, NULL, ch->xn, &po)) return 0;
+ VkcBuf *t = ch->xs; ch->xs = ch->xn; ch->xn = t;
+ return 1;
+}
+static int g53c_mlp(G53Chain *ch, const GModel *m, const Mat *g, const Mat *u, const Mat *d, VkcBuf *out, int n) {
+ int I = g->rows;
+ VkcMhc sw = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, n * I, 0.f, 0.f, m->c.swiglu_limit};
+ return vkc_matmul(g53c_tensor(g), ch->nrm, 0, ch->gs, 0, n) && vkc_matmul(g53c_tensor(u), ch->nrm, 0, ch->us, 0, n) &&
+ vkc_mhc(VKC_SWIGLU_CLAMP, ch->gs, ch->us, NULL, NULL, ch->hs, &sw) && vkc_matmul(g53c_tensor(d), ch->hs, 0, out, 0, n);
+}
+static int g53c_kda(G53Chain *ch, const GModel *m, const GLayer *l, int i, int n) {
+ const Cfg *c = &m->c; int P = c->kda_proj, HK = c->kda_heads, KD = c->kda_hd, r = ch->rows;
+ VkcKdaConv cp = {n, 3 * P, c->conv_k, P, 0, P, r * P, 0, 3 * P, (int)ch->o_conv[i], 0};
+ VkcKdaRec rp = {n, HK, KD, P, 0, 3 * P, 0, P, 0, HK, 0, P, 0, P, 0, (int)ch->o_kda[i], c->gate_lb, 1e-6f, c->eps};
+ return vkc_matmul(g53c_tensor(&l->kq), ch->nrm, 0, ch->qkv3, 0, n) &&
+ vkc_matmul(g53c_tensor(&l->kk), ch->nrm, 0, ch->qkv3, (size_t)r * P, n) &&
+ vkc_matmul(g53c_tensor(&l->kv), ch->nrm, 0, ch->qkv3, (size_t)2 * r * P, n) &&
+ vkc_matmul(g53c_tensor(&l->kfa), ch->nrm, 0, ch->klow, 0, n) &&
+ vkc_matmul(g53c_tensor(&l->kfb), ch->klow, 0, ch->kf, 0, n) &&
+ vkc_matmul(g53c_tensor(&l->kb), ch->nrm, 0, ch->kb, 0, n) &&
+ vkc_matmul(g53c_tensor(&l->kga), ch->nrm, 0, ch->klow, 0, n) &&
+ vkc_matmul(g53c_tensor(&l->kgb), ch->klow, 0, ch->kg, 0, n) &&
+ vkc_kda_conv(ch->qkv3, ch->prm, ch->win[i], ch->cm, &cp) &&
+ vkc_kda_rec(KD, ch->cm, ch->kf, ch->kb, ch->kg, ch->prm, ch->st[i], ch->ky, &rp) &&
+ vkc_matmul(g53c_tensor(&l->ko), ch->ky, 0, ch->br, 0, n);
+}
+static int g53c_mla(G53Chain *ch, const GModel *m, const GLayer *l, int i, int n, int pb) {
+ const Cfg *c = &m->c; int IH = c->index_nh, ID = c->index_hd, pool = c->index_kpool, K = c->kv_lora;
+ int width = coli_sparse_index_width(c->index_topk, pool, c->index_kpool_tail);
+ size_t ko = (size_t)i * ch->kvd_layer;
+ VkcMlaRow ln = {n, 1, 0, ID, 0, 0, ID, 0, pb * ID, ID, 0, 0, 0, (int)ch->o_ikw[i], (int)ch->o_ikb[i], l->ik_nb != NULL, 1e-5f};
+ int done = (pb + n) / pool, p0 = ch->pool_valid[i];
+ VkcDsaPool kp = {done > p0 ? done - p0 : 0, pool, p0, ID, 0, ID, (int)ch->o_ape[i], 0, 0, ID};
+ VkcDsaPick sp = {n, pb, IH, ID, c->index_topk, pool, 0, IH * ID, 0, IH, 0, c->index_kpool_tail, (pb + n) / pool + 1,
+ 1 + width, sqrtf((float)IH), 1.f / sqrtf((float)ID)};
+ int ok = vkc_mla_qkv(&ch->mla[i], &ch->sc, ch->nrm, 0, n, pb, NULL, &ch->kv[i], ch->kvd, ko) &&
+ vkc_matmul(g53c_tensor(&l->iwq), ch->sc.qa, 0, ch->iq, 0, n) &&
+ vkc_matmul(g53c_tensor(&l->iwk), ch->nrm, 0, ch->ikd, 0, n) &&
+ vkc_mla_lnorm(ch->ikd, ch->prm, ch->ik[i], &ln) &&
+ vkc_matmul(g53c_tensor(&l->ikpg), ch->nrm, 0, ch->igd, 0, n) &&
+ vkc_copy(ch->ig[i], (size_t)pb * ID, ch->igd, 0, (size_t)n * ID) &&
+ vkc_matmul(g53c_tensor(&l->iwp), ch->nrm, 0, ch->ihw, 0, n) &&
+ vkc_copy(ch->kvd, ko + (size_t)n * K, ch->ik[i], (size_t)pb * ID, (size_t)n * ID) &&
+ vkc_copy(ch->kvd, ko + (size_t)n * (K + ID), ch->igd, 0, (size_t)n * ID) &&
+ vkc_dsa_pool_keys(ch->ik[i], ch->ig[i], ch->prm, ch->pk[i], &kp) &&
+ vkc_dsa_pool_select(ch->iq, ch->ihw, ch->pk[i], ch->isc, ch->sel, &sp) &&
+ vkc_mla_attn(&ch->mla[i], &ch->sc, n, pb, 0, &ch->kv[i], ch->sel, 0, 1 + width, NULL, 0, ch->br, 0);
+ if (ok && done > ch->pool_valid[i]) ch->pool_valid[i] = done;
+ return ok;
+}
+
+/* The device is gone. If it held s's newest KDA state (dev: the state the forward from
+ * `start` began with was the device's only), the state is rebuilt on the CPU from the
+ * recorded input rows of [rec_base, start), from the host's copy at rec_base. */
+static void g53c_recover(GModel *m, GSession *s, int start, int dev) {
+ G53Chain *ch = g_g53c;
+ g_vk_chain = 0;
+ if (!ch) return;
+ ch->failed = 1;
+ int had = ch->owner == s && dev;
+ ch->owner = NULL; ch->where = G53C_HOST;
+ if (!had) {
+ fprintf(stderr, "[VK] glm53 chain: the device was lost; the host's state is current, the CPU runs from here on\n");
+ return;
+ }
+ const Cfg *c = &m->c; int H = c->hc_mult, D = c->hidden;
+ if (!ch->rec_ok || ch->rec_base + ch->rec_len < start) {
+ fprintf(stderr, "[VK] glm53 chain: the device was lost with a KDA state its recorded rows do not describe -- stopping "
+ "(COLI_VK_CHAIN=0 keeps the state on the CPU)\n");
+ exit(1);
+ }
+ int n = start - ch->rec_base;
+ fprintf(stderr, "[VK] glm53 chain: the device was lost; rebuilding the state of %d positions on the CPU, "
+ "which runs from here on\n", n);
+ if (n <= 0) return;
+ float *xs = malloc((size_t)n * H * D * sizeof(float)), *xn = malloc((size_t)n * H * D * sizeof(float));
+ if (!xs || !xn) { fprintf(stderr, "OOM rebuilding the state\n"); exit(1); }
+ for (int t = 0; t < n; t++) for (int h = 0; h < H; h++)
+ memcpy(xs + ((size_t)t * H + h) * D, ch->rec + (size_t)t * D, (size_t)D * sizeof(float));
+ run_layers(m, s, xs, xn, n, ch->rec_base, 0, c->n_layers);
+ free(xs); free(xn);
+}
+
+/* Every layer for n rows of `streams` (the session's positions start..), the final
+ * streams back into it. 0 = not taken: the CPU runs the layers (streams untouched). */
+static int g53c_forward(GModel *m, GSession *s, float *streams, int n, int start) {
+ G53Chain *ch = g_g53c;
+ if (!g_vk_chain || !ch || !ch->ok || ch->failed) return 0;
+ if (g_vk_chain == COLI_VK_CHAIN_PREFILL && n <= 2) return 0;
+ const Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, H = c->hc_mult, HD = H * D;
+ if (ch->owner != s) { /* another session takes the device */
+ if (ch->owner) g53c_sync_host(m, ch->owner);
+ ch->owner = s; ch->where = G53C_HOST;
+ for (int i = 0; i < L; i++) ch->kv_valid[i] = ch->pool_valid[i] = 0;
+ }
+ int CH = g53c_rows(), rows = n < CH ? n : CH, dev0 = ch->where == G53C_DEV;
+ if (vkc_lost()) { g53c_recover(m, s, start, dev0); return 0; }
+ if (!g53c_mirror(ch, m, start + n, s->cap) || !g53c_scratch(ch, m, rows, start + n)) {
+ fprintf(stderr, "[VK] glm53 chain: device memory for %d rows at %d positions refused; per-matrix path\n", rows, start + n);
+ g53c_cpu_step(m, s, start);
+ ch->failed = 1; g_vk_chain = 0;
+ return 0;
+ }
+ /* the input rows, for a rebuild should the device go: from the position where the
+ * host's KDA state is current (here, when the device takes it up now) */
+ if (ch->where == G53C_HOST) g53c_rec_reset(ch, start);
+ else if (ch->rec_ok && start != ch->rec_base + ch->rec_len) {
+ if (start < ch->rec_base + ch->rec_len && start >= ch->rec_base) ch->rec_len = start - ch->rec_base;
+ else ch->rec_ok = 0;
+ }
+ if (ch->rec_ok) {
+ if (ch->rec_len + n > ch->rec_cap) {
+ int nc = ch->rec_cap ? ch->rec_cap : 256; while (nc < ch->rec_len + n) nc *= 2;
+ float *r = realloc(ch->rec, (size_t)nc * D * sizeof(float));
+ if (!r) ch->rec_ok = 0; else { ch->rec = r; ch->rec_cap = nc; }
+ }
+ if (ch->rec_ok) {
+ for (int t = 0; t < n; t++) memcpy(ch->rec + (size_t)(ch->rec_len + t) * D, streams + (size_t)t * HD, (size_t)D * sizeof(float));
+ ch->rec_len += n;
+ }
+ }
+ vkc_gemm_rows(-1);
+ /* the final streams wait here until every chunk is through: a lost device leaves
+ * the caller's rows the forward's input, for the CPU to run again */
+ float *outs = malloc((size_t)n * HD * sizeof(float));
+ if (!outs) return 0;
+ for (int c0 = 0; c0 < n; c0 += rows) {
+ int nr = n - c0 < rows ? n - c0 : rows, pb = start + c0;
+ if (!vkc_begin() || !g53c_push(ch, m, s, pb) || !vkc_write(ch->xs, 0, streams + (size_t)c0 * HD, (size_t)nr * HD * sizeof(float)))
+ goto lost;
+ int ok = 1, pending = 0, pulled = 0;
+ for (int i = 0; i < L && ok; i++) {
+ const GLayer *l = &m->layer[i];
+ if (pending) { /* the FFN branch of the layer before: shared + routed, written back */
+ VkcEw add = {VKC_EW_ADD, nr * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ ok = vkc_ew(ch->br, ch->ds, ch->routed, NULL, NULL, &add) && g53c_post(ch, m, nr);
+ pending = 0;
+ }
+ double ta = now_s();
+ ok = ok && g53c_pre(ch, m, ch->fna[i], ch->o_hca[i], ch->o_in[i], nr) &&
+ (c->is_full[i] ? g53c_mla(ch, m, l, i, nr, pb) : g53c_kda(ch, m, l, i, nr)) &&
+ g53c_post(ch, m, nr) &&
+ g53c_pre(ch, m, ch->fnf[i], ch->o_hcf[i], ch->o_post[i], nr);
+ if (!ok) break;
+ if (i < c->first_dense) {
+ ok = g53c_mlp(ch, m, &l->dg, &l->du, &l->dd, ch->br, nr) && g53c_post(ch, m, nr);
+ m->t_attn += now_s() - ta;
+ continue;
+ }
+ ok = vkc_copy(ch->h2d, 0, ch->nrm, 0, (size_t)nr * D) && vkc_submit(1); /* A1 */
+ m->t_attn += now_s() - ta;
+ if (!ok) break;
+ g53c_pull(ch, m, s, pulled, i + 1, pb, nr); pulled = i + 1;
+ /* A2: the shared expert, while the host computes the routed experts */
+ ok = vkc_begin() && g53c_mlp(ch, m, &l->rg, &l->ru, &l->rd, ch->ds, nr) && vkc_submit(0);
+ double t1 = now_s();
+ ffn_layer_ex(m, l, i, (const float *)vkc_ptr(ch->h2d), nr, ch->host_routed, 0);
+ memcpy(vkc_ptr(ch->routed), ch->host_routed, (size_t)nr * D * sizeof(float));
+ double dt = now_s() - t1; ch->host_ms += dt * 1e3; m->t_ffn += dt;
+ ok = ok && vkc_begin();
+ pending = 1;
+ }
+ if (ok && pending) {
+ VkcEw add = {VKC_EW_ADD, nr * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ ok = vkc_ew(ch->br, ch->ds, ch->routed, NULL, NULL, &add) && g53c_post(ch, m, nr);
+ }
+ ok = ok && vkc_copy(ch->xd, 0, ch->xs, 0, (size_t)nr * HD) && vkc_submit(1);
+ if (!ok) goto lost;
+ g53c_pull(ch, m, s, pulled, L, pb, nr);
+ memcpy(outs + (size_t)c0 * HD, vkc_ptr(ch->xd), (size_t)nr * HD * sizeof(float));
+ for (int i = 0; i < L; i++) if (c->is_full[i]) ch->kv_valid[i] = pb + nr;
+ ch->where = G53C_DEV;
+ }
+ memcpy(streams, outs, (size_t)n * HD * sizeof(float));
+ free(outs);
+ ch->forwards++;
+ return 1;
+lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
+ free(outs);
+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
+ g53c_recover(m, s, start, dev0);
+ return 0;
+}
+
+static void g53c_report(void) {
+ G53Chain *ch = g_g53c;
+ if (!ch || !ch->ok || !ch->forwards) return;
+ VkcStats st; vkc_stats(&st);
+ fprintf(stderr, "[VK] glm53 chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
+ ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);
+ vkc_prof_print();
+}
+
+/* COLI_VK_CHAIN at startup, before the tier sizes itself: the decision, the pipelines, the tensors. */
+static void g53c_start(GModel *m) {
+ if (!g_vk_ready) return;
+ const Cfg *c = &m->c;
+ int tier_on = vkt_wanted() && m->streaming && c->n_experts > 0 && c->swiglu_limit > 0.f;
+ int on = coli_vk_chain_decide(NULL, tier_on, COLI_VK_CHAIN_OFF);
+ const char *e = getenv("COLI_VK_CHAIN");
+ char why[160];
+ if (e && *e) snprintf(why, sizeof why, "COLI_VK_CHAIN=%s", e);
+ else if (coli_vk_device_integrated())
+ snprintf(why, sizeof why, "an integrated GPU: not measured on a GLM-5.3 checkpoint; COLI_VK_CHAIN=1 on, 2 prompts only");
+ else if (coli_vk_device_shares_ram()) snprintf(why, sizeof why, "a CPU device; COLI_VK_CHAIN=1 turns it on");
+ else snprintf(why, sizeof why, "a discrete GPU%s", tier_on ? ", beside the expert tier" : "");
+ const char *no = NULL;
+ if (on && !(g_g53c_inited = vkc_init())) no = "the chain's pipelines did not come up";
+ if (on && !no && !(vkc_mla_ready() && vkc_kda_ready() && vkc_mhc_ready()))
+ no = "the MLA, KDA or mHC shaders are missing";
+ fprintf(stderr, "[VK] glm53: dense chain %s (%s%s%s)\n",
+ on && !no ? (on == COLI_VK_CHAIN_PREFILL ? "on for prompts" : "on") : "off", why, no ? "; " : "", no ? no : "");
+ if (!on || no) return;
+ if (!g53c_setup(m)) return;
+ g_vk_chain = on;
+}
+/* After the tier's: at exit the chain goes before the device. */
+static void g53c_atexit(void) {
+ if (g_g53c_inited) atexit(vkc_shutdown);
+}
diff --git a/c/glm_chain.h b/c/glm_chain.h
index c3749732..9d0355a5 100644
--- a/c/glm_chain.h
+++ b/c/glm_chain.h
@@ -330,6 +330,10 @@ static int glmc_forward(Model *m, float *xh, int S, int pos_base) {
return 0;
}
vkc_gemm_rows(-1);
+ /* the final rows wait here until every chunk is through: a lost device leaves xh the
+ * forward's input, for the CPU to run again */
+ float *outs = malloc((size_t)S * D * sizeof(float));
+ if (!outs) return 0;
float *inv = R ? malloc((size_t)(R / 2) * sizeof(float)) : NULL;
for (int j = 0; j < R / 2; j++) inv[j] = powf(c->theta, -2.0f * j / R); /* rope_interleave's frequencies */
for (int c0 = 0; c0 < S; c0 += rows) {
@@ -378,14 +382,15 @@ static int glmc_forward(Model *m, float *xh, int S, int pos_base) {
ok = ok && vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D) && vkc_submit(1);
if (!ok) goto lost;
glmc_pull_kv(ch, m, pulled, L, pb, n);
- memcpy(xh + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));
+ memcpy(outs + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));
for (int i = 0; i < L; i++) ch->kv_valid[i] = pb + n;
}
- free(inv);
+ memcpy(xh, outs, (size_t)S * D * sizeof(float));
+ free(inv); free(outs);
ch->forwards++;
return 1;
lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
- free(inv);
+ free(inv); free(outs);
if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
g_vk_chain = 0; ch->failed = 1;
fprintf(stderr, "[VK] colibri chain: the device was lost; the CPU runs this forward again and from here on "