Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

glm53: batch the dense prefill path, bit-identical per token

JackKnifeAIdb7475fc7d38
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index c25acd06..6c4403f1 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -136,6 +136,7 @@ jobs:
- name: Run the GLM-5.3 tiny oracles
run: |
cd c && GLM53_TINY=/tmp/glm53_tiny GLM53_MM_TINY=/tmp/glm53_mm python -m unittest -v tests.test_glm53_oracles
+ GLM53_TINY=/tmp/glm53_tiny COLI_GLM53_FIXTURE=/tmp/glm53_stream-i4 python -m unittest -v tests.test_glm53_prefill_batch
macos:
# clang; libomp for the threaded path (Makefile falls back to
diff --git a/c/glm53.c b/c/glm53.c
index 6f4fffa7..ce2567d8 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -986,6 +986,52 @@ static void mv(float *out, const Mat *w, const float *x) {
}
}
+/* mv for S rows at once: out[S, rows] = x[S, columns] W^T. The kernels compute
+ * each (row, output) pair exactly as the one-row call does, so every token
+ * gets the same bits as S separate mv calls. What changes is the traffic: W is
+ * read once for the whole batch instead of once per token, which is the whole
+ * cost of a prefill, since at one token the dense matrices are bandwidth-bound.
+ * Matrices that live on a GPU keep the per-row path they already have. */
+static void mm(float *out, const Mat *w, const float *x, int S) {
+ int gpu = 0;
+#ifdef COLI_METAL
+ gpu |= g_metal_ready && w->resident && (w->fmt == 1 || w->fmt == 4);
+#endif
+#ifdef COLI_VULKAN
+ gpu |= g_vk_ready && w->resident && (w->fmt == 1 || w->fmt == 4);
+#endif
+ if (S == 1 || gpu) {
+ for (int t = 0; t < S; t++)
+ mv(out + (size_t)t * w->rows, w, x + (size_t)t * w->columns);
+ return;
+ }
+ switch (w->fmt) {
+ case 4: matmul_i4_grouped(out, x, w->q4, w->s, S, w->columns, w->rows, w->gs); break;
+ case 1: matmul_q(out, x, w->q8, w->s, S, w->columns, w->rows); break;
+ default: matmul(out, x, w->f, S, w->columns, w->rows); break;
+ }
+}
+
+/* mv_rows for S rows: out[S, rows] from x[S, columns] and W's rows
+ * [row0, row0 + rows). */
+static void mm_rows(float *out, const Mat *w, const float *x, int S, int row0, int rows) {
+ switch (w->fmt) {
+ case 4: {
+ const int packed = (w->columns + 1) / 2, groups = w->columns / w->gs;
+ matmul_i4_grouped(out, x, w->q4 + (size_t)row0 * packed,
+ w->s + (size_t)row0 * groups, S, w->columns, rows, w->gs);
+ break;
+ }
+ case 1:
+ matmul_q(out, x, w->q8 + (size_t)row0 * w->columns, w->s + row0,
+ S, w->columns, rows);
+ break;
+ default:
+ matmul(out, x, w->f + (size_t)row0 * w->columns, S, w->columns, rows);
+ break;
+ }
+}
+
static void rms(float *out, const float *x, const float *w, int n, float eps) {
float square = 0.0f;
for (int i = 0; i < n; i++) square += x[i] * x[i];
@@ -1029,50 +1075,74 @@ static void mlp3(float *out, const float *x, const Mat *g, const Mat *u, const M
mv(out, d, sg);
}
+/* mlp3 for S rows: sg and su hold S * g->rows floats each. */
+static void mlp3_rows(float *out, const float *x, int S, const Mat *g, const Mat *u,
+ const Mat *d, float limit, float *sg, float *su) {
+ mm(sg, g, x, S); mm(su, u, x, S);
+ for (int t = 0; t < S; t++)
+ swiglu_clamped(sg + (size_t)t * g->rows, su + (size_t)t * g->rows, g->rows, limit);
+ mm(out, d, sg, S);
+}
+
/* ---------- KDA: proiezioni, gate, ricorrenza ---------- */
static void kda_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
float *out, float *state, float *window, float *scratch) {
const int P = c->kda_proj, H = c->kda_heads, D = c->kda_hd;
+ const size_t T = (size_t)tokens;
+ /* Every projection reads only its own token's row, so they all run
+ * batched over the block; only the recurrence walks token by token. */
+ float *q = malloc(T * P * sizeof(float));
+ float *k = malloc(T * P * sizeof(float));
+ float *v = malloc(T * P * sizeof(float));
+ float *low = malloc(T * D * sizeof(float));
+ float *decay = malloc(T * P * sizeof(float));
+ float *beta = malloc(T * H * sizeof(float));
+ float *gate = malloc(T * P * sizeof(float));
+ float *normed = malloc(T * P * sizeof(float));
float *qkv = malloc((size_t)3 * P * sizeof(float));
- float *gate = malloc((size_t)P * sizeof(float));
- float *decay = malloc((size_t)P * sizeof(float));
- float *beta = malloc((size_t)H * sizeof(float));
- float *low = malloc((size_t)D * sizeof(float));
float *core = malloc((size_t)P * sizeof(float));
- for (int t = 0; t < tokens; t++) {
- const float *row = x + (size_t)t * c->hidden;
- mv(qkv, &l->kq, row);
- mv(qkv + P, &l->kk, row);
- mv(qkv + 2 * P, &l->kv, row);
- /* decadimento: gate_lower_bound * sigmoid(exp(A_log[h]) * (W_fb W_fa x + dt_bias)) */
- mv(low, &l->kfa, row);
- mv(decay, &l->kfb, low);
+ if (!q || !k || !v || !low || !decay || !beta || !gate || !normed || !qkv || !core) {
+ fprintf(stderr, "OOM in KDA\n"); exit(1);
+ }
+ mm(q, &l->kq, x, tokens);
+ mm(k, &l->kk, x, tokens);
+ mm(v, &l->kv, x, tokens);
+ /* decadimento: gate_lower_bound * sigmoid(exp(A_log[h]) * (W_fb W_fa x + dt_bias)) */
+ mm(low, &l->kfa, x, tokens);
+ mm(decay, &l->kfb, low, tokens);
+ mm(beta, &l->kb, x, tokens);
+ /* il gate low-rank dell'uscita: stessa forma, altri pesi */
+ mm(low, &l->kga, x, tokens);
+ mm(gate, &l->kgb, low, tokens);
+ for (size_t t = 0; t < T; t++) {
+ float *dk = decay + t * P, *bt = beta + t * H;
for (int h = 0; h < H; h++)
for (int d = 0; d < D; d++) {
int i = h * D + d;
- decay[i] = c->gate_lb * sigmoidf_(expf(l->alog[h]) * (decay[i] + l->dt[i]));
+ dk[i] = c->gate_lb * sigmoidf_(expf(l->alog[h]) * (dk[i] + l->dt[i]));
}
- mv(beta, &l->kb, row);
- for (int h = 0; h < H; h++) beta[h] = sigmoidf_(beta[h]);
- coli_kda_step(core, state, window, qkv, l->conv, decay, beta,
+ for (int h = 0; h < H; h++) bt[h] = sigmoidf_(bt[h]);
+ memcpy(qkv, q + t * P, (size_t)P * sizeof(float));
+ memcpy(qkv + P, k + t * P, (size_t)P * sizeof(float));
+ memcpy(qkv + 2 * P, v + t * P, (size_t)P * sizeof(float));
+ coli_kda_step(core, state, window, qkv, l->conv, dk, bt,
H, D, D, c->conv_k, 1e-6f, scratch);
/* uscita: RMSNorm per testa, pesata da o_norm, moltiplicata dal gate
- * low-rank, poi la proiezione di uscita. */
- mv(low, &l->kga, row);
- mv(gate, &l->kgb, low);
- float *normed = qkv; /* riuso: 3P >= P */
+ * low-rank, poi la proiezione di uscita (dopo il ciclo, in blocco). */
+ const float *gt = gate + t * P;
for (int h = 0; h < H; h++) {
const float *src = core + (size_t)h * D;
- float *dst = normed + (size_t)h * D;
+ float *dst = normed + t * P + (size_t)h * D;
float square = 0.0f;
for (int d = 0; d < D; d++) square += src[d] * src[d];
float inverse = 1.0f / sqrtf(square / D + c->eps);
for (int d = 0; d < D; d++)
- dst[d] = src[d] * inverse * l->onorm[d] * sigmoidf_(gate[(size_t)h * D + d]);
+ dst[d] = src[d] * inverse * l->onorm[d] * sigmoidf_(gt[(size_t)h * D + d]);
}
- mv(out + (size_t)t * c->hidden, &l->ko, normed);
}
- free(core); free(low); free(beta); free(decay); free(gate); free(qkv);
+ mm(out, &l->ko, normed, tokens);
+ free(core); free(qkv); free(normed); free(gate); free(beta); free(decay);
+ free(low); free(v); free(k); free(q);
}
/* ---------- MLA + indexer con k-pool ---------- */
@@ -1093,31 +1163,48 @@ static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
unsigned char *valid = malloc((size_t)seen);
memset(valid, 1, (size_t)seen);
+ /* Le proiezioni leggono solo la riga del proprio token: tutte in blocco. */
+ mm(qa, &l->qa, x, tokens);
for (int t = 0; t < tokens; t++) {
- const int at = base + t; /* posizione assoluta nella cache */
- const float *row = x + (size_t)t * c->hidden;
float *qn = qa + (size_t)t * c->q_lora;
- mv(qn, &l->qa, row);
rms(qn, qn, l->qa_ln, c->q_lora, c->eps);
- mv(queries + (size_t)t * H * QK, &l->qb, qn);
- float *here = latent + (size_t)at * L;
- mv(here, &l->kva, row);
+ }
+ mm(queries, &l->qb, qa, tokens);
+ mm(latent + (size_t)base * L, &l->kva, x, tokens);
+ for (int t = 0; t < tokens; t++) {
+ float *here = latent + (size_t)(base + t) * L;
rms(here, here, l->kva_ln, L, c->eps);
- /* la query entra nello spazio del latente una volta per testa, invece
- * che il latente nello spazio della query una volta per posizione */
- for (int h = 0; h < H; h++)
- mv_rows(absorbed + ((size_t)t * H + h) * L, &l->kvb_kt,
- queries + ((size_t)t * H + h) * QK, h * L, L);
- /* indexer: le query vengono dal q_a normalizzato, le chiavi dall'hidden
- * con LayerNorm (con bias), e i pesi per testa sono scalati da IH^-0.5 */
- mv(iq + (size_t)t * IH * ID, &l->iwq, qn);
- float *kraw = ik + (size_t)at * ID;
- mv(kraw, &l->iwk, row);
+ }
+ /* la query entra nello spazio del latente una volta per testa, invece
+ * che il latente nello spazio della query una volta per posizione; in
+ * blocco, una testa alla volta, raccogliendo la sua fetta di ogni token */
+ {
+ float *qh = malloc((size_t)tokens * QK * sizeof(float));
+ float *ah = malloc((size_t)tokens * L * sizeof(float));
+ if (!qh || !ah) { fprintf(stderr, "OOM in MLA\n"); exit(1); }
+ for (int h = 0; h < H; h++) {
+ for (int t = 0; t < tokens; t++)
+ memcpy(qh + (size_t)t * QK, queries + ((size_t)t * H + h) * QK,
+ (size_t)QK * sizeof(float));
+ mm_rows(ah, &l->kvb_kt, qh, tokens, h * L, L);
+ for (int t = 0; t < tokens; t++)
+ memcpy(absorbed + ((size_t)t * H + h) * L, ah + (size_t)t * L,
+ (size_t)L * sizeof(float));
+ }
+ free(ah); free(qh);
+ }
+ /* indexer: le query vengono dal q_a normalizzato, le chiavi dall'hidden
+ * con LayerNorm (con bias), e i pesi per testa sono scalati da IH^-0.5 */
+ mm(iq, &l->iwq, qa, tokens);
+ mm(ik + (size_t)base * ID, &l->iwk, x, tokens);
+ for (int t = 0; t < tokens; t++) {
+ float *kraw = ik + (size_t)(base + t) * ID;
layer_norm(kraw, kraw, l->ik_nw, l->ik_nb, ID, 1e-5f);
- mv(gates + (size_t)at * ID, &l->ikpg, row);
- mv(head_w + (size_t)t * IH, &l->iwp, row);
- for (int h = 0; h < IH; h++) head_w[(size_t)t * IH + h] /= sqrtf((float)IH);
}
+ mm(gates + (size_t)base * ID, &l->ikpg, x, tokens);
+ mm(head_w, &l->iwp, x, tokens);
+ for (int t = 0; t < tokens; t++)
+ for (int h = 0; h < IH; h++) head_w[(size_t)t * IH + h] /= sqrtf((float)IH);
const int width = coli_sparse_index_width(c->index_topk, c->index_kpool, c->index_kpool_tail);
int *selected = malloc((size_t)tokens * width * sizeof(int));
@@ -1137,14 +1224,18 @@ static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
}
/* Attenzione nello spazio del latente. La scala resta 1/sqrt(qk_nope):
* il prodotto e' lo stesso numero di prima, calcolato in un altro ordine. */
- float *context = malloc((size_t)H * V * sizeof(float));
- float *pooled = malloc((size_t)L * sizeof(float));
+ /* La media pesata dei latenti e' per token e per testa; le due
+ * proiezioni che la seguono (kvb_v per testa, poi o) vanno in blocco. */
+ float *pooled = malloc((size_t)tokens * H * L * sizeof(float));
+ unsigned char *attended = malloc((size_t)tokens * H);
float *score = malloc((size_t)width * sizeof(float));
+ if (!pooled || !attended || !score) { fprintf(stderr, "OOM in MLA\n"); exit(1); }
const float scale = 1.0f / sqrtf((float)QK);
for (int t = 0; t < tokens; t++) {
const int *chosen = selected + (size_t)t * width;
for (int h = 0; h < H; h++) {
const float *q = absorbed + ((size_t)t * H + h) * L;
+ float *pool = pooled + ((size_t)t * H + h) * L;
float top = -INFINITY;
int used = 0;
for (int i = 0; i < width; i++) {
@@ -1157,25 +1248,39 @@ static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
if (score[used] > top) top = score[used];
used++;
}
- float *result = context + (size_t)h * V;
- memset(result, 0, (size_t)V * sizeof(float));
+ memset(pool, 0, (size_t)L * sizeof(float));
+ attended[(size_t)t * H + h] = used > 0;
if (!used) continue;
double total = 0.0;
for (int i = 0; i < used; i++) { score[i] = expf(score[i] - top); total += score[i]; }
- memset(pooled, 0, (size_t)L * sizeof(float));
int seen_slot = 0;
for (int i = 0; i < width; i++) {
const int at = chosen[i];
if (at < 0 || at >= seen) continue;
const float weight = (float)(score[seen_slot++] / total);
const float *c_j = latent + (size_t)at * L;
- for (int d = 0; d < L; d++) pooled[d] += weight * c_j[d];
+ for (int d = 0; d < L; d++) pool[d] += weight * c_j[d];
}
- mv_rows(result, &l->kvb_v, pooled, h * V, V);
}
- mv(out + (size_t)t * c->hidden, &l->o, context);
}
- free(score); free(pooled);
+ float *context = malloc((size_t)tokens * H * V * sizeof(float));
+ float *ph = malloc((size_t)tokens * L * sizeof(float));
+ float *vh = malloc((size_t)tokens * V * sizeof(float));
+ if (!context || !ph || !vh) { fprintf(stderr, "OOM in MLA\n"); exit(1); }
+ for (int h = 0; h < H; h++) {
+ for (int t = 0; t < tokens; t++)
+ memcpy(ph + (size_t)t * L, pooled + ((size_t)t * H + h) * L, (size_t)L * sizeof(float));
+ mm_rows(vh, &l->kvb_v, ph, tokens, h * V, V);
+ for (int t = 0; t < tokens; t++) {
+ float *result = context + (size_t)t * H * V + (size_t)h * V;
+ if (attended[(size_t)t * H + h])
+ memcpy(result, vh + (size_t)t * V, (size_t)V * sizeof(float));
+ else
+ memset(result, 0, (size_t)V * sizeof(float)); /* nessuna posizione: zero, come prima */
+ }
+ }
+ mm(out, &l->o, context, tokens);
+ free(vh); free(ph); free(score); free(attended); free(pooled);
free(context); free(selected); free(valid); free(head_w);
free(iq); free(absorbed); free(queries); free(qa);
@@ -1766,11 +1871,10 @@ static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
const int wide = c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter;
if (index < c->first_dense) { /* layer denso: nessun router */
- float *sg = malloc((size_t)wide * sizeof(float));
- float *su = malloc((size_t)wide * sizeof(float));
- for (int t = 0; t < tokens; t++)
- mlp3(out + (size_t)t * c->hidden, x + (size_t)t * c->hidden,
- &l->dg, &l->du, &l->dd, c->swiglu_limit, sg, su);
+ float *sg = malloc((size_t)tokens * wide * sizeof(float));
+ float *su = malloc((size_t)tokens * wide * sizeof(float));
+ if (!sg || !su) { fprintf(stderr, "OOM in dense FFN\n"); exit(1); }
+ mlp3_rows(out, x, tokens, &l->dg, &l->du, &l->dd, c->swiglu_limit, sg, su);
free(su); free(sg);
return;
}
@@ -1827,15 +1931,18 @@ static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
* che non da' errore, da' numeri sbagliati. Quindi si lavora a blocchi
* grandi al piu' quanto la cache: si legge il blocco in parallelo, si
* applica a tutti i token, si passa al prossimo. */
- float *sg = malloc((size_t)wide * sizeof(float));
- float *su = malloc((size_t)wide * sizeof(float));
- float *tmp = malloc((size_t)c->hidden * sizeof(float));
- if (!sg || !su || !tmp) { fprintf(stderr, "OOM in MoE\n"); exit(1); }
+ /* sg/su/tmp/xg hold a whole block's rows: the shared expert runs on every
+ * token, and a routed expert on every token that chose it, batched. */
+ float *sg = malloc((size_t)tokens * wide * sizeof(float));
+ float *su = malloc((size_t)tokens * wide * sizeof(float));
+ float *tmp = malloc((size_t)tokens * c->hidden * sizeof(float));
+ float *xg = malloc((size_t)tokens * c->hidden * sizeof(float));
+ int *row_t = malloc((size_t)tokens * sizeof(int));
+ float *row_w = malloc((size_t)tokens * sizeof(float));
+ if (!sg || !su || !tmp || !xg || !row_t || !row_w) { fprintf(stderr, "OOM in MoE\n"); exit(1); }
/* l'esperto condiviso e' sempre attivo e non passa dalla cache */
- for (int t = 0; t < tokens; t++)
- mlp3(out + (size_t)t * c->hidden, x + (size_t)t * c->hidden,
- &l->rg, &l->ru, &l->rd, c->swiglu_limit, sg, su);
+ mlp3_rows(out, x, tokens, &l->rg, &l->ru, &l->rd, c->swiglu_limit, sg, su);
if (!m->streaming) {
for (int t = 0; t < tokens; t++)
@@ -1847,7 +1954,7 @@ static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
float *dst = out + (size_t)t * c->hidden;
for (int d = 0; d < c->hidden; d++) dst[d] += scale * tmp[d];
}
- free(tmp); free(su); free(sg); free(weight); free(chosen);
+ free(row_w); free(row_t); free(xg); free(tmp); free(su); free(sg); free(weight); free(chosen);
return;
}
@@ -1959,13 +2066,15 @@ static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
}
#endif
if (!metal_done) {
- /* CPU fallback: one expert at a time, then every token that chose it. */
+ /* CPU fallback: one expert at a time, all the tokens that chose it
+ * in one batch, then added back in token order. */
for (int i = 0; i < here; i++) {
const int eid = union_ids[base + i];
Slot *slot = &cache->s[slot_of[i]];
slot->used = ++m->clock;
Mat gate, up, down;
expert_mats(m, slot, &gate, &up, &down);
+ int R = 0;
for (int t = 0; t < tokens; t++) {
float scale = 0.0f;
for (int k = 0; k < topk; k++)
@@ -1974,16 +2083,23 @@ static void ffn_layer(GModel *m, const GLayer *l, int index, const float *x,
break;
}
if (scale == 0.0f) continue;
- mlp3(tmp, x + (size_t)t * c->hidden, &gate, &up, &down,
- c->swiglu_limit, sg, su);
- float *dst = out + (size_t)t * c->hidden;
- for (int d = 0; d < c->hidden; d++) dst[d] += scale * tmp[d];
+ memcpy(xg + (size_t)R * c->hidden, x + (size_t)t * c->hidden,
+ (size_t)c->hidden * sizeof(float));
+ row_t[R] = t; row_w[R] = scale; R++;
+ }
+ if (!R) continue;
+ mlp3_rows(tmp, xg, R, &gate, &up, &down, c->swiglu_limit, sg, su);
+ for (int r = 0; r < R; r++) {
+ const float *src = tmp + (size_t)r * c->hidden;
+ float *dst = out + (size_t)row_t[r] * c->hidden;
+ const float scale = row_w[r];
+ for (int d = 0; d < c->hidden; d++) dst[d] += scale * src[d];
}
}
}
}
free(to_read); free(slot_of); free(union_ids);
- free(tmp); free(su); free(sg); free(weight); free(chosen);
+ free(row_w); free(row_t); free(xg); free(tmp); free(su); free(sg); free(weight); free(chosen);
}
/* ---------- caricamento ---------- */
@@ -2506,8 +2622,12 @@ static void glm_echo(unsigned long long id, int pos, int token,
putchar('\n'); fflush(stdout);
}
-static float *forward_span(GModel *m, GSession *s, const int *tokens, int n,
- const float *vision, int n_vision) {
+/* `need`: how many of the last rows get logits. The head is the largest
+ * matrix outside the experts (vocab x hidden), and a prefill that keeps only
+ * its last row used to compute it for every token anyway. Rows before
+ * n - need are left unset. */
+static float *forward_span_rows(GModel *m, GSession *s, const int *tokens, int n,
+ const float *vision, int n_vision, int need) {
const Cfg *c = &m->c;
const int H = c->hc_mult;
const int start = s->filled; /* NON 'base': nel ciclo dei layer e' gia' preso */
@@ -2571,8 +2691,8 @@ static float *forward_span(GModel *m, GSession *s, const int *tokens, int n,
float *logits = malloc((size_t)n * c->vocab * sizeof(float));
double t_head0 = now_s();
- for (int t = 0; t < n; t++)
- mv(logits + (size_t)t * c->vocab, &m->head, normed + (size_t)t * D);
+ if (need < 1 || need > n) need = n;
+ mm(logits + (size_t)(n - need) * c->vocab, &m->head, normed + (size_t)(n - need) * D, need);
m->t_head += now_s() - t_head0;
m->forwards++;
@@ -2582,6 +2702,11 @@ static float *forward_span(GModel *m, GSession *s, const int *tokens, int n,
return logits;
}
+static float *forward_span(GModel *m, GSession *s, const int *tokens, int n,
+ const float *vision, int n_vision) {
+ return forward_span_rows(m, s, tokens, n, vision, n_vision, n);
+}
+
/* Prefill a pezzi.
*
* Un prefill in un colpo solo tiene in memoria i temporanei di tutto il
@@ -2619,9 +2744,11 @@ static float *forward_prefill(GModel *m, GSession *s, const int *tokens, int n,
if (vision && c->image_token >= 0)
for (int i = 0; i < here; i++)
if (tokens[at + i] == c->image_token) mine++;
- float *part = forward_span(m, s, tokens + at, here,
- vision ? vision + (size_t)used_vision * c->hidden : NULL,
- mine);
+ /* senza keep_all e senza echo serve solo l'ultima riga del pezzo */
+ const int need = keep_all || (g_echo_k > 0 && g_echo_id) ? here : 1;
+ float *part = forward_span_rows(m, s, tokens + at, here,
+ vision ? vision + (size_t)used_vision * c->hidden : NULL,
+ mine, need);
used_vision += mine;
if (keep_all) {
memcpy(all + (size_t)at * c->vocab, part,
@@ -3652,9 +3779,12 @@ int main(int argc, char **argv) {
GSession *session = session_open(&model, count + (greedy > 0 ? greedy : 0) + 1);
const double prefill_start = now_s();
float *logits = forward_prefill(&model, session, tokens, count, vision, n_vision, 1);
- if (getenv("GLM53_VERBOSE"))
+ if (getenv("GLM53_VERBOSE")) {
fprintf(stderr, "load %.1fs, prefill %d tokens in %.1fs\n",
load_seconds, count, now_s() - prefill_start);
+ fprintf(stderr, "prefill profile: attention %.1fs, ffn %.1fs (disk %.1fs), "
+ "head %.1fs\n", model.t_attn, model.t_ffn, model.t_disk, model.t_head);
+ }
printf("teacher_forcing");
for (int t = 0; t < count; t++)
printf(" %d", argmax(logits + (size_t)t * model.c.vocab, model.c.vocab));
diff --git a/c/tests/test_glm53_prefill_batch.py b/c/tests/test_glm53_prefill_batch.py
new file mode 100644
index 00000000..ec99d245
--- /dev/null
+++ b/c/tests/test_glm53_prefill_batch.py
@@ -0,0 +1,83 @@
+"""GLM-5.3 prefill batching must not change a single bit.
+
+A prefill block runs every dense matrix once over all of its tokens instead of
+once per token. The kernels compute each (token, output) pair exactly as the
+one-row call does, so the block size may change the speed and nothing else:
+GLM53_PREFILL_CHUNK=1 (every matrix called one row at a time) and larger
+chunks must print the same teacher-forcing tokens, greedy tokens and
+last-position logits, byte for byte, at every dense precision.
+
+Runs on whichever fixtures are set, as the GLM-5.3 CI job provides them:
+GLM53_TINY (tools/make_glm53_tiny.py, resident f32 experts) and
+COLI_GLM53_FIXTURE (the int4 streaming fixture, which exercises the routed
+experts' batched CPU loop).
+"""
+import json
+import os
+import subprocess
+import unittest
+from pathlib import Path
+
+C_DIR = Path(__file__).resolve().parents[1]
+BINARY = next((C_DIR / name for name in ("glm53", "glm53.exe")
+ if (C_DIR / name).exists()), None)
+COMPARED = ("teacher_forcing", "last_logits", "greedy")
+
+
+def fixtures():
+ found = []
+ for variable in ("GLM53_TINY", "COLI_GLM53_FIXTURE"):
+ value = os.environ.get(variable)
+ if value:
+ found.append(Path(value))
+ return found
+
+
+def command(fixture):
+ ref = json.loads((fixture / "ref.json").read_text())
+ patches = fixture / "patches.f32"
+ if patches.exists():
+ grid = ref.get("grid", (0, 0))
+ ids = ref["prompt"]
+ extra = ["--patches", str(patches), "--grid", f"{grid[0]}x{grid[1]}"]
+ else:
+ ids = ref.get("prompt_ids") or ref["prompt"]
+ extra = []
+ return [str(BINARY), "--model", str(fixture), "--ids", ",".join(map(str, ids)),
+ *extra, "--greedy", "4", "--logits"]
+
+
+def run(fixture, bits, chunk):
+ env = {**os.environ, "GLM53_BITS": str(bits), "GLM53_PREFILL_CHUNK": str(chunk)}
+ for name in ("CLUSTER_WORKERS", "EXPERT_WORKER"):
+ env.pop(name, None)
+ result = subprocess.run(command(fixture), capture_output=True, text=True,
+ timeout=300, env=env)
+ if result.returncode != 0:
+ raise AssertionError(result.stderr)
+ lines = {}
+ for line in result.stdout.splitlines():
+ if line.strip():
+ lines[line.split()[0]] = line.split()[1:]
+ return lines
+
+
[email protected](fixtures(), "neither GLM53_TINY nor COLI_GLM53_FIXTURE is set")
+class Glm53PrefillBatchTest(unittest.TestCase):
+ def test_chunk_size_never_changes_the_output(self):
+ self.assertIsNotNone(BINARY, "a fixture is set but glm53 is not built")
+ for fixture in fixtures():
+ for bits in (32, 8, 4):
+ one = run(fixture, bits, 1)
+ for key in COMPARED: # a refused run must not pass as "equal"
+ self.assertTrue(one.get(key), f"{fixture.name}: no {key} at chunk 1")
+ for chunk in (3, 7, 128):
+ with self.subTest(fixture=fixture.name, bits=bits, chunk=chunk):
+ got = run(fixture, bits, chunk)
+ for key in COMPARED:
+ self.assertEqual(got.get(key), one[key],
+ f"{key} at chunk {chunk} differs from chunk 1")
+
+
+if __name__ == "__main__":
+ unittest.main()