Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(glm53): absorb kv_b_proj so the cache holds latents, not expanded keys

JustVugg316fb13c465d
diff --git a/c/glm53.c b/c/glm53.c
index 86afc3e2..ee71e3c1 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -609,7 +609,7 @@ typedef struct {
Mat kq, kk, kv, ko, kga, kgb, kfa, kfb, kb;
const float *conv, *dt, *alog, *onorm;
/* MLA + indexer */
- Mat qa, qb, kva, kvb, o, iwq, iwk, iwp, ikpg;
+ Mat qa, qb, kva, kvb_kt, kvb_v, o, iwq, iwk, iwp, ikpg;
const float *qa_ln, *kva_ln, *ik_nw, *ik_nb, *ikpa;
/* FFN */
Mat dg, du, dd; /* denso */
@@ -632,7 +632,7 @@ typedef struct {
typedef struct {
float *kda_state; /* [teste * k * v] */
float *kda_window; /* [3 * proiezione * kernel] */
- float *keys, *values; /* [cap][teste][qk], [cap][teste][v] */
+ float *latent; /* [cap][kv_lora]: MLA assorbita */
float *ikeys, *igates; /* [cap][dim indexer] */
} GLayerState;
@@ -724,6 +724,85 @@ static void quantize_i4_grouped(const float *w, uint8_t *q4, float *scale,
}
}
+/* Un blocco f32 gia' in memoria, portato alla precisione chiesta. Possiede il
+ * buffer: o lo tiene com'e' o lo libera dopo averlo quantizzato. */
+static Mat quantize_loaded(float *buffer, int rows, int columns) {
+ Mat mat; memset(&mat, 0, sizeof(mat));
+ mat.rows = rows; mat.columns = columns;
+ const int bits = glm53_dense_bits();
+ if (bits == 32) { mat.fmt = 0; mat.f = buffer; return mat; }
+ if (bits == 4 && columns % 64 == 0) {
+ const int groups = columns / 64;
+ uint8_t *packed = malloc((size_t)rows * ((columns + 1) / 2));
+ float *step = malloc((size_t)rows * groups * sizeof(float));
+ if (!packed || !step) { fprintf(stderr, "OOM quantizzando %dx%d\n", rows, columns); exit(1); }
+ quantize_i4_grouped(buffer, packed, step, rows, columns, 64);
+ free(buffer);
+ mat.fmt = 4; mat.q4 = packed; mat.s = step; mat.gs = 64;
+ return mat;
+ }
+ /* int8 per riga: e' anche il ripiego quando le colonne non sono multiple
+ * di 64, che capita sulle proiezioni piccole dell'indexer. */
+ int8_t *level = malloc((size_t)rows * columns);
+ float *step = malloc((size_t)rows * sizeof(float));
+ if (!level || !step) { fprintf(stderr, "OOM quantizzando %dx%d\n", rows, columns); exit(1); }
+ quantize_rows(buffer, level, step, rows, columns, 8);
+ free(buffer);
+ mat.fmt = 1; mat.q8 = level; mat.s = step;
+ return mat;
+}
+
+/* Le due forme assorbite di kv_b_proj.
+ *
+ * MLA comprime chiavi e valori in un latente da kv_lora (512 qui) e li
+ * riespande con kv_b_proj. Tenere in cache le chiavi espanse costa
+ * teste*qk_nope*2 float per posizione, cioe' 1,39 MB a token su questo
+ * modello, undici GB a ottomila token: piu' della macchina.
+ *
+ * Il latente invece costa 512 float, quarantatre volte meno, e non serve
+ * riespanderlo se si piegano le proiezioni nei due estremi. Per una testa h,
+ * con W_k e W_v le due meta' del blocco di kv_b_proj:
+ *
+ * punteggio_j = q . (W_k c_j) = (W_k^T q) . c_j
+ * uscita = somma_j a_j (W_v c_j) = W_v (somma_j a_j c_j)
+ *
+ * A sinistra si moltiplica per ogni posizione in cache, a destra una volta per
+ * query. E' un'uguaglianza, non un'approssimazione: cambia solo l'ordine dei
+ * prodotti, e con esso quanta memoria serve.
+ *
+ * Le due matrici si costruiscono qui, da kv_b_proj in f32, e poi passano dallo
+ * stesso quantizzatore di tutto il resto. */
+static void absorb_kvb(GModel *m, GLayer *l, const char *name) {
+ const Cfg *c = &m->c;
+ const int H = c->n_heads, QK = c->qk_nope, V = c->v_head, L = c->kv_lora;
+ st_tensor *t = st_find(&m->S, name);
+ if (!t) { fprintf(stderr, "manca %s\n", name); exit(1); }
+ if (t->numel != (int64_t)H * (QK + V) * L) {
+ fprintf(stderr, "%s: %lld valori, attesi %lld per %d teste\n", name,
+ (long long)t->numel, (long long)H * (QK + V) * L, H);
+ exit(1);
+ }
+ float *whole = malloc((size_t)t->numel * sizeof(float));
+ float *kt = malloc((size_t)H * L * QK * sizeof(float));
+ float *vv = malloc((size_t)H * V * L * sizeof(float));
+ if (!whole || !kt || !vv) { fprintf(stderr, "OOM su %s\n", name); exit(1); }
+ st_read_f32_cap(&m->S, name, whole, t->numel, 1);
+
+ for (int h = 0; h < H; h++) {
+ const float *block = whole + (size_t)h * (QK + V) * L;
+ /* W_k^T: [L, QK], da W_k che e' [QK, L] */
+ for (int d = 0; d < L; d++)
+ for (int i = 0; i < QK; i++)
+ kt[((size_t)h * L + d) * QK + i] = block[(size_t)i * L + d];
+ /* W_v: [V, L], gia' nel verso giusto, e' una copia di righe */
+ memcpy(vv + (size_t)h * V * L, block + (size_t)QK * L,
+ (size_t)V * L * sizeof(float));
+ }
+ free(whole);
+ l->kvb_kt = quantize_loaded(kt, H * L, QK);
+ l->kvb_v = quantize_loaded(vv, H * V, L);
+}
+
static Mat load_mat(GModel *m, const char *fmt, ...) {
char name[512];
va_list args; va_start(args, fmt); vsnprintf(name, sizeof(name), fmt, args); va_end(args);
@@ -786,29 +865,30 @@ static Mat load_mat(GModel *m, const char *fmt, ...) {
mat.rows = (int)t->shape[0];
mat.columns = (int)t->shape[1];
- const int bits = glm53_dense_bits();
- if (bits == 32) { mat.fmt = 0; mat.f = buffer; return mat; }
- if (bits == 4 && mat.columns % 64 == 0) {
- const int groups = mat.columns / 64;
- uint8_t *packed = malloc((size_t)mat.rows * ((mat.columns + 1) / 2));
- float *step = malloc((size_t)mat.rows * groups * sizeof(float));
- if (!packed || !step) { fprintf(stderr, "OOM su %s\n", name); exit(1); }
- quantize_i4_grouped(buffer, packed, step, mat.rows, mat.columns, 64);
- free(buffer);
- mat.fmt = 4; mat.q4 = packed; mat.s = step; mat.gs = 64;
- return mat;
- }
- /* int8 per riga: e' anche il ripiego quando le colonne non sono multiple
- * di 64, che capita sulle proiezioni piccole dell'indexer. */
- int8_t *level = malloc((size_t)mat.rows * mat.columns);
- float *step = malloc((size_t)mat.rows * sizeof(float));
- if (!level || !step) { fprintf(stderr, "OOM su %s\n", name); exit(1); }
- quantize_rows(buffer, level, step, mat.rows, mat.columns, 8);
- free(buffer);
- mat.fmt = 1; mat.q8 = level; mat.s = step;
+ mat = quantize_loaded(buffer, mat.rows, mat.columns);
return mat;
}
+/* Come mv ma su un blocco di righe contigue: serve alle matrici che tengono
+ * una testa dopo l'altra in un unico tensore. */
+static void mv_rows(float *out, const Mat *w, const float *x, int row0, int rows) {
+ switch (w->fmt) {
+ case 4: {
+ const int packed = (w->columns + 1) / 2, groups = w->columns / w->gs;
+ matmul_i4_grouped(out, x, w->q4 + (size_t)row0 * packed,
+ w->s + (size_t)row0 * groups, 1, w->columns, rows, w->gs);
+ break;
+ }
+ case 1:
+ matmul_q(out, x, w->q8 + (size_t)row0 * w->columns, w->s + row0,
+ 1, w->columns, rows);
+ break;
+ default:
+ matmul(out, x, w->f + (size_t)row0 * w->columns, 1, w->columns, rows);
+ break;
+ }
+}
+
/* out[rows] = W x, con W in layout [rows, columns] come transformers. */
static void mv(float *out, const Mat *w, const float *x) {
switch (w->fmt) {
@@ -914,11 +994,10 @@ static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
const int IH = c->index_nh, ID = c->index_hd;
const int seen = base + tokens;
float *qa = malloc((size_t)tokens * c->q_lora * sizeof(float));
+ const int L = c->kv_lora;
float *queries = malloc((size_t)tokens * H * QK * sizeof(float));
- float *keys = st->keys; /* tutto il prefisso, non solo i nuovi */
- float *values = st->values;
- float *latent = malloc((size_t)c->kv_lora * sizeof(float));
- float *expanded = malloc((size_t)H * (QK + V) * sizeof(float));
+ float *absorbed = malloc((size_t)tokens * H * L * sizeof(float));
+ float *latent = st->latent; /* tutto il prefisso, non solo i nuovi */
float *iq = malloc((size_t)tokens * IH * ID * sizeof(float));
float *ik = st->ikeys;
float *gates = st->igates;
@@ -933,14 +1012,14 @@ static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
mv(qn, &l->qa, row);
rms(qn, qn, l->qa_ln, c->q_lora, c->eps);
mv(queries + (size_t)t * H * QK, &l->qb, qn);
- mv(latent, &l->kva, row);
- rms(latent, latent, l->kva_ln, c->kv_lora, c->eps);
- mv(expanded, &l->kvb, latent);
- for (int h = 0; h < H; h++) {
- const float *src = expanded + (size_t)h * (QK + V);
- memcpy(keys + ((size_t)at * H + h) * QK, src, (size_t)QK * sizeof(float));
- memcpy(values + ((size_t)at * H + h) * V, src + QK, (size_t)V * sizeof(float));
- }
+ float *here = latent + (size_t)at * L;
+ mv(here, &l->kva, row);
+ rms(here, here, l->kva_ln, L, c->eps);
+ /* la query entra nello spazio del latente una volta per testa, invece
+ * che il latente nello spazio della query una volta per posizione */
+ for (int h = 0; h < H; h++)
+ mv_rows(absorbed + ((size_t)t * H + h) * L, &l->kvb_kt,
+ queries + ((size_t)t * H + h) * QK, h * L, L);
/* indexer: le query vengono dal q_a normalizzato, le chiavi dall'hidden
* con LayerNorm (con bias), e i pesi per testa sono scalati da IH^-0.5 */
mv(iq + (size_t)t * IH * ID, &l->iwq, qn);
@@ -968,16 +1047,50 @@ static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
fprintf(stderr, "\n");
}
}
- float *context = malloc((size_t)tokens * H * V * sizeof(float));
- if (coli_sparse_attention_range(context, queries, keys, values, selected,
- seen, width, H, QK, V, base, seen)) {
- fprintf(stderr, "attenzione sparsa fallita\n"); exit(1);
+ /* Attenzione nello spazio del latente. La scala resta 1/sqrt(qk_nope):
+ * il prodotto e' lo stesso numero di prima, calcolato in un altro ordine. */
+ float *context = malloc((size_t)H * V * sizeof(float));
+ float *pooled = malloc((size_t)L * sizeof(float));
+ float *score = malloc((size_t)width * sizeof(float));
+ const float scale = 1.0f / sqrtf((float)QK);
+ for (int t = 0; t < tokens; t++) {
+ const int *chosen = selected + (size_t)t * width;
+ for (int h = 0; h < H; h++) {
+ const float *q = absorbed + ((size_t)t * H + h) * L;
+ float top = -INFINITY;
+ int used = 0;
+ for (int i = 0; i < width; i++) {
+ const int at = chosen[i];
+ if (at < 0 || at >= seen) continue;
+ const float *c_j = latent + (size_t)at * L;
+ float dot = 0.0f;
+ for (int d = 0; d < L; d++) dot += q[d] * c_j[d];
+ score[used] = dot * scale;
+ if (score[used] > top) top = score[used];
+ used++;
+ }
+ float *result = context + (size_t)h * V;
+ memset(result, 0, (size_t)V * sizeof(float));
+ if (!used) continue;
+ double total = 0.0;
+ for (int i = 0; i < used; i++) { score[i] = expf(score[i] - top); total += score[i]; }
+ memset(pooled, 0, (size_t)L * sizeof(float));
+ int seen_slot = 0;
+ for (int i = 0; i < width; i++) {
+ const int at = chosen[i];
+ if (at < 0 || at >= seen) continue;
+ const float weight = (float)(score[seen_slot++] / total);
+ const float *c_j = latent + (size_t)at * L;
+ for (int d = 0; d < L; d++) pooled[d] += weight * c_j[d];
+ }
+ mv_rows(result, &l->kvb_v, pooled, h * V, V);
+ }
+ mv(out + (size_t)t * c->hidden, &l->o, context);
}
- for (int t = 0; t < tokens; t++)
- mv(out + (size_t)t * c->hidden, &l->o, context + (size_t)t * H * V);
+ free(score); free(pooled);
free(context); free(selected); free(valid); free(head_w);
- free(iq); free(expanded); free(latent); free(queries); free(qa);
+ free(iq); free(absorbed); free(queries); free(qa);
}
/* ---------- FFN: denso oppure MoE ---------- */
@@ -1273,7 +1386,12 @@ static void model_load(GModel *m, const char *dir) {
l->qb = load_mat(m, "%slayers.%d.self_attn.q_b_proj.weight", P, i);
l->kva = load_mat(m, "%slayers.%d.self_attn.kv_a_proj_with_mqa.weight", P, i);
l->kva_ln = load_f32(m, "%slayers.%d.self_attn.kv_a_layernorm.weight", P, i);
- l->kvb = load_mat(m, "%slayers.%d.self_attn.kv_b_proj.weight", P, i);
+ {
+ char kvb_name[512];
+ snprintf(kvb_name, sizeof(kvb_name),
+ "%slayers.%d.self_attn.kv_b_proj.weight", P, i);
+ absorb_kvb(m, l, kvb_name);
+ }
l->o = load_mat(m, "%slayers.%d.self_attn.o_proj.weight", P, i);
l->iwq = load_mat(m, "%slayers.%d.self_attn.indexer.wq_b.weight", P, i);
l->iwk = load_mat(m, "%slayers.%d.self_attn.indexer.wk.weight", P, i);
@@ -1448,11 +1566,10 @@ static GSession *session_open(const GModel *m, int cap) {
for (int i = 0; i < c->n_layers; i++) {
GLayerState *st = &s->layer[i];
if (c->is_full[i]) {
- st->keys = malloc((size_t)cap * c->n_heads * c->qk_nope * sizeof(float));
- st->values = malloc((size_t)cap * c->n_heads * c->v_head * sizeof(float));
+ st->latent = malloc((size_t)cap * c->kv_lora * sizeof(float));
st->ikeys = malloc((size_t)cap * c->index_hd * sizeof(float));
st->igates = malloc((size_t)cap * c->index_hd * sizeof(float));
- if (!st->keys || !st->values || !st->ikeys || !st->igates) {
+ if (!st->latent || !st->ikeys || !st->igates) {
fprintf(stderr, "OOM sulla cache del layer %d\n", i); exit(1);
}
} else if (c->kda_proj) {
@@ -1464,6 +1581,14 @@ static GSession *session_open(const GModel *m, int cap) {
}
}
}
+ if (getenv("GLM53_VERBOSE")) {
+ int full = 0;
+ for (int i = 0; i < c->n_layers; i++) if (c->is_full[i]) full++;
+ const double per_token = (double)full * (c->kv_lora + 2 * c->index_hd) * sizeof(float);
+ fprintf(stderr, "cache: %.1f KB per token su %d layer DSA "
+ "(%.2f GB a %d posizioni)\n",
+ per_token / 1024.0, full, per_token * cap / 1e9, cap);
+ }
return s;
}
@@ -1471,7 +1596,7 @@ static void session_close(const GModel *m, GSession *s) {
if (!s) return;
for (int i = 0; i < m->c.n_layers; i++) {
GLayerState *st = &s->layer[i];
- free(st->keys); free(st->values); free(st->ikeys); free(st->igates);
+ free(st->latent); free(st->ikeys); free(st->igates);
free(st->kda_state); free(st->kda_window);
}
free(s->kda_scratch);