Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(glm53): complete CPU text engine, token-exact on the tiny oracle

JustVugg69fc89ee56f0
diff --git a/c/glm53.c b/c/glm53.c
index af71b9b4..fcbf39db 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -63,6 +63,7 @@
#include <string.h>
#include <math.h>
#include <stdint.h>
+#include <stdarg.h>
#include "json.h"
#include "st.h"
@@ -77,7 +78,7 @@ typedef struct {
int hidden, n_layers, vocab, first_dense, dense_inter;
int n_heads, q_lora, kv_lora, qk_nope, qk_rope, v_head, qk_head;
int n_experts, topk, moe_inter, n_shared;
- float routed_scale, eps;
+ float routed_scale, eps, swiglu_limit;
int n_mtp; /* num_nextn_predict_layers (1) */
/* KDA */
int kda_heads, kda_hd, kda_proj, conv_k;
@@ -229,7 +230,29 @@ static void load_cfg(Cfg *c, const char *snap) {
c->hidden = (int)req_num(tc, "hidden_size");
c->n_layers = (int)req_num(tc, "num_hidden_layers");
c->vocab = (int)req_num(tc, "vocab_size");
- c->first_dense = (int)req_num(tc, "first_k_dense_replace");
+ /* Il checkpoint reale dichiara first_k_dense_replace; la fixture tiny usa
+ * mlp_layer_types. Accettiamo entrambi invece di imporne uno: sono la
+ * stessa informazione detta in due modi, e rifiutare la fixture
+ * significherebbe non poter provare il motore senza 194 GB. */
+ {
+ jval *fkd = json_get(tc, "first_k_dense_replace");
+ if (fkd && fkd->t == J_NUM) {
+ c->first_dense = (int)fkd->num;
+ } else {
+ jval *kinds = json_get(tc, "mlp_layer_types");
+ if (!kinds || kinds->t != J_ARR) {
+ fprintf(stderr, "config.json: serve first_k_dense_replace "
+ "oppure mlp_layer_types\n");
+ exit(1);
+ }
+ c->first_dense = kinds->len;
+ for (int i = 0; i < kinds->len; i++)
+ if (kinds->kids[i]->t == J_STR && strstr(kinds->kids[i]->str, "sparse")) {
+ c->first_dense = i;
+ break;
+ }
+ }
+ }
c->dense_inter = (int)req_num(tc, "intermediate_size");
c->n_heads = (int)req_num(tc, "num_attention_heads");
c->q_lora = (int)req_num(tc, "q_lora_rank");
@@ -243,6 +266,7 @@ static void load_cfg(Cfg *c, const char *snap) {
c->n_shared = (int)opt_num(tc, "n_shared_experts", 1);
c->routed_scale= (float)opt_num(tc, "routed_scaling_factor", 1.0);
c->eps = (float)opt_num(tc, "rms_norm_eps", 1e-6);
+ c->swiglu_limit= (float)opt_num(tc, "swiglu_limit", 0.0);
c->n_mtp = (int)opt_num(tc, "num_nextn_predict_layers", 0);
c->qk_head = c->qk_nope + c->qk_rope;
@@ -324,7 +348,7 @@ static void cfg_report(const Cfg *c) {
c->vis_image_size, c->vis_merge, c->vis_out_hidden);
}
-#ifdef GLM53_CFG_MAIN
+#ifdef GLM53_CFG_MAIN_UNUSED
/* Config-only entry point: `glm53_cfg <model_dir>` parses and reports, so the
* parser can be checked against a real checkpoint before any weight exists. */
int main(int argc, char **argv) {
@@ -526,7 +550,7 @@ static void glm53_walk_tensors(const Cfg *c, const Names *n,
#undef EMIT
}
-#ifdef GLM53_INVENTORY_MAIN
+#ifdef GLM53_INVENTORY_MAIN_UNUSED
/* Inventory entry point: prints every tensor the engine would load, in load
* order. Checked against a real checkpoint's index offline, so a naming
* mistake surfaces before 180 GB have been converted rather than after. */
@@ -543,3 +567,519 @@ int main(int argc, char **argv) {
return 0;
}
#endif
+
+/* ================= forward =================
+ * Il modello caricato in RAM come f32. Lo streaming degli esperti e la
+ * quantizzazione a load time arrivano col modello vero; qui la priorita' e'
+ * che i token siano quelli giusti, provati contro l'oracolo. */
+#include "delta_attention.h"
+#include "sparse_index.h"
+
+typedef struct { const float *w; int rows, columns; } Mat;
+
+typedef struct {
+ /* comune */
+ const float *in_ln, *post_ln;
+ const float *hc_attn_fn, *hc_attn_base, *hc_attn_scale;
+ const float *hc_ffn_fn, *hc_ffn_base, *hc_ffn_scale;
+ /* KDA */
+ Mat kq, kk, kv, ko, kga, kgb, kfa, kfb, kb;
+ const float *conv, *dt, *alog, *onorm;
+ /* MLA + indexer */
+ Mat qa, qb, kva, kvb, o, iwq, iwk, iwp, ikpg;
+ const float *qa_ln, *kva_ln, *ik_nw, *ik_nb, *ikpa;
+ /* FFN */
+ Mat dg, du, dd; /* denso */
+ Mat rg, ru, rd; /* router / shared: gate,up,down */
+ const float *router, *rbias;
+ Mat *eg, *eu, *ed; /* esperti routed */
+} GLayer;
+
+typedef struct {
+ Cfg c;
+ shards S;
+ const float *embed, *final_norm, *head;
+ GLayer *layer;
+ char prefix[64];
+} GModel;
+
+static const float *load_f32(GModel *m, const char *fmt, ...) {
+ char name[512];
+ va_list args; va_start(args, fmt); vsnprintf(name, sizeof(name), fmt, args); va_end(args);
+ st_tensor *t = st_find(&m->S, name);
+ if (!t) { fprintf(stderr, "manca il tensore %s\n", name); exit(1); }
+ float *buffer = malloc((size_t)t->numel * sizeof(float));
+ if (!buffer) { fprintf(stderr, "OOM su %s\n", name); exit(1); }
+ st_read_f32_cap(&m->S, name, buffer, t->numel, 0);
+ return buffer;
+}
+
+static Mat load_mat(GModel *m, const char *fmt, ...) {
+ char name[512];
+ va_list args; va_start(args, fmt); vsnprintf(name, sizeof(name), fmt, args); va_end(args);
+ st_tensor *t = st_find(&m->S, name);
+ if (!t) { fprintf(stderr, "manca la matrice %s\n", name); exit(1); }
+ if (t->rank != 2) { fprintf(stderr, "%s: rank %d, attesa 2\n", name, t->rank); exit(1); }
+ float *buffer = malloc((size_t)t->numel * sizeof(float));
+ st_read_f32_cap(&m->S, name, buffer, t->numel, 0);
+ Mat mat = { buffer, (int)t->shape[0], (int)t->shape[1] };
+ return mat;
+}
+
+/* out[rows] = W x, con W in layout [rows, columns] come transformers. */
+static void mv(float *out, const Mat *w, const float *x) {
+ for (int r = 0; r < w->rows; r++) {
+ const float *row = w->w + (size_t)r * w->columns;
+ float sum = 0.0f;
+ for (int c = 0; c < w->columns; c++) sum += row[c] * x[c];
+ out[r] = sum;
+ }
+}
+
+static void rms(float *out, const float *x, const float *w, int n, float eps) {
+ float square = 0.0f;
+ for (int i = 0; i < n; i++) square += x[i] * x[i];
+ float inverse = 1.0f / sqrtf(square / n + eps);
+ for (int i = 0; i < n; i++) out[i] = x[i] * inverse * w[i];
+}
+
+static void layer_norm(float *out, const float *x, const float *w, const float *b,
+ int n, float eps) {
+ float mean = 0.0f;
+ for (int i = 0; i < n; i++) mean += x[i];
+ mean /= n;
+ float variance = 0.0f;
+ for (int i = 0; i < n; i++) variance += (x[i] - mean) * (x[i] - mean);
+ variance /= n;
+ float inverse = 1.0f / sqrtf(variance + eps);
+ for (int i = 0; i < n; i++) out[i] = (x[i] - mean) * inverse * w[i] + (b ? b[i] : 0.0f);
+}
+
+static float sigmoidf_(float x) {
+ return x >= 0.0f ? 1.0f / (1.0f + expf(-x)) : expf(x) / (1.0f + expf(x));
+}
+static float siluf_(float x) { return x / (1.0f + expf(-x)); }
+
+/* SwiGLU clampata: il gate ha solo il tetto, up e' limitato da entrambi i lati.
+ * Vale sia per l'MLP denso che per gli esperti -- il testo di GLM-5.3 NON usa
+ * la SiLU semplice, ed e' un errore che darebbe un modello che parla bene e
+ * sbaglia. */
+static void swiglu_clamped(float *gate, const float *up, int n, float limit) {
+ for (int i = 0; i < n; i++) {
+ float g = gate[i] > limit ? limit : gate[i];
+ float u = up[i] < -limit ? -limit : (up[i] > limit ? limit : up[i]);
+ gate[i] = siluf_(g) * u;
+ }
+}
+
+static void mlp3(float *out, const float *x, const Mat *g, const Mat *u, const Mat *d,
+ float limit, float *sg, float *su) {
+ mv(sg, g, x); mv(su, u, x);
+ swiglu_clamped(sg, su, g->rows, limit);
+ mv(out, d, sg);
+}
+
+/* ---------- KDA: proiezioni, gate, ricorrenza ---------- */
+static void kda_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,
+ float *out, float *state, float *window, float *scratch) {
+ const int P = c->kda_proj, H = c->kda_heads, D = c->kda_hd;
+ float *qkv = malloc((size_t)3 * P * sizeof(float));
+ float *gate = malloc((size_t)P * sizeof(float));
+ float *decay = malloc((size_t)P * sizeof(float));
+ float *beta = malloc((size_t)H * sizeof(float));
+ float *low = malloc((size_t)D * sizeof(float));
+ float *core = malloc((size_t)P * sizeof(float));
+ for (int t = 0; t < tokens; t++) {
+ const float *row = x + (size_t)t * c->hidden;
+ mv(qkv, &l->kq, row);
+ mv(qkv + P, &l->kk, row);
+ mv(qkv + 2 * P, &l->kv, row);
+ /* decadimento: gate_lower_bound * sigmoid(exp(A_log[h]) * (W_fb W_fa x + dt_bias)) */
+ mv(low, &l->kfa, row);
+ mv(decay, &l->kfb, low);
+ for (int h = 0; h < H; h++)
+ for (int d = 0; d < D; d++) {
+ int i = h * D + d;
+ decay[i] = c->gate_lb * sigmoidf_(expf(l->alog[h]) * (decay[i] + l->dt[i]));
+ }
+ mv(beta, &l->kb, row);
+ for (int h = 0; h < H; h++) beta[h] = sigmoidf_(beta[h]);
+ coli_kda_step(core, state, window, qkv, l->conv, decay, beta,
+ H, D, D, c->conv_k, 1e-6f, scratch);
+ /* uscita: RMSNorm per testa, pesata da o_norm, moltiplicata dal gate
+ * low-rank, poi la proiezione di uscita. */
+ mv(low, &l->kga, row);
+ mv(gate, &l->kgb, low);
+ float *normed = qkv; /* riuso: 3P >= P */
+ for (int h = 0; h < H; h++) {
+ const float *src = core + (size_t)h * D;
+ float *dst = normed + (size_t)h * D;
+ float square = 0.0f;
+ for (int d = 0; d < D; d++) square += src[d] * src[d];
+ float inverse = 1.0f / sqrtf(square / D + c->eps);
+ for (int d = 0; d < D; d++)
+ dst[d] = src[d] * inverse * l->onorm[d] * sigmoidf_(gate[(size_t)h * D + d]);
+ }
+ mv(out + (size_t)t * c->hidden, &l->ko, normed);
+ }
+ free(core); free(low); free(beta); free(decay); free(gate); free(qkv);
+}
+
+/* ---------- MLA + indexer con k-pool ---------- */
+static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens, float *out) {
+ const int H = c->n_heads, QK = c->qk_nope, V = c->v_head;
+ const int IH = c->index_nh, ID = c->index_hd;
+ float *qa = malloc((size_t)tokens * c->q_lora * sizeof(float));
+ float *queries = malloc((size_t)tokens * H * QK * sizeof(float));
+ float *keys = malloc((size_t)tokens * H * QK * sizeof(float));
+ float *values = malloc((size_t)tokens * H * V * sizeof(float));
+ float *latent = malloc((size_t)c->kv_lora * sizeof(float));
+ float *expanded = malloc((size_t)H * (QK + V) * sizeof(float));
+ float *iq = malloc((size_t)tokens * IH * ID * sizeof(float));
+ float *ik = malloc((size_t)tokens * ID * sizeof(float));
+ float *gates = malloc((size_t)tokens * ID * sizeof(float));
+ float *head_w = malloc((size_t)tokens * IH * sizeof(float));
+ unsigned char *valid = malloc((size_t)tokens);
+ memset(valid, 1, (size_t)tokens);
+
+ for (int t = 0; t < tokens; t++) {
+ const float *row = x + (size_t)t * c->hidden;
+ float *qn = qa + (size_t)t * c->q_lora;
+ mv(qn, &l->qa, row);
+ rms(qn, qn, l->qa_ln, c->q_lora, c->eps);
+ mv(queries + (size_t)t * H * QK, &l->qb, qn);
+ mv(latent, &l->kva, row);
+ rms(latent, latent, l->kva_ln, c->kv_lora, c->eps);
+ mv(expanded, &l->kvb, latent);
+ for (int h = 0; h < H; h++) {
+ const float *src = expanded + (size_t)h * (QK + V);
+ memcpy(keys + ((size_t)t * H + h) * QK, src, (size_t)QK * sizeof(float));
+ memcpy(values + ((size_t)t * H + h) * V, src + QK, (size_t)V * sizeof(float));
+ }
+ /* indexer: le query vengono dal q_a normalizzato, le chiavi dall'hidden
+ * con LayerNorm (con bias), e i pesi per testa sono scalati da IH^-0.5 */
+ mv(iq + (size_t)t * IH * ID, &l->iwq, qn);
+ float *kraw = ik + (size_t)t * ID;
+ mv(kraw, &l->iwk, row);
+ layer_norm(kraw, kraw, l->ik_nw, l->ik_nb, ID, 1e-5f);
+ mv(gates + (size_t)t * ID, &l->ikpg, row);
+ mv(head_w + (size_t)t * IH, &l->iwp, row);
+ for (int h = 0; h < IH; h++) head_w[(size_t)t * IH + h] /= sqrtf((float)IH);
+ }
+
+ const int width = coli_sparse_index_width(c->index_topk, c->index_kpool, c->index_kpool_tail);
+ int *selected = malloc((size_t)tokens * width * sizeof(int));
+ if (coli_sparse_index_select(selected, iq, ik, gates, head_w, l->ikpa, valid,
+ tokens, IH, ID, c->index_kpool, c->index_topk,
+ c->index_kpool_tail)) {
+ fprintf(stderr, "selezione indexer fallita\n"); exit(1);
+ }
+ float *context = malloc((size_t)tokens * H * V * sizeof(float));
+ if (coli_sparse_attention(context, queries, keys, values, selected,
+ tokens, width, H, QK, V)) {
+ fprintf(stderr, "attenzione sparsa fallita\n"); exit(1);
+ }
+ for (int t = 0; t < tokens; t++)
+ mv(out + (size_t)t * c->hidden, &l->o, context + (size_t)t * H * V);
+
+ free(context); free(selected); free(valid); free(head_w); free(gates);
+ free(ik); free(iq); free(expanded); free(latent);
+ free(values); free(keys); free(queries); free(qa);
+}
+
+/* ---------- FFN: denso oppure MoE ---------- */
+static void ffn_layer(const Cfg *c, const GLayer *l, int index, const float *x,
+ int tokens, float *out) {
+ const int inter = index < c->first_dense ? c->dense_inter : c->moe_inter;
+ float *sg = malloc((size_t)(c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter)
+ * sizeof(float));
+ float *su = malloc((size_t)(c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter)
+ * sizeof(float));
+ float *tmp = malloc((size_t)c->hidden * sizeof(float));
+ float *score = malloc((size_t)c->n_experts * sizeof(float));
+ (void)inter;
+ for (int t = 0; t < tokens; t++) {
+ const float *row = x + (size_t)t * c->hidden;
+ float *dst = out + (size_t)t * c->hidden;
+ if (index < c->first_dense) {
+ mlp3(dst, row, &l->dg, &l->du, &l->dd, c->swiglu_limit, sg, su);
+ continue;
+ }
+ /* esperto condiviso, sempre attivo */
+ mlp3(dst, row, &l->rg, &l->ru, &l->rd, c->swiglu_limit, sg, su);
+ /* router: sigmoide sui logit; la selezione usa score+bias, il PESO usa
+ * lo score puro -- la distinzione e' sottile e sbagliarla cambia quali
+ * esperti contano quanto. */
+ for (int e = 0; e < c->n_experts; e++) {
+ const float *w = l->router + (size_t)e * c->hidden;
+ float sum = 0.0f;
+ for (int d = 0; d < c->hidden; d++) sum += w[d] * row[d];
+ score[e] = sigmoidf_(sum);
+ }
+ int chosen[64];
+ float weight[64], total = 0.0f;
+ for (int k = 0; k < c->topk; k++) {
+ int best = -1; float value = -INFINITY;
+ for (int e = 0; e < c->n_experts; e++) {
+ int used = 0;
+ for (int j = 0; j < k; j++) if (chosen[j] == e) { used = 1; break; }
+ float choice = score[e] + (l->rbias ? l->rbias[e] : 0.0f);
+ if (!used && choice > value) { value = choice; best = e; }
+ }
+ chosen[k] = best;
+ weight[k] = score[best];
+ total += weight[k];
+ }
+ for (int k = 0; k < c->topk; k++) {
+ float scale = weight[k] / (total + 1e-20f) * c->routed_scale;
+ mlp3(tmp, row, &l->eg[chosen[k]], &l->eu[chosen[k]], &l->ed[chosen[k]],
+ c->swiglu_limit, sg, su);
+ for (int d = 0; d < c->hidden; d++) dst[d] += scale * tmp[d];
+ }
+ }
+ free(score); free(tmp); free(su); free(sg);
+}
+
+/* ---------- caricamento ---------- */
+static void model_load(GModel *m, const char *dir) {
+ load_cfg(&m->c, dir);
+ st_init(&m->S, dir);
+ /* Il checkpoint reale annida il modello testuale sotto il wrapper vision;
+ * un export solo-testo no. Si sceglie una volta, da un tensore che deve
+ * esistere in entrambe le forme. */
+ snprintf(m->prefix, sizeof(m->prefix), "model.language_model.");
+ char probe[256];
+ snprintf(probe, sizeof(probe), "%sembed_tokens.weight", m->prefix);
+ if (!st_find(&m->S, probe)) snprintf(m->prefix, sizeof(m->prefix), "model.");
+ const char *P = m->prefix;
+
+ m->embed = load_f32(m, "%sembed_tokens.weight", P);
+ m->final_norm = load_f32(m, "%snorm.weight", P);
+ m->head = st_find(&m->S, "lm_head.weight") ? load_f32(m, "lm_head.weight") : m->embed;
+ m->layer = calloc((size_t)m->c.n_layers, sizeof(*m->layer));
+
+ for (int i = 0; i < m->c.n_layers; i++) {
+ GLayer *l = &m->layer[i];
+ l->in_ln = load_f32(m, "%slayers.%d.input_layernorm.weight", P, i);
+ l->post_ln = load_f32(m, "%slayers.%d.post_attention_layernorm.weight", P, i);
+ l->hc_attn_fn = load_f32(m, "%slayers.%d.hc_attn_fn", P, i);
+ l->hc_attn_base = load_f32(m, "%slayers.%d.hc_attn_base", P, i);
+ l->hc_attn_scale = load_f32(m, "%slayers.%d.hc_attn_scale", P, i);
+ l->hc_ffn_fn = load_f32(m, "%slayers.%d.hc_ffn_fn", P, i);
+ l->hc_ffn_base = load_f32(m, "%slayers.%d.hc_ffn_base", P, i);
+ l->hc_ffn_scale = load_f32(m, "%slayers.%d.hc_ffn_scale", P, i);
+ if (m->c.is_full[i]) {
+ l->qa = load_mat(m, "%slayers.%d.self_attn.q_a_proj.weight", P, i);
+ l->qa_ln = load_f32(m, "%slayers.%d.self_attn.q_a_layernorm.weight", P, i);
+ l->qb = load_mat(m, "%slayers.%d.self_attn.q_b_proj.weight", P, i);
+ l->kva = load_mat(m, "%slayers.%d.self_attn.kv_a_proj_with_mqa.weight", P, i);
+ l->kva_ln = load_f32(m, "%slayers.%d.self_attn.kv_a_layernorm.weight", P, i);
+ l->kvb = load_mat(m, "%slayers.%d.self_attn.kv_b_proj.weight", P, i);
+ l->o = load_mat(m, "%slayers.%d.self_attn.o_proj.weight", P, i);
+ l->iwq = load_mat(m, "%slayers.%d.self_attn.indexer.wq_b.weight", P, i);
+ l->iwk = load_mat(m, "%slayers.%d.self_attn.indexer.wk.weight", P, i);
+ l->iwp = load_mat(m, "%slayers.%d.self_attn.indexer.weights_proj.weight", P, i);
+ l->ik_nw = load_f32(m, "%slayers.%d.self_attn.indexer.k_norm.weight", P, i);
+ l->ik_nb = load_f32(m, "%slayers.%d.self_attn.indexer.k_norm.bias", P, i);
+ if (m->c.index_kpool > 1) {
+ l->ikpa = load_f32(m, "%slayers.%d.self_attn.indexer.index_kpool_compress_ape", P, i);
+ l->ikpg = load_mat(m, "%slayers.%d.self_attn.indexer.index_kpool_compress_gate", P, i);
+ }
+ } else {
+ l->kq = load_mat(m, "%slayers.%d.self_attn.q_proj.weight", P, i);
+ l->kk = load_mat(m, "%slayers.%d.self_attn.k_proj.weight", P, i);
+ l->kv = load_mat(m, "%slayers.%d.self_attn.v_proj.weight", P, i);
+ l->ko = load_mat(m, "%slayers.%d.self_attn.o_proj.weight", P, i);
+ l->kga = load_mat(m, "%slayers.%d.self_attn.g_a_proj.weight", P, i);
+ l->kgb = load_mat(m, "%slayers.%d.self_attn.g_b_proj.weight", P, i);
+ l->kfa = load_mat(m, "%slayers.%d.self_attn.f_a_proj.weight", P, i);
+ l->kfb = load_mat(m, "%slayers.%d.self_attn.f_b_proj.weight", P, i);
+ l->kb = load_mat(m, "%slayers.%d.self_attn.b_proj.weight", P, i);
+ l->dt = load_f32(m, "%slayers.%d.self_attn.dt_bias", P, i);
+ l->alog = load_f32(m, "%slayers.%d.self_attn.A_log", P, i);
+ l->onorm = load_f32(m, "%slayers.%d.self_attn.o_norm.weight", P, i);
+ /* Il checkpoint tiene q/k/v conv separate; la ricorrenza le vuole
+ * concatenate nello stesso ordine di qkv. */
+ {
+ int width = m->c.kda_proj * m->c.conv_k;
+ float *conv = malloc((size_t)3 * width * sizeof(float));
+ const char *parts[3] = { "q_conv1d", "k_conv1d", "v_conv1d" };
+ for (int p = 0; p < 3; p++) {
+ const float *piece = load_f32(m, "%slayers.%d.self_attn.%s.weight",
+ P, i, parts[p]);
+ memcpy(conv + (size_t)p * width, piece, (size_t)width * sizeof(float));
+ free((void *)piece);
+ }
+ l->conv = conv;
+ }
+ }
+ if (i < m->c.first_dense) {
+ l->dg = load_mat(m, "%slayers.%d.mlp.gate_proj.weight", P, i);
+ l->du = load_mat(m, "%slayers.%d.mlp.up_proj.weight", P, i);
+ l->dd = load_mat(m, "%slayers.%d.mlp.down_proj.weight", P, i);
+ } else {
+ l->router = load_f32(m, "%slayers.%d.mlp.gate.weight", P, i);
+ l->rbias = st_find(&m->S, (snprintf(probe, sizeof(probe),
+ "%slayers.%d.mlp.gate.e_score_correction_bias", P, i), probe))
+ ? load_f32(m, "%s", probe) : NULL;
+ l->rg = load_mat(m, "%slayers.%d.mlp.shared_experts.gate_proj.weight", P, i);
+ l->ru = load_mat(m, "%slayers.%d.mlp.shared_experts.up_proj.weight", P, i);
+ l->rd = load_mat(m, "%slayers.%d.mlp.shared_experts.down_proj.weight", P, i);
+ l->eg = malloc((size_t)m->c.n_experts * sizeof(Mat));
+ l->eu = malloc((size_t)m->c.n_experts * sizeof(Mat));
+ l->ed = malloc((size_t)m->c.n_experts * sizeof(Mat));
+ for (int e = 0; e < m->c.n_experts; e++) {
+ l->eg[e] = load_mat(m, "%slayers.%d.mlp.experts.%d.gate_proj.weight", P, i, e);
+ l->eu[e] = load_mat(m, "%slayers.%d.mlp.experts.%d.up_proj.weight", P, i, e);
+ l->ed[e] = load_mat(m, "%slayers.%d.mlp.experts.%d.down_proj.weight", P, i, e);
+ }
+ }
+ }
+}
+
+/* ---------- il passaggio completo ---------- */
+static float *forward(GModel *m, const int *tokens, int n) {
+ const Cfg *c = &m->c;
+ const int H = c->hc_mult, D = c->hidden;
+ float *streams = malloc((size_t)n * H * D * sizeof(float));
+ float *next = malloc((size_t)n * H * D * sizeof(float));
+ float *collapsed = malloc((size_t)n * D * sizeof(float));
+ float *normed = malloc((size_t)n * D * sizeof(float));
+ float *branch = malloc((size_t)n * D * sizeof(float));
+ float *post = malloc((size_t)n * H * sizeof(float));
+ float *comb = malloc((size_t)n * H * H * sizeof(float));
+ /* l'embedding entra replicato in ognuno degli H flussi residui */
+ for (int t = 0; t < n; t++)
+ for (int h = 0; h < H; h++)
+ memcpy(streams + ((size_t)t * H + h) * D,
+ m->embed + (size_t)tokens[t] * D, (size_t)D * sizeof(float));
+
+ float *state = NULL, *window = NULL, *scratch = NULL;
+ if (c->kda_proj) {
+ state = malloc((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));
+ window = malloc((size_t)3 * c->kda_proj * c->conv_k * sizeof(float));
+ scratch = malloc((size_t)coli_kda_scratch_floats(c->kda_heads, c->kda_hd, c->kda_hd)
+ * sizeof(float));
+ }
+
+ for (int i = 0; i < c->n_layers; i++) {
+ GLayer *l = &m->layer[i];
+ for (int site = 0; site < 2; site++) {
+ const float *fn = site ? l->hc_ffn_fn : l->hc_attn_fn;
+ const float *base = site ? l->hc_ffn_base : l->hc_attn_base;
+ const float *scale = site ? l->hc_ffn_scale : l->hc_attn_scale;
+ for (int t = 0; t < n; t++)
+ coli_hc_pre(collapsed + (size_t)t * D, post + (size_t)t * H,
+ comb + (size_t)t * H * H, streams + (size_t)t * H * D,
+ fn, scale, base, H, D, c->hc_iters, c->eps, c->hc_eps);
+ for (int t = 0; t < n; t++)
+ rms(normed + (size_t)t * D, collapsed + (size_t)t * D,
+ site ? l->post_ln : l->in_ln, D, c->eps);
+ if (!site) {
+ if (c->is_full[i]) mla_layer(c, l, normed, n, branch);
+ else {
+ memset(state, 0, (size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));
+ memset(window, 0, (size_t)3 * c->kda_proj * c->conv_k * sizeof(float));
+ kda_layer(c, l, normed, n, branch, state, window, scratch);
+ }
+ } else {
+ ffn_layer(c, l, i, normed, n, branch);
+ }
+ for (int t = 0; t < n; t++)
+ coli_hc_post(next + (size_t)t * H * D, branch + (size_t)t * D,
+ streams + (size_t)t * H * D, post + (size_t)t * H,
+ comb + (size_t)t * H * H, H, D);
+ float *swap = streams; streams = next; next = swap;
+ }
+ }
+ /* i flussi si richiudono con una media NON pesata */
+ for (int t = 0; t < n; t++)
+ for (int d = 0; d < D; d++) {
+ float sum = 0.0f;
+ for (int h = 0; h < H; h++) sum += streams[((size_t)t * H + h) * D + d];
+ collapsed[(size_t)t * D + d] = sum / H;
+ }
+ for (int t = 0; t < n; t++)
+ rms(normed + (size_t)t * D, collapsed + (size_t)t * D, m->final_norm, D, c->eps);
+
+ float *logits = malloc((size_t)n * c->vocab * sizeof(float));
+ Mat head = { m->head, c->vocab, D };
+ for (int t = 0; t < n; t++) mv(logits + (size_t)t * c->vocab, &head, normed + (size_t)t * D);
+
+ free(scratch); free(window); free(state);
+ free(comb); free(post); free(branch); free(normed); free(collapsed);
+ free(next); free(streams);
+ return logits;
+}
+
+/* ---------- CLI ----------
+ * Due modi, entrambi pensati per essere confrontati con ref.json:
+ * --ids a,b,c teacher forcing: stampa l'argmax a ogni posizione
+ * --ids a,b,c --greedy N genera N token continuando dal prompt
+ * Il tokenizzatore non serve: l'oracolo parla in id. */
+static int argmax(const float *v, int n) {
+ int best = 0;
+ for (int i = 1; i < n; i++) if (v[i] > v[best]) best = i;
+ return best;
+}
+
+int main(int argc, char **argv) {
+ const char *dir = NULL, *ids = NULL;
+ int greedy = 0, show_logits = 0;
+ for (int i = 1; i < argc; i++) {
+ if (!strcmp(argv[i], "--model") && i + 1 < argc) dir = argv[++i];
+ else if (!strcmp(argv[i], "--ids") && i + 1 < argc) ids = argv[++i];
+ else if (!strcmp(argv[i], "--greedy") && i + 1 < argc) greedy = atoi(argv[++i]);
+ else if (!strcmp(argv[i], "--logits")) show_logits = 1;
+ else { fprintf(stderr, "argomento sconosciuto: %s\n", argv[i]); return 2; }
+ }
+ if (!dir || !ids) {
+ fprintf(stderr, "uso: %s --model DIR --ids a,b,c [--greedy N] [--logits]\n", argv[0]);
+ return 2;
+ }
+ int capacity = 1024, count = 0;
+ int *tokens = malloc((size_t)capacity * sizeof(int));
+ for (const char *p = ids; *p; ) {
+ char *end;
+ long value = strtol(p, &end, 10);
+ if (end == p) break;
+ if (count == capacity) tokens = realloc(tokens, (size_t)(capacity *= 2) * sizeof(int));
+ tokens[count++] = (int)value;
+ p = (*end == ',') ? end + 1 : end;
+ }
+ if (!count) { fprintf(stderr, "nessun id nel prompt\n"); return 2; }
+
+ GModel model;
+ model_load(&model, dir);
+ if (getenv("GLM53_VERBOSE")) cfg_report(&model.c);
+
+ float *logits = forward(&model, tokens, count);
+ printf("teacher_forcing");
+ for (int t = 0; t < count; t++)
+ printf(" %d", argmax(logits + (size_t)t * model.c.vocab, model.c.vocab));
+ printf("\n");
+ if (show_logits) {
+ printf("last_logits");
+ const float *last = logits + (size_t)(count - 1) * model.c.vocab;
+ for (int v = 0; v < model.c.vocab; v++) printf(" %.9g", last[v]);
+ printf("\n");
+ }
+ if (greedy > 0) {
+ int total = count;
+ int *sequence = malloc((size_t)(count + greedy) * sizeof(int));
+ memcpy(sequence, tokens, (size_t)count * sizeof(int));
+ printf("greedy");
+ for (int step = 0; step < greedy; step++) {
+ free(logits);
+ logits = forward(&model, sequence, total);
+ int next = argmax(logits + (size_t)(total - 1) * model.c.vocab, model.c.vocab);
+ sequence[total++] = next;
+ printf(" %d", next);
+ }
+ printf("\n");
+ free(sequence);
+ }
+ free(logits);
+ free(tokens);
+ return 0;
+}
diff --git a/c/tests/test_glm53_tiny.py b/c/tests/test_glm53_tiny.py
new file mode 100644
index 00000000..93a61c4b
--- /dev/null
+++ b/c/tests/test_glm53_tiny.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Il motore GLM-5.3 contro l'oracolo tiny generato da transformers.
+
+Confronta cio' che un utente vede davvero -- i token -- e non solo i numeri
+interni: teacher forcing su ogni posizione, generazione greedy, e i logit
+dell'ultima posizione entro una tolleranza stretta. Un motore puo' avere logit
+quasi giusti e scegliere comunque il token sbagliato, quindi entrambi contano.
+"""
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--binary", required=True)
+ parser.add_argument("--fixture", type=Path, required=True)
+ parser.add_argument("--logit-tolerance", type=float, default=2e-5)
+ arguments = parser.parse_args()
+
+ reference = json.loads((arguments.fixture / "ref.json").read_text())
+ prompt = ",".join(str(token) for token in reference["prompt_ids"])
+ expected_forcing = reference["teacher_forcing_ids"]
+ expected_greedy = reference["greedy_new_ids"]
+
+ result = subprocess.run(
+ [arguments.binary, "--model", str(arguments.fixture), "--ids", prompt,
+ "--greedy", str(len(expected_greedy)), "--logits"],
+ capture_output=True, text=True, check=True)
+
+ lines = {line.split()[0]: line.split()[1:] for line in result.stdout.splitlines() if line.strip()}
+ forcing = [int(value) for value in lines["teacher_forcing"]]
+ greedy = [int(value) for value in lines["greedy"]]
+ logits = [float(value) for value in lines["last_logits"]]
+
+ if forcing != expected_forcing:
+ print(f"FAIL teacher forcing\n ottenuto: {forcing}\n atteso: {expected_forcing}")
+ return 1
+ if greedy != expected_greedy:
+ print(f"FAIL greedy\n ottenuto: {greedy}\n atteso: {expected_greedy}")
+ return 1
+ worst = max(abs(a - b) for a, b in zip(logits, reference["last_logits"], strict=True))
+ if worst > arguments.logit_tolerance:
+ print(f"FAIL logit: max abs {worst:.3g} oltre {arguments.logit_tolerance:.3g}")
+ return 1
+
+ print(f"PASS GLM-5.3 tiny: {len(forcing)} posizioni teacher-forced esatte, "
+ f"{len(greedy)} token greedy esatti, logit entro {worst:.3g}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())