Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(glm53): complete CPU text engine, token-exact on the tiny oracle
JustVugg69fc89ee56f0
diff --git a/c/glm53.c b/c/glm53.cindex af71b9b4..fcbf39db 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -63,6 +63,7 @@#include <string.h>#include <math.h>#include <stdint.h>+#include <stdarg.h>#include "json.h"#include "st.h"@@ -77,7 +78,7 @@ typedef struct {int hidden, n_layers, vocab, first_dense, dense_inter;int n_heads, q_lora, kv_lora, qk_nope, qk_rope, v_head, qk_head;int n_experts, topk, moe_inter, n_shared;- float routed_scale, eps;+ float routed_scale, eps, swiglu_limit;int n_mtp; /* num_nextn_predict_layers (1) *//* KDA */int kda_heads, kda_hd, kda_proj, conv_k;@@ -229,7 +230,29 @@ static void load_cfg(Cfg *c, const char *snap) {c->hidden = (int)req_num(tc, "hidden_size");c->n_layers = (int)req_num(tc, "num_hidden_layers");c->vocab = (int)req_num(tc, "vocab_size");- c->first_dense = (int)req_num(tc, "first_k_dense_replace");+ /* Il checkpoint reale dichiara first_k_dense_replace; la fixture tiny usa+ * mlp_layer_types. Accettiamo entrambi invece di imporne uno: sono la+ * stessa informazione detta in due modi, e rifiutare la fixture+ * significherebbe non poter provare il motore senza 194 GB. */+ {+ jval *fkd = json_get(tc, "first_k_dense_replace");+ if (fkd && fkd->t == J_NUM) {+ c->first_dense = (int)fkd->num;+ } else {+ jval *kinds = json_get(tc, "mlp_layer_types");+ if (!kinds || kinds->t != J_ARR) {+ fprintf(stderr, "config.json: serve first_k_dense_replace "+ "oppure mlp_layer_types\n");+ exit(1);+ }+ c->first_dense = kinds->len;+ for (int i = 0; i < kinds->len; i++)+ if (kinds->kids[i]->t == J_STR && strstr(kinds->kids[i]->str, "sparse")) {+ c->first_dense = i;+ break;+ }+ }+ }c->dense_inter = (int)req_num(tc, "intermediate_size");c->n_heads = (int)req_num(tc, "num_attention_heads");c->q_lora = (int)req_num(tc, "q_lora_rank");@@ -243,6 +266,7 @@ static void load_cfg(Cfg *c, const char *snap) {c->n_shared = (int)opt_num(tc, "n_shared_experts", 1);c->routed_scale= (float)opt_num(tc, "routed_scaling_factor", 1.0);c->eps = (float)opt_num(tc, "rms_norm_eps", 1e-6);+ c->swiglu_limit= (float)opt_num(tc, "swiglu_limit", 0.0);c->n_mtp = (int)opt_num(tc, "num_nextn_predict_layers", 0);c->qk_head = c->qk_nope + c->qk_rope;@@ -324,7 +348,7 @@ static void cfg_report(const Cfg *c) {c->vis_image_size, c->vis_merge, c->vis_out_hidden);}-#ifdef GLM53_CFG_MAIN+#ifdef GLM53_CFG_MAIN_UNUSED/* Config-only entry point: `glm53_cfg <model_dir>` parses and reports, so the* parser can be checked against a real checkpoint before any weight exists. */int main(int argc, char **argv) {@@ -526,7 +550,7 @@ static void glm53_walk_tensors(const Cfg *c, const Names *n,#undef EMIT}-#ifdef GLM53_INVENTORY_MAIN+#ifdef GLM53_INVENTORY_MAIN_UNUSED/* Inventory entry point: prints every tensor the engine would load, in load* order. Checked against a real checkpoint's index offline, so a naming* mistake surfaces before 180 GB have been converted rather than after. */@@ -543,3 +567,519 @@ int main(int argc, char **argv) {return 0;}#endif++/* ================= forward =================+ * Il modello caricato in RAM come f32. Lo streaming degli esperti e la+ * quantizzazione a load time arrivano col modello vero; qui la priorita' e'+ * che i token siano quelli giusti, provati contro l'oracolo. */+#include "delta_attention.h"+#include "sparse_index.h"++typedef struct { const float *w; int rows, columns; } Mat;++typedef struct {+ /* comune */+ const float *in_ln, *post_ln;+ const float *hc_attn_fn, *hc_attn_base, *hc_attn_scale;+ const float *hc_ffn_fn, *hc_ffn_base, *hc_ffn_scale;+ /* KDA */+ Mat kq, kk, kv, ko, kga, kgb, kfa, kfb, kb;+ const float *conv, *dt, *alog, *onorm;+ /* MLA + indexer */+ Mat qa, qb, kva, kvb, o, iwq, iwk, iwp, ikpg;+ const float *qa_ln, *kva_ln, *ik_nw, *ik_nb, *ikpa;+ /* FFN */+ Mat dg, du, dd; /* denso */+ Mat rg, ru, rd; /* router / shared: gate,up,down */+ const float *router, *rbias;+ Mat *eg, *eu, *ed; /* esperti routed */+} GLayer;++typedef struct {+ Cfg c;+ shards S;+ const float *embed, *final_norm, *head;+ GLayer *layer;+ char prefix[64];+} GModel;++static const float *load_f32(GModel *m, const char *fmt, ...) {+ char name[512];+ va_list args; va_start(args, fmt); vsnprintf(name, sizeof(name), fmt, args); va_end(args);+ st_tensor *t = st_find(&m->S, name);+ if (!t) { fprintf(stderr, "manca il tensore %s\n", name); exit(1); }+ float *buffer = malloc((size_t)t->numel * sizeof(float));+ if (!buffer) { fprintf(stderr, "OOM su %s\n", name); exit(1); }+ st_read_f32_cap(&m->S, name, buffer, t->numel, 0);+ return buffer;+}++static Mat load_mat(GModel *m, const char *fmt, ...) {+ char name[512];+ va_list args; va_start(args, fmt); vsnprintf(name, sizeof(name), fmt, args); va_end(args);+ st_tensor *t = st_find(&m->S, name);+ if (!t) { fprintf(stderr, "manca la matrice %s\n", name); exit(1); }+ if (t->rank != 2) { fprintf(stderr, "%s: rank %d, attesa 2\n", name, t->rank); exit(1); }+ float *buffer = malloc((size_t)t->numel * sizeof(float));+ st_read_f32_cap(&m->S, name, buffer, t->numel, 0);+ Mat mat = { buffer, (int)t->shape[0], (int)t->shape[1] };+ return mat;+}++/* out[rows] = W x, con W in layout [rows, columns] come transformers. */+static void mv(float *out, const Mat *w, const float *x) {+ for (int r = 0; r < w->rows; r++) {+ const float *row = w->w + (size_t)r * w->columns;+ float sum = 0.0f;+ for (int c = 0; c < w->columns; c++) sum += row[c] * x[c];+ out[r] = sum;+ }+}++static void rms(float *out, const float *x, const float *w, int n, float eps) {+ float square = 0.0f;+ for (int i = 0; i < n; i++) square += x[i] * x[i];+ float inverse = 1.0f / sqrtf(square / n + eps);+ for (int i = 0; i < n; i++) out[i] = x[i] * inverse * w[i];+}++static void layer_norm(float *out, const float *x, const float *w, const float *b,+ int n, float eps) {+ float mean = 0.0f;+ for (int i = 0; i < n; i++) mean += x[i];+ mean /= n;+ float variance = 0.0f;+ for (int i = 0; i < n; i++) variance += (x[i] - mean) * (x[i] - mean);+ variance /= n;+ float inverse = 1.0f / sqrtf(variance + eps);+ for (int i = 0; i < n; i++) out[i] = (x[i] - mean) * inverse * w[i] + (b ? b[i] : 0.0f);+}++static float sigmoidf_(float x) {+ return x >= 0.0f ? 1.0f / (1.0f + expf(-x)) : expf(x) / (1.0f + expf(x));+}+static float siluf_(float x) { return x / (1.0f + expf(-x)); }++/* SwiGLU clampata: il gate ha solo il tetto, up e' limitato da entrambi i lati.+ * Vale sia per l'MLP denso che per gli esperti -- il testo di GLM-5.3 NON usa+ * la SiLU semplice, ed e' un errore che darebbe un modello che parla bene e+ * sbaglia. */+static void swiglu_clamped(float *gate, const float *up, int n, float limit) {+ for (int i = 0; i < n; i++) {+ float g = gate[i] > limit ? limit : gate[i];+ float u = up[i] < -limit ? -limit : (up[i] > limit ? limit : up[i]);+ gate[i] = siluf_(g) * u;+ }+}++static void mlp3(float *out, const float *x, const Mat *g, const Mat *u, const Mat *d,+ float limit, float *sg, float *su) {+ mv(sg, g, x); mv(su, u, x);+ swiglu_clamped(sg, su, g->rows, limit);+ mv(out, d, sg);+}++/* ---------- KDA: proiezioni, gate, ricorrenza ---------- */+static void kda_layer(const Cfg *c, const GLayer *l, const float *x, int tokens,+ float *out, float *state, float *window, float *scratch) {+ const int P = c->kda_proj, H = c->kda_heads, D = c->kda_hd;+ float *qkv = malloc((size_t)3 * P * sizeof(float));+ float *gate = malloc((size_t)P * sizeof(float));+ float *decay = malloc((size_t)P * sizeof(float));+ float *beta = malloc((size_t)H * sizeof(float));+ float *low = malloc((size_t)D * sizeof(float));+ float *core = malloc((size_t)P * sizeof(float));+ for (int t = 0; t < tokens; t++) {+ const float *row = x + (size_t)t * c->hidden;+ mv(qkv, &l->kq, row);+ mv(qkv + P, &l->kk, row);+ mv(qkv + 2 * P, &l->kv, row);+ /* decadimento: gate_lower_bound * sigmoid(exp(A_log[h]) * (W_fb W_fa x + dt_bias)) */+ mv(low, &l->kfa, row);+ mv(decay, &l->kfb, low);+ for (int h = 0; h < H; h++)+ for (int d = 0; d < D; d++) {+ int i = h * D + d;+ decay[i] = c->gate_lb * sigmoidf_(expf(l->alog[h]) * (decay[i] + l->dt[i]));+ }+ mv(beta, &l->kb, row);+ for (int h = 0; h < H; h++) beta[h] = sigmoidf_(beta[h]);+ coli_kda_step(core, state, window, qkv, l->conv, decay, beta,+ H, D, D, c->conv_k, 1e-6f, scratch);+ /* uscita: RMSNorm per testa, pesata da o_norm, moltiplicata dal gate+ * low-rank, poi la proiezione di uscita. */+ mv(low, &l->kga, row);+ mv(gate, &l->kgb, low);+ float *normed = qkv; /* riuso: 3P >= P */+ for (int h = 0; h < H; h++) {+ const float *src = core + (size_t)h * D;+ float *dst = normed + (size_t)h * D;+ float square = 0.0f;+ for (int d = 0; d < D; d++) square += src[d] * src[d];+ float inverse = 1.0f / sqrtf(square / D + c->eps);+ for (int d = 0; d < D; d++)+ dst[d] = src[d] * inverse * l->onorm[d] * sigmoidf_(gate[(size_t)h * D + d]);+ }+ mv(out + (size_t)t * c->hidden, &l->ko, normed);+ }+ free(core); free(low); free(beta); free(decay); free(gate); free(qkv);+}++/* ---------- MLA + indexer con k-pool ---------- */+static void mla_layer(const Cfg *c, const GLayer *l, const float *x, int tokens, float *out) {+ const int H = c->n_heads, QK = c->qk_nope, V = c->v_head;+ const int IH = c->index_nh, ID = c->index_hd;+ float *qa = malloc((size_t)tokens * c->q_lora * sizeof(float));+ float *queries = malloc((size_t)tokens * H * QK * sizeof(float));+ float *keys = malloc((size_t)tokens * H * QK * sizeof(float));+ float *values = malloc((size_t)tokens * H * V * sizeof(float));+ float *latent = malloc((size_t)c->kv_lora * sizeof(float));+ float *expanded = malloc((size_t)H * (QK + V) * sizeof(float));+ float *iq = malloc((size_t)tokens * IH * ID * sizeof(float));+ float *ik = malloc((size_t)tokens * ID * sizeof(float));+ float *gates = malloc((size_t)tokens * ID * sizeof(float));+ float *head_w = malloc((size_t)tokens * IH * sizeof(float));+ unsigned char *valid = malloc((size_t)tokens);+ memset(valid, 1, (size_t)tokens);++ for (int t = 0; t < tokens; t++) {+ const float *row = x + (size_t)t * c->hidden;+ float *qn = qa + (size_t)t * c->q_lora;+ mv(qn, &l->qa, row);+ rms(qn, qn, l->qa_ln, c->q_lora, c->eps);+ mv(queries + (size_t)t * H * QK, &l->qb, qn);+ mv(latent, &l->kva, row);+ rms(latent, latent, l->kva_ln, c->kv_lora, c->eps);+ mv(expanded, &l->kvb, latent);+ for (int h = 0; h < H; h++) {+ const float *src = expanded + (size_t)h * (QK + V);+ memcpy(keys + ((size_t)t * H + h) * QK, src, (size_t)QK * sizeof(float));+ memcpy(values + ((size_t)t * H + h) * V, src + QK, (size_t)V * sizeof(float));+ }+ /* indexer: le query vengono dal q_a normalizzato, le chiavi dall'hidden+ * con LayerNorm (con bias), e i pesi per testa sono scalati da IH^-0.5 */+ mv(iq + (size_t)t * IH * ID, &l->iwq, qn);+ float *kraw = ik + (size_t)t * ID;+ mv(kraw, &l->iwk, row);+ layer_norm(kraw, kraw, l->ik_nw, l->ik_nb, ID, 1e-5f);+ mv(gates + (size_t)t * ID, &l->ikpg, row);+ mv(head_w + (size_t)t * IH, &l->iwp, row);+ for (int h = 0; h < IH; h++) head_w[(size_t)t * IH + h] /= sqrtf((float)IH);+ }++ const int width = coli_sparse_index_width(c->index_topk, c->index_kpool, c->index_kpool_tail);+ int *selected = malloc((size_t)tokens * width * sizeof(int));+ if (coli_sparse_index_select(selected, iq, ik, gates, head_w, l->ikpa, valid,+ tokens, IH, ID, c->index_kpool, c->index_topk,+ c->index_kpool_tail)) {+ fprintf(stderr, "selezione indexer fallita\n"); exit(1);+ }+ float *context = malloc((size_t)tokens * H * V * sizeof(float));+ if (coli_sparse_attention(context, queries, keys, values, selected,+ tokens, width, H, QK, V)) {+ fprintf(stderr, "attenzione sparsa fallita\n"); exit(1);+ }+ for (int t = 0; t < tokens; t++)+ mv(out + (size_t)t * c->hidden, &l->o, context + (size_t)t * H * V);++ free(context); free(selected); free(valid); free(head_w); free(gates);+ free(ik); free(iq); free(expanded); free(latent);+ free(values); free(keys); free(queries); free(qa);+}++/* ---------- FFN: denso oppure MoE ---------- */+static void ffn_layer(const Cfg *c, const GLayer *l, int index, const float *x,+ int tokens, float *out) {+ const int inter = index < c->first_dense ? c->dense_inter : c->moe_inter;+ float *sg = malloc((size_t)(c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter)+ * sizeof(float));+ float *su = malloc((size_t)(c->dense_inter > c->moe_inter ? c->dense_inter : c->moe_inter)+ * sizeof(float));+ float *tmp = malloc((size_t)c->hidden * sizeof(float));+ float *score = malloc((size_t)c->n_experts * sizeof(float));+ (void)inter;+ for (int t = 0; t < tokens; t++) {+ const float *row = x + (size_t)t * c->hidden;+ float *dst = out + (size_t)t * c->hidden;+ if (index < c->first_dense) {+ mlp3(dst, row, &l->dg, &l->du, &l->dd, c->swiglu_limit, sg, su);+ continue;+ }+ /* esperto condiviso, sempre attivo */+ mlp3(dst, row, &l->rg, &l->ru, &l->rd, c->swiglu_limit, sg, su);+ /* router: sigmoide sui logit; la selezione usa score+bias, il PESO usa+ * lo score puro -- la distinzione e' sottile e sbagliarla cambia quali+ * esperti contano quanto. */+ for (int e = 0; e < c->n_experts; e++) {+ const float *w = l->router + (size_t)e * c->hidden;+ float sum = 0.0f;+ for (int d = 0; d < c->hidden; d++) sum += w[d] * row[d];+ score[e] = sigmoidf_(sum);+ }+ int chosen[64];+ float weight[64], total = 0.0f;+ for (int k = 0; k < c->topk; k++) {+ int best = -1; float value = -INFINITY;+ for (int e = 0; e < c->n_experts; e++) {+ int used = 0;+ for (int j = 0; j < k; j++) if (chosen[j] == e) { used = 1; break; }+ float choice = score[e] + (l->rbias ? l->rbias[e] : 0.0f);+ if (!used && choice > value) { value = choice; best = e; }+ }+ chosen[k] = best;+ weight[k] = score[best];+ total += weight[k];+ }+ for (int k = 0; k < c->topk; k++) {+ float scale = weight[k] / (total + 1e-20f) * c->routed_scale;+ mlp3(tmp, row, &l->eg[chosen[k]], &l->eu[chosen[k]], &l->ed[chosen[k]],+ c->swiglu_limit, sg, su);+ for (int d = 0; d < c->hidden; d++) dst[d] += scale * tmp[d];+ }+ }+ free(score); free(tmp); free(su); free(sg);+}++/* ---------- caricamento ---------- */+static void model_load(GModel *m, const char *dir) {+ load_cfg(&m->c, dir);+ st_init(&m->S, dir);+ /* Il checkpoint reale annida il modello testuale sotto il wrapper vision;+ * un export solo-testo no. Si sceglie una volta, da un tensore che deve+ * esistere in entrambe le forme. */+ snprintf(m->prefix, sizeof(m->prefix), "model.language_model.");+ char probe[256];+ snprintf(probe, sizeof(probe), "%sembed_tokens.weight", m->prefix);+ if (!st_find(&m->S, probe)) snprintf(m->prefix, sizeof(m->prefix), "model.");+ const char *P = m->prefix;++ m->embed = load_f32(m, "%sembed_tokens.weight", P);+ m->final_norm = load_f32(m, "%snorm.weight", P);+ m->head = st_find(&m->S, "lm_head.weight") ? load_f32(m, "lm_head.weight") : m->embed;+ m->layer = calloc((size_t)m->c.n_layers, sizeof(*m->layer));++ for (int i = 0; i < m->c.n_layers; i++) {+ GLayer *l = &m->layer[i];+ l->in_ln = load_f32(m, "%slayers.%d.input_layernorm.weight", P, i);+ l->post_ln = load_f32(m, "%slayers.%d.post_attention_layernorm.weight", P, i);+ l->hc_attn_fn = load_f32(m, "%slayers.%d.hc_attn_fn", P, i);+ l->hc_attn_base = load_f32(m, "%slayers.%d.hc_attn_base", P, i);+ l->hc_attn_scale = load_f32(m, "%slayers.%d.hc_attn_scale", P, i);+ l->hc_ffn_fn = load_f32(m, "%slayers.%d.hc_ffn_fn", P, i);+ l->hc_ffn_base = load_f32(m, "%slayers.%d.hc_ffn_base", P, i);+ l->hc_ffn_scale = load_f32(m, "%slayers.%d.hc_ffn_scale", P, i);+ if (m->c.is_full[i]) {+ l->qa = load_mat(m, "%slayers.%d.self_attn.q_a_proj.weight", P, i);+ l->qa_ln = load_f32(m, "%slayers.%d.self_attn.q_a_layernorm.weight", P, i);+ l->qb = load_mat(m, "%slayers.%d.self_attn.q_b_proj.weight", P, i);+ l->kva = load_mat(m, "%slayers.%d.self_attn.kv_a_proj_with_mqa.weight", P, i);+ l->kva_ln = load_f32(m, "%slayers.%d.self_attn.kv_a_layernorm.weight", P, i);+ l->kvb = load_mat(m, "%slayers.%d.self_attn.kv_b_proj.weight", P, i);+ l->o = load_mat(m, "%slayers.%d.self_attn.o_proj.weight", P, i);+ l->iwq = load_mat(m, "%slayers.%d.self_attn.indexer.wq_b.weight", P, i);+ l->iwk = load_mat(m, "%slayers.%d.self_attn.indexer.wk.weight", P, i);+ l->iwp = load_mat(m, "%slayers.%d.self_attn.indexer.weights_proj.weight", P, i);+ l->ik_nw = load_f32(m, "%slayers.%d.self_attn.indexer.k_norm.weight", P, i);+ l->ik_nb = load_f32(m, "%slayers.%d.self_attn.indexer.k_norm.bias", P, i);+ if (m->c.index_kpool > 1) {+ l->ikpa = load_f32(m, "%slayers.%d.self_attn.indexer.index_kpool_compress_ape", P, i);+ l->ikpg = load_mat(m, "%slayers.%d.self_attn.indexer.index_kpool_compress_gate", P, i);+ }+ } else {+ l->kq = load_mat(m, "%slayers.%d.self_attn.q_proj.weight", P, i);+ l->kk = load_mat(m, "%slayers.%d.self_attn.k_proj.weight", P, i);+ l->kv = load_mat(m, "%slayers.%d.self_attn.v_proj.weight", P, i);+ l->ko = load_mat(m, "%slayers.%d.self_attn.o_proj.weight", P, i);+ l->kga = load_mat(m, "%slayers.%d.self_attn.g_a_proj.weight", P, i);+ l->kgb = load_mat(m, "%slayers.%d.self_attn.g_b_proj.weight", P, i);+ l->kfa = load_mat(m, "%slayers.%d.self_attn.f_a_proj.weight", P, i);+ l->kfb = load_mat(m, "%slayers.%d.self_attn.f_b_proj.weight", P, i);+ l->kb = load_mat(m, "%slayers.%d.self_attn.b_proj.weight", P, i);+ l->dt = load_f32(m, "%slayers.%d.self_attn.dt_bias", P, i);+ l->alog = load_f32(m, "%slayers.%d.self_attn.A_log", P, i);+ l->onorm = load_f32(m, "%slayers.%d.self_attn.o_norm.weight", P, i);+ /* Il checkpoint tiene q/k/v conv separate; la ricorrenza le vuole+ * concatenate nello stesso ordine di qkv. */+ {+ int width = m->c.kda_proj * m->c.conv_k;+ float *conv = malloc((size_t)3 * width * sizeof(float));+ const char *parts[3] = { "q_conv1d", "k_conv1d", "v_conv1d" };+ for (int p = 0; p < 3; p++) {+ const float *piece = load_f32(m, "%slayers.%d.self_attn.%s.weight",+ P, i, parts[p]);+ memcpy(conv + (size_t)p * width, piece, (size_t)width * sizeof(float));+ free((void *)piece);+ }+ l->conv = conv;+ }+ }+ if (i < m->c.first_dense) {+ l->dg = load_mat(m, "%slayers.%d.mlp.gate_proj.weight", P, i);+ l->du = load_mat(m, "%slayers.%d.mlp.up_proj.weight", P, i);+ l->dd = load_mat(m, "%slayers.%d.mlp.down_proj.weight", P, i);+ } else {+ l->router = load_f32(m, "%slayers.%d.mlp.gate.weight", P, i);+ l->rbias = st_find(&m->S, (snprintf(probe, sizeof(probe),+ "%slayers.%d.mlp.gate.e_score_correction_bias", P, i), probe))+ ? load_f32(m, "%s", probe) : NULL;+ l->rg = load_mat(m, "%slayers.%d.mlp.shared_experts.gate_proj.weight", P, i);+ l->ru = load_mat(m, "%slayers.%d.mlp.shared_experts.up_proj.weight", P, i);+ l->rd = load_mat(m, "%slayers.%d.mlp.shared_experts.down_proj.weight", P, i);+ l->eg = malloc((size_t)m->c.n_experts * sizeof(Mat));+ l->eu = malloc((size_t)m->c.n_experts * sizeof(Mat));+ l->ed = malloc((size_t)m->c.n_experts * sizeof(Mat));+ for (int e = 0; e < m->c.n_experts; e++) {+ l->eg[e] = load_mat(m, "%slayers.%d.mlp.experts.%d.gate_proj.weight", P, i, e);+ l->eu[e] = load_mat(m, "%slayers.%d.mlp.experts.%d.up_proj.weight", P, i, e);+ l->ed[e] = load_mat(m, "%slayers.%d.mlp.experts.%d.down_proj.weight", P, i, e);+ }+ }+ }+}++/* ---------- il passaggio completo ---------- */+static float *forward(GModel *m, const int *tokens, int n) {+ const Cfg *c = &m->c;+ const int H = c->hc_mult, D = c->hidden;+ float *streams = malloc((size_t)n * H * D * sizeof(float));+ float *next = malloc((size_t)n * H * D * sizeof(float));+ float *collapsed = malloc((size_t)n * D * sizeof(float));+ float *normed = malloc((size_t)n * D * sizeof(float));+ float *branch = malloc((size_t)n * D * sizeof(float));+ float *post = malloc((size_t)n * H * sizeof(float));+ float *comb = malloc((size_t)n * H * H * sizeof(float));+ /* l'embedding entra replicato in ognuno degli H flussi residui */+ for (int t = 0; t < n; t++)+ for (int h = 0; h < H; h++)+ memcpy(streams + ((size_t)t * H + h) * D,+ m->embed + (size_t)tokens[t] * D, (size_t)D * sizeof(float));++ float *state = NULL, *window = NULL, *scratch = NULL;+ if (c->kda_proj) {+ state = malloc((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));+ window = malloc((size_t)3 * c->kda_proj * c->conv_k * sizeof(float));+ scratch = malloc((size_t)coli_kda_scratch_floats(c->kda_heads, c->kda_hd, c->kda_hd)+ * sizeof(float));+ }++ for (int i = 0; i < c->n_layers; i++) {+ GLayer *l = &m->layer[i];+ for (int site = 0; site < 2; site++) {+ const float *fn = site ? l->hc_ffn_fn : l->hc_attn_fn;+ const float *base = site ? l->hc_ffn_base : l->hc_attn_base;+ const float *scale = site ? l->hc_ffn_scale : l->hc_attn_scale;+ for (int t = 0; t < n; t++)+ coli_hc_pre(collapsed + (size_t)t * D, post + (size_t)t * H,+ comb + (size_t)t * H * H, streams + (size_t)t * H * D,+ fn, scale, base, H, D, c->hc_iters, c->eps, c->hc_eps);+ for (int t = 0; t < n; t++)+ rms(normed + (size_t)t * D, collapsed + (size_t)t * D,+ site ? l->post_ln : l->in_ln, D, c->eps);+ if (!site) {+ if (c->is_full[i]) mla_layer(c, l, normed, n, branch);+ else {+ memset(state, 0, (size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));+ memset(window, 0, (size_t)3 * c->kda_proj * c->conv_k * sizeof(float));+ kda_layer(c, l, normed, n, branch, state, window, scratch);+ }+ } else {+ ffn_layer(c, l, i, normed, n, branch);+ }+ for (int t = 0; t < n; t++)+ coli_hc_post(next + (size_t)t * H * D, branch + (size_t)t * D,+ streams + (size_t)t * H * D, post + (size_t)t * H,+ comb + (size_t)t * H * H, H, D);+ float *swap = streams; streams = next; next = swap;+ }+ }+ /* i flussi si richiudono con una media NON pesata */+ for (int t = 0; t < n; t++)+ for (int d = 0; d < D; d++) {+ float sum = 0.0f;+ for (int h = 0; h < H; h++) sum += streams[((size_t)t * H + h) * D + d];+ collapsed[(size_t)t * D + d] = sum / H;+ }+ for (int t = 0; t < n; t++)+ rms(normed + (size_t)t * D, collapsed + (size_t)t * D, m->final_norm, D, c->eps);++ float *logits = malloc((size_t)n * c->vocab * sizeof(float));+ Mat head = { m->head, c->vocab, D };+ for (int t = 0; t < n; t++) mv(logits + (size_t)t * c->vocab, &head, normed + (size_t)t * D);++ free(scratch); free(window); free(state);+ free(comb); free(post); free(branch); free(normed); free(collapsed);+ free(next); free(streams);+ return logits;+}++/* ---------- CLI ----------+ * Due modi, entrambi pensati per essere confrontati con ref.json:+ * --ids a,b,c teacher forcing: stampa l'argmax a ogni posizione+ * --ids a,b,c --greedy N genera N token continuando dal prompt+ * Il tokenizzatore non serve: l'oracolo parla in id. */+static int argmax(const float *v, int n) {+ int best = 0;+ for (int i = 1; i < n; i++) if (v[i] > v[best]) best = i;+ return best;+}++int main(int argc, char **argv) {+ const char *dir = NULL, *ids = NULL;+ int greedy = 0, show_logits = 0;+ for (int i = 1; i < argc; i++) {+ if (!strcmp(argv[i], "--model") && i + 1 < argc) dir = argv[++i];+ else if (!strcmp(argv[i], "--ids") && i + 1 < argc) ids = argv[++i];+ else if (!strcmp(argv[i], "--greedy") && i + 1 < argc) greedy = atoi(argv[++i]);+ else if (!strcmp(argv[i], "--logits")) show_logits = 1;+ else { fprintf(stderr, "argomento sconosciuto: %s\n", argv[i]); return 2; }+ }+ if (!dir || !ids) {+ fprintf(stderr, "uso: %s --model DIR --ids a,b,c [--greedy N] [--logits]\n", argv[0]);+ return 2;+ }+ int capacity = 1024, count = 0;+ int *tokens = malloc((size_t)capacity * sizeof(int));+ for (const char *p = ids; *p; ) {+ char *end;+ long value = strtol(p, &end, 10);+ if (end == p) break;+ if (count == capacity) tokens = realloc(tokens, (size_t)(capacity *= 2) * sizeof(int));+ tokens[count++] = (int)value;+ p = (*end == ',') ? end + 1 : end;+ }+ if (!count) { fprintf(stderr, "nessun id nel prompt\n"); return 2; }++ GModel model;+ model_load(&model, dir);+ if (getenv("GLM53_VERBOSE")) cfg_report(&model.c);++ float *logits = forward(&model, tokens, count);+ printf("teacher_forcing");+ for (int t = 0; t < count; t++)+ printf(" %d", argmax(logits + (size_t)t * model.c.vocab, model.c.vocab));+ printf("\n");+ if (show_logits) {+ printf("last_logits");+ const float *last = logits + (size_t)(count - 1) * model.c.vocab;+ for (int v = 0; v < model.c.vocab; v++) printf(" %.9g", last[v]);+ printf("\n");+ }+ if (greedy > 0) {+ int total = count;+ int *sequence = malloc((size_t)(count + greedy) * sizeof(int));+ memcpy(sequence, tokens, (size_t)count * sizeof(int));+ printf("greedy");+ for (int step = 0; step < greedy; step++) {+ free(logits);+ logits = forward(&model, sequence, total);+ int next = argmax(logits + (size_t)(total - 1) * model.c.vocab, model.c.vocab);+ sequence[total++] = next;+ printf(" %d", next);+ }+ printf("\n");+ free(sequence);+ }+ free(logits);+ free(tokens);+ return 0;+}diff --git a/c/tests/test_glm53_tiny.py b/c/tests/test_glm53_tiny.pynew file mode 100644index 00000000..93a61c4b--- /dev/null+++ b/c/tests/test_glm53_tiny.py@@ -0,0 +1,55 @@+#!/usr/bin/env python3+"""Il motore GLM-5.3 contro l'oracolo tiny generato da transformers.++Confronta cio' che un utente vede davvero -- i token -- e non solo i numeri+interni: teacher forcing su ogni posizione, generazione greedy, e i logit+dell'ultima posizione entro una tolleranza stretta. Un motore puo' avere logit+quasi giusti e scegliere comunque il token sbagliato, quindi entrambi contano.+"""+import argparse+import json+import subprocess+import sys+from pathlib import Path+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--binary", required=True)+ parser.add_argument("--fixture", type=Path, required=True)+ parser.add_argument("--logit-tolerance", type=float, default=2e-5)+ arguments = parser.parse_args()++ reference = json.loads((arguments.fixture / "ref.json").read_text())+ prompt = ",".join(str(token) for token in reference["prompt_ids"])+ expected_forcing = reference["teacher_forcing_ids"]+ expected_greedy = reference["greedy_new_ids"]++ result = subprocess.run(+ [arguments.binary, "--model", str(arguments.fixture), "--ids", prompt,+ "--greedy", str(len(expected_greedy)), "--logits"],+ capture_output=True, text=True, check=True)++ lines = {line.split()[0]: line.split()[1:] for line in result.stdout.splitlines() if line.strip()}+ forcing = [int(value) for value in lines["teacher_forcing"]]+ greedy = [int(value) for value in lines["greedy"]]+ logits = [float(value) for value in lines["last_logits"]]++ if forcing != expected_forcing:+ print(f"FAIL teacher forcing\n ottenuto: {forcing}\n atteso: {expected_forcing}")+ return 1+ if greedy != expected_greedy:+ print(f"FAIL greedy\n ottenuto: {greedy}\n atteso: {expected_greedy}")+ return 1+ worst = max(abs(a - b) for a, b in zip(logits, reference["last_logits"], strict=True))+ if worst > arguments.logit_tolerance:+ print(f"FAIL logit: max abs {worst:.3g} oltre {arguments.logit_tolerance:.3g}")+ return 1++ print(f"PASS GLM-5.3 tiny: {len(forcing)} posizioni teacher-forced esatte, "+ f"{len(greedy)} token greedy esatti, logit entro {worst:.3g}")+ return 0+++if __name__ == "__main__":+ sys.exit(main())