Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(glm53): vision tower, validated against a transformers oracle
JustVugge884da10f463
diff --git a/c/glm53.c b/c/glm53.cnew file mode 100644index 00000000..af71b9b4--- /dev/null+++ b/c/glm53.c@@ -0,0 +1,545 @@+/* GLM-5.3-Flash inference engine in pure C β sibling of kimi_k3.c / colibri.c /+ * deepseek_v4.c / inkling.c / qwen36.c / olmoe.c, sharing st.h / json.h / tok.h /+ * quant.h / compat.h.+ *+ * The name says GLM, but the skeleton is Kimi K3's, not GLM-5.2's: a hybrid+ * stack of linear-attention layers punctuated by full attention, over a+ * streamed MoE. That is why this file starts from kimi_k3.c and borrows the+ * indexer from colibri.c and the hyper-connections from deepseek_v4.c, instead+ * of extending the GLM engine.+ *+ * Architecture, read off the released checkpoint (321.34 B parameters measured+ * from the shard headers; the card says 320 B total / 18 B active):+ *+ * - 45 text layers plus one MTP layer (index 45), hidden 4096, vocab 154880.+ * - Hybrid attention, pattern (KDA KDA KDA FULL) repeating: 34 KDA linear+ * layers and 11 DeepSeek-sparse-attention layers at 3, 7, ... 43, plus the+ * MTP layer's own full-attention block. config.layer_types is explicit and+ * is what this engine follows; linear_attn_config.kda_layers agrees.+ * - MLA with kv_lora 512, q_lora 1536, qk_nope 256 and **qk_rope 0**: the+ * full-attention layers are NoPE, exactly like K3. Position lives in the+ * KDA decay and short convolutions, not in a rotation.+ * - DSA lightning indexer on every full-attention layer, with k-pooling:+ * keys are grouped into pools of index_kpool=4, the pools are scored+ * instead of the tokens, the top index_topk/kpool pools are expanded back+ * into token indices and the incomplete tail pool is always appended+ * (index_kpool_always_select_tail). Two tensors carry it that GLM-5.2's+ * indexer does not have: index_kpool_compress_ape and _compress_gate.+ * - Manifold-Constrained Hyper-Connections (mHC, hc_mult 4, 20 Sinkhorn+ * iterations) replace the plain residual at both sites of every layer β+ * the same mHC DeepSeek V4 uses, down to the config keys, so the split+ * into pre/post/comb and the Sinkhorn projection are shared code.+ * - MoE: 288 routed experts (top-8) plus 1 shared expert per layer from+ * layer 3 on, moe_intermediate 2048, sigmoid scoring with noaux_tc and+ * e_score_correction_bias, routed_scaling_factor 2.5. The first three+ * layers are dense (intermediate 12288).+ * - Natively multimodal: a 24-block ViT (hidden 1024, patch 14, 448 px,+ * spatial merge 2) whose patches are projected to 4096 and substituted at+ * the image-token positions of the text stream. Text-only prompts never+ * touch it.+ *+ * KDA recurrence β the same Kimi Delta Attention kimi_k3.c already reproduces+ * token-exact against the vendor, verified line by line against+ * transformers' recurrent_kimi_delta_attention:+ * q,k,v = SiLU(ShortConv4(W{q,k,v} x)); q,k L2-normalized, q *= d^-0.5+ * z = W_fb(W_fa x) + dt_bias+ * gk = gmin * sigmoid(exp(A_log[h]) * z), gmin = gate_lower_bound = -5+ * S = (I - beta k k^T) Diag(exp(gk)) S + beta k v^T, beta = sigmoid(W_b x)+ * o = S^T q; out = W_o [ sigmoid(W_gb(W_ga x)) * RMSNorm_head(o) ]+ * One difference from K3: the output gate is LOW-RANK here (g_a_proj into+ * head_dim, then g_b_proj back out), where K3 has a single full g_proj.+ *+ * Container: tools/convert_glm53.py writes routed experts as int4 group-scaled+ * gs64 (`name` U8 + `name.qs` F32, fmt=4 β the same container GLM-5.2 uses,+ * measured cosine 0.994 against the fp8 source on real weights) and everything+ * else as BF16, quantized at LOAD TIME here. That split is deliberate: the+ * non-expert weights are 3% of the bytes, so keeping them exact on disk costs+ * ~14 GB and means retuning dense precision never requires re-downloading the+ * checkpoint.+ */+#define _GNU_SOURCE+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <math.h>+#include <stdint.h>++#include "json.h"+#include "st.h"+#include "hyper_connections.h" /* mHC, condiviso con deepseek_v4.c */++/* ---------- config ----------+ * Nested like Kimi K3's: the root carries the vision wrapper and `text_config`+ * carries the language model. A text-only export therefore has the text keys at+ * the root, and both shapes are accepted. */+typedef struct {+ /* text */+ int hidden, n_layers, vocab, first_dense, dense_inter;+ int n_heads, q_lora, kv_lora, qk_nope, qk_rope, v_head, qk_head;+ int n_experts, topk, moe_inter, n_shared;+ float routed_scale, eps;+ int n_mtp; /* num_nextn_predict_layers (1) */+ /* KDA */+ int kda_heads, kda_hd, kda_proj, conv_k;+ float gate_lb;+ /* DSA indexer with k-pooling */+ int index_topk, index_nh, index_hd, index_kpool, index_kpool_tail;+ /* mHC */+ int hc_mult, hc_iters;+ float hc_eps;+ /* per-layer kind: 1 = full attention (MLA + indexer), 0 = KDA */+ unsigned char is_full[128];+ /* vision (0 = text-only checkpoint) */+ int vis_layers, vis_hidden, vis_heads, vis_inter, vis_patch, vis_temporal;+ int vis_merge, vis_out_hidden, vis_proj_inter, vis_image_size, vis_in_ch;+ float vis_swiglu_limit, vis_eps;+ int image_token, image_start_token, image_end_token;+ int video_token, video_start_token, video_end_token;+} Cfg;++static double req_num(jval *object, const char *key) {+ jval *value = json_get(object, key);+ if (!value || value->t != J_NUM) {+ fprintf(stderr, "config.json: missing or non-numeric \"%s\"\n", key);+ exit(1);+ }+ return value->num;+}++static double opt_num(jval *object, const char *key, double fallback) {+ jval *value = object ? json_get(object, key) : NULL;+ return (value && value->t == J_NUM) ? value->num : fallback;+}++static int opt_bool(jval *object, const char *key, int fallback) {+ jval *value = object ? json_get(object, key) : NULL;+ if (!value) return fallback;+ if (value->t == J_BOOL) return value->boolean;+ if (value->t == J_NUM) return value->num != 0.0;+ return fallback;+}++/* layer_types is the authority on which layers are full attention. The+ * linear_attn_config.{kda_layers,full_attn_layers} lists say the same thing;+ * disagreeing checkpoints are refused rather than guessed at, because picking+ * the wrong kind for one layer produces plausible-looking garbage. */+static void load_layer_kinds(Cfg *c, jval *text) {+ memset(c->is_full, 0, sizeof(c->is_full));+ jval *types = json_get(text, "layer_types");+ if (!types || types->t != J_ARR || types->len != c->n_layers) {+ fprintf(stderr, "config.json: layer_types must list %d entries\n", c->n_layers);+ exit(1);+ }+ int full = 0;+ for (int i = 0; i < types->len; i++) {+ jval *entry = types->kids[i];+ if (!entry || entry->t != J_STR) {+ fprintf(stderr, "config.json: layer_types[%d] is not a string\n", i);+ exit(1);+ }+ if (strstr(entry->str, "linear")) {+ c->is_full[i] = 0;+ } else if (strstr(entry->str, "attention") || strstr(entry->str, "full")) {+ c->is_full[i] = 1;+ full++;+ } else {+ fprintf(stderr, "config.json: unknown layer type \"%s\" at %d\n", entry->str, i);+ exit(1);+ }+ }+ jval *linear = json_get(text, "linear_attn_config");+ jval *full_list = linear ? json_get(linear, "full_attn_layers") : NULL;+ if (full_list && full_list->t == J_ARR) {+ if (full_list->len != full) {+ fprintf(stderr, "config.json: full_attn_layers lists %d layers, "+ "layer_types marks %d\n", full_list->len, full);+ exit(1);+ }+ for (int i = 0; i < full_list->len; i++) {+ int index = (int)full_list->kids[i]->num;+ if (index < 0 || index >= c->n_layers || !c->is_full[index]) {+ fprintf(stderr, "config.json: full_attn_layers disagrees with "+ "layer_types at %d\n", index);+ exit(1);+ }+ }+ }+ /* The MTP block is a full-attention layer that lives past num_hidden_layers+ * and is not described by layer_types. */+ if (c->n_layers < (int)sizeof(c->is_full)) c->is_full[c->n_layers] = 1;+}++static void load_vision(Cfg *c, jval *root) {+ jval *vision = json_get(root, "vision_config");+ if (!vision || vision->t != J_OBJ) { c->vis_layers = 0; return; }+ c->vis_layers = (int)req_num(vision, "depth");+ c->vis_hidden = (int)req_num(vision, "hidden_size");+ c->vis_heads = (int)req_num(vision, "num_heads");+ c->vis_inter = (int)req_num(vision, "intermediate_size");+ c->vis_patch = (int)req_num(vision, "patch_size");+ c->vis_temporal = (int)opt_num(vision, "temporal_patch_size", 2);+ c->vis_merge = (int)opt_num(vision, "spatial_merge_size", 2);+ c->vis_out_hidden = (int)opt_num(vision, "out_hidden_size", c->hidden);+ c->vis_proj_inter = (int)opt_num(vision, "projection_intermediate_size", 0);+ c->vis_image_size = (int)opt_num(vision, "image_size", 448);+ c->vis_in_ch = (int)opt_num(vision, "in_channels", 3);+ c->vis_swiglu_limit= (float)opt_num(vision, "swiglu_limit", 10.0);+ c->vis_eps = (float)opt_num(vision, "rms_norm_eps", 1e-5);+ c->image_token = (int)opt_num(root, "image_token_id", -1);+ c->image_start_token = (int)opt_num(root, "image_start_token_id", -1);+ c->image_end_token = (int)opt_num(root, "image_end_token_id", -1);+ c->video_token = (int)opt_num(root, "video_token_id", -1);+ c->video_start_token = (int)opt_num(root, "video_start_token_id", -1);+ c->video_end_token = (int)opt_num(root, "video_end_token_id", -1);+ if (c->vis_layers < 1 || c->vis_layers > 128 || c->vis_hidden < 1 ||+ c->vis_heads < 1 || c->vis_hidden % c->vis_heads ||+ c->vis_patch < 1 || c->vis_merge < 1 || c->vis_out_hidden != c->hidden) {+ fprintf(stderr, "config.json: vision_config out of range "+ "(out_hidden_size must equal the text hidden size)\n");+ exit(1);+ }+}++static void load_cfg(Cfg *c, const char *snap) {+ char path[4096];+ snprintf(path, sizeof(path), "%s/config.json", snap);+ char *buffer = NULL;+ {+ FILE *stream = fopen(path, "rb");+ if (!stream) { fprintf(stderr, "cannot read %s\n", path); exit(1); }+ if (fseek(stream, 0, SEEK_END)) { fprintf(stderr, "%s: not seekable\n", path); exit(1); }+ long length = ftell(stream);+ if (length < 2 || fseek(stream, 0, SEEK_SET)) {+ fprintf(stderr, "%s: unusable\n", path); exit(1); }+ buffer = malloc((size_t)length + 1);+ if (!buffer) { fprintf(stderr, "OOM reading config\n"); exit(1); }+ if (fread(buffer, 1, (size_t)length, stream) != (size_t)length) {+ fprintf(stderr, "%s: short read\n", path); exit(1); }+ buffer[length] = 0;+ fclose(stream);+ }+ char *arena = NULL;+ jval *root = json_parse(buffer, &arena);+ if (!root || root->t != J_OBJ) { fprintf(stderr, "%s: not a JSON object\n", path); exit(1); }+ memset(c, 0, sizeof(*c));++ jval *tc = json_get(root, "text_config");+ if (!tc || tc->t != J_OBJ) tc = root; /* text-only export */++ c->hidden = (int)req_num(tc, "hidden_size");+ c->n_layers = (int)req_num(tc, "num_hidden_layers");+ c->vocab = (int)req_num(tc, "vocab_size");+ c->first_dense = (int)req_num(tc, "first_k_dense_replace");+ c->dense_inter = (int)req_num(tc, "intermediate_size");+ c->n_heads = (int)req_num(tc, "num_attention_heads");+ c->q_lora = (int)req_num(tc, "q_lora_rank");+ c->kv_lora = (int)req_num(tc, "kv_lora_rank");+ c->qk_nope = (int)req_num(tc, "qk_nope_head_dim");+ c->qk_rope = (int)opt_num(tc, "qk_rope_head_dim", 0);+ c->v_head = (int)req_num(tc, "v_head_dim");+ c->n_experts = (int)req_num(tc, "n_routed_experts");+ c->topk = (int)req_num(tc, "num_experts_per_tok");+ c->moe_inter = (int)req_num(tc, "moe_intermediate_size");+ c->n_shared = (int)opt_num(tc, "n_shared_experts", 1);+ c->routed_scale= (float)opt_num(tc, "routed_scaling_factor", 1.0);+ c->eps = (float)opt_num(tc, "rms_norm_eps", 1e-6);+ c->n_mtp = (int)opt_num(tc, "num_nextn_predict_layers", 0);+ c->qk_head = c->qk_nope + c->qk_rope;++ jval *linear = json_get(tc, "linear_attn_config");+ if (!linear || linear->t != J_OBJ) {+ fprintf(stderr, "config.json: missing linear_attn_config\n"); exit(1);+ }+ c->kda_heads = (int)req_num(linear, "num_heads");+ c->kda_hd = (int)req_num(linear, "head_dim");+ c->conv_k = (int)req_num(linear, "short_conv_kernel_size");+ c->gate_lb = (float)opt_num(linear, "gate_lower_bound", -5.0);+ c->kda_proj = c->kda_heads * c->kda_hd;++ c->index_topk = (int)opt_num(tc, "index_topk", 0);+ c->index_nh = (int)opt_num(tc, "index_n_heads", 0);+ c->index_hd = (int)opt_num(tc, "index_head_dim", 0);+ c->index_kpool = (int)opt_num(tc, "index_kpool", 1);+ c->index_kpool_tail = opt_bool(tc, "index_kpool_always_select_tail", 0);++ c->hc_mult = (int)opt_num(tc, "hc_mult", 1);+ c->hc_iters = (int)opt_num(tc, "hc_sinkhorn_iters", 0);+ c->hc_eps = (float)opt_num(tc, "hc_eps", 1e-6);++ load_layer_kinds(c, tc);+ load_vision(c, root);++ if (c->hidden < 1 || c->hidden > 65536 ||+ c->n_layers < 1 || c->n_layers > 120 ||+ c->vocab < 1 || c->vocab > (1 << 22) ||+ c->n_experts < 1 || c->n_experts > 4096 ||+ c->topk < 1 || c->topk > 64 || c->topk > c->n_experts ||+ c->kda_proj < 1 || c->kda_proj > (1 << 20) ||+ c->conv_k < 1 || c->conv_k > 8 ||+ c->moe_inter % 32 || c->kda_hd > 512 || c->kv_lora > 4096 ||+ c->first_dense < 0 || c->first_dense > c->n_layers ||+ c->index_kpool < 1 || c->index_kpool > 64 ||+ c->hc_mult < 1 || c->hc_mult > 8) {+ fprintf(stderr, "config.json: dimension out of range\n"); exit(1);+ }+ /* qk_rope must be zero: a rotary GLM-5.3 would need position handling this+ * engine deliberately does not have, and silently ignoring the rotation+ * would produce a model that answers fluently and wrongly. */+ if (c->qk_rope != 0) {+ fprintf(stderr, "config.json: qk_rope_head_dim=%d, but this engine "+ "implements the NoPE full-attention of GLM-5.3\n", c->qk_rope);+ exit(1);+ }+ free(arena);+ free(buffer);+}++static void cfg_report(const Cfg *c) {+ int full = 0, kda = 0;+ for (int i = 0; i < c->n_layers; i++) { if (c->is_full[i]) full++; else kda++; }+ double expert_params = (double)c->n_experts * 3.0 * c->hidden * c->moe_inter *+ (c->n_layers - c->first_dense + (c->n_mtp ? 1 : 0));+ fprintf(stderr,+ "GLM-5.3-Flash: %d layers (%d KDA + %d full) + %d MTP, hidden %d, vocab %d\n"+ " MoE : %d routed (top-%d) + %d shared, inter %d, from layer %d, scale %.2f\n"+ " KDA : %d heads x %d, conv %d, gate floor %.1f\n"+ " MLA : q_lora %d, kv_lora %d, qk %d (nope, NoPE), v %d, %d heads\n"+ " indexer : top-%d, %d heads x %d, kpool %d%s\n"+ " mHC : mult %d, %d Sinkhorn iterations\n"+ " vision : %s\n"+ " routed experts: %.1f B parameters (%.0f GB at int4-g64)\n",+ c->n_layers, kda, full, c->n_mtp, c->hidden, c->vocab,+ c->n_experts, c->topk, c->n_shared, c->moe_inter, c->first_dense, c->routed_scale,+ c->kda_heads, c->kda_hd, c->conv_k, (double)c->gate_lb,+ c->q_lora, c->kv_lora, c->qk_nope, c->v_head, c->n_heads,+ c->index_topk, c->index_nh, c->index_hd, c->index_kpool,+ c->index_kpool_tail ? " (+tail)" : "",+ c->hc_mult, c->hc_iters,+ c->vis_layers ? "yes" : "text-only checkpoint",+ expert_params / 1e9, expert_params * 0.5625 / 1e9);+ if (c->vis_layers)+ fprintf(stderr,+ " %d blocks x %d, %d heads, patch %d, %dpx, merge %d -> %d\n",+ c->vis_layers, c->vis_hidden, c->vis_heads, c->vis_patch,+ c->vis_image_size, c->vis_merge, c->vis_out_hidden);+}++#ifdef GLM53_CFG_MAIN+/* Config-only entry point: `glm53_cfg <model_dir>` parses and reports, so the+ * parser can be checked against a real checkpoint before any weight exists. */+int main(int argc, char **argv) {+ if (argc != 2) { fprintf(stderr, "usage: %s <model_dir>\n", argv[0]); return 2; }+ Cfg c;+ load_cfg(&c, argv[1]);+ cfg_report(&c);+ return 0;+}+#endif++/* ---------- RAM-resident weight ----------+ * Same shape as kimi_k3.c's: either f32, int8 per-row, or int4 group-scaled.+ * The container written by tools/convert_glm53.py is U8 + `.qs` F32, which is+ * exactly what K3's loader already reads, so the two engines share a format+ * rather than each inventing one. */+typedef struct {+ int fmt; /* 0 = f32, 8 = int8 per-row, 4 = int4-g64 */+ float *f;+ int8_t *q8;+ uint8_t *q4;+ float *s; /* scales: [O] per-row, or [O*ngroups] */+ int O, I, gs;+} W;++/* ---------- layer structures ----------+ * Mirrors kimi_k3.c's shapes, with the GLM-5.3 differences called out where+ * they bite: the KDA output gate is low-rank here, the full-attention layers+ * are NOT gated (K3's are), and every layer carries two mHC sites instead of a+ * plain residual. */+typedef struct { /* KDA (linear attention) layer */+ W q, k, v, o; /* [proj x hidden] x3, [hidden x proj] */+ W ga, gb; /* low-rank output gate: hidden->hd->proj */+ float *conv_q, *conv_k, *conv_v; /* [proj*conv_k] depthwise taps */+ float *fa, *fb; /* decay low-rank: [hd,hidden], [proj,hd] */+ float *bp; /* beta projection [heads,hidden] */+ float *dt, *A, *onw; /* dt_bias[proj], exp(A_log)[heads], o_norm[hd] */+} Kda;++typedef struct { /* MLA + DSA indexer (full attention) */+ W qa, qb, kva, kvb, o;+ float *qa_ln, *kva_ln;+ W wq, wk, wp; /* indexer: wq_b, wk, weights_proj */+ float *knw, *knb; /* indexer key LayerNorm (weight + bias) */+ float *kpool_ape; /* [kpool, index_hd] pool position bias */+ W kpool_gate; /* [index_hd, hidden] compression gate */+} Mla;++typedef struct { /* MoE (routed streamed + shared resident) */+ float *router, *rbias; /* [E,hidden] f32, [E] correction bias */+ W sh_gate, sh_up, sh_down;+} Moe;++typedef struct {+ int full; /* 1 = MLA + indexer, 0 = KDA */+ int dense; /* 1 = plain MLP (layers < first_dense) */+ Kda a;+ Mla m;+ Moe moe;+ W d_gate, d_up, d_down; /* dense layers only */+ float *in_ln, *post_ln;+ /* mHC, two sites per layer. fn is [(2+H)*H, H*hidden], base [(2+H)*H],+ * scale [3]; the split into pre/post/comb and the Sinkhorn projection are+ * the same as DeepSeek V4's. Absent on the MTP layer. */+ float *hc_attn_fn, *hc_attn_base, *hc_attn_scale;+ float *hc_ffn_fn, *hc_ffn_base, *hc_ffn_scale;+ /* MTP layer only */+ W mtp_eh;+ float *mtp_enorm, *mtp_hnorm, *mtp_head_norm;+} Layer;++/* ---------- tensor names ----------+ * One place builds every name the engine asks for. The checkpoint prefixes the+ * language model with `model.language_model.` because the root is the vision+ * wrapper; a text-only export drops it. Both are probed, and the choice is made+ * once from a tensor that must exist either way. */+typedef struct {+ const char *prefix; /* "model.language_model." or "model." */+ const char *visual; /* "model.visual." */+} Names;++#define GLM53_NAME(dst, fmt, ...) snprintf((dst), sizeof(dst), (fmt), __VA_ARGS__)++/* Emits, in load order, every tensor this engine will look for. `sink` is+ * called with (name, required); a NULL model just prints them, which is how+ * the mapping gets checked against a real checkpoint index before any weight+ * has been downloaded. */+static void glm53_walk_tensors(const Cfg *c, const Names *n,+ void (*sink)(void *, const char *, int),+ void *user) {+ char name[512];+#define EMIT(required, fmt, ...) do { \+ GLM53_NAME(name, fmt, __VA_ARGS__); sink(user, name, (required)); } while (0)+ EMIT(1, "%sembed_tokens.weight", n->prefix);+ EMIT(1, "%snorm.weight", n->prefix);+ sink(user, "lm_head.weight", 1);++ int last = c->n_layers + (c->n_mtp ? 1 : 0);+ for (int i = 0; i < last; i++) {+ int mtp = i >= c->n_layers;+ int full = c->is_full[i];+ EMIT(1, "%slayers.%d.input_layernorm.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.post_attention_layernorm.weight", n->prefix, i);+ if (!mtp) { /* mHC lives on the 45 real layers */+ EMIT(1, "%slayers.%d.hc_attn_fn", n->prefix, i);+ EMIT(1, "%slayers.%d.hc_attn_base", n->prefix, i);+ EMIT(1, "%slayers.%d.hc_attn_scale", n->prefix, i);+ EMIT(1, "%slayers.%d.hc_ffn_fn", n->prefix, i);+ EMIT(1, "%slayers.%d.hc_ffn_base", n->prefix, i);+ EMIT(1, "%slayers.%d.hc_ffn_scale", n->prefix, i);+ } else {+ EMIT(1, "%slayers.%d.eh_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.enorm.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.hnorm.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.shared_head.norm.weight", n->prefix, i);+ }+ if (full) {+ EMIT(1, "%slayers.%d.self_attn.q_a_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.q_a_layernorm.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.q_b_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.kv_a_proj_with_mqa.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.kv_a_layernorm.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.kv_b_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.o_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.indexer.wq_b.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.indexer.wk.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.indexer.weights_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.indexer.k_norm.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.indexer.k_norm.bias", n->prefix, i);+ if (c->index_kpool > 1) {+ EMIT(1, "%slayers.%d.self_attn.indexer.index_kpool_compress_ape", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.indexer.index_kpool_compress_gate", n->prefix, i);+ }+ } else {+ EMIT(1, "%slayers.%d.self_attn.q_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.k_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.v_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.o_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.g_a_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.g_b_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.q_conv1d.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.k_conv1d.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.v_conv1d.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.f_a_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.f_b_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.b_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.dt_bias", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.A_log", n->prefix, i);+ EMIT(1, "%slayers.%d.self_attn.o_norm.weight", n->prefix, i);+ }+ if (i < c->first_dense) {+ EMIT(1, "%slayers.%d.mlp.gate_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.mlp.up_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.mlp.down_proj.weight", n->prefix, i);+ } else {+ EMIT(1, "%slayers.%d.mlp.gate.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.mlp.gate.e_score_correction_bias", n->prefix, i);+ EMIT(1, "%slayers.%d.mlp.shared_experts.gate_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.mlp.shared_experts.up_proj.weight", n->prefix, i);+ EMIT(1, "%slayers.%d.mlp.shared_experts.down_proj.weight", n->prefix, i);+ /* Routed experts are streamed, not resident: named here so the+ * inventory can prove they are all present and correctly shaped. */+ for (int e = 0; e < c->n_experts; e++) {+ EMIT(1, "%slayers.%d.mlp.experts.%d.gate_proj.weight", n->prefix, i, e);+ EMIT(1, "%slayers.%d.mlp.experts.%d.up_proj.weight", n->prefix, i, e);+ EMIT(1, "%slayers.%d.mlp.experts.%d.down_proj.weight", n->prefix, i, e);+ }+ }+ }+ if (c->vis_layers) {+ EMIT(1, "%spatch_embed.proj.weight", n->visual);+ EMIT(1, "%spatch_embed.proj.bias", n->visual);+ EMIT(1, "%spost_layernorm.weight", n->visual);+ EMIT(1, "%sdownsample.weight", n->visual);+ EMIT(1, "%sdownsample.bias", n->visual);+ EMIT(1, "%smerger.proj.weight", n->visual);+ EMIT(1, "%smerger.post_projection_norm.weight", n->visual);+ EMIT(1, "%smerger.post_projection_norm.bias", n->visual);+ EMIT(1, "%smerger.gate_proj.weight", n->visual);+ EMIT(1, "%smerger.up_proj.weight", n->visual);+ EMIT(1, "%smerger.down_proj.weight", n->visual);+ for (int b = 0; b < c->vis_layers; b++) {+ EMIT(1, "%sblocks.%d.norm1.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.norm2.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.attn.qkv.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.attn.qkv.bias", n->visual, b);+ EMIT(1, "%sblocks.%d.attn.proj.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.attn.proj.bias", n->visual, b);+ EMIT(1, "%sblocks.%d.attn.q_norm.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.attn.k_norm.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.mlp.gate_proj.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.mlp.gate_proj.bias", n->visual, b);+ EMIT(1, "%sblocks.%d.mlp.up_proj.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.mlp.up_proj.bias", n->visual, b);+ EMIT(1, "%sblocks.%d.mlp.down_proj.weight", n->visual, b);+ EMIT(1, "%sblocks.%d.mlp.down_proj.bias", n->visual, b);+ }+ }+#undef EMIT+}++#ifdef GLM53_INVENTORY_MAIN+/* Inventory entry point: prints every tensor the engine would load, in load+ * order. Checked against a real checkpoint's index offline, so a naming+ * mistake surfaces before 180 GB have been converted rather than after. */+static void print_name(void *user, const char *name, int required) {+ (void)user; printf("%s\t%d\n", name, required);+}++int main(int argc, char **argv) {+ if (argc != 2) { fprintf(stderr, "usage: %s <model_dir>\n", argv[0]); return 2; }+ Cfg c;+ load_cfg(&c, argv[1]);+ Names n = { "model.language_model.", "model.visual." };+ glm53_walk_tensors(&c, &n, print_name, NULL);+ return 0;+}+#endifdiff --git a/c/tests/test_vision_tower.c b/c/tests/test_vision_tower.cnew file mode 100644index 00000000..9e546fd9--- /dev/null+++ b/c/tests/test_vision_tower.c@@ -0,0 +1,126 @@+/* Il tower vision contro l'oracolo generato da transformers+ * (tools/make_glm53_vision_tiny.py). Legge la fixture con st.h, cosi' il test+ * esercita anche il percorso di caricamento reale invece di dati incollati. */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <math.h>++#include "../json.h"+#include "../st.h"+#include "../vision_tower.h"++static const float *need(shards *S, const char *name, int64_t expect) {+ st_tensor *t = st_find(S, name);+ if (!t) { fprintf(stderr, "manca il tensore %s\n", name); exit(1); }+ if (expect > 0 && t->numel != expect) {+ fprintf(stderr, "%s: %lld elementi, attesi %lld\n", name,+ (long long)t->numel, (long long)expect); exit(1);+ }+ float *buffer = malloc((size_t)t->numel * sizeof(float));+ if (!buffer) { fprintf(stderr, "OOM su %s\n", name); exit(1); }+ st_read_f32_cap(S, name, buffer, t->numel, 0);+ return buffer;+}++static char *slurp(const char *path) {+ FILE *f = fopen(path, "rb");+ if (!f) { fprintf(stderr, "non riesco ad aprire %s\n", path); exit(1); }+ fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);+ char *b = malloc((size_t)n + 1);+ if (fread(b, 1, (size_t)n, f) != (size_t)n) { fprintf(stderr, "%s: lettura corta\n", path); exit(1); }+ b[n] = 0; fclose(f);+ return b;+}++int main(int argc, char **argv) {+ const char *dir = argc > 1 ? argv[1] : "glm53_vision_tiny";+ char path[1024];+ shards S;+ st_init(&S, dir);++ snprintf(path, sizeof(path), "%s/config.json", dir);+ char *text = slurp(path), *arena = NULL;+ jval *root = json_parse(text, &arena);+ jval *vc = json_get(root, "vision_config");+ if (!vc) { fprintf(stderr, "config.json senza vision_config\n"); return 1; }+#define NUM(key) ((int)json_get(vc, key)->num)+ ColiVisionConfig cfg = {+ .depth = NUM("depth"), .hidden = NUM("hidden_size"), .heads = NUM("num_heads"),+ .intermediate = NUM("intermediate_size"), .patch = NUM("patch_size"),+ .temporal = NUM("temporal_patch_size"), .merge = NUM("spatial_merge_size"),+ .in_channels = NUM("in_channels"), .out_hidden = NUM("out_hidden_size"),+ .proj_intermediate = NUM("projection_intermediate_size"),+ .eps = (float)json_get(vc, "rms_norm_eps")->num,+ .swiglu_limit = (float)json_get(vc, "swiglu_limit")->num,+ .rope_theta = 10000.0f,+ };+#undef NUM+ cfg.head_dim = cfg.hidden / cfg.heads;++ ColiVisionBlock *blocks = calloc((size_t)cfg.depth, sizeof(*blocks));+ for (int b = 0; b < cfg.depth; b++) {+ char n[256];+#define GET(field, suffix, expect) \+ snprintf(n, sizeof(n), "blocks.%d." suffix, b); blocks[b].field = need(&S, n, expect)+ GET(norm1, "norm1.weight", cfg.hidden);+ GET(norm2, "norm2.weight", cfg.hidden);+ GET(qkv_w, "attn.qkv.weight", (int64_t)3 * cfg.hidden * cfg.hidden);+ GET(qkv_b, "attn.qkv.bias", 3 * cfg.hidden);+ GET(q_norm, "attn.q_norm.weight", cfg.head_dim);+ GET(k_norm, "attn.k_norm.weight", cfg.head_dim);+ GET(proj_w, "attn.proj.weight", (int64_t)cfg.hidden * cfg.hidden);+ GET(proj_b, "attn.proj.bias", cfg.hidden);+ GET(gate_w, "mlp.gate_proj.weight", (int64_t)cfg.intermediate * cfg.hidden);+ GET(gate_b, "mlp.gate_proj.bias", cfg.intermediate);+ GET(up_w, "mlp.up_proj.weight", (int64_t)cfg.intermediate * cfg.hidden);+ GET(up_b, "mlp.up_proj.bias", cfg.intermediate);+ GET(down_w, "mlp.down_proj.weight", (int64_t)cfg.hidden * cfg.intermediate);+ GET(down_b, "mlp.down_proj.bias", cfg.hidden);+#undef GET+ }+ ColiVisionTower tower = {+ .config = cfg, .blocks = blocks,+ .patch_w = need(&S, "patch_embed.proj.weight", 0),+ .patch_b = need(&S, "patch_embed.proj.bias", cfg.hidden),+ .post_norm = need(&S, "post_layernorm.weight", cfg.hidden),+ .down_w = need(&S, "downsample.weight", 0),+ .down_b = need(&S, "downsample.bias", cfg.out_hidden),+ .merger_proj = need(&S, "merger.proj.weight", (int64_t)cfg.out_hidden * cfg.out_hidden),+ .merger_norm_w = need(&S, "merger.post_projection_norm.weight", cfg.out_hidden),+ .merger_norm_b = need(&S, "merger.post_projection_norm.bias", cfg.out_hidden),+ .merger_gate = need(&S, "merger.gate_proj.weight", 0),+ .merger_up = need(&S, "merger.up_proj.weight", 0),+ .merger_down = need(&S, "merger.down_proj.weight", 0),+ };++ snprintf(path, sizeof(path), "%s/ref.json", dir);+ char *rtext = slurp(path), *rarena = NULL;+ jval *ref = json_parse(rtext, &rarena);+ jval *grid = json_get(ref, "grid_thw");+ int gh = (int)grid->kids[0]->kids[1]->num, gw = (int)grid->kids[0]->kids[2]->num;+ jval *want = json_get(ref, "output");++ const float *pixels = need(&S, "input.pixel_values", 0);+ int out_tokens = coli_vision_output_tokens(&cfg, gh, gw);+ float *out = malloc((size_t)out_tokens * cfg.out_hidden * sizeof(float));+ if (coli_vision_forward(out, &tower, pixels, gh, gw)) {+ fprintf(stderr, "coli_vision_forward ha fallito\n"); return 1;+ }++ int n = out_tokens * cfg.out_hidden;+ if (want->len != n) {+ fprintf(stderr, "ref ha %d valori, il tower ne produce %d\n", want->len, n);+ return 1;+ }+ float worst = 0.0f; int at = 0;+ for (int i = 0; i < n; i++) {+ float d = fabsf(out[i] - (float)want->kids[i]->num);+ if (d > worst) { worst = d; at = i; }+ }+ printf("vision tower: %d token x %d, max|delta| = %.3e (indice %d: %.9g vs %.9g)\n",+ out_tokens, cfg.out_hidden, worst, at, out[at], want->kids[at]->num);+ if (worst > 2e-5f) { printf("FALLITO\n"); return 1; }+ printf("PASS tower vision: combacia con l'oracolo transformers\n");+ return 0;+}diff --git a/c/tools/make_glm53_vision_tiny.py b/c/tools/make_glm53_vision_tiny.pynew file mode 100644index 00000000..6ef1f137--- /dev/null+++ b/c/tools/make_glm53_vision_tiny.py@@ -0,0 +1,136 @@+#!/usr/bin/env python3+"""Oracolo tiny del tower vision di GLM-5.3-Flash.++Il testo ha gia' il suo oracolo; questo copre l'altra meta' del modello, che+nessun test raggiungeva: patch embed, RoPE 2D, i blocchi ViT con SwiGLU+clampata, la norma finale, il downsample 2x2 e il patch merger.++Il fixture e' prodotto dal `Glm5NextVisionModel` ufficiale di Hugging Face, non+da noi: il motore C ha cosi' un bersaglio indipendente invece di validare se+stesso. Pesi e input sono deterministici (linspace su un seed fisso), quindi il+file e' riproducibile bit per bit e la CI non ha bisogno della rete.++Perche' un tower minuscolo e non quello vero: 24 blocchi da 1024 sono 563 M+parametri, inutili per provare che la matematica e' giusta. Qui bastano 2+blocchi e un'immagine da 4x4 patch, e ogni meccanismo viene comunque esercitato+almeno una volta - inclusa la fusione 2x2, che con una griglia 4x4 produce+esattamente 4 token di uscita.++USO:+ python3 tools/make_glm53_vision_tiny.py --output c/glm53_vision_tiny+"""+from __future__ import annotations++import argparse+import json+from pathlib import Path++EXPECTED_TRANSFORMERS = "5.16.1"++DEPTH = 2+HIDDEN = 64+HEADS = 4+INTERMEDIATE = 128+PATCH = 4+TEMPORAL = 2+MERGE = 2+OUT_HIDDEN = 96+PROJ_INTERMEDIATE = 128+IN_CHANNELS = 3+GRID_H = 4+GRID_W = 4+SEED = 1253+++def deterministic_(tensor, torch, spread=0.05):+ """Riempie in modo riproducibile e indipendente dall'ordine di init."""+ flat = torch.linspace(-spread, spread, tensor.numel(), dtype=torch.float32)+ with torch.no_grad():+ tensor.copy_(flat.view_as(tensor))+++def build(output: Path) -> int:+ import torch+ import transformers+ from safetensors.torch import save_file+ from transformers.models.glm5_next.configuration_glm5_next import Glm5NextVisionConfig+ from transformers.models.glm5_next.modeling_glm5_next import Glm5NextVisionModel++ if transformers.__version__ != EXPECTED_TRANSFORMERS:+ print(f"expected Transformers {EXPECTED_TRANSFORMERS}, "+ f"found {transformers.__version__}")+ return 2++ torch.manual_seed(SEED)+ config = Glm5NextVisionConfig(+ depth=DEPTH,+ hidden_size=HIDDEN,+ num_heads=HEADS,+ intermediate_size=INTERMEDIATE,+ patch_size=PATCH,+ temporal_patch_size=TEMPORAL,+ spatial_merge_size=MERGE,+ out_hidden_size=OUT_HIDDEN,+ projection_intermediate_size=PROJ_INTERMEDIATE,+ in_channels=IN_CHANNELS,+ image_size=PATCH * GRID_H,+ )+ model = Glm5NextVisionModel(config).eval()+ for parameter in model.parameters():+ deterministic_(parameter, torch)+ for name, buffer in model.named_buffers():+ if buffer.dtype.is_floating_point and "inv_freq" not in name:+ deterministic_(buffer, torch)++ patches = GRID_H * GRID_W+ width = IN_CHANNELS * TEMPORAL * PATCH * PATCH+ pixels = torch.linspace(-1.0, 1.0, patches * width, dtype=torch.float32).view(patches, width)+ grid_thw = torch.tensor([[1, GRID_H, GRID_W]], dtype=torch.long)++ with torch.no_grad():+ outputs = model(pixels, grid_thw=grid_thw)+ merged = outputs.pooler_output.float()++ output.mkdir(parents=True, exist_ok=True)+ tensors = {name: parameter.detach().contiguous().to(torch.float32)+ for name, parameter in model.state_dict().items()+ if parameter.dtype.is_floating_point}+ tensors["input.pixel_values"] = pixels.contiguous()+ save_file(tensors, str(output / "model.safetensors"), metadata={"format": "pt"})+ (output / "config.json").write_text(+ json.dumps({"model_type": "glm5_next_vision", "vision_config": config.to_dict()},+ indent=2, sort_keys=True, default=str),+ encoding="utf-8")++ reference = {+ "schema_version": 1,+ "source": "transformers",+ "transformers_version": transformers.__version__,+ "torch_version": torch.__version__,+ "seed": SEED,+ "grid_thw": grid_thw.tolist(),+ "input_shape": list(pixels.shape),+ "output_shape": list(merged.shape),+ "output_sum": float(merged.sum()),+ "output_square_sum": float(merged.square().sum()),+ # Ogni token di uscita per intero: un confronto per statistiche sole+ # nasconderebbe un errore che sposta valori fra token senza cambiarne+ # somma e norma, ed e' esattamente cosi' che sbaglia una fusione 2x2.+ "output": merged.flatten().tolist(),+ }+ (output / "ref.json").write_text(json.dumps(reference, indent=2), encoding="utf-8")+ print(f"wrote {output} ({len(tensors)} tensors, "+ f"{merged.shape[0]} merged tokens x {merged.shape[1]})")+ return 0+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--output", type=Path,+ default=Path(__file__).resolve().parents[1] / "glm53_vision_tiny")+ arguments = parser.parse_args()+ return build(arguments.output)+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/c/vision_tower.h b/c/vision_tower.hnew file mode 100644index 00000000..ceb27401--- /dev/null+++ b/c/vision_tower.h@@ -0,0 +1,327 @@+#ifndef COLIBRI_VISION_TOWER_H+#define COLIBRI_VISION_TOWER_H++/*+ * Vision transformer tower, shared by the engines whose checkpoint carries one.+ *+ * GLM-5.3-Flash is the first, and the shape is the common one for this+ * generation of vision-language models: patches are projected, run through a+ * plain pre-norm transformer with 2-D rotary positions, then merged in m x m+ * spatial blocks and projected into the language model's width. Nothing here is+ * GLM-specific except the SwiGLU clamp, which is passed in.+ *+ * Two details decide whether the output is right, and both are easy to get+ * subtly wrong:+ *+ * - Token order is BLOCK-MAJOR over the m x m merge blocks, not row-major+ * over the image. The reference builds position ids as+ * reshape(h/m, m, w/m, m).transpose(1,2), so the tokens of one merge block+ * are adjacent. The final merge relies on that adjacency, which is why the+ * two must agree: get the order wrong and the picture is still "processed",+ * just scrambled in a way no assertion would catch.+ * - The 2-D rotary rotates each head's first half against the h axis and the+ * second against w: freqs are [h*inv_freq, w*inv_freq] concatenated, then+ * duplicated to the full head width, exactly like the text-side RoPE.+ *+ * Weights are plain f32 here. Quantized towers can come later; a 563 M+ * parameter tower is about 1 GB in bf16, which is not where this model's memory+ * goes.+ */++#include <math.h>+#include <stdlib.h>+#include <string.h>++typedef struct {+ int depth, hidden, heads, head_dim, intermediate;+ int patch, temporal, merge, in_channels;+ int out_hidden, proj_intermediate;+ float eps, swiglu_limit, rope_theta;+} ColiVisionConfig;++/* Row-major weights, transformers layout: a [out, in] matrix multiplies a+ * column vector of `in` values. Biases may be NULL where the checkpoint has+ * none. */+typedef struct {+ const float *norm1, *norm2;+ const float *qkv_w, *qkv_b; /* [3*hidden, hidden], [3*hidden] */+ const float *q_norm, *k_norm; /* [head_dim] each */+ const float *proj_w, *proj_b; /* [hidden, hidden], [hidden] */+ const float *gate_w, *gate_b; /* [intermediate, hidden] */+ const float *up_w, *up_b;+ const float *down_w, *down_b; /* [hidden, intermediate] */+} ColiVisionBlock;++typedef struct {+ ColiVisionConfig config;+ const float *patch_w, *patch_b; /* [hidden, in_channels*temporal*patch*patch] */+ const ColiVisionBlock *blocks; /* depth entries */+ const float *post_norm; /* [hidden] */+ const float *down_w, *down_b; /* Conv2d [out_hidden, hidden, merge, merge] */+ const float *merger_proj; /* [out_hidden, out_hidden] */+ const float *merger_norm_w, *merger_norm_b;+ const float *merger_gate, *merger_up;/* [proj_intermediate, out_hidden] */+ const float *merger_down; /* [out_hidden, proj_intermediate] */+} ColiVisionTower;++static inline void coli_vision_matvec(float *out, const float *w, const float *b,+ const float *in, int rows, int columns) {+ for (int row = 0; row < rows; row++) {+ float sum = b ? b[row] : 0.0f;+ const float *weight = w + (size_t)row * columns;+ for (int column = 0; column < columns; column++) sum += weight[column] * in[column];+ out[row] = sum;+ }+}++static inline void coli_vision_rmsnorm(float *out, const float *in, const float *weight,+ int dimension, float eps) {+ float square = 0.0f;+ for (int i = 0; i < dimension; i++) square += in[i] * in[i];+ float inverse = 1.0f / sqrtf(square / dimension + eps);+ for (int i = 0; i < dimension; i++) out[i] = in[i] * inverse * weight[i];+}++/* Classic LayerNorm (mean and variance), which is what the patch merger uses -+ * the rest of the tower is RMSNorm. */+static inline void coli_vision_layernorm(float *out, const float *in, const float *weight,+ const float *bias, int dimension, float eps) {+ float mean = 0.0f;+ for (int i = 0; i < dimension; i++) mean += in[i];+ mean /= dimension;+ float variance = 0.0f;+ for (int i = 0; i < dimension; i++) variance += (in[i] - mean) * (in[i] - mean);+ variance /= dimension;+ float inverse = 1.0f / sqrtf(variance + eps);+ for (int i = 0; i < dimension; i++)+ out[i] = (in[i] - mean) * inverse * weight[i] + (bias ? bias[i] : 0.0f);+}++static inline float coli_vision_silu(float value) {+ return value / (1.0f + expf(-value));+}++/* Exact GELU, the torch nn.GELU() default: erf-based, not the tanh+ * approximation. On a two-block tower the difference is invisible; on a+ * 24-block one it is not, and the two are not interchangeable. */+static inline float coli_vision_gelu(float value) {+ return 0.5f * value * (1.0f + erff(value * 0.70710678118654752f));+}++static inline float coli_vision_clamp(float value, float low, float high) {+ if (value < low) return low;+ if (value > high) return high;+ return value;+}++/* Output tokens for a grid: one per m x m block. */+static inline int coli_vision_output_tokens(const ColiVisionConfig *config,+ int grid_h, int grid_w) {+ if (config->merge < 1 || grid_h % config->merge || grid_w % config->merge) return -1;+ return (grid_h / config->merge) * (grid_w / config->merge);+}++/* pixels: [grid_h*grid_w, in_channels*temporal*patch*patch], already in the+ * block-major token order the reference produces.+ * out: [output_tokens, out_hidden]. Returns 0, or -1 on bad arguments or+ * allocation failure. */+static inline int coli_vision_forward(float *out, const ColiVisionTower *tower,+ const float *pixels, int grid_h, int grid_w) {+ const ColiVisionConfig *c = &tower->config;+ if (!out || !tower || !pixels || grid_h < 1 || grid_w < 1 ||+ c->heads < 1 || c->hidden != c->heads * c->head_dim ||+ c->head_dim % 4 || coli_vision_output_tokens(c, grid_h, grid_w) < 0)+ return -1;++ const int tokens = grid_h * grid_w;+ const int hidden = c->hidden, heads = c->heads, hd = c->head_dim;+ const int patch_width = c->in_channels * c->temporal * c->patch * c->patch;+ const int rot = hd / 2; /* rotary width before duplication */+ const int half = rot / 2; /* entries per spatial axis */++ float *state = malloc((size_t)tokens * hidden * sizeof(float));+ float *cos_table = malloc((size_t)tokens * hd * sizeof(float));+ float *sin_table = malloc((size_t)tokens * hd * sizeof(float));+ float *qkv = malloc((size_t)tokens * 3 * hidden * sizeof(float));+ float *scores = malloc((size_t)tokens * sizeof(float));+ float *scratch = malloc((size_t)(hidden > c->intermediate ? hidden : c->intermediate) *+ sizeof(float));+ float *branch = malloc((size_t)tokens * hidden * sizeof(float));+ if (!state || !cos_table || !sin_table || !qkv || !scores || !scratch || !branch) {+ free(branch); free(scratch); free(scores); free(qkv);+ free(sin_table); free(cos_table); free(state);+ return -1;+ }++ /* Patch embedding: the checkpoint's Conv3d has kernel == stride, so it is a+ * matrix applied to each flattened patch, not a convolution. */+ for (int t = 0; t < tokens; t++)+ coli_vision_matvec(state + (size_t)t * hidden, tower->patch_w, tower->patch_b,+ pixels + (size_t)t * patch_width, hidden, patch_width);++ /* 2-D rotary tables, in the same block-major order as the tokens. */+ {+ int m = c->merge, index = 0;+ for (int bh = 0; bh < grid_h / m; bh++)+ for (int bw = 0; bw < grid_w / m; bw++)+ for (int ih = 0; ih < m; ih++)+ for (int iw = 0; iw < m; iw++, index++) {+ float position[2] = { (float)(bh * m + ih), (float)(bw * m + iw) };+ float *cosine = cos_table + (size_t)index * hd;+ float *sine = sin_table + (size_t)index * hd;+ for (int axis = 0; axis < 2; axis++)+ for (int j = 0; j < half; j++) {+ float inv_freq = powf(c->rope_theta,+ -(float)(2 * j) / (float)rot);+ float angle = position[axis] * inv_freq;+ int slot = axis * half + j;+ cosine[slot] = cosf(angle);+ sine[slot] = sinf(angle);+ cosine[slot + rot] = cosine[slot]; /* cat(emb, emb) */+ sine[slot + rot] = sine[slot];+ }+ }+ }++ for (int layer = 0; layer < c->depth; layer++) {+ const ColiVisionBlock *block = &tower->blocks[layer];+ /* ---- attention ---- */+ for (int t = 0; t < tokens; t++) {+ coli_vision_rmsnorm(scratch, state + (size_t)t * hidden, block->norm1, hidden, c->eps);+ coli_vision_matvec(qkv + (size_t)t * 3 * hidden, block->qkv_w, block->qkv_b,+ scratch, 3 * hidden, hidden);+ }+ /* q/k head norms, then rotate. The reference reshapes qkv to+ * [seq, 3, heads, head_dim], so q, k and v are contiguous thirds. */+ for (int t = 0; t < tokens; t++) {+ float *row = qkv + (size_t)t * 3 * hidden;+ const float *cosine = cos_table + (size_t)t * hd;+ const float *sine = sin_table + (size_t)t * hd;+ for (int h = 0; h < heads; h++) {+ float *q = row + (size_t)h * hd;+ float *k = row + hidden + (size_t)h * hd;+ coli_vision_rmsnorm(q, q, block->q_norm, hd, c->eps);+ coli_vision_rmsnorm(k, k, block->k_norm, hd, c->eps);+ for (int pass = 0; pass < 2; pass++) {+ float *vec = pass ? k : q;+ float rotated[512];+ int limit = hd < 512 ? hd : 512;+ for (int i = 0; i < limit; i++)+ rotated[i] = i < hd / 2 ? -vec[i + hd / 2] : vec[i - hd / 2];+ for (int i = 0; i < limit; i++)+ vec[i] = vec[i] * cosine[i] + rotated[i] * sine[i];+ }+ }+ }+ const float scaling = 1.0f / sqrtf((float)hd);+ for (int t = 0; t < tokens; t++) {+ float *result = branch + (size_t)t * hidden;+ for (int h = 0; h < heads; h++) {+ const float *q = qkv + (size_t)t * 3 * hidden + (size_t)h * hd;+ float maximum = -INFINITY;+ for (int s = 0; s < tokens; s++) {+ const float *k = qkv + (size_t)s * 3 * hidden + hidden + (size_t)h * hd;+ float dot = 0.0f;+ for (int i = 0; i < hd; i++) dot += q[i] * k[i];+ scores[s] = dot * scaling;+ if (scores[s] > maximum) maximum = scores[s];+ }+ float total = 0.0f;+ for (int s = 0; s < tokens; s++) {+ scores[s] = expf(scores[s] - maximum);+ total += scores[s];+ }+ float *slot = result + (size_t)h * hd;+ for (int i = 0; i < hd; i++) slot[i] = 0.0f;+ for (int s = 0; s < tokens; s++) {+ const float *v = qkv + (size_t)s * 3 * hidden + 2 * hidden + (size_t)h * hd;+ float weight = scores[s] / total;+ for (int i = 0; i < hd; i++) slot[i] += weight * v[i];+ }+ }+ }+ for (int t = 0; t < tokens; t++) {+ coli_vision_matvec(scratch, block->proj_w, block->proj_b,+ branch + (size_t)t * hidden, hidden, hidden);+ for (int i = 0; i < hidden; i++) state[(size_t)t * hidden + i] += scratch[i];+ }+ /* ---- MLP ---- */+ for (int t = 0; t < tokens; t++) {+ float *gate = malloc((size_t)c->intermediate * 2 * sizeof(float));+ if (!gate) { free(branch); free(scratch); free(scores); free(qkv);+ free(sin_table); free(cos_table); free(state); return -1; }+ float *up = gate + c->intermediate;+ coli_vision_rmsnorm(scratch, state + (size_t)t * hidden, block->norm2, hidden, c->eps);+ coli_vision_matvec(gate, block->gate_w, block->gate_b, scratch, c->intermediate, hidden);+ coli_vision_matvec(up, block->up_w, block->up_b, scratch, c->intermediate, hidden);+ for (int i = 0; i < c->intermediate; i++) {+ float g = gate[i] > c->swiglu_limit ? c->swiglu_limit : gate[i];+ float u = coli_vision_clamp(up[i], -c->swiglu_limit, c->swiglu_limit);+ gate[i] = coli_vision_silu(g) * u;+ }+ coli_vision_matvec(scratch, block->down_w, block->down_b, gate, hidden, c->intermediate);+ for (int i = 0; i < hidden; i++) state[(size_t)t * hidden + i] += scratch[i];+ free(gate);+ }+ }++ for (int t = 0; t < tokens; t++)+ coli_vision_rmsnorm(state + (size_t)t * hidden, state + (size_t)t * hidden,+ tower->post_norm, hidden, c->eps);++ /* Spatial merge. The reference views the tokens as [-1, m, m, hidden],+ * permutes to [N, hidden, m, m] and applies a Conv2d whose kernel covers+ * the whole block, so each output token is one dot product over the+ * block's m*m*hidden values - which only lines up because the tokens of a+ * block are adjacent (see the header comment). */+ const int m = c->merge, blocks = coli_vision_output_tokens(c, grid_h, grid_w);+ float *merged = malloc((size_t)c->out_hidden * sizeof(float));+ float *gated = malloc((size_t)c->proj_intermediate * 2 * sizeof(float));+ if (!merged || !gated) {+ free(gated); free(merged); free(branch); free(scratch); free(scores);+ free(qkv); free(sin_table); free(cos_table); free(state);+ return -1;+ }+ for (int n = 0; n < blocks; n++) {+ for (int o = 0; o < c->out_hidden; o++) {+ float sum = tower->down_b ? tower->down_b[o] : 0.0f;+ for (int ch = 0; ch < hidden; ch++)+ for (int kh = 0; kh < m; kh++)+ for (int kw = 0; kw < m; kw++) {+ size_t weight_index = (((size_t)o * hidden + ch) * m + kh) * m + kw;+ size_t token = (size_t)n * m * m + (size_t)kh * m + kw;+ sum += tower->down_w[weight_index] * state[token * hidden + ch];+ }+ merged[o] = sum;+ }+ /* Patch merger: proj, LayerNorm, GELU, then a clamped SwiGLU. */+ float *tmp = branch; /* reuse: >= out_hidden */+ coli_vision_matvec(tmp, tower->merger_proj, NULL, merged, c->out_hidden, c->out_hidden);+ coli_vision_layernorm(merged, tmp, tower->merger_norm_w, tower->merger_norm_b,+ c->out_hidden, 1e-5f);+ for (int i = 0; i < c->out_hidden; i++) merged[i] = coli_vision_gelu(merged[i]);+ float *up = gated + c->proj_intermediate;+ coli_vision_matvec(gated, tower->merger_gate, NULL, merged, c->proj_intermediate, c->out_hidden);+ coli_vision_matvec(up, tower->merger_up, NULL, merged, c->proj_intermediate, c->out_hidden);+ for (int i = 0; i < c->proj_intermediate; i++) {+ float g = gated[i] > c->swiglu_limit ? c->swiglu_limit : gated[i];+ float u = coli_vision_clamp(up[i], -c->swiglu_limit, c->swiglu_limit);+ gated[i] = coli_vision_silu(g) * u;+ }+ coli_vision_matvec(out + (size_t)n * c->out_hidden, tower->merger_down, NULL,+ gated, c->out_hidden, c->proj_intermediate);+ }++ free(gated);+ free(merged);+ free(branch);+ free(scratch);+ free(scores);+ free(qkv);+ free(sin_table);+ free(cos_table);+ free(state);+ return 0;+}++#endif /* COLIBRI_VISION_TOWER_H */