Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(glm53): vision tower, validated against a transformers oracle

JustVugge884da10f463
diff --git a/c/glm53.c b/c/glm53.c
new file mode 100644
index 00000000..af71b9b4
--- /dev/null
+++ b/c/glm53.c
@@ -0,0 +1,545 @@
+/* GLM-5.3-Flash inference engine in pure C β€” sibling of kimi_k3.c / colibri.c /
+ * deepseek_v4.c / inkling.c / qwen36.c / olmoe.c, sharing st.h / json.h / tok.h /
+ * quant.h / compat.h.
+ *
+ * The name says GLM, but the skeleton is Kimi K3's, not GLM-5.2's: a hybrid
+ * stack of linear-attention layers punctuated by full attention, over a
+ * streamed MoE. That is why this file starts from kimi_k3.c and borrows the
+ * indexer from colibri.c and the hyper-connections from deepseek_v4.c, instead
+ * of extending the GLM engine.
+ *
+ * Architecture, read off the released checkpoint (321.34 B parameters measured
+ * from the shard headers; the card says 320 B total / 18 B active):
+ *
+ * - 45 text layers plus one MTP layer (index 45), hidden 4096, vocab 154880.
+ * - Hybrid attention, pattern (KDA KDA KDA FULL) repeating: 34 KDA linear
+ * layers and 11 DeepSeek-sparse-attention layers at 3, 7, ... 43, plus the
+ * MTP layer's own full-attention block. config.layer_types is explicit and
+ * is what this engine follows; linear_attn_config.kda_layers agrees.
+ * - MLA with kv_lora 512, q_lora 1536, qk_nope 256 and **qk_rope 0**: the
+ * full-attention layers are NoPE, exactly like K3. Position lives in the
+ * KDA decay and short convolutions, not in a rotation.
+ * - DSA lightning indexer on every full-attention layer, with k-pooling:
+ * keys are grouped into pools of index_kpool=4, the pools are scored
+ * instead of the tokens, the top index_topk/kpool pools are expanded back
+ * into token indices and the incomplete tail pool is always appended
+ * (index_kpool_always_select_tail). Two tensors carry it that GLM-5.2's
+ * indexer does not have: index_kpool_compress_ape and _compress_gate.
+ * - Manifold-Constrained Hyper-Connections (mHC, hc_mult 4, 20 Sinkhorn
+ * iterations) replace the plain residual at both sites of every layer β€”
+ * the same mHC DeepSeek V4 uses, down to the config keys, so the split
+ * into pre/post/comb and the Sinkhorn projection are shared code.
+ * - MoE: 288 routed experts (top-8) plus 1 shared expert per layer from
+ * layer 3 on, moe_intermediate 2048, sigmoid scoring with noaux_tc and
+ * e_score_correction_bias, routed_scaling_factor 2.5. The first three
+ * layers are dense (intermediate 12288).
+ * - Natively multimodal: a 24-block ViT (hidden 1024, patch 14, 448 px,
+ * spatial merge 2) whose patches are projected to 4096 and substituted at
+ * the image-token positions of the text stream. Text-only prompts never
+ * touch it.
+ *
+ * KDA recurrence β€” the same Kimi Delta Attention kimi_k3.c already reproduces
+ * token-exact against the vendor, verified line by line against
+ * transformers' recurrent_kimi_delta_attention:
+ * q,k,v = SiLU(ShortConv4(W{q,k,v} x)); q,k L2-normalized, q *= d^-0.5
+ * z = W_fb(W_fa x) + dt_bias
+ * gk = gmin * sigmoid(exp(A_log[h]) * z), gmin = gate_lower_bound = -5
+ * S = (I - beta k k^T) Diag(exp(gk)) S + beta k v^T, beta = sigmoid(W_b x)
+ * o = S^T q; out = W_o [ sigmoid(W_gb(W_ga x)) * RMSNorm_head(o) ]
+ * One difference from K3: the output gate is LOW-RANK here (g_a_proj into
+ * head_dim, then g_b_proj back out), where K3 has a single full g_proj.
+ *
+ * Container: tools/convert_glm53.py writes routed experts as int4 group-scaled
+ * gs64 (`name` U8 + `name.qs` F32, fmt=4 β€” the same container GLM-5.2 uses,
+ * measured cosine 0.994 against the fp8 source on real weights) and everything
+ * else as BF16, quantized at LOAD TIME here. That split is deliberate: the
+ * non-expert weights are 3% of the bytes, so keeping them exact on disk costs
+ * ~14 GB and means retuning dense precision never requires re-downloading the
+ * checkpoint.
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <stdint.h>
+
+#include "json.h"
+#include "st.h"
+#include "hyper_connections.h" /* mHC, condiviso con deepseek_v4.c */
+
+/* ---------- config ----------
+ * Nested like Kimi K3's: the root carries the vision wrapper and `text_config`
+ * carries the language model. A text-only export therefore has the text keys at
+ * the root, and both shapes are accepted. */
+typedef struct {
+ /* text */
+ int hidden, n_layers, vocab, first_dense, dense_inter;
+ int n_heads, q_lora, kv_lora, qk_nope, qk_rope, v_head, qk_head;
+ int n_experts, topk, moe_inter, n_shared;
+ float routed_scale, eps;
+ int n_mtp; /* num_nextn_predict_layers (1) */
+ /* KDA */
+ int kda_heads, kda_hd, kda_proj, conv_k;
+ float gate_lb;
+ /* DSA indexer with k-pooling */
+ int index_topk, index_nh, index_hd, index_kpool, index_kpool_tail;
+ /* mHC */
+ int hc_mult, hc_iters;
+ float hc_eps;
+ /* per-layer kind: 1 = full attention (MLA + indexer), 0 = KDA */
+ unsigned char is_full[128];
+ /* vision (0 = text-only checkpoint) */
+ int vis_layers, vis_hidden, vis_heads, vis_inter, vis_patch, vis_temporal;
+ int vis_merge, vis_out_hidden, vis_proj_inter, vis_image_size, vis_in_ch;
+ float vis_swiglu_limit, vis_eps;
+ int image_token, image_start_token, image_end_token;
+ int video_token, video_start_token, video_end_token;
+} Cfg;
+
+static double req_num(jval *object, const char *key) {
+ jval *value = json_get(object, key);
+ if (!value || value->t != J_NUM) {
+ fprintf(stderr, "config.json: missing or non-numeric \"%s\"\n", key);
+ exit(1);
+ }
+ return value->num;
+}
+
+static double opt_num(jval *object, const char *key, double fallback) {
+ jval *value = object ? json_get(object, key) : NULL;
+ return (value && value->t == J_NUM) ? value->num : fallback;
+}
+
+static int opt_bool(jval *object, const char *key, int fallback) {
+ jval *value = object ? json_get(object, key) : NULL;
+ if (!value) return fallback;
+ if (value->t == J_BOOL) return value->boolean;
+ if (value->t == J_NUM) return value->num != 0.0;
+ return fallback;
+}
+
+/* layer_types is the authority on which layers are full attention. The
+ * linear_attn_config.{kda_layers,full_attn_layers} lists say the same thing;
+ * disagreeing checkpoints are refused rather than guessed at, because picking
+ * the wrong kind for one layer produces plausible-looking garbage. */
+static void load_layer_kinds(Cfg *c, jval *text) {
+ memset(c->is_full, 0, sizeof(c->is_full));
+ jval *types = json_get(text, "layer_types");
+ if (!types || types->t != J_ARR || types->len != c->n_layers) {
+ fprintf(stderr, "config.json: layer_types must list %d entries\n", c->n_layers);
+ exit(1);
+ }
+ int full = 0;
+ for (int i = 0; i < types->len; i++) {
+ jval *entry = types->kids[i];
+ if (!entry || entry->t != J_STR) {
+ fprintf(stderr, "config.json: layer_types[%d] is not a string\n", i);
+ exit(1);
+ }
+ if (strstr(entry->str, "linear")) {
+ c->is_full[i] = 0;
+ } else if (strstr(entry->str, "attention") || strstr(entry->str, "full")) {
+ c->is_full[i] = 1;
+ full++;
+ } else {
+ fprintf(stderr, "config.json: unknown layer type \"%s\" at %d\n", entry->str, i);
+ exit(1);
+ }
+ }
+ jval *linear = json_get(text, "linear_attn_config");
+ jval *full_list = linear ? json_get(linear, "full_attn_layers") : NULL;
+ if (full_list && full_list->t == J_ARR) {
+ if (full_list->len != full) {
+ fprintf(stderr, "config.json: full_attn_layers lists %d layers, "
+ "layer_types marks %d\n", full_list->len, full);
+ exit(1);
+ }
+ for (int i = 0; i < full_list->len; i++) {
+ int index = (int)full_list->kids[i]->num;
+ if (index < 0 || index >= c->n_layers || !c->is_full[index]) {
+ fprintf(stderr, "config.json: full_attn_layers disagrees with "
+ "layer_types at %d\n", index);
+ exit(1);
+ }
+ }
+ }
+ /* The MTP block is a full-attention layer that lives past num_hidden_layers
+ * and is not described by layer_types. */
+ if (c->n_layers < (int)sizeof(c->is_full)) c->is_full[c->n_layers] = 1;
+}
+
+static void load_vision(Cfg *c, jval *root) {
+ jval *vision = json_get(root, "vision_config");
+ if (!vision || vision->t != J_OBJ) { c->vis_layers = 0; return; }
+ c->vis_layers = (int)req_num(vision, "depth");
+ c->vis_hidden = (int)req_num(vision, "hidden_size");
+ c->vis_heads = (int)req_num(vision, "num_heads");
+ c->vis_inter = (int)req_num(vision, "intermediate_size");
+ c->vis_patch = (int)req_num(vision, "patch_size");
+ c->vis_temporal = (int)opt_num(vision, "temporal_patch_size", 2);
+ c->vis_merge = (int)opt_num(vision, "spatial_merge_size", 2);
+ c->vis_out_hidden = (int)opt_num(vision, "out_hidden_size", c->hidden);
+ c->vis_proj_inter = (int)opt_num(vision, "projection_intermediate_size", 0);
+ c->vis_image_size = (int)opt_num(vision, "image_size", 448);
+ c->vis_in_ch = (int)opt_num(vision, "in_channels", 3);
+ c->vis_swiglu_limit= (float)opt_num(vision, "swiglu_limit", 10.0);
+ c->vis_eps = (float)opt_num(vision, "rms_norm_eps", 1e-5);
+ c->image_token = (int)opt_num(root, "image_token_id", -1);
+ c->image_start_token = (int)opt_num(root, "image_start_token_id", -1);
+ c->image_end_token = (int)opt_num(root, "image_end_token_id", -1);
+ c->video_token = (int)opt_num(root, "video_token_id", -1);
+ c->video_start_token = (int)opt_num(root, "video_start_token_id", -1);
+ c->video_end_token = (int)opt_num(root, "video_end_token_id", -1);
+ if (c->vis_layers < 1 || c->vis_layers > 128 || c->vis_hidden < 1 ||
+ c->vis_heads < 1 || c->vis_hidden % c->vis_heads ||
+ c->vis_patch < 1 || c->vis_merge < 1 || c->vis_out_hidden != c->hidden) {
+ fprintf(stderr, "config.json: vision_config out of range "
+ "(out_hidden_size must equal the text hidden size)\n");
+ exit(1);
+ }
+}
+
+static void load_cfg(Cfg *c, const char *snap) {
+ char path[4096];
+ snprintf(path, sizeof(path), "%s/config.json", snap);
+ char *buffer = NULL;
+ {
+ FILE *stream = fopen(path, "rb");
+ if (!stream) { fprintf(stderr, "cannot read %s\n", path); exit(1); }
+ if (fseek(stream, 0, SEEK_END)) { fprintf(stderr, "%s: not seekable\n", path); exit(1); }
+ long length = ftell(stream);
+ if (length < 2 || fseek(stream, 0, SEEK_SET)) {
+ fprintf(stderr, "%s: unusable\n", path); exit(1); }
+ buffer = malloc((size_t)length + 1);
+ if (!buffer) { fprintf(stderr, "OOM reading config\n"); exit(1); }
+ if (fread(buffer, 1, (size_t)length, stream) != (size_t)length) {
+ fprintf(stderr, "%s: short read\n", path); exit(1); }
+ buffer[length] = 0;
+ fclose(stream);
+ }
+ char *arena = NULL;
+ jval *root = json_parse(buffer, &arena);
+ if (!root || root->t != J_OBJ) { fprintf(stderr, "%s: not a JSON object\n", path); exit(1); }
+ memset(c, 0, sizeof(*c));
+
+ jval *tc = json_get(root, "text_config");
+ if (!tc || tc->t != J_OBJ) tc = root; /* text-only export */
+
+ c->hidden = (int)req_num(tc, "hidden_size");
+ c->n_layers = (int)req_num(tc, "num_hidden_layers");
+ c->vocab = (int)req_num(tc, "vocab_size");
+ c->first_dense = (int)req_num(tc, "first_k_dense_replace");
+ c->dense_inter = (int)req_num(tc, "intermediate_size");
+ c->n_heads = (int)req_num(tc, "num_attention_heads");
+ c->q_lora = (int)req_num(tc, "q_lora_rank");
+ c->kv_lora = (int)req_num(tc, "kv_lora_rank");
+ c->qk_nope = (int)req_num(tc, "qk_nope_head_dim");
+ c->qk_rope = (int)opt_num(tc, "qk_rope_head_dim", 0);
+ c->v_head = (int)req_num(tc, "v_head_dim");
+ c->n_experts = (int)req_num(tc, "n_routed_experts");
+ c->topk = (int)req_num(tc, "num_experts_per_tok");
+ c->moe_inter = (int)req_num(tc, "moe_intermediate_size");
+ c->n_shared = (int)opt_num(tc, "n_shared_experts", 1);
+ c->routed_scale= (float)opt_num(tc, "routed_scaling_factor", 1.0);
+ c->eps = (float)opt_num(tc, "rms_norm_eps", 1e-6);
+ c->n_mtp = (int)opt_num(tc, "num_nextn_predict_layers", 0);
+ c->qk_head = c->qk_nope + c->qk_rope;
+
+ jval *linear = json_get(tc, "linear_attn_config");
+ if (!linear || linear->t != J_OBJ) {
+ fprintf(stderr, "config.json: missing linear_attn_config\n"); exit(1);
+ }
+ c->kda_heads = (int)req_num(linear, "num_heads");
+ c->kda_hd = (int)req_num(linear, "head_dim");
+ c->conv_k = (int)req_num(linear, "short_conv_kernel_size");
+ c->gate_lb = (float)opt_num(linear, "gate_lower_bound", -5.0);
+ c->kda_proj = c->kda_heads * c->kda_hd;
+
+ c->index_topk = (int)opt_num(tc, "index_topk", 0);
+ c->index_nh = (int)opt_num(tc, "index_n_heads", 0);
+ c->index_hd = (int)opt_num(tc, "index_head_dim", 0);
+ c->index_kpool = (int)opt_num(tc, "index_kpool", 1);
+ c->index_kpool_tail = opt_bool(tc, "index_kpool_always_select_tail", 0);
+
+ c->hc_mult = (int)opt_num(tc, "hc_mult", 1);
+ c->hc_iters = (int)opt_num(tc, "hc_sinkhorn_iters", 0);
+ c->hc_eps = (float)opt_num(tc, "hc_eps", 1e-6);
+
+ load_layer_kinds(c, tc);
+ load_vision(c, root);
+
+ if (c->hidden < 1 || c->hidden > 65536 ||
+ c->n_layers < 1 || c->n_layers > 120 ||
+ c->vocab < 1 || c->vocab > (1 << 22) ||
+ c->n_experts < 1 || c->n_experts > 4096 ||
+ c->topk < 1 || c->topk > 64 || c->topk > c->n_experts ||
+ c->kda_proj < 1 || c->kda_proj > (1 << 20) ||
+ c->conv_k < 1 || c->conv_k > 8 ||
+ c->moe_inter % 32 || c->kda_hd > 512 || c->kv_lora > 4096 ||
+ c->first_dense < 0 || c->first_dense > c->n_layers ||
+ c->index_kpool < 1 || c->index_kpool > 64 ||
+ c->hc_mult < 1 || c->hc_mult > 8) {
+ fprintf(stderr, "config.json: dimension out of range\n"); exit(1);
+ }
+ /* qk_rope must be zero: a rotary GLM-5.3 would need position handling this
+ * engine deliberately does not have, and silently ignoring the rotation
+ * would produce a model that answers fluently and wrongly. */
+ if (c->qk_rope != 0) {
+ fprintf(stderr, "config.json: qk_rope_head_dim=%d, but this engine "
+ "implements the NoPE full-attention of GLM-5.3\n", c->qk_rope);
+ exit(1);
+ }
+ free(arena);
+ free(buffer);
+}
+
+static void cfg_report(const Cfg *c) {
+ int full = 0, kda = 0;
+ for (int i = 0; i < c->n_layers; i++) { if (c->is_full[i]) full++; else kda++; }
+ double expert_params = (double)c->n_experts * 3.0 * c->hidden * c->moe_inter *
+ (c->n_layers - c->first_dense + (c->n_mtp ? 1 : 0));
+ fprintf(stderr,
+ "GLM-5.3-Flash: %d layers (%d KDA + %d full) + %d MTP, hidden %d, vocab %d\n"
+ " MoE : %d routed (top-%d) + %d shared, inter %d, from layer %d, scale %.2f\n"
+ " KDA : %d heads x %d, conv %d, gate floor %.1f\n"
+ " MLA : q_lora %d, kv_lora %d, qk %d (nope, NoPE), v %d, %d heads\n"
+ " indexer : top-%d, %d heads x %d, kpool %d%s\n"
+ " mHC : mult %d, %d Sinkhorn iterations\n"
+ " vision : %s\n"
+ " routed experts: %.1f B parameters (%.0f GB at int4-g64)\n",
+ c->n_layers, kda, full, c->n_mtp, c->hidden, c->vocab,
+ c->n_experts, c->topk, c->n_shared, c->moe_inter, c->first_dense, c->routed_scale,
+ c->kda_heads, c->kda_hd, c->conv_k, (double)c->gate_lb,
+ c->q_lora, c->kv_lora, c->qk_nope, c->v_head, c->n_heads,
+ c->index_topk, c->index_nh, c->index_hd, c->index_kpool,
+ c->index_kpool_tail ? " (+tail)" : "",
+ c->hc_mult, c->hc_iters,
+ c->vis_layers ? "yes" : "text-only checkpoint",
+ expert_params / 1e9, expert_params * 0.5625 / 1e9);
+ if (c->vis_layers)
+ fprintf(stderr,
+ " %d blocks x %d, %d heads, patch %d, %dpx, merge %d -> %d\n",
+ c->vis_layers, c->vis_hidden, c->vis_heads, c->vis_patch,
+ c->vis_image_size, c->vis_merge, c->vis_out_hidden);
+}
+
+#ifdef GLM53_CFG_MAIN
+/* Config-only entry point: `glm53_cfg <model_dir>` parses and reports, so the
+ * parser can be checked against a real checkpoint before any weight exists. */
+int main(int argc, char **argv) {
+ if (argc != 2) { fprintf(stderr, "usage: %s <model_dir>\n", argv[0]); return 2; }
+ Cfg c;
+ load_cfg(&c, argv[1]);
+ cfg_report(&c);
+ return 0;
+}
+#endif
+
+/* ---------- RAM-resident weight ----------
+ * Same shape as kimi_k3.c's: either f32, int8 per-row, or int4 group-scaled.
+ * The container written by tools/convert_glm53.py is U8 + `.qs` F32, which is
+ * exactly what K3's loader already reads, so the two engines share a format
+ * rather than each inventing one. */
+typedef struct {
+ int fmt; /* 0 = f32, 8 = int8 per-row, 4 = int4-g64 */
+ float *f;
+ int8_t *q8;
+ uint8_t *q4;
+ float *s; /* scales: [O] per-row, or [O*ngroups] */
+ int O, I, gs;
+} W;
+
+/* ---------- layer structures ----------
+ * Mirrors kimi_k3.c's shapes, with the GLM-5.3 differences called out where
+ * they bite: the KDA output gate is low-rank here, the full-attention layers
+ * are NOT gated (K3's are), and every layer carries two mHC sites instead of a
+ * plain residual. */
+typedef struct { /* KDA (linear attention) layer */
+ W q, k, v, o; /* [proj x hidden] x3, [hidden x proj] */
+ W ga, gb; /* low-rank output gate: hidden->hd->proj */
+ float *conv_q, *conv_k, *conv_v; /* [proj*conv_k] depthwise taps */
+ float *fa, *fb; /* decay low-rank: [hd,hidden], [proj,hd] */
+ float *bp; /* beta projection [heads,hidden] */
+ float *dt, *A, *onw; /* dt_bias[proj], exp(A_log)[heads], o_norm[hd] */
+} Kda;
+
+typedef struct { /* MLA + DSA indexer (full attention) */
+ W qa, qb, kva, kvb, o;
+ float *qa_ln, *kva_ln;
+ W wq, wk, wp; /* indexer: wq_b, wk, weights_proj */
+ float *knw, *knb; /* indexer key LayerNorm (weight + bias) */
+ float *kpool_ape; /* [kpool, index_hd] pool position bias */
+ W kpool_gate; /* [index_hd, hidden] compression gate */
+} Mla;
+
+typedef struct { /* MoE (routed streamed + shared resident) */
+ float *router, *rbias; /* [E,hidden] f32, [E] correction bias */
+ W sh_gate, sh_up, sh_down;
+} Moe;
+
+typedef struct {
+ int full; /* 1 = MLA + indexer, 0 = KDA */
+ int dense; /* 1 = plain MLP (layers < first_dense) */
+ Kda a;
+ Mla m;
+ Moe moe;
+ W d_gate, d_up, d_down; /* dense layers only */
+ float *in_ln, *post_ln;
+ /* mHC, two sites per layer. fn is [(2+H)*H, H*hidden], base [(2+H)*H],
+ * scale [3]; the split into pre/post/comb and the Sinkhorn projection are
+ * the same as DeepSeek V4's. Absent on the MTP layer. */
+ float *hc_attn_fn, *hc_attn_base, *hc_attn_scale;
+ float *hc_ffn_fn, *hc_ffn_base, *hc_ffn_scale;
+ /* MTP layer only */
+ W mtp_eh;
+ float *mtp_enorm, *mtp_hnorm, *mtp_head_norm;
+} Layer;
+
+/* ---------- tensor names ----------
+ * One place builds every name the engine asks for. The checkpoint prefixes the
+ * language model with `model.language_model.` because the root is the vision
+ * wrapper; a text-only export drops it. Both are probed, and the choice is made
+ * once from a tensor that must exist either way. */
+typedef struct {
+ const char *prefix; /* "model.language_model." or "model." */
+ const char *visual; /* "model.visual." */
+} Names;
+
+#define GLM53_NAME(dst, fmt, ...) snprintf((dst), sizeof(dst), (fmt), __VA_ARGS__)
+
+/* Emits, in load order, every tensor this engine will look for. `sink` is
+ * called with (name, required); a NULL model just prints them, which is how
+ * the mapping gets checked against a real checkpoint index before any weight
+ * has been downloaded. */
+static void glm53_walk_tensors(const Cfg *c, const Names *n,
+ void (*sink)(void *, const char *, int),
+ void *user) {
+ char name[512];
+#define EMIT(required, fmt, ...) do { \
+ GLM53_NAME(name, fmt, __VA_ARGS__); sink(user, name, (required)); } while (0)
+ EMIT(1, "%sembed_tokens.weight", n->prefix);
+ EMIT(1, "%snorm.weight", n->prefix);
+ sink(user, "lm_head.weight", 1);
+
+ int last = c->n_layers + (c->n_mtp ? 1 : 0);
+ for (int i = 0; i < last; i++) {
+ int mtp = i >= c->n_layers;
+ int full = c->is_full[i];
+ EMIT(1, "%slayers.%d.input_layernorm.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.post_attention_layernorm.weight", n->prefix, i);
+ if (!mtp) { /* mHC lives on the 45 real layers */
+ EMIT(1, "%slayers.%d.hc_attn_fn", n->prefix, i);
+ EMIT(1, "%slayers.%d.hc_attn_base", n->prefix, i);
+ EMIT(1, "%slayers.%d.hc_attn_scale", n->prefix, i);
+ EMIT(1, "%slayers.%d.hc_ffn_fn", n->prefix, i);
+ EMIT(1, "%slayers.%d.hc_ffn_base", n->prefix, i);
+ EMIT(1, "%slayers.%d.hc_ffn_scale", n->prefix, i);
+ } else {
+ EMIT(1, "%slayers.%d.eh_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.enorm.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.hnorm.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.shared_head.norm.weight", n->prefix, i);
+ }
+ if (full) {
+ EMIT(1, "%slayers.%d.self_attn.q_a_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.q_a_layernorm.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.q_b_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.kv_a_proj_with_mqa.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.kv_a_layernorm.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.kv_b_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.o_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.indexer.wq_b.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.indexer.wk.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.indexer.weights_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.indexer.k_norm.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.indexer.k_norm.bias", n->prefix, i);
+ if (c->index_kpool > 1) {
+ EMIT(1, "%slayers.%d.self_attn.indexer.index_kpool_compress_ape", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.indexer.index_kpool_compress_gate", n->prefix, i);
+ }
+ } else {
+ EMIT(1, "%slayers.%d.self_attn.q_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.k_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.v_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.o_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.g_a_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.g_b_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.q_conv1d.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.k_conv1d.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.v_conv1d.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.f_a_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.f_b_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.b_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.dt_bias", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.A_log", n->prefix, i);
+ EMIT(1, "%slayers.%d.self_attn.o_norm.weight", n->prefix, i);
+ }
+ if (i < c->first_dense) {
+ EMIT(1, "%slayers.%d.mlp.gate_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.mlp.up_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.mlp.down_proj.weight", n->prefix, i);
+ } else {
+ EMIT(1, "%slayers.%d.mlp.gate.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.mlp.gate.e_score_correction_bias", n->prefix, i);
+ EMIT(1, "%slayers.%d.mlp.shared_experts.gate_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.mlp.shared_experts.up_proj.weight", n->prefix, i);
+ EMIT(1, "%slayers.%d.mlp.shared_experts.down_proj.weight", n->prefix, i);
+ /* Routed experts are streamed, not resident: named here so the
+ * inventory can prove they are all present and correctly shaped. */
+ for (int e = 0; e < c->n_experts; e++) {
+ EMIT(1, "%slayers.%d.mlp.experts.%d.gate_proj.weight", n->prefix, i, e);
+ EMIT(1, "%slayers.%d.mlp.experts.%d.up_proj.weight", n->prefix, i, e);
+ EMIT(1, "%slayers.%d.mlp.experts.%d.down_proj.weight", n->prefix, i, e);
+ }
+ }
+ }
+ if (c->vis_layers) {
+ EMIT(1, "%spatch_embed.proj.weight", n->visual);
+ EMIT(1, "%spatch_embed.proj.bias", n->visual);
+ EMIT(1, "%spost_layernorm.weight", n->visual);
+ EMIT(1, "%sdownsample.weight", n->visual);
+ EMIT(1, "%sdownsample.bias", n->visual);
+ EMIT(1, "%smerger.proj.weight", n->visual);
+ EMIT(1, "%smerger.post_projection_norm.weight", n->visual);
+ EMIT(1, "%smerger.post_projection_norm.bias", n->visual);
+ EMIT(1, "%smerger.gate_proj.weight", n->visual);
+ EMIT(1, "%smerger.up_proj.weight", n->visual);
+ EMIT(1, "%smerger.down_proj.weight", n->visual);
+ for (int b = 0; b < c->vis_layers; b++) {
+ EMIT(1, "%sblocks.%d.norm1.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.norm2.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.attn.qkv.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.attn.qkv.bias", n->visual, b);
+ EMIT(1, "%sblocks.%d.attn.proj.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.attn.proj.bias", n->visual, b);
+ EMIT(1, "%sblocks.%d.attn.q_norm.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.attn.k_norm.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.mlp.gate_proj.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.mlp.gate_proj.bias", n->visual, b);
+ EMIT(1, "%sblocks.%d.mlp.up_proj.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.mlp.up_proj.bias", n->visual, b);
+ EMIT(1, "%sblocks.%d.mlp.down_proj.weight", n->visual, b);
+ EMIT(1, "%sblocks.%d.mlp.down_proj.bias", n->visual, b);
+ }
+ }
+#undef EMIT
+}
+
+#ifdef GLM53_INVENTORY_MAIN
+/* Inventory entry point: prints every tensor the engine would load, in load
+ * order. Checked against a real checkpoint's index offline, so a naming
+ * mistake surfaces before 180 GB have been converted rather than after. */
+static void print_name(void *user, const char *name, int required) {
+ (void)user; printf("%s\t%d\n", name, required);
+}
+
+int main(int argc, char **argv) {
+ if (argc != 2) { fprintf(stderr, "usage: %s <model_dir>\n", argv[0]); return 2; }
+ Cfg c;
+ load_cfg(&c, argv[1]);
+ Names n = { "model.language_model.", "model.visual." };
+ glm53_walk_tensors(&c, &n, print_name, NULL);
+ return 0;
+}
+#endif
diff --git a/c/tests/test_vision_tower.c b/c/tests/test_vision_tower.c
new file mode 100644
index 00000000..9e546fd9
--- /dev/null
+++ b/c/tests/test_vision_tower.c
@@ -0,0 +1,126 @@
+/* Il tower vision contro l'oracolo generato da transformers
+ * (tools/make_glm53_vision_tiny.py). Legge la fixture con st.h, cosi' il test
+ * esercita anche il percorso di caricamento reale invece di dati incollati. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+#include "../json.h"
+#include "../st.h"
+#include "../vision_tower.h"
+
+static const float *need(shards *S, const char *name, int64_t expect) {
+ st_tensor *t = st_find(S, name);
+ if (!t) { fprintf(stderr, "manca il tensore %s\n", name); exit(1); }
+ if (expect > 0 && t->numel != expect) {
+ fprintf(stderr, "%s: %lld elementi, attesi %lld\n", name,
+ (long long)t->numel, (long long)expect); exit(1);
+ }
+ float *buffer = malloc((size_t)t->numel * sizeof(float));
+ if (!buffer) { fprintf(stderr, "OOM su %s\n", name); exit(1); }
+ st_read_f32_cap(S, name, buffer, t->numel, 0);
+ return buffer;
+}
+
+static char *slurp(const char *path) {
+ FILE *f = fopen(path, "rb");
+ if (!f) { fprintf(stderr, "non riesco ad aprire %s\n", path); exit(1); }
+ fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);
+ char *b = malloc((size_t)n + 1);
+ if (fread(b, 1, (size_t)n, f) != (size_t)n) { fprintf(stderr, "%s: lettura corta\n", path); exit(1); }
+ b[n] = 0; fclose(f);
+ return b;
+}
+
+int main(int argc, char **argv) {
+ const char *dir = argc > 1 ? argv[1] : "glm53_vision_tiny";
+ char path[1024];
+ shards S;
+ st_init(&S, dir);
+
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ char *text = slurp(path), *arena = NULL;
+ jval *root = json_parse(text, &arena);
+ jval *vc = json_get(root, "vision_config");
+ if (!vc) { fprintf(stderr, "config.json senza vision_config\n"); return 1; }
+#define NUM(key) ((int)json_get(vc, key)->num)
+ ColiVisionConfig cfg = {
+ .depth = NUM("depth"), .hidden = NUM("hidden_size"), .heads = NUM("num_heads"),
+ .intermediate = NUM("intermediate_size"), .patch = NUM("patch_size"),
+ .temporal = NUM("temporal_patch_size"), .merge = NUM("spatial_merge_size"),
+ .in_channels = NUM("in_channels"), .out_hidden = NUM("out_hidden_size"),
+ .proj_intermediate = NUM("projection_intermediate_size"),
+ .eps = (float)json_get(vc, "rms_norm_eps")->num,
+ .swiglu_limit = (float)json_get(vc, "swiglu_limit")->num,
+ .rope_theta = 10000.0f,
+ };
+#undef NUM
+ cfg.head_dim = cfg.hidden / cfg.heads;
+
+ ColiVisionBlock *blocks = calloc((size_t)cfg.depth, sizeof(*blocks));
+ for (int b = 0; b < cfg.depth; b++) {
+ char n[256];
+#define GET(field, suffix, expect) \
+ snprintf(n, sizeof(n), "blocks.%d." suffix, b); blocks[b].field = need(&S, n, expect)
+ GET(norm1, "norm1.weight", cfg.hidden);
+ GET(norm2, "norm2.weight", cfg.hidden);
+ GET(qkv_w, "attn.qkv.weight", (int64_t)3 * cfg.hidden * cfg.hidden);
+ GET(qkv_b, "attn.qkv.bias", 3 * cfg.hidden);
+ GET(q_norm, "attn.q_norm.weight", cfg.head_dim);
+ GET(k_norm, "attn.k_norm.weight", cfg.head_dim);
+ GET(proj_w, "attn.proj.weight", (int64_t)cfg.hidden * cfg.hidden);
+ GET(proj_b, "attn.proj.bias", cfg.hidden);
+ GET(gate_w, "mlp.gate_proj.weight", (int64_t)cfg.intermediate * cfg.hidden);
+ GET(gate_b, "mlp.gate_proj.bias", cfg.intermediate);
+ GET(up_w, "mlp.up_proj.weight", (int64_t)cfg.intermediate * cfg.hidden);
+ GET(up_b, "mlp.up_proj.bias", cfg.intermediate);
+ GET(down_w, "mlp.down_proj.weight", (int64_t)cfg.hidden * cfg.intermediate);
+ GET(down_b, "mlp.down_proj.bias", cfg.hidden);
+#undef GET
+ }
+ ColiVisionTower tower = {
+ .config = cfg, .blocks = blocks,
+ .patch_w = need(&S, "patch_embed.proj.weight", 0),
+ .patch_b = need(&S, "patch_embed.proj.bias", cfg.hidden),
+ .post_norm = need(&S, "post_layernorm.weight", cfg.hidden),
+ .down_w = need(&S, "downsample.weight", 0),
+ .down_b = need(&S, "downsample.bias", cfg.out_hidden),
+ .merger_proj = need(&S, "merger.proj.weight", (int64_t)cfg.out_hidden * cfg.out_hidden),
+ .merger_norm_w = need(&S, "merger.post_projection_norm.weight", cfg.out_hidden),
+ .merger_norm_b = need(&S, "merger.post_projection_norm.bias", cfg.out_hidden),
+ .merger_gate = need(&S, "merger.gate_proj.weight", 0),
+ .merger_up = need(&S, "merger.up_proj.weight", 0),
+ .merger_down = need(&S, "merger.down_proj.weight", 0),
+ };
+
+ snprintf(path, sizeof(path), "%s/ref.json", dir);
+ char *rtext = slurp(path), *rarena = NULL;
+ jval *ref = json_parse(rtext, &rarena);
+ jval *grid = json_get(ref, "grid_thw");
+ int gh = (int)grid->kids[0]->kids[1]->num, gw = (int)grid->kids[0]->kids[2]->num;
+ jval *want = json_get(ref, "output");
+
+ const float *pixels = need(&S, "input.pixel_values", 0);
+ int out_tokens = coli_vision_output_tokens(&cfg, gh, gw);
+ float *out = malloc((size_t)out_tokens * cfg.out_hidden * sizeof(float));
+ if (coli_vision_forward(out, &tower, pixels, gh, gw)) {
+ fprintf(stderr, "coli_vision_forward ha fallito\n"); return 1;
+ }
+
+ int n = out_tokens * cfg.out_hidden;
+ if (want->len != n) {
+ fprintf(stderr, "ref ha %d valori, il tower ne produce %d\n", want->len, n);
+ return 1;
+ }
+ float worst = 0.0f; int at = 0;
+ for (int i = 0; i < n; i++) {
+ float d = fabsf(out[i] - (float)want->kids[i]->num);
+ if (d > worst) { worst = d; at = i; }
+ }
+ printf("vision tower: %d token x %d, max|delta| = %.3e (indice %d: %.9g vs %.9g)\n",
+ out_tokens, cfg.out_hidden, worst, at, out[at], want->kids[at]->num);
+ if (worst > 2e-5f) { printf("FALLITO\n"); return 1; }
+ printf("PASS tower vision: combacia con l'oracolo transformers\n");
+ return 0;
+}
diff --git a/c/tools/make_glm53_vision_tiny.py b/c/tools/make_glm53_vision_tiny.py
new file mode 100644
index 00000000..6ef1f137
--- /dev/null
+++ b/c/tools/make_glm53_vision_tiny.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Oracolo tiny del tower vision di GLM-5.3-Flash.
+
+Il testo ha gia' il suo oracolo; questo copre l'altra meta' del modello, che
+nessun test raggiungeva: patch embed, RoPE 2D, i blocchi ViT con SwiGLU
+clampata, la norma finale, il downsample 2x2 e il patch merger.
+
+Il fixture e' prodotto dal `Glm5NextVisionModel` ufficiale di Hugging Face, non
+da noi: il motore C ha cosi' un bersaglio indipendente invece di validare se
+stesso. Pesi e input sono deterministici (linspace su un seed fisso), quindi il
+file e' riproducibile bit per bit e la CI non ha bisogno della rete.
+
+Perche' un tower minuscolo e non quello vero: 24 blocchi da 1024 sono 563 M
+parametri, inutili per provare che la matematica e' giusta. Qui bastano 2
+blocchi e un'immagine da 4x4 patch, e ogni meccanismo viene comunque esercitato
+almeno una volta - inclusa la fusione 2x2, che con una griglia 4x4 produce
+esattamente 4 token di uscita.
+
+USO:
+ python3 tools/make_glm53_vision_tiny.py --output c/glm53_vision_tiny
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+EXPECTED_TRANSFORMERS = "5.16.1"
+
+DEPTH = 2
+HIDDEN = 64
+HEADS = 4
+INTERMEDIATE = 128
+PATCH = 4
+TEMPORAL = 2
+MERGE = 2
+OUT_HIDDEN = 96
+PROJ_INTERMEDIATE = 128
+IN_CHANNELS = 3
+GRID_H = 4
+GRID_W = 4
+SEED = 1253
+
+
+def deterministic_(tensor, torch, spread=0.05):
+ """Riempie in modo riproducibile e indipendente dall'ordine di init."""
+ flat = torch.linspace(-spread, spread, tensor.numel(), dtype=torch.float32)
+ with torch.no_grad():
+ tensor.copy_(flat.view_as(tensor))
+
+
+def build(output: Path) -> int:
+ import torch
+ import transformers
+ from safetensors.torch import save_file
+ from transformers.models.glm5_next.configuration_glm5_next import Glm5NextVisionConfig
+ from transformers.models.glm5_next.modeling_glm5_next import Glm5NextVisionModel
+
+ if transformers.__version__ != EXPECTED_TRANSFORMERS:
+ print(f"expected Transformers {EXPECTED_TRANSFORMERS}, "
+ f"found {transformers.__version__}")
+ return 2
+
+ torch.manual_seed(SEED)
+ config = Glm5NextVisionConfig(
+ depth=DEPTH,
+ hidden_size=HIDDEN,
+ num_heads=HEADS,
+ intermediate_size=INTERMEDIATE,
+ patch_size=PATCH,
+ temporal_patch_size=TEMPORAL,
+ spatial_merge_size=MERGE,
+ out_hidden_size=OUT_HIDDEN,
+ projection_intermediate_size=PROJ_INTERMEDIATE,
+ in_channels=IN_CHANNELS,
+ image_size=PATCH * GRID_H,
+ )
+ model = Glm5NextVisionModel(config).eval()
+ for parameter in model.parameters():
+ deterministic_(parameter, torch)
+ for name, buffer in model.named_buffers():
+ if buffer.dtype.is_floating_point and "inv_freq" not in name:
+ deterministic_(buffer, torch)
+
+ patches = GRID_H * GRID_W
+ width = IN_CHANNELS * TEMPORAL * PATCH * PATCH
+ pixels = torch.linspace(-1.0, 1.0, patches * width, dtype=torch.float32).view(patches, width)
+ grid_thw = torch.tensor([[1, GRID_H, GRID_W]], dtype=torch.long)
+
+ with torch.no_grad():
+ outputs = model(pixels, grid_thw=grid_thw)
+ merged = outputs.pooler_output.float()
+
+ output.mkdir(parents=True, exist_ok=True)
+ tensors = {name: parameter.detach().contiguous().to(torch.float32)
+ for name, parameter in model.state_dict().items()
+ if parameter.dtype.is_floating_point}
+ tensors["input.pixel_values"] = pixels.contiguous()
+ save_file(tensors, str(output / "model.safetensors"), metadata={"format": "pt"})
+ (output / "config.json").write_text(
+ json.dumps({"model_type": "glm5_next_vision", "vision_config": config.to_dict()},
+ indent=2, sort_keys=True, default=str),
+ encoding="utf-8")
+
+ reference = {
+ "schema_version": 1,
+ "source": "transformers",
+ "transformers_version": transformers.__version__,
+ "torch_version": torch.__version__,
+ "seed": SEED,
+ "grid_thw": grid_thw.tolist(),
+ "input_shape": list(pixels.shape),
+ "output_shape": list(merged.shape),
+ "output_sum": float(merged.sum()),
+ "output_square_sum": float(merged.square().sum()),
+ # Ogni token di uscita per intero: un confronto per statistiche sole
+ # nasconderebbe un errore che sposta valori fra token senza cambiarne
+ # somma e norma, ed e' esattamente cosi' che sbaglia una fusione 2x2.
+ "output": merged.flatten().tolist(),
+ }
+ (output / "ref.json").write_text(json.dumps(reference, indent=2), encoding="utf-8")
+ print(f"wrote {output} ({len(tensors)} tensors, "
+ f"{merged.shape[0]} merged tokens x {merged.shape[1]})")
+ return 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output", type=Path,
+ default=Path(__file__).resolve().parents[1] / "glm53_vision_tiny")
+ arguments = parser.parse_args()
+ return build(arguments.output)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/c/vision_tower.h b/c/vision_tower.h
new file mode 100644
index 00000000..ceb27401
--- /dev/null
+++ b/c/vision_tower.h
@@ -0,0 +1,327 @@
+#ifndef COLIBRI_VISION_TOWER_H
+#define COLIBRI_VISION_TOWER_H
+
+/*
+ * Vision transformer tower, shared by the engines whose checkpoint carries one.
+ *
+ * GLM-5.3-Flash is the first, and the shape is the common one for this
+ * generation of vision-language models: patches are projected, run through a
+ * plain pre-norm transformer with 2-D rotary positions, then merged in m x m
+ * spatial blocks and projected into the language model's width. Nothing here is
+ * GLM-specific except the SwiGLU clamp, which is passed in.
+ *
+ * Two details decide whether the output is right, and both are easy to get
+ * subtly wrong:
+ *
+ * - Token order is BLOCK-MAJOR over the m x m merge blocks, not row-major
+ * over the image. The reference builds position ids as
+ * reshape(h/m, m, w/m, m).transpose(1,2), so the tokens of one merge block
+ * are adjacent. The final merge relies on that adjacency, which is why the
+ * two must agree: get the order wrong and the picture is still "processed",
+ * just scrambled in a way no assertion would catch.
+ * - The 2-D rotary rotates each head's first half against the h axis and the
+ * second against w: freqs are [h*inv_freq, w*inv_freq] concatenated, then
+ * duplicated to the full head width, exactly like the text-side RoPE.
+ *
+ * Weights are plain f32 here. Quantized towers can come later; a 563 M
+ * parameter tower is about 1 GB in bf16, which is not where this model's memory
+ * goes.
+ */
+
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+ int depth, hidden, heads, head_dim, intermediate;
+ int patch, temporal, merge, in_channels;
+ int out_hidden, proj_intermediate;
+ float eps, swiglu_limit, rope_theta;
+} ColiVisionConfig;
+
+/* Row-major weights, transformers layout: a [out, in] matrix multiplies a
+ * column vector of `in` values. Biases may be NULL where the checkpoint has
+ * none. */
+typedef struct {
+ const float *norm1, *norm2;
+ const float *qkv_w, *qkv_b; /* [3*hidden, hidden], [3*hidden] */
+ const float *q_norm, *k_norm; /* [head_dim] each */
+ const float *proj_w, *proj_b; /* [hidden, hidden], [hidden] */
+ const float *gate_w, *gate_b; /* [intermediate, hidden] */
+ const float *up_w, *up_b;
+ const float *down_w, *down_b; /* [hidden, intermediate] */
+} ColiVisionBlock;
+
+typedef struct {
+ ColiVisionConfig config;
+ const float *patch_w, *patch_b; /* [hidden, in_channels*temporal*patch*patch] */
+ const ColiVisionBlock *blocks; /* depth entries */
+ const float *post_norm; /* [hidden] */
+ const float *down_w, *down_b; /* Conv2d [out_hidden, hidden, merge, merge] */
+ const float *merger_proj; /* [out_hidden, out_hidden] */
+ const float *merger_norm_w, *merger_norm_b;
+ const float *merger_gate, *merger_up;/* [proj_intermediate, out_hidden] */
+ const float *merger_down; /* [out_hidden, proj_intermediate] */
+} ColiVisionTower;
+
+static inline void coli_vision_matvec(float *out, const float *w, const float *b,
+ const float *in, int rows, int columns) {
+ for (int row = 0; row < rows; row++) {
+ float sum = b ? b[row] : 0.0f;
+ const float *weight = w + (size_t)row * columns;
+ for (int column = 0; column < columns; column++) sum += weight[column] * in[column];
+ out[row] = sum;
+ }
+}
+
+static inline void coli_vision_rmsnorm(float *out, const float *in, const float *weight,
+ int dimension, float eps) {
+ float square = 0.0f;
+ for (int i = 0; i < dimension; i++) square += in[i] * in[i];
+ float inverse = 1.0f / sqrtf(square / dimension + eps);
+ for (int i = 0; i < dimension; i++) out[i] = in[i] * inverse * weight[i];
+}
+
+/* Classic LayerNorm (mean and variance), which is what the patch merger uses -
+ * the rest of the tower is RMSNorm. */
+static inline void coli_vision_layernorm(float *out, const float *in, const float *weight,
+ const float *bias, int dimension, float eps) {
+ float mean = 0.0f;
+ for (int i = 0; i < dimension; i++) mean += in[i];
+ mean /= dimension;
+ float variance = 0.0f;
+ for (int i = 0; i < dimension; i++) variance += (in[i] - mean) * (in[i] - mean);
+ variance /= dimension;
+ float inverse = 1.0f / sqrtf(variance + eps);
+ for (int i = 0; i < dimension; i++)
+ out[i] = (in[i] - mean) * inverse * weight[i] + (bias ? bias[i] : 0.0f);
+}
+
+static inline float coli_vision_silu(float value) {
+ return value / (1.0f + expf(-value));
+}
+
+/* Exact GELU, the torch nn.GELU() default: erf-based, not the tanh
+ * approximation. On a two-block tower the difference is invisible; on a
+ * 24-block one it is not, and the two are not interchangeable. */
+static inline float coli_vision_gelu(float value) {
+ return 0.5f * value * (1.0f + erff(value * 0.70710678118654752f));
+}
+
+static inline float coli_vision_clamp(float value, float low, float high) {
+ if (value < low) return low;
+ if (value > high) return high;
+ return value;
+}
+
+/* Output tokens for a grid: one per m x m block. */
+static inline int coli_vision_output_tokens(const ColiVisionConfig *config,
+ int grid_h, int grid_w) {
+ if (config->merge < 1 || grid_h % config->merge || grid_w % config->merge) return -1;
+ return (grid_h / config->merge) * (grid_w / config->merge);
+}
+
+/* pixels: [grid_h*grid_w, in_channels*temporal*patch*patch], already in the
+ * block-major token order the reference produces.
+ * out: [output_tokens, out_hidden]. Returns 0, or -1 on bad arguments or
+ * allocation failure. */
+static inline int coli_vision_forward(float *out, const ColiVisionTower *tower,
+ const float *pixels, int grid_h, int grid_w) {
+ const ColiVisionConfig *c = &tower->config;
+ if (!out || !tower || !pixels || grid_h < 1 || grid_w < 1 ||
+ c->heads < 1 || c->hidden != c->heads * c->head_dim ||
+ c->head_dim % 4 || coli_vision_output_tokens(c, grid_h, grid_w) < 0)
+ return -1;
+
+ const int tokens = grid_h * grid_w;
+ const int hidden = c->hidden, heads = c->heads, hd = c->head_dim;
+ const int patch_width = c->in_channels * c->temporal * c->patch * c->patch;
+ const int rot = hd / 2; /* rotary width before duplication */
+ const int half = rot / 2; /* entries per spatial axis */
+
+ float *state = malloc((size_t)tokens * hidden * sizeof(float));
+ float *cos_table = malloc((size_t)tokens * hd * sizeof(float));
+ float *sin_table = malloc((size_t)tokens * hd * sizeof(float));
+ float *qkv = malloc((size_t)tokens * 3 * hidden * sizeof(float));
+ float *scores = malloc((size_t)tokens * sizeof(float));
+ float *scratch = malloc((size_t)(hidden > c->intermediate ? hidden : c->intermediate) *
+ sizeof(float));
+ float *branch = malloc((size_t)tokens * hidden * sizeof(float));
+ if (!state || !cos_table || !sin_table || !qkv || !scores || !scratch || !branch) {
+ free(branch); free(scratch); free(scores); free(qkv);
+ free(sin_table); free(cos_table); free(state);
+ return -1;
+ }
+
+ /* Patch embedding: the checkpoint's Conv3d has kernel == stride, so it is a
+ * matrix applied to each flattened patch, not a convolution. */
+ for (int t = 0; t < tokens; t++)
+ coli_vision_matvec(state + (size_t)t * hidden, tower->patch_w, tower->patch_b,
+ pixels + (size_t)t * patch_width, hidden, patch_width);
+
+ /* 2-D rotary tables, in the same block-major order as the tokens. */
+ {
+ int m = c->merge, index = 0;
+ for (int bh = 0; bh < grid_h / m; bh++)
+ for (int bw = 0; bw < grid_w / m; bw++)
+ for (int ih = 0; ih < m; ih++)
+ for (int iw = 0; iw < m; iw++, index++) {
+ float position[2] = { (float)(bh * m + ih), (float)(bw * m + iw) };
+ float *cosine = cos_table + (size_t)index * hd;
+ float *sine = sin_table + (size_t)index * hd;
+ for (int axis = 0; axis < 2; axis++)
+ for (int j = 0; j < half; j++) {
+ float inv_freq = powf(c->rope_theta,
+ -(float)(2 * j) / (float)rot);
+ float angle = position[axis] * inv_freq;
+ int slot = axis * half + j;
+ cosine[slot] = cosf(angle);
+ sine[slot] = sinf(angle);
+ cosine[slot + rot] = cosine[slot]; /* cat(emb, emb) */
+ sine[slot + rot] = sine[slot];
+ }
+ }
+ }
+
+ for (int layer = 0; layer < c->depth; layer++) {
+ const ColiVisionBlock *block = &tower->blocks[layer];
+ /* ---- attention ---- */
+ for (int t = 0; t < tokens; t++) {
+ coli_vision_rmsnorm(scratch, state + (size_t)t * hidden, block->norm1, hidden, c->eps);
+ coli_vision_matvec(qkv + (size_t)t * 3 * hidden, block->qkv_w, block->qkv_b,
+ scratch, 3 * hidden, hidden);
+ }
+ /* q/k head norms, then rotate. The reference reshapes qkv to
+ * [seq, 3, heads, head_dim], so q, k and v are contiguous thirds. */
+ for (int t = 0; t < tokens; t++) {
+ float *row = qkv + (size_t)t * 3 * hidden;
+ const float *cosine = cos_table + (size_t)t * hd;
+ const float *sine = sin_table + (size_t)t * hd;
+ for (int h = 0; h < heads; h++) {
+ float *q = row + (size_t)h * hd;
+ float *k = row + hidden + (size_t)h * hd;
+ coli_vision_rmsnorm(q, q, block->q_norm, hd, c->eps);
+ coli_vision_rmsnorm(k, k, block->k_norm, hd, c->eps);
+ for (int pass = 0; pass < 2; pass++) {
+ float *vec = pass ? k : q;
+ float rotated[512];
+ int limit = hd < 512 ? hd : 512;
+ for (int i = 0; i < limit; i++)
+ rotated[i] = i < hd / 2 ? -vec[i + hd / 2] : vec[i - hd / 2];
+ for (int i = 0; i < limit; i++)
+ vec[i] = vec[i] * cosine[i] + rotated[i] * sine[i];
+ }
+ }
+ }
+ const float scaling = 1.0f / sqrtf((float)hd);
+ for (int t = 0; t < tokens; t++) {
+ float *result = branch + (size_t)t * hidden;
+ for (int h = 0; h < heads; h++) {
+ const float *q = qkv + (size_t)t * 3 * hidden + (size_t)h * hd;
+ float maximum = -INFINITY;
+ for (int s = 0; s < tokens; s++) {
+ const float *k = qkv + (size_t)s * 3 * hidden + hidden + (size_t)h * hd;
+ float dot = 0.0f;
+ for (int i = 0; i < hd; i++) dot += q[i] * k[i];
+ scores[s] = dot * scaling;
+ if (scores[s] > maximum) maximum = scores[s];
+ }
+ float total = 0.0f;
+ for (int s = 0; s < tokens; s++) {
+ scores[s] = expf(scores[s] - maximum);
+ total += scores[s];
+ }
+ float *slot = result + (size_t)h * hd;
+ for (int i = 0; i < hd; i++) slot[i] = 0.0f;
+ for (int s = 0; s < tokens; s++) {
+ const float *v = qkv + (size_t)s * 3 * hidden + 2 * hidden + (size_t)h * hd;
+ float weight = scores[s] / total;
+ for (int i = 0; i < hd; i++) slot[i] += weight * v[i];
+ }
+ }
+ }
+ for (int t = 0; t < tokens; t++) {
+ coli_vision_matvec(scratch, block->proj_w, block->proj_b,
+ branch + (size_t)t * hidden, hidden, hidden);
+ for (int i = 0; i < hidden; i++) state[(size_t)t * hidden + i] += scratch[i];
+ }
+ /* ---- MLP ---- */
+ for (int t = 0; t < tokens; t++) {
+ float *gate = malloc((size_t)c->intermediate * 2 * sizeof(float));
+ if (!gate) { free(branch); free(scratch); free(scores); free(qkv);
+ free(sin_table); free(cos_table); free(state); return -1; }
+ float *up = gate + c->intermediate;
+ coli_vision_rmsnorm(scratch, state + (size_t)t * hidden, block->norm2, hidden, c->eps);
+ coli_vision_matvec(gate, block->gate_w, block->gate_b, scratch, c->intermediate, hidden);
+ coli_vision_matvec(up, block->up_w, block->up_b, scratch, c->intermediate, hidden);
+ for (int i = 0; i < c->intermediate; i++) {
+ float g = gate[i] > c->swiglu_limit ? c->swiglu_limit : gate[i];
+ float u = coli_vision_clamp(up[i], -c->swiglu_limit, c->swiglu_limit);
+ gate[i] = coli_vision_silu(g) * u;
+ }
+ coli_vision_matvec(scratch, block->down_w, block->down_b, gate, hidden, c->intermediate);
+ for (int i = 0; i < hidden; i++) state[(size_t)t * hidden + i] += scratch[i];
+ free(gate);
+ }
+ }
+
+ for (int t = 0; t < tokens; t++)
+ coli_vision_rmsnorm(state + (size_t)t * hidden, state + (size_t)t * hidden,
+ tower->post_norm, hidden, c->eps);
+
+ /* Spatial merge. The reference views the tokens as [-1, m, m, hidden],
+ * permutes to [N, hidden, m, m] and applies a Conv2d whose kernel covers
+ * the whole block, so each output token is one dot product over the
+ * block's m*m*hidden values - which only lines up because the tokens of a
+ * block are adjacent (see the header comment). */
+ const int m = c->merge, blocks = coli_vision_output_tokens(c, grid_h, grid_w);
+ float *merged = malloc((size_t)c->out_hidden * sizeof(float));
+ float *gated = malloc((size_t)c->proj_intermediate * 2 * sizeof(float));
+ if (!merged || !gated) {
+ free(gated); free(merged); free(branch); free(scratch); free(scores);
+ free(qkv); free(sin_table); free(cos_table); free(state);
+ return -1;
+ }
+ for (int n = 0; n < blocks; n++) {
+ for (int o = 0; o < c->out_hidden; o++) {
+ float sum = tower->down_b ? tower->down_b[o] : 0.0f;
+ for (int ch = 0; ch < hidden; ch++)
+ for (int kh = 0; kh < m; kh++)
+ for (int kw = 0; kw < m; kw++) {
+ size_t weight_index = (((size_t)o * hidden + ch) * m + kh) * m + kw;
+ size_t token = (size_t)n * m * m + (size_t)kh * m + kw;
+ sum += tower->down_w[weight_index] * state[token * hidden + ch];
+ }
+ merged[o] = sum;
+ }
+ /* Patch merger: proj, LayerNorm, GELU, then a clamped SwiGLU. */
+ float *tmp = branch; /* reuse: >= out_hidden */
+ coli_vision_matvec(tmp, tower->merger_proj, NULL, merged, c->out_hidden, c->out_hidden);
+ coli_vision_layernorm(merged, tmp, tower->merger_norm_w, tower->merger_norm_b,
+ c->out_hidden, 1e-5f);
+ for (int i = 0; i < c->out_hidden; i++) merged[i] = coli_vision_gelu(merged[i]);
+ float *up = gated + c->proj_intermediate;
+ coli_vision_matvec(gated, tower->merger_gate, NULL, merged, c->proj_intermediate, c->out_hidden);
+ coli_vision_matvec(up, tower->merger_up, NULL, merged, c->proj_intermediate, c->out_hidden);
+ for (int i = 0; i < c->proj_intermediate; i++) {
+ float g = gated[i] > c->swiglu_limit ? c->swiglu_limit : gated[i];
+ float u = coli_vision_clamp(up[i], -c->swiglu_limit, c->swiglu_limit);
+ gated[i] = coli_vision_silu(g) * u;
+ }
+ coli_vision_matvec(out + (size_t)n * c->out_hidden, tower->merger_down, NULL,
+ gated, c->out_hidden, c->proj_intermediate);
+ }
+
+ free(gated);
+ free(merged);
+ free(branch);
+ free(scratch);
+ free(scores);
+ free(qkv);
+ free(sin_table);
+ free(cos_table);
+ free(state);
+ return 0;
+}
+
+#endif /* COLIBRI_VISION_TOWER_H */