Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* mimo_vision.h -- the MiMo ViT (681M on the release), images only.
*
* Included by mimo.c after the Model and DW helpers. Mirrors
* MiMoVisionTransformer in modeling_mimo_v2.py, with the two points where the
* release and that module disagree resolved as the release (and vLLM's
* mimo_v2_omni.py) has them:
* - the merger stores no biases: its two Linears and its LayerNorm bias are
* zero (vLLM builds them with bias=False; a bias present in a container is
* still used);
* - after a run of column-order blocks that reaches the end of the tower, the
* tokens go back to row order before the merger (vLLM does; the release's
* last block is a full-attention one, so the two readings never differ on it).
*
* The tower, per block: x += proj(attn(rms(x))), x += mlp(rms(x))
* patches [N, 3*T*16*16] in 2x2-merge-block order (the Qwen2-VL processor)
* 2D RoPE head_dim/2 dims: h then w frequencies (theta 1e4), cat(e, e),
* rotate-half over the whole head
* attention GQA; full over the image in fullatt_block_indexes, else a 1D
* window |i - j| <= visual_token_window_size in the CURRENT token
* order, plus a sink logit added to key 0 of that order
* order vit_window_attn_types: -1 full, 0 row order, 1 column order of
* the merge units (entering a type-1 run permutes the tokens and
* their rotary angles, leaving it permutes them back)
* merger LayerNorm, the 2x2 units concatenated, Linear, exact GELU, Linear
*/
typedef struct {
float *norm1, *norm2, *sinks;
DW qkv, proj, gate, up, down;
float *qkv_b, *proj_b, *gate_b, *up_b, *down_b;
int full, type;
} VBlock;
struct Vision {
int depth, d, inter, heads, kvh, hd, out, patch, temporal, merge, in_ch, window;
float eps;
DW embed; /* [d, in_ch*T*P*P] */
VBlock *b;
float *ln_w, *ln_b, *fc1_b, *fc2_b;
DW fc1, fc2;
int patch_in;
};
static void vis_mat(DW *d, shards *S, const char *name, int O, int I) {
/* the tower stays BF16 (or f32 for the oracle): int8 is a dense-trunk option */
int saved = g_dense_bits;
g_dense_bits = g_dense_bits == 32 ? 32 : 0;
dw_load_bf16(d, S, name, O, I);
g_dense_bits = saved;
}
static float *vis_vec_opt(shards *S, const char *name, int n) {
return st_has(S, name) ? load_vec(S, name, n) : NULL;
}
static int vjson_int(jval *o, const char *key, int fallback) {
jval *v = o ? json_get(o, key) : NULL;
return (v && v->t == J_NUM) ? (int)v->num : fallback;
}
static Vision *vision_load(Model *m, const char *dir) {
if (!st_has(&m->S, "visual.patch_embed.proj.weight")) return NULL;
char path[4096];
snprintf(path, sizeof(path), "%s/config.json", dir);
char *text = read_text(path, NULL);
if (!text) return NULL;
char *arena = NULL;
jval *root = json_parse(text, &arena);
jval *vc = root ? json_get(root, "vision_config") : NULL;
if (!vc || vc->t != J_OBJ) {
fprintf(stderr, "[mimo] visual.* tensors but no vision_config: text only\n");
json_free(root); free(arena); free(text);
return NULL;
}
Vision *v = xcalloc(1, sizeof(Vision), "vision");
v->depth = vjson_int(vc, "depth", 0);
v->d = vjson_int(vc, "hidden_size", 0);
v->inter = vjson_int(vc, "intermediate_size", 0);
v->heads = vjson_int(vc, "num_heads", 0);
v->kvh = vjson_int(vc, "num_key_value_heads", v->heads);
v->hd = vjson_int(vc, "qk_channels", 64);
v->out = vjson_int(vc, "out_hidden_size", 0);
v->patch = vjson_int(vc, "patch_size", 16);
v->temporal = vjson_int(vc, "temporal_patch_size", 2);
v->merge = vjson_int(vc, "spatial_merge_size", 2);
v->in_ch = vjson_int(vc, "in_chans", 3);
v->window = vjson_int(vc, "visual_token_window_size", -1);
v->eps = (float)jnum(vc, "rms_norm_eps", 1e-6);
int use_sink = jbool(vc, "use_sink", 0);
if (v->depth < 1 || v->depth > 256 || v->d < 8 || v->d > 16384 || v->inter < 1 || v->inter > 65536 ||
v->heads < 1 || v->kvh < 1 || v->heads % v->kvh || v->hd < 4 || v->hd % 4 || v->hd > 1024 ||
v->out != m->c.hidden || v->merge != 2 || v->patch < 1 || v->patch > 64 ||
v->temporal < 1 || v->temporal > 4 || v->in_ch != 3) {
fprintf(stderr, "[mimo] vision_config out of range (or out_hidden_size != hidden_size) -- refusing\n");
exit(1);
}
v->patch_in = v->in_ch * v->temporal * v->patch * v->patch;
v->b = xcalloc((size_t)v->depth, sizeof(VBlock), "vision blocks");
jval *full = json_get(vc, "fullatt_block_indexes"), *types = json_get(vc, "vit_window_attn_types");
for (int i = 0; i < v->depth; i++) v->b[i].type = -1;
if (types && types->t == J_ARR) {
if (types->len != v->depth) { fprintf(stderr, "[mimo] vit_window_attn_types must list every block\n"); exit(1); }
for (int i = 0; i < v->depth; i++) v->b[i].type = (int)types->kids[i]->num;
}
if (full && full->t == J_ARR)
for (int i = 0; i < full->len; i++) {
int b = (int)full->kids[i]->num;
if (b >= 0 && b < v->depth) v->b[b].full = 1;
}
json_free(root); free(arena); free(text);
shards *S = &m->S;
st_tensor *t = st_find(S, "visual.patch_embed.proj.weight");
if (t->dtype != 0 || t->numel != (int64_t)v->d * v->patch_in || t->nbytes != t->numel * 2) {
fprintf(stderr, "[mimo] visual.patch_embed.proj.weight is not BF16 [%d, %d] -- refusing\n", v->d, v->patch_in);
exit(1);
}
/* Conv3d [d, C, T, P, P] with stride = kernel is a Linear over the flattened
* patch, in the processor's C, T, row, col order */
v->embed.O = v->d; v->embed.I = v->patch_in;
if (g_dense_bits == 32) {
float *w = xmalloc((size_t)v->d * v->patch_in * sizeof(float), "patch embed");
st_read_f32(S, "visual.patch_embed.proj.weight", w, 1);
v->embed.fmt = DW_F32; v->embed.w = w;
} else {
uint16_t *b16 = xmalloc((size_t)v->d * v->patch_in * sizeof(uint16_t), "patch embed");
st_read_raw_cap(S, "visual.patch_embed.proj.weight", b16, (int64_t)v->d * v->patch_in * 2, 1);
v->embed.fmt = DW_BF16; v->embed.w = b16;
}
int qkv_rows = (v->heads + 2 * v->kvh) * v->hd;
for (int i = 0; i < v->depth; i++) {
VBlock *b = &v->b[i];
char nm[512];
#define VN(fmt_) (snprintf(nm, sizeof(nm), "visual.blocks.%d." fmt_, i), nm)
b->norm1 = load_vec(S, VN("norm1.weight"), v->d);
b->norm2 = load_vec(S, VN("norm2.weight"), v->d);
vis_mat(&b->qkv, S, VN("attn.qkv.weight"), qkv_rows, v->d);
b->qkv_b = load_vec(S, VN("attn.qkv.bias"), qkv_rows);
vis_mat(&b->proj, S, VN("attn.proj.weight"), v->d, v->heads * v->hd);
b->proj_b = load_vec(S, VN("attn.proj.bias"), v->d);
vis_mat(&b->gate, S, VN("mlp.gate_proj.weight"), v->inter, v->d);
b->gate_b = load_vec(S, VN("mlp.gate_proj.bias"), v->inter);
vis_mat(&b->up, S, VN("mlp.up_proj.weight"), v->inter, v->d);
b->up_b = load_vec(S, VN("mlp.up_proj.bias"), v->inter);
vis_mat(&b->down, S, VN("mlp.down_proj.weight"), v->d, v->inter);
b->down_b = load_vec(S, VN("mlp.down_proj.bias"), v->d);
if (use_sink && !b->full) b->sinks = load_vec(S, VN("attn.sinks"), v->heads);
#undef VN
}
int merged = v->d * v->merge * v->merge;
v->ln_w = load_vec(S, "visual.merger.ln_q.weight", v->d);
v->ln_b = vis_vec_opt(S, "visual.merger.ln_q.bias", v->d);
vis_mat(&v->fc1, S, "visual.merger.mlp.0.weight", merged, merged);
v->fc1_b = vis_vec_opt(S, "visual.merger.mlp.0.bias", merged);
vis_mat(&v->fc2, S, "visual.merger.mlp.2.weight", v->out, merged);
v->fc2_b = vis_vec_opt(S, "visual.merger.mlp.2.bias", v->out);
return v;
}
static void add_bias(float *y, const float *b, int n, int dim) {
if (!b) return;
for (int t = 0; t < n; t++) for (int i = 0; i < dim; i++) y[(size_t)t * dim + i] += b[i];
}
/* new[unit k] = old[unit idx[k]], units of 4 tokens (the 2x2 merge) */
static void permute_units(float *x, const int *idx, int units, int width) {
size_t unit = (size_t)4 * width;
float *tmp = xmalloc((size_t)units * unit * sizeof(float), "vision permute");
for (int k = 0; k < units; k++) memcpy(tmp + (size_t)k * unit, x + (size_t)idx[k] * unit, unit * sizeof(float));
memcpy(x, tmp, (size_t)units * unit * sizeof(float));
free(tmp);
}
static void vis_rope(float *x, int nh, int hd, const float *ang) {
int half = hd / 2;
for (int h = 0; h < nh; h++) {
float *v = x + (size_t)h * hd;
for (int i = 0; i < half; i++) {
float cs = cosf(ang[i]), sn = sinf(ang[i]);
float a = v[i], b = v[i + half];
v[i] = a * cs - b * sn;
v[i + half] = b * cs + a * sn;
}
}
}
static float *vision_run(Model *m, const float *patches, int gh, int gw, int *n_rows) {
Vision *v = g_vision;
(void)m;
*n_rows = 0;
if (!v || gh < 2 || gw < 2 || gh % 2 || gw % 2) return NULL;
double t0 = now_s();
int N = gh * gw, d = v->d, hd = v->hd, nh = v->heads, kvh = v->kvh, group = nh / kvh;
int units = N / 4, uh = gh / 2, uw = gw / 2;
float *x = xmalloc((size_t)N * d * sizeof(float), "vision tokens");
dw_matmul(x, patches, N, &v->embed);
/* rotary angles per token, in the processor's block-major order */
int rdim = hd / 2, nf = rdim / 2;
float *ang = xmalloc((size_t)N * rdim * sizeof(float), "vision angles");
float *inv = xmalloc((size_t)nf * sizeof(float), "vision inv_freq");
for (int j = 0; j < nf; j++) inv[j] = 1.0f / powf(10000.0f, (float)(2 * j) / (float)rdim);
for (int n = 0; n < N; n++) {
int u = n / 4, r = (n % 4) / 2, cc = n % 2;
int hp = (u / uw) * 2 + r, wp = (u % uw) * 2 + cc;
for (int j = 0; j < nf; j++) {
ang[(size_t)n * rdim + j] = (float)hp * inv[j];
ang[(size_t)n * rdim + nf + j] = (float)wp * inv[j];
}
}
free(inv);
/* column order of the merge units and its inverse */
int *col = xmalloc((size_t)units * sizeof(int), "vision col order");
int *rev = xmalloc((size_t)units * sizeof(int), "vision row order");
for (int k = 0; k < units; k++) col[k] = (k % uh) * uw + k / uh;
for (int k = 0; k < units; k++) rev[col[k]] = k;
float *ang_col = xmalloc((size_t)N * rdim * sizeof(float), "vision col angles");
memcpy(ang_col, ang, (size_t)N * rdim * sizeof(float));
permute_units(ang_col, col, units, rdim);
int qd = nh * hd, kd = kvh * hd, rows = qd + 2 * kd;
float *xn = xmalloc((size_t)N * d * sizeof(float), "vision normed");
float *qkv = xmalloc((size_t)N * rows * sizeof(float), "vision qkv");
float *ctx = xmalloc((size_t)N * qd * sizeof(float), "vision attention");
float *tmp = xmalloc((size_t)N * d * sizeof(float), "vision block out");
float *g = xmalloc((size_t)N * v->inter * sizeof(float), "vision mlp gate");
float *u = xmalloc((size_t)N * v->inter * sizeof(float), "vision mlp up");
float scale = 1.0f / sqrtf((float)hd);
int prev = -2;
for (int bi = 0; bi < v->depth; bi++) {
VBlock *b = &v->b[bi];
if (b->type == 1 && prev != 1) permute_units(x, col, units, d);
if (b->type != 1 && prev == 1) permute_units(x, rev, units, d);
prev = b->type;
const float *A = b->type == 1 ? ang_col : ang;
for (int n = 0; n < N; n++) rmsnorm(xn + (size_t)n * d, x + (size_t)n * d, b->norm1, d, v->eps);
dw_matmul(qkv, xn, N, &b->qkv);
add_bias(qkv, b->qkv_b, N, rows);
for (int n = 0; n < N; n++) {
vis_rope(qkv + (size_t)n * rows, nh, hd, A + (size_t)n * rdim);
vis_rope(qkv + (size_t)n * rows + qd, kvh, hd, A + (size_t)n * rdim);
}
int window = b->full ? -1 : v->window;
#pragma omp parallel
{
float *score = xmalloc((size_t)N * sizeof(float), "vision scores");
#pragma omp for collapse(2) schedule(static)
for (int i = 0; i < N; i++) {
for (int h = 0; h < nh; h++) {
int kh = h / group;
int lo = 0, hi = N - 1;
if (window >= 0) { lo = i - window > 0 ? i - window : 0; hi = i + window < N - 1 ? i + window : N - 1; }
const float *q = qkv + (size_t)i * rows + (size_t)h * hd;
float best = -INFINITY;
for (int j = lo; j <= hi; j++) {
const float *kr = qkv + (size_t)j * rows + qd + (size_t)kh * hd;
float s = 0;
for (int e = 0; e < hd; e++) s += q[e] * kr[e];
s *= scale;
if (j == 0 && b->sinks) s += b->sinks[h];
score[j - lo] = s;
if (s > best) best = s;
}
float den = 0;
for (int j = lo; j <= hi; j++) { score[j - lo] = expf(score[j - lo] - best); den += score[j - lo]; }
float *o = ctx + (size_t)i * qd + (size_t)h * hd;
for (int e = 0; e < hd; e++) o[e] = 0;
for (int j = lo; j <= hi; j++) {
const float *vr = qkv + (size_t)j * rows + qd + kd + (size_t)kh * hd;
float w = score[j - lo] / den;
for (int e = 0; e < hd; e++) o[e] += w * vr[e];
}
}
}
free(score);
}
dw_matmul(tmp, ctx, N, &b->proj);
add_bias(tmp, b->proj_b, N, d);
for (size_t i = 0; i < (size_t)N * d; i++) x[i] += tmp[i];
for (int n = 0; n < N; n++) rmsnorm(xn + (size_t)n * d, x + (size_t)n * d, b->norm2, d, v->eps);
dw_matmul(g, xn, N, &b->gate);
add_bias(g, b->gate_b, N, v->inter);
dw_matmul(u, xn, N, &b->up);
add_bias(u, b->up_b, N, v->inter);
for (size_t i = 0; i < (size_t)N * v->inter; i++) g[i] = silu(g[i]) * u[i];
dw_matmul(tmp, g, N, &b->down);
add_bias(tmp, b->down_b, N, d);
for (size_t i = 0; i < (size_t)N * d; i++) x[i] += tmp[i];
}
if (prev == 1) permute_units(x, rev, units, d);
/* merger: LayerNorm per token, 2x2 units concatenated, Linear, GELU, Linear */
for (int n = 0; n < N; n++) {
float *r = x + (size_t)n * d;
float mean = 0, var = 0;
for (int i = 0; i < d; i++) mean += r[i];
mean /= d;
for (int i = 0; i < d; i++) { float z = r[i] - mean; var += z * z; }
var /= d;
float inv_std = 1.0f / sqrtf(var + 1e-6f);
for (int i = 0; i < d; i++) r[i] = (r[i] - mean) * inv_std * v->ln_w[i] + (v->ln_b ? v->ln_b[i] : 0.0f);
}
int merged = 4 * d;
float *h1 = xmalloc((size_t)units * merged * sizeof(float), "vision merger");
dw_matmul(h1, x, units, &v->fc1);
add_bias(h1, v->fc1_b, units, merged);
for (size_t i = 0; i < (size_t)units * merged; i++) h1[i] = 0.5f * h1[i] * (1.0f + erff(h1[i] * 0.70710678118654752f));
float *rows_out = xmalloc((size_t)units * v->out * sizeof(float), "image rows");
dw_matmul(rows_out, h1, units, &v->fc2);
add_bias(rows_out, v->fc2_b, units, v->out);
free(h1); free(x); free(ang); free(ang_col); free(col); free(rev);
free(xn); free(qkv); free(ctx); free(tmp); free(g); free(u);
*n_rows = units;
/* MIMO_VISION_DUMP=<file>: the rows the tower produced, f32 [units, hidden],
* for tools/mimo_real_check.py to hold against Xiaomi's ViT */
const char *dump = getenv("MIMO_VISION_DUMP");
if (dump) {
FILE *f = fopen(dump, "wb");
if (f) { fwrite(rows_out, sizeof(float), (size_t)units * v->out, f); fclose(f); }
}
if (env_int("MIMO_STATS", 0))
fprintf(stderr, "[mimo] image %dx%d patches -> %d rows in %.2fs\n", gh, gw, units, now_s() - t0);
return rows_out;
}