Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

mimo: MiMo-V2.6 engine (Flash, Pro), text and vision, read as released

JustVuggb290d5eb1808
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 6c4403f1..6270965a 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -83,6 +83,28 @@ jobs:
- name: Generate fixture and run vendor oracle
run: make -C c kimi-k3-tiny-check
+ mimo-tiny:
+ name: Linux (generated tiny MiMo-V2.6 vendor oracle, text + vision)
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.13'
+ cache: pip
+ cache-dependency-path: c/tools/requirements-mimo-tiny.txt
+ - name: Install pinned tiny-fixture dependencies
+ run: python -m pip install -r c/tools/requirements-mimo-tiny.txt
+ - name: Generate fixture and run vendor oracle
+ run: make -C c mimo-tiny-check
+ - name: Same oracle under ASan and UBSan
+ run: |
+ make -C c clean >/dev/null
+ make -C c mimo EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
+ ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \
+ python c/tests/mimo_tiny_harness.py --binary c/mimo --fixture c/mimo_tiny
+
v4-tiny:
name: Linux (generated tiny V4 target oracle)
runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index eeba7b07..ff65ffa1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -158,3 +158,12 @@ c/kimi_k3_tiny/.coli_usage
c/qwen38_vision_tiny/
c/qwen38_mm_tiny/
c/qwen38_tiny_fp8/
+# MiMo tiny fixture: config.json and ref.json are committed, the rest is regenerated
+c/mimo_tiny/*.safetensors
+c/mimo_tiny/model.safetensors.index.json
+c/mimo_tiny/.vendor_mimo_v2/
+c/mimo_tiny/image.png
+c/mimo_tiny/image.json
+c/mimo_tiny/patches.f32
+c/mimo
+c/mimo.exe
diff --git a/c/Makefile b/c/Makefile
index 1530e1be..d9d97be6 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1236,6 +1236,28 @@ deepseek_v41$(EXE): deepseek_v41.c sse41_kernels.h cli_args.h st.h json.h tok.h
sparse_attn.h hyper_connections.h serve_codec.h serve_poll.h .build-config
$(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)
+# Xiaomi MiMo-V2.6 (Flash 309B, Pro 1.02T): one file plus its ViT header, reading the
+# release as it is -- MXFP4 experts streamed per layer, FP8/BF16 dense resident.
+ifneq ($(EXE),)
+.PHONY: mimo
+mimo: mimo$(EXE)
+endif
+mimo$(EXE): mimo.c mimo_vision.h sse41_kernels.h cli_args.h compat.h json.h st.h quant.h idot.h fp8_format.h \
+ omp_tune.h kv_prefix.h tok.h tok_unicode.h tok_unicode_o200k.h tok_unicode_deepseek.h serve_codec.h \
+ decode_batch.h serve_poll.h stop_ids.h .build-config
+ $(CC) $(CFLAGS) mimo.c -o mimo$(EXE) $(LDFLAGS)
+
+# Tiny MiMo fixture + vendor oracle, the Kimi K3 pattern: the generator is numpy (and
+# Pillow for the test picture) and deterministic; ref.json is committed and comes from
+# Xiaomi's own modeling code (tools/make_mimo_ref.py, maintainer-run, SHA-256-pinned),
+# so CI needs neither torch nor network. The fixture carries the vision tower too.
+.PHONY: mimo-tiny-generate mimo-tiny-check
+mimo-tiny-generate:
+ $(PYTHON) tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
+
+mimo-tiny-check: mimo-tiny-generate mimo$(EXE)
+ $(PYTHON) tests/mimo_tiny_harness.py --binary ./mimo$(EXE) --fixture ./mimo_tiny
+
# Qwen3.8-Flash-Next text-only sibling. qwen38.c owns the direct checkpoint
# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as
# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);
@@ -2453,7 +2475,7 @@ check:
$(MAKE) portable
$(MAKE) test
-install: colibri$(EXE) glm53$(EXE) inkling$(EXE) kimi_k3$(EXE) olmoe$(EXE) qwen36$(EXE) qwen38$(EXE) deepseek_v41$(EXE) qwenimage$(EXE) \
+install: colibri$(EXE) glm53$(EXE) inkling$(EXE) kimi_k3$(EXE) olmoe$(EXE) qwen36$(EXE) qwen38$(EXE) deepseek_v41$(EXE) mimo$(EXE) qwenimage$(EXE) \
$(if $(filter 1,$(COLI_V4_SUPPORTED)),deepseek-v4,)
$(INSTALL) -d $(DESTDIR)$(BINDIR)
$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)
@@ -2468,6 +2490,7 @@ install: colibri$(EXE) glm53$(EXE) inkling$(EXE) kimi_k3$(EXE) olmoe$(EXE) qwen3
$(INSTALL) -m 755 qwen38$(EXE) $(DESTDIR)$(LIBEXECDIR)/qwen38$(EXE)
$(INSTALL) -m 755 qwenimage$(EXE) $(DESTDIR)$(LIBEXECDIR)/qwenimage$(EXE)
$(INSTALL) -m 755 deepseek_v41$(EXE) $(DESTDIR)$(LIBEXECDIR)/deepseek_v41$(EXE)
+ $(INSTALL) -m 755 mimo$(EXE) $(DESTDIR)$(LIBEXECDIR)/mimo$(EXE)
@if [ -f deepseek_v4$(EXE) ]; then \
$(INSTALL) -m 755 deepseek_v4$(EXE) $(DESTDIR)$(LIBEXECDIR)/deepseek_v4$(EXE); \
fi
diff --git a/c/mimo.c b/c/mimo.c
new file mode 100644
index 00000000..10d569e8
--- /dev/null
+++ b/c/mimo.c
@@ -0,0 +1,1427 @@
+/* mimo.c -- Xiaomi MiMo-V2.6 (Flash 309B/15B, Pro 1.02T/42B) on a machine that
+ * cannot hold it.
+ *
+ * The released checkpoint is read as it is, no conversion step:
+ *
+ * experts MXFP4 g32 (e2m1 nibble pairs, low nibble = even column, one e8m0
+ * byte per 32 columns), the six tensors of an expert back to back in
+ * the shard, so an expert is ONE read. 256 of them per layer on
+ * Flash, 8 routed per token, cached in RAM by a per-layer LRU.
+ * dense qkv_proj and the layer-0 MLP in FP8 e4m3 with a F32 scale per
+ * 128x128 block; o_proj, router, norms, sinks, embed, lm_head BF16.
+ *
+ * What makes MiMo MiMo, each mirroring modeling_mimo_v2.py:
+ * hybrid attention hybrid_layer_pattern: 1 = sliding window (128 keys,
+ * itself included), 0 = full attention; the two kinds have
+ * their own KV head count and their own RoPE theta
+ * partial RoPE on the first int(head_dim * 0.334) dims of each head,
+ * rotate-half style; head_dim 192 for q/k, 128 for v
+ * value scale v *= attention_value_scale before attention
+ * attention sink SWA layers carry one logit per head that joins the
+ * softmax denominator and nothing else
+ * router sigmoid scores, e_score_correction_bias picks but does
+ * not weigh, top-k renormalised; no shared expert
+ *
+ * One storage detail the modeling code does not describe: the fused qkv_proj is
+ * PRE-SHARDED in num_key_value_heads chunks [Q_0|K_0|V_0|Q_1|K_1|V_1|...] and
+ * its FP8 scale grid is tiled per chunk (a Flash full-attention layer has 108
+ * scale rows, not the 106 a contiguous grid would have). The layout is vLLM's
+ * (vllm/model_executor/models/mimo_v2.py, _shard_fp8_qkv_proj); the loader
+ * undoes it and the tiny oracle stores it the same way so a wrong reading fails.
+ *
+ * Held to Xiaomi's own modeling code through tools/make_mimo_ref.py (vendor
+ * files pinned by SHA-256) on the tiny fixture of tools/make_mimo_tiny.py.
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <math.h>
+#include <errno.h>
+#include <time.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
+#include <sys/resource.h>
+#endif
+#ifdef _OPENMP
+#include <omp.h>
+#endif
+
+#include "cli_args.h"
+#include "compat.h"
+#include "json.h"
+#include "st.h"
+#include "quant.h"
+#include "omp_tune.h"
+#include "kv_prefix.h"
+#include "tok.h"
+#include "serve_codec.h"
+#include "serve_poll.h"
+#include "stop_ids.h"
+#if defined(__AVX2__)
+#include <immintrin.h>
+#endif
+
+#define MIMO_MAX_LAYERS 128
+#define MIMO_MAX_TOPK 16
+#define MIMO_MAX_CHUNKS 16
+
+/* ------------------------------------------------------------------ utils ---- */
+
+static double now_s(void) {
+ struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t);
+ return t.tv_sec + t.tv_nsec * 1e-9;
+}
+
+static double rss_gb(void) {
+ struct rusage usage;
+ getrusage(RUSAGE_SELF, &usage);
+#ifdef __APPLE__
+ return usage.ru_maxrss / 1e9;
+#else
+ return usage.ru_maxrss / 1e6;
+#endif
+}
+
+static void *xmalloc(size_t bytes, const char *what) {
+ void *p = malloc(bytes ? bytes : 1);
+ if (!p) { fprintf(stderr, "[mimo] out of memory allocating %s (%zu bytes)\n", what, bytes); exit(1); }
+ return p;
+}
+
+static void *xcalloc(size_t n, size_t size, const char *what) {
+ void *p = calloc(n ? n : 1, size ? size : 1);
+ if (!p) { fprintf(stderr, "[mimo] out of memory allocating %s\n", what); exit(1); }
+ return p;
+}
+
+static int env_int(const char *name, int fallback) {
+ const char *v = getenv(name);
+ return (v && *v) ? atoi(v) : fallback;
+}
+
+static inline float silu(float x) { return x / (1.0f + expf(-x)); }
+static inline float sigmoid(float x) { return 1.0f / (1.0f + expf(-x)); }
+
+/* ---------------------------------------------------------------- config ---- */
+
+/* kind 0 = full attention, 1 = sliding window */
+typedef struct {
+ int vocab, hidden, n_layers, dense_inter, moe_inter, n_experts, topk;
+ int heads[2], kv_heads[2], head_dim[2], v_dim[2], rope_dim[2];
+ float theta[2];
+ int sink[2];
+ int swa[MIMO_MAX_LAYERS], moe[MIMO_MAX_LAYERS];
+ int window, chunks, norm_topk;
+ float v_scale, eps, route_scale;
+ int max_positions;
+ int image_token_id, vision_start_id, vision_end_id;
+} Cfg;
+
+static double jnum(jval *o, const char *key, double fallback) {
+ jval *v = o ? json_get(o, key) : NULL;
+ return (v && v->t == J_NUM) ? v->num : fallback;
+}
+
+static int jbool(jval *o, const char *key, int fallback) {
+ jval *v = o ? json_get(o, key) : NULL;
+ if (!v) return fallback;
+ if (v->t == J_BOOL) return v->boolean != 0;
+ if (v->t == J_NUM) return v->num != 0;
+ return fallback;
+}
+
+static char *read_text(const char *path, long *size_out) {
+ FILE *f = fopen(path, "rb");
+ if (!f) return NULL;
+ fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);
+ if (size < 0 || size > (64l << 20)) { fclose(f); return NULL; }
+ char *text = xmalloc((size_t)size + 1, path);
+ if (fread(text, 1, (size_t)size, f) != (size_t)size) { fclose(f); free(text); return NULL; }
+ text[size] = 0; fclose(f);
+ if (size_out) *size_out = size;
+ return text;
+}
+
+static void die_cfg(const char *what) {
+ fprintf(stderr, "[mimo] config.json: %s -- refusing\n", what);
+ exit(1);
+}
+
+static void load_cfg(Cfg *c, const char *dir) {
+ char path[4096];
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ char *text = read_text(path, NULL);
+ if (!text) { fprintf(stderr, "[mimo] cannot read %s\n", path); exit(1); }
+ char *arena = NULL;
+ jval *root = json_parse(text, &arena);
+ if (!root) die_cfg("not JSON");
+ jval *mt = json_get(root, "model_type");
+ if (!mt || mt->t != J_STR || strcmp(mt->str, "mimo_v2"))
+ die_cfg("model_type is not mimo_v2");
+ memset(c, 0, sizeof(*c));
+ c->vocab = (int)jnum(root, "vocab_size", 0);
+ c->hidden = (int)jnum(root, "hidden_size", 0);
+ c->n_layers = (int)jnum(root, "num_hidden_layers", 0);
+ c->dense_inter = (int)jnum(root, "intermediate_size", 0);
+ c->moe_inter = (int)jnum(root, "moe_intermediate_size", 0);
+ c->n_experts = (int)jnum(root, "n_routed_experts", 0);
+ c->topk = (int)jnum(root, "num_experts_per_tok", 0);
+ c->heads[0] = (int)jnum(root, "num_attention_heads", 0);
+ c->kv_heads[0] = (int)jnum(root, "num_key_value_heads", 0);
+ c->head_dim[0] = (int)jnum(root, "head_dim", 0);
+ c->v_dim[0] = (int)jnum(root, "v_head_dim", c->head_dim[0]);
+ c->heads[1] = (int)jnum(root, "swa_num_attention_heads", c->heads[0]);
+ c->kv_heads[1] = (int)jnum(root, "swa_num_key_value_heads", c->kv_heads[0]);
+ c->head_dim[1] = (int)jnum(root, "swa_head_dim", c->head_dim[0]);
+ c->v_dim[1] = (int)jnum(root, "swa_v_head_dim", c->v_dim[0]);
+ double prf = jnum(root, "partial_rotary_factor", 1.0);
+ for (int k = 0; k < 2; k++) c->rope_dim[k] = (int)(c->head_dim[k] * prf);
+ c->theta[0] = (float)jnum(root, "rope_theta", 10000.0);
+ c->theta[1] = (float)jnum(root, "swa_rope_theta", c->theta[0]);
+ c->sink[0] = jbool(root, "add_full_attention_sink_bias", 0);
+ c->sink[1] = jbool(root, "add_swa_attention_sink_bias", 0);
+ c->window = (int)jnum(root, "sliding_window", 0);
+ c->v_scale = (float)jnum(root, "attention_value_scale", 1.0);
+ c->eps = (float)jnum(root, "layernorm_epsilon", 1e-6);
+ c->route_scale = (float)jnum(root, "routed_scaling_factor", 1.0);
+ c->norm_topk = jbool(root, "norm_topk_prob", 1);
+ c->max_positions = (int)jnum(root, "max_position_embeddings", 32768);
+ c->image_token_id = (int)jnum(root, "image_token_id", -1);
+ c->vision_start_id = (int)jnum(root, "vision_start_token_id", -1);
+ c->vision_end_id = (int)jnum(root, "vision_end_token_id", -1);
+ c->chunks = c->kv_heads[0];
+ /* MIMO_LAYERS=N: only the first N layers, then the final norm and head. A
+ * checking aid (tools/mimo_real_check.py compares it with Xiaomi's code on
+ * the real weights), never an answer anyone should read. */
+ int keep = env_int("MIMO_LAYERS", 0);
+
+ jval *pat = json_get(root, "hybrid_layer_pattern");
+ jval *freq = json_get(root, "moe_layer_freq");
+ if (c->n_layers < 1 || c->n_layers > MIMO_MAX_LAYERS) die_cfg("num_hidden_layers out of range");
+ if (!pat || pat->t != J_ARR || pat->len != c->n_layers) die_cfg("hybrid_layer_pattern must list every layer");
+ if (!freq || freq->t != J_ARR || freq->len != c->n_layers) die_cfg("moe_layer_freq must list every layer");
+ for (int i = 0; i < c->n_layers; i++) {
+ c->swa[i] = (int)pat->kids[i]->num == 1;
+ c->moe[i] = (int)freq->kids[i]->num != 0;
+ }
+ if (keep > 0 && keep < c->n_layers) {
+ fprintf(stderr, "[mimo] MIMO_LAYERS=%d: a truncated model for checking, not for answers\n", keep);
+ c->n_layers = keep;
+ }
+ /* Everything below sizes a buffer or indexes one: refuse a config that
+ * would make any of it overflow or divide by zero, before reading a byte. */
+ if (c->vocab < 2 || c->vocab > (1 << 22)) die_cfg("vocab_size out of range");
+ if (c->hidden < 32 || c->hidden > 65536 || c->hidden % 32) die_cfg("hidden_size must be a positive multiple of 32");
+ if (c->moe_inter < 32 || c->moe_inter % 32 || c->moe_inter > 65536) die_cfg("moe_intermediate_size must be a positive multiple of 32");
+ if (c->n_experts < 1 || c->n_experts > 4096) die_cfg("n_routed_experts out of range");
+ if (c->topk < 1 || c->topk > MIMO_MAX_TOPK || c->topk > c->n_experts) die_cfg("num_experts_per_tok out of range");
+ for (int k = 0; k < 2; k++) {
+ if (c->heads[k] < 1 || c->kv_heads[k] < 1 || c->heads[k] % c->kv_heads[k]) die_cfg("attention heads must be a multiple of kv heads");
+ if (c->heads[k] > 1024 || c->kv_heads[k] > 1024) die_cfg("too many attention heads");
+ if (c->head_dim[k] < 2 || c->head_dim[k] > 1024 || c->v_dim[k] < 1 || c->v_dim[k] > 1024) die_cfg("head dims out of range");
+ if (c->rope_dim[k] < 0 || c->rope_dim[k] > c->head_dim[k] || c->rope_dim[k] % 2) die_cfg("rotary dims must be even and fit the head");
+ }
+ if (c->chunks < 1 || c->chunks > MIMO_MAX_CHUNKS) die_cfg("num_key_value_heads (the qkv pre-shard count) out of range");
+ for (int k = 0; k < 2; k++)
+ if (c->heads[k] % c->chunks || c->kv_heads[k] % c->chunks) die_cfg("head counts do not divide into the qkv pre-shard chunks");
+ int any_swa = 0, any_moe = 0;
+ for (int i = 0; i < c->n_layers; i++) { any_swa |= c->swa[i]; any_moe |= c->moe[i]; }
+ if (any_swa && (c->window < 1 || c->window > (1 << 20))) die_cfg("sliding_window out of range");
+ for (int i = 0; i < c->n_layers; i++)
+ if (!c->moe[i] && (c->dense_inter < 1 || c->dense_inter > (1 << 20))) die_cfg("intermediate_size out of range");
+ if (!any_moe && keep <= 0) die_cfg("no MoE layer");
+ if (!(c->eps > 0) || !(c->eps < 1)) die_cfg("layernorm_epsilon out of range");
+ if (c->max_positions < 1) c->max_positions = 32768;
+ json_free(root); free(arena); free(text);
+}
+
+/* ----------------------------------------------------------- dense weights ---- */
+
+/* A dense matrix [O, I] in one of four forms. FP8 keeps the checkpoint bytes and
+ * expands the 128x128 block grid to one scale per (row, column block), which is
+ * what lets a row move (the qkv de-interleave) without touching its bytes. */
+enum { DW_F32 = 0, DW_BF16 = 1, DW_FP8 = 2, DW_I8 = 3 };
+typedef struct {
+ int fmt, O, I, nblk;
+ void *w; /* f32 / bf16 / e4m3 / int8 */
+ float *s; /* FP8: [O][nblk]; I8: [O] */
+} DW;
+
+/* 0 (default): the checkpoint's own FP8 and BF16 bytes, exact.
+ * 8: int8 per row, about 30% less RAM for the dense part, not exact.
+ * 32: f32, exact and four times the RAM: the oracle's configuration. */
+static int g_dense_bits;
+
+static void matmul_bf16(float *y, const float *x, const uint16_t *W, int S, int I, int O) {
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++) {
+ const uint16_t *w = W + (int64_t)o * I;
+ for (int s = 0; s < S; s++) {
+ const float *xs = x + (int64_t)s * I;
+ float a = 0; int i = 0;
+#ifdef __AVX2__
+ __m256 acc = _mm256_setzero_ps();
+ for (; i + 8 <= I; i += 8)
+ acc = _mm256_fmadd_ps(_mm256_loadu_ps(xs + i), bf16_decode8(w + i), acc);
+ a = hsum256(acc);
+#endif
+ for (; i < I; i++) a += xs[i] * bf16_to_f32(w[i]);
+ y[(int64_t)s * O + o] = a;
+ }
+ }
+}
+
+/* FP8 with a scale per (row, 128-column block): float inside a block, double
+ * across blocks, the convention of quant.h's matmul_fp8. */
+static void matmul_fp8_rows(float *y, const float *x, const uint8_t *q8, const float *rs,
+ int nblk, int S, int I, int O) {
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++) {
+ const uint8_t *w = q8 + (int64_t)o * I;
+ const float *scl = rs + (int64_t)o * nblk;
+ for (int s = 0; s < S; s++) {
+ const float *xs = x + (int64_t)s * I;
+ double a = 0;
+ for (int b = 0; b < nblk; b++) {
+ int base = b * 128, end = base + 128 < I ? base + 128 : I, i = base;
+ float acc = 0;
+#ifdef __AVX2__
+ __m256 v = _mm256_setzero_ps();
+ for (; i + 8 <= end; i += 8)
+ v = _mm256_fmadd_ps(_mm256_loadu_ps(xs + i), e4m3_decode8(w + i), v);
+ acc = hsum256(v);
+#endif
+ for (; i < end; i++) acc += e4m3_decode(w[i]) * xs[i];
+ a += (double)acc * scl[b];
+ }
+ y[(int64_t)s * O + o] = (float)a;
+ }
+ }
+}
+
+static void dw_matmul(float *y, const float *x, int S, const DW *d) {
+ switch (d->fmt) {
+ case DW_F32: matmul(y, x, (const float *)d->w, S, d->I, d->O); break;
+ case DW_BF16: matmul_bf16(y, x, (const uint16_t *)d->w, S, d->I, d->O); break;
+ case DW_FP8: matmul_fp8_rows(y, x, (const uint8_t *)d->w, d->s, d->nblk, S, d->I, d->O); break;
+ case DW_I8: matmul_q(y, x, (const int8_t *)d->w, d->s, S, d->I, d->O); break;
+ }
+}
+
+static st_tensor *need(shards *S, const char *name, int dtype, int O, int I) {
+ st_tensor *t = st_find(S, name);
+ if (!t) st_die_missing(S, name);
+ int esz = st_dtype_esz(dtype);
+ int64_t numel = I > 0 ? (int64_t)O * I : O;
+ int rank_ok = I > 0 ? (t->rank == 2 && t->shape[0] == O && t->shape[1] == I)
+ : (t->rank == 1 && t->shape[0] == O);
+ if (t->dtype != dtype || !rank_ok || t->nbytes != numel * esz) {
+ fprintf(stderr, "[mimo] %s: %s %lld bytes, expected %s [%d%s%d] -- refusing (untrusted container)\n",
+ name, st_dtype_name(t->dtype), (long long)t->nbytes, st_dtype_name(dtype), O,
+ I > 0 ? ", " : "", I > 0 ? I : 0);
+ exit(1);
+ }
+ return t;
+}
+
+/* f32 copy of a BF16 or F32 vector */
+static float *load_vec(shards *S, const char *name, int n) {
+ st_tensor *t = st_find(S, name);
+ if (!t) st_die_missing(S, name);
+ int dtype = t->dtype == 2 ? 2 : 0;
+ need(S, name, dtype, n, 0);
+ float *out = xmalloc((size_t)n * sizeof(float), name);
+ st_read_f32(S, name, out, 1);
+ return out;
+}
+
+/* Rows of an f32 matrix [O, I] into the configured dense form. Takes ownership. */
+static void dw_from_f32(DW *d, float *w, int O, int I) {
+ d->O = O; d->I = I;
+ if (g_dense_bits == 8) {
+ d->fmt = DW_I8;
+ d->w = xmalloc((size_t)O * I, "int8 weights");
+ d->s = xmalloc((size_t)O * sizeof(float), "int8 scales");
+ quantize_rows(w, (int8_t *)d->w, d->s, O, I, 8);
+ free(w);
+ } else {
+ d->fmt = DW_F32; d->w = w; d->s = NULL;
+ }
+}
+
+static void dw_load_bf16(DW *d, shards *S, const char *name, int O, int I) {
+ st_tensor *t = need(S, name, 0, O, I);
+ if (g_dense_bits == 0) {
+ d->fmt = DW_BF16; d->O = O; d->I = I; d->s = NULL;
+ d->w = xmalloc((size_t)t->nbytes, name);
+ st_read_raw_cap(S, name, d->w, t->nbytes, 1);
+ return;
+ }
+ float *w = xmalloc((size_t)O * I * sizeof(float), name);
+ st_read_f32(S, name, w, 1);
+ dw_from_f32(d, w, O, I);
+}
+
+/* An FP8 [O, I] with its block grid; `row_block[r]` names the scale row of
+ * stored row r (NULL: r / 128). `order[r]` names which stored row becomes row r
+ * of the result (NULL: identity). */
+static void dw_load_fp8(DW *d, shards *S, const char *name, int O, int I,
+ const int *row_block, int scale_rows, const int *order) {
+ char sname[600];
+ snprintf(sname, sizeof(sname), "%s_scale_inv", name);
+ int nblk = (I + 127) / 128;
+ if (scale_rows <= 0) scale_rows = (O + 127) / 128;
+ st_tensor *t = need(S, name, 4, O, I);
+ need(S, sname, 2, scale_rows, nblk);
+ uint8_t *raw = xmalloc((size_t)t->nbytes, name);
+ st_read_raw_cap(S, name, raw, t->nbytes, 1);
+ float *grid = xmalloc((size_t)scale_rows * nblk * sizeof(float), sname);
+ st_read_f32(S, sname, grid, 1);
+ for (int64_t i = 0; i < (int64_t)scale_rows * nblk; i++)
+ if (!isfinite(grid[i])) { fprintf(stderr, "[mimo] %s: non-finite scale -- refusing\n", sname); exit(1); }
+ d->O = O; d->I = I; d->nblk = nblk;
+ if (g_dense_bits == 0) {
+ d->fmt = DW_FP8;
+ uint8_t *w = xmalloc((size_t)O * I, name);
+ float *rs = xmalloc((size_t)O * nblk * sizeof(float), sname);
+ for (int r = 0; r < O; r++) {
+ int src = order ? order[r] : r;
+ int sb = row_block ? row_block[src] : src / 128;
+ memcpy(w + (int64_t)r * I, raw + (int64_t)src * I, (size_t)I);
+ memcpy(rs + (int64_t)r * nblk, grid + (int64_t)sb * nblk, (size_t)nblk * sizeof(float));
+ }
+ d->w = w; d->s = rs;
+ free(raw); free(grid);
+ return;
+ }
+ float *w = xmalloc((size_t)O * I * sizeof(float), name);
+ #pragma omp parallel for schedule(static)
+ for (int r = 0; r < O; r++) {
+ int src = order ? order[r] : r;
+ int sb = row_block ? row_block[src] : src / 128;
+ const uint8_t *q = raw + (int64_t)src * I;
+ float *dst = w + (int64_t)r * I;
+ for (int i = 0; i < I; i++) dst[i] = e4m3_decode(q[i]) * grid[(int64_t)sb * nblk + i / 128];
+ }
+ free(raw); free(grid);
+ dw_from_f32(d, w, O, I);
+}
+
+/* The fused qkv of layer `li`, read in its pre-sharded order and handed back as
+ * [Q | K | V] rows. Two scale grids are legal (vLLM accepts the same two): one
+ * tiled per chunk, ceil(rows_per_chunk / 128) rows each, or one continuous grid;
+ * they coincide when a chunk is a whole number of blocks. */
+static void load_qkv(DW *d, shards *S, const Cfg *c, int li) {
+ int k = c->swa[li];
+ int T = c->chunks, hd = c->head_dim[k], vd = c->v_dim[k];
+ int q_rows = (c->heads[k] / T) * hd, k_rows = (c->kv_heads[k] / T) * hd,
+ v_rows = (c->kv_heads[k] / T) * vd;
+ int rpc = q_rows + k_rows + v_rows, O = T * rpc;
+ int per_chunk = (rpc + 127) / 128, continuous = (O + 127) / 128;
+ char name[512], sname[600];
+ snprintf(name, sizeof(name), "model.layers.%d.self_attn.qkv_proj.weight", li);
+ snprintf(sname, sizeof(sname), "%s_scale_inv", name);
+ st_tensor *st = st_find(S, sname);
+ if (!st) st_die_missing(S, sname);
+ int scale_rows = st->rank == 2 ? (int)st->shape[0] : -1;
+ int *row_block = xmalloc((size_t)O * sizeof(int), "qkv scale rows");
+ for (int r = 0; r < O; r++) {
+ if (scale_rows == T * per_chunk) row_block[r] = (r / rpc) * per_chunk + (r % rpc) / 128;
+ else if (scale_rows == continuous) row_block[r] = r / 128;
+ else {
+ fprintf(stderr, "[mimo] %s: %d scale rows fit neither the per-chunk grid (%d) nor a "
+ "continuous one (%d) -- refusing\n", sname, scale_rows, T * per_chunk, continuous);
+ exit(1);
+ }
+ }
+ int *order = xmalloc((size_t)O * sizeof(int), "qkv order");
+ int w = 0;
+ for (int ch = 0; ch < T; ch++) for (int r = 0; r < q_rows; r++) order[w++] = ch * rpc + r;
+ for (int ch = 0; ch < T; ch++) for (int r = 0; r < k_rows; r++) order[w++] = ch * rpc + q_rows + r;
+ for (int ch = 0; ch < T; ch++) for (int r = 0; r < v_rows; r++) order[w++] = ch * rpc + q_rows + k_rows + r;
+ dw_load_fp8(d, S, name, O, c->hidden, row_block, scale_rows, order);
+ free(row_block); free(order);
+}
+
+/* ------------------------------------------------------------------ model ---- */
+
+typedef struct { int eid; uint64_t used; uint8_t *base, *buf; } Slot;
+typedef struct { Slot *s; int n, cap; int *by_expert; } LCache;
+typedef struct { int fd[6]; int64_t off[6]; int contig; } ERef;
+
+typedef struct {
+ float *ln1, *ln2, *sink;
+ DW qkv, o;
+ DW gate, up, down; /* dense MLP (layer 0) */
+ float *router, *bias; /* [E, H], [E] */
+ /* KV: full layers [ctx][kvh][hd]; SWA layers a ring of `window` rows */
+ float *K, *V;
+ int *ring_pos;
+ int rows;
+} Layer;
+
+typedef struct {
+ Cfg c;
+ shards S;
+ Layer L[MIMO_MAX_LAYERS];
+ uint16_t *embed; /* bf16 [V, H], rows widened on use */
+ DW head;
+ float *norm;
+ ERef *eref;
+ int64_t part[6], e_bytes; /* down, down scale, gate, gate scale, up, up scale */
+ LCache cache[MIMO_MAX_LAYERS];
+ int ctx, pos;
+ kv_prefix kvp;
+ uint64_t clock, hits, miss, bytes_read, forwards;
+ double t_disk, t_expert, t_attn;
+ int idot, direct;
+ uint8_t **ehit; /* [layer][expert] routed this turn, for HITS */
+} Model;
+
+static void expert_table_init(Model *m) {
+ Cfg *c = &m->c;
+ int H = c->hidden, MI = c->moe_inter;
+ m->part[0] = (int64_t)H * (MI / 2); m->part[1] = (int64_t)H * (MI / 32);
+ m->part[2] = (int64_t)MI * (H / 2); m->part[3] = (int64_t)MI * (H / 32);
+ m->part[4] = m->part[2]; m->part[5] = m->part[3];
+ m->e_bytes = 0;
+ for (int k = 0; k < 6; k++) m->e_bytes += m->part[k];
+ m->eref = xcalloc((size_t)c->n_layers * c->n_experts, sizeof(ERef), "expert table");
+ static const char *mat[3] = {"down_proj", "gate_proj", "up_proj"};
+ int missing = 0, split = 0;
+ for (int li = 0; li < c->n_layers; li++) {
+ if (!c->moe[li]) continue;
+ for (int e = 0; e < c->n_experts; e++) {
+ ERef *er = &m->eref[(int64_t)li * c->n_experts + e];
+ for (int k = 0; k < 6; k++) {
+ char nm[512];
+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.%s.%s", li, e,
+ mat[k / 2], (k & 1) ? "weight_scale" : "weight");
+ st_tensor *t = st_find(&m->S, nm);
+ if (!t) { missing++; er->fd[k] = -1; continue; }
+ if (t->dtype != 3 || t->nbytes != m->part[k]) {
+ fprintf(stderr, "[mimo] %s: %s %lld bytes, expected U8 %lld -- refusing "
+ "(untrusted container)\n", nm, st_dtype_name(t->dtype),
+ (long long)t->nbytes, (long long)m->part[k]);
+ exit(1);
+ }
+ er->fd[k] = t->fd; er->off[k] = t->off;
+ }
+ /* The release stores an expert's six tensors back to back in this
+ * order (down, down scale, gate, gate scale, up, up scale): one read. */
+ er->contig = er->fd[0] >= 0;
+ for (int k = 0; k < 5 && er->contig; k++)
+ if (er->fd[k] != er->fd[k + 1] || er->off[k] + m->part[k] != er->off[k + 1]) er->contig = 0;
+ split += !er->contig;
+ }
+ }
+ if (missing) fprintf(stderr, "[mimo] WARNING: %d expert tensors missing (incomplete download?) -- touching one aborts\n", missing);
+ if (split) fprintf(stderr, "[mimo] %d experts are not contiguous on disk: six reads each\n", split);
+}
+
+static void cache_init(Model *m, int cap) {
+ Cfg *c = &m->c;
+ if (cap < c->topk) cap = c->topk;
+ if (cap > c->n_experts) cap = c->n_experts;
+ for (int li = 0; li < c->n_layers; li++) {
+ if (!c->moe[li]) continue;
+ LCache *lc = &m->cache[li];
+ lc->cap = cap; lc->n = 0;
+ lc->s = xcalloc((size_t)cap, sizeof(Slot), "expert slots");
+ lc->by_expert = xmalloc((size_t)c->n_experts * sizeof(int), "expert index");
+ for (int e = 0; e < c->n_experts; e++) lc->by_expert[e] = -1;
+ for (int k = 0; k < cap; k++) lc->s[k].eid = -1;
+ }
+}
+
+static void expert_read(Model *m, int li, int eid, Slot *s) {
+ if (!s->base && posix_memalign((void **)&s->base, 4096, (size_t)m->e_bytes + 8192)) {
+ fprintf(stderr, "[mimo] out of memory for an expert slot\n"); exit(1);
+ }
+ ERef *er = &m->eref[(int64_t)li * m->c.n_experts + eid];
+ if (er->fd[0] < 0) { fprintf(stderr, "[mimo] expert L%d E%d missing on disk\n", li, eid); exit(1); }
+ if (er->contig) {
+ int dfd = m->direct ? st_direct_fd(&m->S, er->fd[0]) : -1;
+ if (dfd >= 0) {
+ int64_t a0 = er->off[0] & ~4095LL, pad = er->off[0] - a0, want = pad + m->e_bytes;
+ int64_t dlen = (want + 4095) & ~4095LL;
+ struct stat sb;
+ if (fstat(dfd, &sb) == 0 && a0 + dlen > sb.st_size) dlen = (sb.st_size - a0) & ~4095LL;
+ if (dlen > 0) st_pread_full(dfd, s->base, dlen, a0, "pread expert direct");
+ if (dlen < want) st_pread_full(er->fd[0], s->base + dlen, want - dlen, a0 + dlen, "pread expert tail");
+ s->buf = s->base + pad;
+ } else {
+ st_pread_full(er->fd[0], s->base, m->e_bytes, er->off[0], "pread expert");
+ s->buf = s->base;
+ }
+ } else {
+ uint8_t *dst = s->base;
+ for (int k = 0; k < 6; k++) {
+ if (er->fd[k] < 0) { fprintf(stderr, "[mimo] expert L%d E%d tensor %d missing\n", li, eid, k); exit(1); }
+ st_pread_full(er->fd[k], dst, m->part[k], er->off[k], "pread expert");
+ dst += m->part[k];
+ }
+ s->buf = s->base;
+ }
+}
+
+/* Make the experts `want[0..n)` of layer li resident, n <= cap, and return their
+ * slots in `out`. Hits keep their slot; misses take the least recently used
+ * slots not in `want`, and are read in parallel. */
+static void experts_ensure(Model *m, int li, const int *want, int n, Slot **out) {
+ LCache *lc = &m->cache[li];
+ int miss_idx[4096], nmiss = 0;
+ uint64_t stamp = ++m->clock;
+ for (int j = 0; j < n; j++) {
+ int si = lc->by_expert[want[j]];
+ if (si >= 0) { out[j] = &lc->s[si]; out[j]->used = stamp; m->hits++; }
+ else { out[j] = NULL; miss_idx[nmiss++] = j; }
+ }
+ for (int k = 0; k < nmiss; k++) {
+ int best = -1;
+ if (lc->n < lc->cap) best = lc->n++;
+ else {
+ uint64_t oldest = UINT64_MAX;
+ for (int s = 0; s < lc->n; s++)
+ if (lc->s[s].used < stamp && lc->s[s].used < oldest) { oldest = lc->s[s].used; best = s; }
+ }
+ if (best < 0) { fprintf(stderr, "[mimo] expert cache smaller than one routing step\n"); exit(1); }
+ Slot *s = &lc->s[best];
+ if (s->eid >= 0) lc->by_expert[s->eid] = -1;
+ s->eid = want[miss_idx[k]];
+ s->used = stamp;
+ lc->by_expert[s->eid] = best;
+ out[miss_idx[k]] = s;
+ }
+ if (!nmiss) return;
+ double t0 = now_s();
+ int threads = env_int("MIMO_READ_THREADS", 8);
+ #pragma omp parallel for num_threads(threads) schedule(dynamic, 1)
+ for (int k = 0; k < nmiss; k++) {
+ Slot *s = out[miss_idx[k]];
+ expert_read(m, li, s->eid, s);
+ }
+ m->miss += (uint64_t)nmiss;
+ m->bytes_read += (uint64_t)nmiss * (uint64_t)m->e_bytes;
+ m->t_disk += now_s() - t0;
+}
+
+static void kv_alloc(Model *m, int ctx) {
+ Cfg *c = &m->c;
+ m->ctx = ctx;
+ for (int li = 0; li < c->n_layers; li++) {
+ Layer *l = &m->L[li];
+ int k = c->swa[li];
+ l->rows = k ? (c->window < ctx ? c->window : ctx) : ctx;
+ free(l->K); free(l->V); free(l->ring_pos);
+ l->K = xmalloc((size_t)l->rows * c->kv_heads[k] * c->head_dim[k] * sizeof(float), "K cache");
+ l->V = xmalloc((size_t)l->rows * c->kv_heads[k] * c->v_dim[k] * sizeof(float), "V cache");
+ l->ring_pos = xmalloc((size_t)l->rows * sizeof(int), "ring positions");
+ for (int r = 0; r < l->rows; r++) l->ring_pos[r] = -1;
+ }
+ kv_prefix_alloc(&m->kvp, ctx);
+}
+
+static void model_reset(Model *m) {
+ Cfg *c = &m->c;
+ for (int li = 0; li < c->n_layers; li++)
+ for (int r = 0; r < m->L[li].rows; r++) m->L[li].ring_pos[r] = -1;
+ m->pos = 0;
+ kv_prefix_clear(&m->kvp);
+}
+
+static void model_load(Model *m, const char *dir, int cap) {
+ Cfg *c = &m->c;
+ load_cfg(c, dir);
+ st_init_multi(&m->S, dir, getenv("MIMO_DIRS"));
+ g_dense_bits = env_int("MIMO_DENSE_BITS", 0);
+ if (g_dense_bits != 0 && g_dense_bits != 8 && g_dense_bits != 32) {
+ fprintf(stderr, "[mimo] MIMO_DENSE_BITS must be 0 (native FP8/BF16), 8 or 32\n"); exit(1);
+ }
+ m->idot = env_int("MIMO_IDOT", 0);
+ m->direct = env_int("MIMO_DIRECT", 1);
+ int H = c->hidden;
+ need(&m->S, "model.embed_tokens.weight", 0, c->vocab, H);
+ m->embed = xmalloc((size_t)c->vocab * H * sizeof(uint16_t), "embeddings");
+ st_read_raw_cap(&m->S, "model.embed_tokens.weight", m->embed, (int64_t)c->vocab * H * 2, 1);
+ m->norm = load_vec(&m->S, "model.norm.weight", H);
+ dw_load_bf16(&m->head, &m->S, "lm_head.weight", c->vocab, H);
+ for (int li = 0; li < c->n_layers; li++) {
+ Layer *l = &m->L[li];
+ int k = c->swa[li];
+ char nm[512];
+ snprintf(nm, sizeof(nm), "model.layers.%d.input_layernorm.weight", li);
+ l->ln1 = load_vec(&m->S, nm, H);
+ snprintf(nm, sizeof(nm), "model.layers.%d.post_attention_layernorm.weight", li);
+ l->ln2 = load_vec(&m->S, nm, H);
+ load_qkv(&l->qkv, &m->S, c, li);
+ snprintf(nm, sizeof(nm), "model.layers.%d.self_attn.o_proj.weight", li);
+ dw_load_bf16(&l->o, &m->S, nm, H, c->heads[k] * c->v_dim[k]);
+ if (c->sink[k]) {
+ snprintf(nm, sizeof(nm), "model.layers.%d.self_attn.attention_sink_bias", li);
+ l->sink = load_vec(&m->S, nm, c->heads[k]);
+ }
+ if (!c->moe[li]) {
+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate_proj.weight", li);
+ dw_load_fp8(&l->gate, &m->S, nm, c->dense_inter, H, NULL, 0, NULL);
+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.up_proj.weight", li);
+ dw_load_fp8(&l->up, &m->S, nm, c->dense_inter, H, NULL, 0, NULL);
+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.down_proj.weight", li);
+ dw_load_fp8(&l->down, &m->S, nm, H, c->dense_inter, NULL, 0, NULL);
+ } else {
+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate.weight", li);
+ need(&m->S, nm, 0, c->n_experts, H);
+ l->router = xmalloc((size_t)c->n_experts * H * sizeof(float), nm);
+ st_read_f32(&m->S, nm, l->router, 1);
+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate.e_score_correction_bias", li);
+ l->bias = load_vec(&m->S, nm, c->n_experts);
+ }
+ }
+ expert_table_init(m);
+ cache_init(m, cap);
+ /* coli --ctx arrives as CTX (the family's context_env); MIMO_CTX wins over it */
+ const char *ctx_text = getenv("MIMO_CTX");
+ if (!ctx_text || !*ctx_text) ctx_text = getenv("CTX");
+ int ctx = (ctx_text && *ctx_text) ? atoi(ctx_text) : 8192;
+ if (ctx > c->max_positions) ctx = c->max_positions;
+ if (ctx < 16) ctx = 16;
+ kv_alloc(m, ctx);
+ m->ehit = xmalloc((size_t)c->n_layers * sizeof(uint8_t *), "hits");
+ for (int li = 0; li < c->n_layers; li++) m->ehit[li] = xcalloc((size_t)c->n_experts, 1, "hits row");
+}
+
+/* ------------------------------------------------------------ the forward ---- */
+
+static void rmsnorm(float *out, const float *x, const float *w, int n, float eps) {
+ float ss = 0;
+ for (int i = 0; i < n; i++) ss += x[i] * x[i];
+ float r = 1.0f / sqrtf(ss / n + eps);
+ for (int i = 0; i < n; i++) out[i] = w[i] * (x[i] * r);
+}
+
+/* rotate-half RoPE on the first `rd` dims of a head, as the reference builds it:
+ * inv_freq and the angle in float32, cos/sin of the float angle */
+static void rope(float *v, int rd, float theta, int pos) {
+ int half = rd / 2;
+ for (int i = 0; i < half; i++) {
+ float inv = 1.0f / powf(theta, (float)(2 * i) / (float)rd);
+ float ang = inv * (float)pos;
+ float cs = cosf(ang), sn = sinf(ang);
+ float a = v[i], b = v[i + half];
+ v[i] = a * cs - b * sn;
+ v[i + half] = b * cs + a * sn;
+ }
+}
+
+/* Attention of `n` rows at positions p0.. of layer li. `xn` is the normed input
+ * [n, H], `out` receives o_proj's output [n, H]. */
+static void attention(Model *m, int li, const float *xn, int n, int p0, float *out) {
+ Cfg *c = &m->c;
+ Layer *l = &m->L[li];
+ int k = c->swa[li];
+ int nh = c->heads[k], kvh = c->kv_heads[k], hd = c->head_dim[k], vd = c->v_dim[k];
+ int qd = nh * hd, kd = kvh * hd, vdd = kvh * vd, rows = qd + kd + vdd;
+ double t0 = now_s();
+ float *qkv = xmalloc((size_t)n * rows * sizeof(float), "qkv");
+ dw_matmul(qkv, xn, n, &l->qkv);
+ for (int t = 0; t < n; t++) {
+ float *q = qkv + (size_t)t * rows, *kk = q + qd, *vv = kk + kd;
+ for (int h = 0; h < nh; h++) rope(q + h * hd, c->rope_dim[k], c->theta[k], p0 + t);
+ for (int h = 0; h < kvh; h++) rope(kk + h * hd, c->rope_dim[k], c->theta[k], p0 + t);
+ for (int i = 0; i < vdd; i++) vv[i] *= c->v_scale;
+ }
+ /* Keys this block can see: [lo, p0 + n). For a sliding window the ring still
+ * holds the ones before p0; for full attention the cache holds everything,
+ * and the block is written in first. */
+ int window = k ? c->window : 0;
+ int lo = k ? (p0 - (window - 1) > 0 ? p0 - (window - 1) : 0) : 0;
+ int nkeys = p0 + n - lo;
+ float *Kc = NULL, *Vc = NULL;
+ if (k) {
+ Kc = xmalloc((size_t)nkeys * kd * sizeof(float), "window keys");
+ Vc = xmalloc((size_t)nkeys * vdd * sizeof(float), "window values");
+ for (int p = lo; p < p0; p++) {
+ int slot = p % l->rows;
+ if (l->ring_pos[slot] != p) { fprintf(stderr, "[mimo] window ring lost position %d\n", p); exit(1); }
+ memcpy(Kc + (size_t)(p - lo) * kd, l->K + (size_t)slot * kd, (size_t)kd * sizeof(float));
+ memcpy(Vc + (size_t)(p - lo) * vdd, l->V + (size_t)slot * vdd, (size_t)vdd * sizeof(float));
+ }
+ for (int t = 0; t < n; t++) {
+ const float *src = qkv + (size_t)t * rows + qd;
+ memcpy(Kc + (size_t)(p0 + t - lo) * kd, src, (size_t)kd * sizeof(float));
+ memcpy(Vc + (size_t)(p0 + t - lo) * vdd, src + kd, (size_t)vdd * sizeof(float));
+ }
+ } else {
+ for (int t = 0; t < n; t++) {
+ const float *src = qkv + (size_t)t * rows + qd;
+ memcpy(l->K + (size_t)(p0 + t) * kd, src, (size_t)kd * sizeof(float));
+ memcpy(l->V + (size_t)(p0 + t) * vdd, src + kd, (size_t)vdd * sizeof(float));
+ }
+ Kc = l->K; Vc = l->V;
+ }
+ float *ctxv = xmalloc((size_t)n * nh * vd * sizeof(float), "attention output");
+ float scale = 1.0f / sqrtf((float)hd);
+ int group = nh / kvh;
+ #pragma omp parallel
+ {
+ float *score = xmalloc((size_t)nkeys * sizeof(float), "scores");
+ #pragma omp for collapse(2) schedule(static)
+ for (int t = 0; t < n; t++) {
+ for (int h = 0; h < nh; h++) {
+ int p = p0 + t, kh = h / group;
+ int first = window ? (p - (window - 1) > lo ? p - (window - 1) : lo) : 0;
+ const float *q = qkv + (size_t)t * rows + (size_t)h * hd;
+ float best = -INFINITY;
+ for (int j = first; j <= p; j++) {
+ const float *kr = Kc + (size_t)(j - lo) * kd + (size_t)kh * hd;
+ float s = 0;
+ for (int i = 0; i < hd; i++) s += q[i] * kr[i];
+ s *= scale;
+ score[j - first] = s;
+ if (s > best) best = s;
+ }
+ float sink = l->sink ? l->sink[h] : -INFINITY;
+ if (sink > best) best = sink;
+ float den = l->sink ? expf(sink - best) : 0.0f;
+ for (int j = first; j <= p; j++) { score[j - first] = expf(score[j - first] - best); den += score[j - first]; }
+ float *o = ctxv + ((size_t)t * nh + h) * vd;
+ for (int i = 0; i < vd; i++) o[i] = 0;
+ for (int j = first; j <= p; j++) {
+ const float *vr = Vc + (size_t)(j - lo) * vdd + (size_t)kh * vd;
+ float w = score[j - first] / den;
+ for (int i = 0; i < vd; i++) o[i] += w * vr[i];
+ }
+ }
+ }
+ free(score);
+ }
+ if (k) {
+ /* the ring keeps the last `window` positions of the block */
+ int from = n > l->rows ? n - l->rows : 0;
+ for (int t = from; t < n; t++) {
+ int p = p0 + t, slot = p % l->rows;
+ const float *src = qkv + (size_t)t * rows + qd;
+ memcpy(l->K + (size_t)slot * kd, src, (size_t)kd * sizeof(float));
+ memcpy(l->V + (size_t)slot * vdd, src + kd, (size_t)vdd * sizeof(float));
+ l->ring_pos[slot] = p;
+ }
+ free(Kc); free(Vc);
+ }
+ dw_matmul(out, ctxv, n, &l->o);
+ free(ctxv); free(qkv);
+ m->t_attn += now_s() - t0;
+}
+
+static void dense_mlp(Model *m, Layer *l, const float *xn, int n, float *out) {
+ int F = l->gate.O;
+ float *g = xmalloc((size_t)n * F * sizeof(float), "mlp gate");
+ float *u = xmalloc((size_t)n * F * sizeof(float), "mlp up");
+ dw_matmul(g, xn, n, &l->gate);
+ dw_matmul(u, xn, n, &l->up);
+ for (size_t i = 0; i < (size_t)n * F; i++) g[i] = silu(g[i]) * u[i];
+ dw_matmul(out, g, n, &l->down);
+ free(g); free(u);
+ (void)m;
+}
+
+/* Route every row, then run each chosen expert once over the rows that chose it.
+ * `out` [n, H] receives the weighted sum. */
+static void moe(Model *m, int li, const float *xn, int n, float *out) {
+ Cfg *c = &m->c;
+ Layer *l = &m->L[li];
+ int H = c->hidden, E = c->n_experts, K = c->topk, MI = c->moe_inter;
+ int *sel = xmalloc((size_t)n * K * sizeof(int), "routing");
+ float *wt = xmalloc((size_t)n * K * sizeof(float), "routing weights");
+ float *logits = xmalloc((size_t)n * E * sizeof(float), "router logits");
+ matmul(logits, xn, l->router, n, H, E);
+ for (int t = 0; t < n; t++) {
+ float *score = logits + (size_t)t * E;
+ for (int e = 0; e < E; e++) score[e] = sigmoid(score[e]);
+ int *s = sel + (size_t)t * K;
+ float *w = wt + (size_t)t * K;
+ for (int j = 0; j < K; j++) {
+ int best = -1; float bv = -INFINITY;
+ for (int e = 0; e < E; e++) {
+ int taken = 0;
+ for (int q = 0; q < j; q++) if (s[q] == e) { taken = 1; break; }
+ if (taken) continue;
+ float v = score[e] + l->bias[e];
+ if (v > bv) { bv = v; best = e; }
+ }
+ s[j] = best; w[j] = score[best];
+ }
+ float sum = 0;
+ for (int j = 0; j < K; j++) sum += w[j];
+ for (int j = 0; j < K; j++) w[j] = (c->norm_topk && K > 1 ? w[j] / (sum + 1e-20f) : w[j]) * c->route_scale;
+ for (int j = 0; j < K; j++) m->ehit[li][s[j]] = 1;
+ }
+ free(logits);
+ /* the union of chosen experts, in first-use order */
+ int *uni = xmalloc((size_t)E * sizeof(int), "expert union");
+ int *seen = xcalloc((size_t)E, sizeof(int), "expert seen");
+ int nu = 0;
+ for (int i = 0; i < n * K; i++) if (!seen[sel[i]]) { seen[sel[i]] = 1; uni[nu++] = sel[i]; }
+ free(seen);
+ memset(out, 0, (size_t)n * H * sizeof(float));
+ /* Each group of resident experts is computed in two parallel passes -- gate and
+ * up of every expert, then down of every expert -- split in row blocks, instead
+ * of three OpenMP regions per expert (a thousand a token on Flash, and every
+ * barrier costs a scheduling round on a busy machine). The kernel inside a
+ * task is the same matmul_mxfp4 on the same rows, so the numbers do not move. */
+ int cap = m->cache[li].cap;
+ int *row = xmalloc((size_t)n * K * sizeof(int), "expert rows index");
+ float *rw = xmalloc((size_t)n * K * sizeof(float), "expert row weights");
+ int *first = xmalloc((size_t)(cap + 1) * sizeof(int), "expert row offsets");
+ float *xg = xmalloc((size_t)n * K * H * sizeof(float), "expert inputs");
+ float *g = xmalloc((size_t)n * K * MI * sizeof(float), "expert gate");
+ float *u = xmalloc((size_t)n * K * MI * sizeof(float), "expert up");
+ float *y = xmalloc((size_t)n * K * H * sizeof(float), "expert out");
+ void (*mm)(float *, const float *, const uint8_t *, const uint8_t *, int, int, int)
+ = m->idot ? matmul_mxfp4_i8 : matmul_mxfp4;
+ const int BLOCK_ROWS = 256;
+ Slot *slots[4096];
+ for (int b = 0; b < nu; b += cap) {
+ int nb = nu - b < cap ? nu - b : cap;
+ experts_ensure(m, li, uni + b, nb, slots);
+ double t0 = now_s();
+ int total = 0;
+ for (int j = 0; j < nb; j++) {
+ int e = uni[b + j];
+ first[j] = total;
+ for (int t = 0; t < n; t++)
+ for (int q = 0; q < K; q++)
+ if (sel[(size_t)t * K + q] == e) { row[total] = t; rw[total] = wt[(size_t)t * K + q]; total++; }
+ }
+ first[nb] = total;
+ for (int i = 0; i < total; i++) memcpy(xg + (size_t)i * H, xn + (size_t)row[i] * H, (size_t)H * sizeof(float));
+ int up_blocks = (MI + BLOCK_ROWS - 1) / BLOCK_ROWS, down_blocks = (H + BLOCK_ROWS - 1) / BLOCK_ROWS;
+ int rb_mi = MI / 2, gb_mi = MI / 32, rb_h = H / 2, gb_h = H / 32;
+ #pragma omp parallel for schedule(dynamic, 1)
+ for (int task = 0; task < nb * 2 * up_blocks; task++) {
+ int j = task / (2 * up_blocks), which = (task / up_blocks) % 2, blk = task % up_blocks;
+ int r = first[j + 1] - first[j];
+ if (!r) continue;
+ int o0 = blk * BLOCK_ROWS, rows = MI - o0 < BLOCK_ROWS ? MI - o0 : BLOCK_ROWS;
+ const uint8_t *buf = slots[j]->buf;
+ const uint8_t *q = buf + m->part[0] + m->part[1] + (which ? m->part[2] + m->part[3] : 0);
+ const uint8_t *sc = q + m->part[2];
+ float *dst = (which ? u : g) + (size_t)first[j] * MI;
+ /* rows [o0, o0+rows) of this expert's gate or up, for its r inputs; the
+ * kernel writes [r, rows] contiguously, so it goes through a scratch */
+ float tmp[4 * 256];
+ float *scratch = r * rows <= (int)(sizeof(tmp) / sizeof(tmp[0])) ? tmp
+ : xmalloc((size_t)r * rows * sizeof(float), "expert scratch");
+ mm(scratch, xg + (size_t)first[j] * H, q + (size_t)o0 * rb_h, sc + (size_t)o0 * gb_h, r, H, rows);
+ for (int i = 0; i < r; i++)
+ memcpy(dst + (size_t)i * MI + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));
+ if (scratch != tmp) free(scratch);
+ }
+ for (size_t i = 0; i < (size_t)total * MI; i++) g[i] = silu(g[i]) * u[i];
+ #pragma omp parallel for schedule(dynamic, 1)
+ for (int task = 0; task < nb * down_blocks; task++) {
+ int j = task / down_blocks, blk = task % down_blocks;
+ int r = first[j + 1] - first[j];
+ if (!r) continue;
+ int o0 = blk * BLOCK_ROWS, rows = H - o0 < BLOCK_ROWS ? H - o0 : BLOCK_ROWS;
+ const uint8_t *dq = slots[j]->buf, *ds = dq + m->part[0];
+ float tmp[4 * 256];
+ float *scratch = r * rows <= (int)(sizeof(tmp) / sizeof(tmp[0])) ? tmp
+ : xmalloc((size_t)r * rows * sizeof(float), "expert scratch");
+ mm(scratch, g + (size_t)first[j] * MI, dq + (size_t)o0 * rb_mi, ds + (size_t)o0 * gb_mi, r, MI, rows);
+ float *dst = y + (size_t)first[j] * H;
+ for (int i = 0; i < r; i++)
+ memcpy(dst + (size_t)i * H + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));
+ if (scratch != tmp) free(scratch);
+ }
+ /* the weighted sum, expert by expert in the order they were chosen */
+ for (int j = 0; j < nb; j++)
+ for (int i = first[j]; i < first[j + 1]; i++) {
+ float *o = out + (size_t)row[i] * H;
+ const float *yy = y + (size_t)i * H;
+ for (int d = 0; d < H; d++) o[d] += rw[i] * yy[d];
+ }
+ m->t_expert += now_s() - t0;
+ }
+ free(first);
+ free(xg); free(g); free(u); free(y); free(row); free(rw); free(uni); free(sel); free(wt);
+}
+
+static void embed_rows(Model *m, const int *ids, int n, float *h) {
+ int H = m->c.hidden;
+ for (int t = 0; t < n; t++) {
+ int id = ids[t];
+ if (id < 0 || id >= m->c.vocab) { fprintf(stderr, "[mimo] token id %d out of range\n", id); exit(1); }
+ for (int i = 0; i < H; i++) h[(size_t)t * H + i] = bf16_to_f32(m->embed[(size_t)id * H + i]);
+ }
+}
+
+/* Image rows to splice in: `rows[r]` replaces the embedding of the r-th
+ * image-pad position of the WHOLE prompt (the first at `first_pad`). */
+typedef struct { const float *rows; int n_rows; } ImageRows;
+
+/* One block of n tokens at m->pos. logits: NULL, or [n, V] (all rows) when
+ * all_rows, else [V] for the last row. */
+static void forward(Model *m, const int *ids, int n, float *logits, int all_rows,
+ const ImageRows *img, int *img_used) {
+ Cfg *c = &m->c;
+ int H = c->hidden;
+ if (m->pos + n > m->ctx) { fprintf(stderr, "[mimo] context full (%d + %d > %d)\n", m->pos, n, m->ctx); exit(1); }
+ float *h = xmalloc((size_t)n * H * sizeof(float), "residual");
+ float *xn = xmalloc((size_t)n * H * sizeof(float), "normed");
+ float *tmp = xmalloc((size_t)n * H * sizeof(float), "block out");
+ embed_rows(m, ids, n, h);
+ if (img && img->rows && c->image_token_id >= 0)
+ for (int t = 0; t < n; t++)
+ if (ids[t] == c->image_token_id && *img_used < img->n_rows) {
+ memcpy(h + (size_t)t * H, img->rows + (size_t)(*img_used) * H, (size_t)H * sizeof(float));
+ (*img_used)++;
+ }
+ /* MIMO_TRACE=<file>: the residual after the embedding and after every
+ * attention and every MLP of the first block, f32 [2L+1][n][H] -- what
+ * tools/make_mimo_ref.py --trace compares against the vendor's hooks. */
+ static FILE *trace;
+ static int traced;
+ if (!traced && getenv("MIMO_TRACE")) { trace = fopen(getenv("MIMO_TRACE"), "wb"); }
+ if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);
+ for (int li = 0; li < c->n_layers; li++) {
+ Layer *l = &m->L[li];
+ for (int t = 0; t < n; t++) rmsnorm(xn + (size_t)t * H, h + (size_t)t * H, l->ln1, H, c->eps);
+ attention(m, li, xn, n, m->pos, tmp);
+ for (size_t i = 0; i < (size_t)n * H; i++) h[i] += tmp[i];
+ if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);
+ for (int t = 0; t < n; t++) rmsnorm(xn + (size_t)t * H, h + (size_t)t * H, l->ln2, H, c->eps);
+ if (c->moe[li]) moe(m, li, xn, n, tmp);
+ else dense_mlp(m, l, xn, n, tmp);
+ for (size_t i = 0; i < (size_t)n * H; i++) h[i] += tmp[i];
+ if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);
+ }
+ if (trace) { fclose(trace); trace = NULL; }
+ traced = 1;
+ if (logits) {
+ int from = all_rows ? 0 : n - 1, rows = n - from;
+ for (int t = from; t < n; t++) rmsnorm(xn + (size_t)(t - from) * H, h + (size_t)t * H, m->norm, H, c->eps);
+ dw_matmul(logits, xn, rows, &m->head);
+ }
+ kv_prefix_record(&m->kvp, ids, m->pos, n);
+ m->pos += n;
+ m->forwards++;
+ free(h); free(xn); free(tmp);
+}
+
+/* A prompt in blocks of MIMO_CHUNK rows (default 64): the last block's last
+ * row (or every row, for the teacher-forcing dump) comes back in `logits`. */
+static void prefill(Model *m, const int *ids, int n, float *logits, float *all_logits,
+ const ImageRows *img) {
+ int chunk = env_int("MIMO_CHUNK", 64);
+ if (chunk < 1) chunk = 1;
+ int used = 0;
+ for (int b = 0; b < n; b += chunk) {
+ int nb = n - b < chunk ? n - b : chunk;
+ if (all_logits) forward(m, ids + b, nb, all_logits + (size_t)b * m->c.vocab, 1, img, &used);
+ else forward(m, ids + b, nb, b + nb == n ? logits : NULL, 0, img, &used);
+ }
+ if (all_logits && logits) memcpy(logits, all_logits + (size_t)(n - 1) * m->c.vocab, (size_t)m->c.vocab * sizeof(float));
+}
+
+/* ----------------------------------------------------------------- vision ---- */
+
+/* The tower, when the container carries one (visual.*): vision_run turns the
+ * gateway's patches into the rows that replace the image-pad embeddings. */
+typedef struct Vision Vision;
+static Vision *g_vision;
+#include "mimo_vision.h"
+
+/* --------------------------------------------------------------- sampling ---- */
+
+static int argmax(const float *x, int n) {
+ int best = 0;
+ for (int i = 1; i < n; i++) if (x[i] > x[best]) best = i;
+ return best;
+}
+
+typedef struct { float p; int id; } SampleProb;
+static int sample_desc(const void *a, const void *b) {
+ float pa = ((const SampleProb *)a)->p, pb = ((const SampleProb *)b)->p;
+ return (pb > pa) - (pa > pb);
+}
+
+static int sample(const float *logits, int vocab, float temperature, float top_p) {
+ if (temperature <= 0.0f) return argmax(logits, vocab);
+ SampleProb *rank = xmalloc((size_t)vocab * sizeof(SampleProb), "sampling");
+ float top = logits[0];
+ for (int i = 1; i < vocab; i++) if (logits[i] > top) top = logits[i];
+ double total = 0.0;
+ for (int i = 0; i < vocab; i++) {
+ float p = expf((logits[i] - top) / temperature);
+ total += p; rank[i].p = p; rank[i].id = i;
+ }
+ qsort(rank, (size_t)vocab, sizeof(SampleProb), sample_desc);
+ double cut = (top_p > 0.0f && top_p < 1.0f) ? top_p * total : total, kept = 0.0;
+ int n = 0;
+ while (n < vocab && kept < cut) kept += rank[n++].p;
+ double draw = ((double)rand() / RAND_MAX) * kept, acc = 0.0;
+ int pick = rank[0].id;
+ for (int i = 0; i < n; i++) { acc += rank[i].p; if (acc >= draw) { pick = rank[i].id; break; } }
+ free(rank);
+ return pick;
+}
+
+/* ------------------------------------------------------------------ serve ---- */
+
+static const ColiServeWireProfile mimo_wire = {
+ .max_header_bytes = 511,
+ .max_payload_bytes = 1u << 26,
+ .max_tokens = 1 << 20,
+ .require_exact_lf = 1,
+ .require_finite_sampling = 1,
+};
+
+static void serve_line(const char *format, ...) {
+ va_list args;
+ va_start(args, format);
+ vfprintf(stdout, format, args);
+ va_end(args);
+ fflush(stdout);
+}
+
+/* the dashboard's grid: rows are the MoE layers, columns the routed experts */
+static void serve_emap(Model *m) {
+ Cfg *c = &m->c;
+ int rows = 0;
+ for (int li = 0; li < c->n_layers; li++) rows += c->moe[li];
+ int cols = c->n_experts;
+ char *hex = xmalloc((size_t)rows * cols * 2 + 1, "emap");
+ int w = 0;
+ for (int li = 0; li < c->n_layers; li++) {
+ if (!c->moe[li]) continue;
+ for (int e = 0; e < cols; e++) {
+ int byte = (m->cache[li].by_expert[e] >= 0) << 6;
+ hex[w++] = "0123456789abcdef"[byte >> 4];
+ hex[w++] = "0123456789abcdef"[byte & 15];
+ }
+ }
+ hex[w] = 0;
+ serve_line("EMAP %d %d %s\n", rows, cols, hex);
+ free(hex);
+}
+
+static void serve_hits(Model *m) {
+ Cfg *c = &m->c;
+ int rows = 0;
+ for (int li = 0; li < c->n_layers; li++) rows += c->moe[li];
+ int cols = c->n_experts, nb = (rows * cols + 7) / 8, bit = 0;
+ uint8_t *bitmap = xcalloc((size_t)nb, 1, "hits bitmap");
+ for (int li = 0; li < c->n_layers; li++) {
+ if (!c->moe[li]) continue;
+ for (int e = 0; e < cols; e++, bit++)
+ if (m->ehit[li][e]) { bitmap[bit >> 3] |= (uint8_t)(1 << (bit & 7)); m->ehit[li][e] = 0; }
+ }
+ char *hex = xmalloc((size_t)nb * 2 + 1, "hits hex");
+ for (int b = 0; b < nb; b++) {
+ hex[2 * b] = "0123456789abcdef"[bitmap[b] >> 4];
+ hex[2 * b + 1] = "0123456789abcdef"[bitmap[b] & 15];
+ }
+ hex[2 * nb] = 0;
+ serve_line("HITS %d %d %s\n", rows, cols, hex);
+ free(hex); free(bitmap);
+}
+
+/* max_tokens is a ceiling; generation needs at least one free position */
+static int serve_budget(int prompt, int requested, int context) {
+ if (prompt < 1 || prompt > context) return -1;
+ int budget = requested > 0 ? requested : 256;
+ int room = context - prompt;
+ if (room == 0) return -1;
+ return budget < room ? budget : room;
+}
+
+static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
+ Cfg *c = &m->c;
+ coli_serve_stdio_init();
+ int eos[8];
+ int n_eos = coli_load_stop_ids(dir, eos, 8, NULL);
+ coli_serve_write_ready(stdout, rss_gb());
+ serve_emap(m);
+ float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");
+ int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "prompt ids");
+ float *pending = NULL;
+ int pending_h = 0, pending_w = 0;
+ for (;;) {
+ ColiServeCommand command;
+ ColiServeReadResult result = coli_serve_read_command(stdin, &mimo_wire, &command);
+ if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) break;
+ if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); break; }
+ if (result == COLI_SERVE_READ_BAD_REQUEST) {
+ if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ if (command.kind == COLI_SERVE_COMMAND_IMAGE) {
+ /* One image waits for the SUBMIT that names it; a second replaces it. */
+ if (!g_vision) {
+ coli_serve_write_error(stdout, command.id, "this container has no vision tower");
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ uint64_t expected = (uint64_t)command.grid_h * command.grid_w * g_vision->patch_in * sizeof(float);
+ if (command.grid_h < 2 || command.grid_w < 2 || command.grid_h % 2 || command.grid_w % 2 ||
+ command.payload_bytes != expected) {
+ coli_serve_write_error(stdout, command.id, "BAD_IMAGE payload does not match its grid");
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ free(pending);
+ pending = (float *)coli_serve_command_take_payload(&command);
+ pending_h = command.grid_h; pending_w = command.grid_w;
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {
+ if (command.kind == COLI_SERVE_COMMAND_CANCEL) coli_serve_write_error(stdout, command.id, "NOT_FOUND");
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ double started = now_s();
+ double disk0 = m->t_disk, expert0 = m->t_expert, attn0 = m->t_attn;
+ uint64_t hits0 = m->hits, miss0 = m->miss, fw0 = m->forwards;
+ int n_prompt = tok_encode(tokenizer, (const char *)command.payload, (int)command.payload_bytes,
+ ids, m->ctx + 1);
+ int budget = serve_budget(n_prompt, command.max_tokens, m->ctx);
+ if (budget < 0) {
+ char message[160];
+ snprintf(message, sizeof(message), "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
+ n_prompt, command.max_tokens, m->ctx);
+ coli_serve_write_error(stdout, command.id, n_prompt < 1 ? "EMPTY_PROMPT" : message);
+ free(pending); pending = NULL;
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ /* An image: its rows replace the pad ids, which must be exactly as many
+ * as the tower produces. Checked before ACCEPT so a mismatch is a clean
+ * 400, not a half-answered turn. */
+ float *image = NULL;
+ int image_rows = 0;
+ if (pending) {
+ int pads = 0;
+ for (int t = 0; t < n_prompt; t++) pads += ids[t] == c->image_token_id;
+ int want = (pending_h / 2) * (pending_w / 2);
+ if (pads != want) {
+ char message[160];
+ snprintf(message, sizeof(message), "BAD_IMAGE prompt has %d image pads, the %dx%d grid needs %d",
+ pads, pending_h, pending_w, want);
+ coli_serve_write_error(stdout, command.id, message);
+ free(pending); pending = NULL;
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ }
+ /* A chat client resends the whole transcript: if this prompt begins with
+ * the ids the state was built from, only the tail is fed. An image
+ * refuses it -- the pad ids do not describe the picture. */
+ int prefix_on = env_int("COLI_KV_PREFIX", 1) != 0;
+ int reuse = (!pending && prefix_on) ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;
+ if (getenv("COLI_PREFIX_LOG"))
+ fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens\n", reuse ? "reusing" : "no reuse,", reuse, n_prompt);
+ if (!reuse) model_reset(m);
+ coli_serve_write_accept(stdout, command.id, n_prompt);
+ if (pending) {
+ image = vision_run(m, pending, pending_h, pending_w, &image_rows);
+ free(pending); pending = NULL;
+ }
+ ImageRows img = { image, image_rows };
+ prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL);
+ if (image) { kv_prefix_taint(&m->kvp); free(image); }
+ int emitted = 0, limited = 1, cancelled = 0, done_early = 0;
+ char piece[512];
+ while (emitted < budget && !cancelled && !done_early) {
+ int token = sample(logits, c->vocab, command.temperature, command.top_p);
+ int stop = 0;
+ for (int i = 0; i < n_eos; i++) if (token == eos[i]) stop = 1;
+ if (stop) { limited = 0; break; }
+ int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
+ if (written > 0) coli_serve_write_data(stdout, command.id, piece, (size_t)written);
+ emitted++;
+ while (coli_serve_stdin_ready()) {
+ ColiServeCommand control;
+ ColiServeReadResult inner = coli_serve_read_command(stdin, &mimo_wire, &control);
+ if (inner == COLI_SERVE_READ_EOF) { done_early = 1; break; }
+ if (control.kind == COLI_SERVE_COMMAND_CANCEL && !strcmp(control.id, command.id)) cancelled = 1;
+ else if (control.kind == COLI_SERVE_COMMAND_STOP && !strcmp(control.id, command.id)) { limited = 0; done_early = 1; }
+ else if (control.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, control.id, "SLOT_BUSY");
+ coli_serve_command_dispose(&control);
+ }
+ if (cancelled || done_early || emitted >= budget) break;
+ prefill(m, &token, 1, logits, NULL, NULL);
+ }
+ if (cancelled) {
+ coli_serve_write_error(stdout, command.id, "CANCELLED");
+ coli_serve_command_dispose(&command);
+ continue;
+ }
+ double wall = now_s() - started;
+ uint64_t th = m->hits - hits0, tm = m->miss - miss0;
+ ColiServeDone done = { emitted, wall > 0 ? emitted / wall : 0.0,
+ (th + tm) ? 100.0 * th / (double)(th + tm) : 0.0, rss_gb(), n_prompt, limited };
+ coli_serve_write_done(stdout, command.id, &done);
+ serve_line("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n", wall, n_prompt, emitted,
+ m->t_disk - disk0, 0.0, m->t_expert - expert0, m->t_attn - attn0, 0.0,
+ (unsigned long long)(m->forwards - fw0));
+ serve_hits(m);
+ serve_emap(m);
+ coli_serve_command_dispose(&command);
+ }
+ free(ids); free(logits); free(pending);
+}
+
+/* ------------------------------------------------------------------- main ---- */
+
+static int parse_ids(const char *text, int *out, int cap) {
+ int n = 0;
+ const char *p = text;
+ while (*p && n < cap) {
+ while (*p == ' ' || *p == ',' || *p == '\t' || *p == '\n') p++;
+ if (!*p) break;
+ char *end;
+ long v = strtol(p, &end, 10);
+ if (end == p) break;
+ out[n++] = (int)v;
+ p = end;
+ }
+ return n;
+}
+
+static int is_dir(const char *path) {
+ struct stat sb;
+ return stat(path, &sb) == 0 && S_ISDIR(sb.st_mode);
+}
+
+int main(int argc, char **argv) {
+ coli_omp_tune_threads("mimo");
+ /* Two ways in. The gateway: SNAP=<dir> mimo <cap>, SERVE=1. By hand and in
+ * the tests: mimo <dir> [--ids "..." | --prompt "..."] [--ngen N] [--cap N]. */
+ const char *dir = getenv("SNAP");
+ int cap = env_int("MIMO_CAP", 0);
+ const char *ids_text = NULL, *prompt = NULL, *image_path = NULL;
+ int ngen = 32, grid_h = 0, grid_w = 0;
+ for (int i = 1; i < argc; i++) {
+ if (!strcmp(argv[i], "--ids") && i + 1 < argc) ids_text = argv[++i];
+ else if (!strcmp(argv[i], "--prompt") && i + 1 < argc) prompt = argv[++i];
+ else if (!strcmp(argv[i], "--ngen") && i + 1 < argc) ngen = coli_arg_int(argv[++i], "--ngen");
+ else if (!strcmp(argv[i], "--cap") && i + 1 < argc) cap = coli_arg_int(argv[++i], "--cap");
+ else if (!strcmp(argv[i], "--image") && i + 1 < argc) image_path = argv[++i];
+ else if (!strcmp(argv[i], "--grid") && i + 2 < argc) {
+ grid_h = coli_arg_int(argv[++i], "--grid h"); grid_w = coli_arg_int(argv[++i], "--grid w"); }
+ else if (is_dir(argv[i])) dir = argv[i];
+ else cap = coli_arg_int(argv[i], "cache/layer");
+ }
+ if (!dir) {
+ fprintf(stderr, "usage: %s <model dir> [--ids \"1 2 3\" | --prompt TEXT] [--ngen N] [--cap N]\n"
+ " SNAP=<model dir> SERVE=1 %s <cap>\n", argv[0], argv[0]);
+ return 2;
+ }
+ Model *m = xcalloc(1, sizeof(Model), "model");
+ double t0 = now_s();
+ if (cap <= 0) cap = 64;
+ model_load(m, dir, cap);
+ Cfg *c = &m->c;
+ g_vision = vision_load(m, dir);
+ int swa = 0, shown_cap = 0;
+ for (int i = 0; i < c->n_layers; i++) { swa += c->swa[i]; if (c->moe[i] && !shown_cap) shown_cap = m->cache[i].cap; }
+ fprintf(stderr, "[mimo] %d layers (%d sliding window %d, %d full), %d experts top-%d, cache %d/layer, "
+ "dense %s, ctx %d%s -- loaded in %.2fs\n",
+ c->n_layers, swa, c->window, c->n_layers - swa, c->n_experts, c->topk, shown_cap,
+ g_dense_bits == 0 ? "native" : g_dense_bits == 8 ? "int8" : "f32", m->ctx,
+ g_vision ? ", vision" : "", now_s() - t0);
+
+ char tok_path[4096];
+ snprintf(tok_path, sizeof(tok_path), "%s/tokenizer.json", dir);
+ int have_tok = access(tok_path, R_OK) == 0;
+ Tok tokenizer;
+ if (have_tok) tok_load(&tokenizer, tok_path);
+ const char *seed = getenv("SEED");
+ srand(seed ? (unsigned)strtoul(seed, NULL, 10) : (unsigned)time(NULL));
+
+ if (env_int("SERVE", 0)) {
+ if (!have_tok) { fprintf(stderr, "[mimo] SERVE needs %s\n", tok_path); return 1; }
+ serve_loop(m, &tokenizer, dir);
+ return 0;
+ }
+ int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "ids");
+ int n = 0;
+ if (ids_text) n = parse_ids(ids_text, ids, m->ctx);
+ else if (prompt && have_tok) n = tok_encode(&tokenizer, prompt, (int)strlen(prompt), ids, m->ctx);
+ else { fprintf(stderr, "[mimo] give --ids, or --prompt with a tokenizer.json in the model dir\n"); return 2; }
+ if (n < 1) { fprintf(stderr, "[mimo] empty prompt\n"); return 2; }
+ if (n + ngen > m->ctx) { fprintf(stderr, "[mimo] prompt + ngen exceed the context %d\n", m->ctx); return 2; }
+ float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");
+ const char *dump = getenv("MIMO_LOGITS");
+ float *all = dump ? xmalloc((size_t)n * c->vocab * sizeof(float), "all logits") : NULL;
+ /* --image patches.f32 --grid H W: the gateway's patches for the image-pad
+ * ids already in the prompt */
+ float *image = NULL;
+ int image_rows = 0;
+ if (image_path) {
+ if (!g_vision) { fprintf(stderr, "[mimo] --image: this container has no vision tower\n"); return 2; }
+ size_t want = (size_t)grid_h * grid_w * g_vision->patch_in;
+ float *patches = xmalloc(want * sizeof(float), "patches");
+ FILE *f = fopen(image_path, "rb");
+ if (!f || fread(patches, sizeof(float), want, f) != want || fgetc(f) != EOF) {
+ fprintf(stderr, "[mimo] %s does not hold %dx%d patches\n", image_path, grid_h, grid_w); return 2;
+ }
+ fclose(f);
+ int pads = 0;
+ for (int t = 0; t < n; t++) pads += ids[t] == c->image_token_id;
+ if (pads != (grid_h / 2) * (grid_w / 2)) {
+ fprintf(stderr, "[mimo] the prompt has %d image pads, the grid needs %d\n", pads, (grid_h / 2) * (grid_w / 2));
+ return 2;
+ }
+ image = vision_run(m, patches, grid_h, grid_w, &image_rows);
+ free(patches);
+ }
+ ImageRows img = { image, image_rows };
+ double tp = now_s();
+ prefill(m, ids, n, logits, all, image ? &img : NULL);
+ double prefill_s = now_s() - tp;
+ if (dump) {
+ FILE *f = fopen(dump, "wb");
+ if (!f || fwrite(all, sizeof(float), (size_t)n * c->vocab, f) != (size_t)n * c->vocab) {
+ fprintf(stderr, "[mimo] cannot write %s\n", dump); return 1;
+ }
+ fclose(f);
+ }
+ float temperature = getenv("COLI_TEMP") ? (float)atof(getenv("COLI_TEMP")) : 0.0f;
+ int eos[8], n_eos = coli_load_stop_ids(dir, eos, 8, NULL);
+ double td = now_s();
+ int produced = 0;
+ char piece[512];
+ for (int g = 0; g < ngen; g++) {
+ int token = sample(logits, c->vocab, temperature, 1.0f);
+ int stop = 0;
+ for (int i = 0; i < n_eos; i++) stop |= token == eos[i];
+ if (have_tok && !ids_text) {
+ int w = tok_decode(&tokenizer, &token, 1, piece, (int)sizeof(piece));
+ fwrite(piece, 1, (size_t)(w > 0 ? w : 0), stdout);
+ } else printf("%d ", token);
+ fflush(stdout);
+ produced++;
+ if (stop && have_tok && !ids_text) break;
+ if (g + 1 < ngen) prefill(m, &token, 1, logits, NULL, NULL);
+ }
+ printf("\n");
+ fflush(stdout);
+ double decode_s = now_s() - td;
+ double total = prefill_s + decode_s;
+ fprintf(stderr, "[mimo] prefill %d tokens %.2fs, decode %d tokens %.2fs (%.2f tok/s), experts %llu hits "
+ "%llu misses %.1f MB read, rss %.2f GB\n"
+ "[mimo] time: expert reads %.2fs (%.2f GB/s), expert matmuls %.2fs, attention %.2fs, "
+ "the rest (dense MLP, router, head) %.2fs\n",
+ n, prefill_s, produced, decode_s, decode_s > 0 ? produced / decode_s : 0.0,
+ (unsigned long long)m->hits, (unsigned long long)m->miss, m->bytes_read / 1e6, rss_gb(),
+ m->t_disk, m->t_disk > 0 ? m->bytes_read / 1e9 / m->t_disk : 0.0, m->t_expert, m->t_attn,
+ total - m->t_disk - m->t_expert - m->t_attn);
+ return 0;
+}
diff --git a/c/mimo_tiny/config.json b/c/mimo_tiny/config.json
new file mode 100644
index 00000000..4f325d36
--- /dev/null
+++ b/c/mimo_tiny/config.json
@@ -0,0 +1,124 @@
+{
+ "architectures": [
+ "MiMoV2ForCausalLM"
+ ],
+ "model_type": "mimo_v2",
+ "vocab_size": 320,
+ "hidden_size": 256,
+ "intermediate_size": 256,
+ "num_hidden_layers": 6,
+ "hybrid_layer_pattern": [
+ 0,
+ 1,
+ 1,
+ 0,
+ 1,
+ 1
+ ],
+ "moe_layer_freq": [
+ 0,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "num_attention_heads": 4,
+ "num_key_value_heads": 2,
+ "head_dim": 48,
+ "v_head_dim": 32,
+ "swa_num_attention_heads": 4,
+ "swa_num_key_value_heads": 4,
+ "swa_head_dim": 48,
+ "swa_v_head_dim": 32,
+ "sliding_window": 8,
+ "sliding_window_size": 8,
+ "attention_chunk_size": 8,
+ "add_full_attention_sink_bias": false,
+ "add_swa_attention_sink_bias": true,
+ "attention_value_scale": 0.707,
+ "attention_projection_layout": "fused_qkv",
+ "attention_bias": false,
+ "partial_rotary_factor": 0.334,
+ "rope_theta": 10000000.0,
+ "swa_rope_theta": 10000.0,
+ "rope_parameters": {
+ "partial_rotary_factor": 0.334,
+ "rope_theta": 10000000.0,
+ "rope_type": "default",
+ "type": "default"
+ },
+ "n_routed_experts": 16,
+ "num_experts_per_tok": 4,
+ "moe_intermediate_size": 64,
+ "n_shared_experts": null,
+ "n_group": 1,
+ "topk_group": 1,
+ "topk_method": "noaux_tc",
+ "scoring_func": "sigmoid",
+ "norm_topk_prob": true,
+ "routed_scaling_factor": null,
+ "moe_router_dtype": "bfloat16",
+ "hidden_act": "silu",
+ "layernorm_epsilon": 1e-06,
+ "max_position_embeddings": 256,
+ "tie_word_embeddings": false,
+ "bos_token_id": null,
+ "eos_token_id": 1,
+ "pad_token_id": 0,
+ "dtype": "bfloat16",
+ "quantization_config": {
+ "activation_scheme": "dynamic",
+ "fmt": "e4m3",
+ "quant_method": "fp8",
+ "store_dtype": "mxfp4",
+ "mxfp4_block_size": 32,
+ "weight_block_size": [
+ 128,
+ 128
+ ],
+ "ignored_layers": [
+ "model.layers.0.self_attn.o_proj",
+ "model.layers.1.self_attn.o_proj",
+ "model.layers.2.self_attn.o_proj",
+ "model.layers.3.self_attn.o_proj",
+ "model.layers.4.self_attn.o_proj",
+ "model.layers.5.self_attn.o_proj"
+ ]
+ },
+ "vision_config": {
+ "depth": 6,
+ "hidden_size": 64,
+ "intermediate_size": 96,
+ "num_heads": 4,
+ "num_key_value_heads": 2,
+ "qk_channels": 16,
+ "out_hidden_size": 256,
+ "patch_size": 16,
+ "spatial_patch_size": 16,
+ "temporal_patch_size": 2,
+ "spatial_merge_size": 2,
+ "in_chans": 3,
+ "hidden_act": "silu",
+ "fullatt_block_indexes": [
+ 0,
+ 5
+ ],
+ "vit_window_attn_types": [
+ -1,
+ 0,
+ 1,
+ 1,
+ 0,
+ -1
+ ],
+ "visual_token_window_size": 5,
+ "use_sink": true,
+ "window_size": 128,
+ "tokens_per_second": 2
+ },
+ "image_token_id": 300,
+ "vision_start_token_id": 301,
+ "vision_end_token_id": 302,
+ "vision_model_type": "mimovl"
+}
diff --git a/c/mimo_tiny/preprocessor_config.json b/c/mimo_tiny/preprocessor_config.json
new file mode 100644
index 00000000..0467ea20
--- /dev/null
+++ b/c/mimo_tiny/preprocessor_config.json
@@ -0,0 +1,19 @@
+{
+ "min_pixels": 3136,
+ "max_pixels": 12845056,
+ "patch_size": 16,
+ "temporal_patch_size": 2,
+ "merge_size": 2,
+ "image_mean": [
+ 0.48145466,
+ 0.4578275,
+ 0.40821073
+ ],
+ "image_std": [
+ 0.26862954,
+ 0.26130258,
+ 0.27577711
+ ],
+ "image_processor_type": "Qwen2VLImageProcessor",
+ "processor_class": "Qwen2_5_VLProcessor"
+}
diff --git a/c/mimo_tiny/ref.json b/c/mimo_tiny/ref.json
new file mode 100644
index 00000000..9fb5b0f2
--- /dev/null
+++ b/c/mimo_tiny/ref.json
@@ -0,0 +1,638 @@
+{
+ "schema_version": 1,
+ "source": "xiaomi-vendor",
+ "vendor_repo": "XiaomiMiMo/MiMo-V2.6-Flash-MOPD",
+ "vendor_revision": "2479e2d0029eca9a34cc7e7f55a121925f81908e",
+ "vendor_files": {
+ "configuration_mimo_v2.py": "773062ac9850b908eb54751b3e4dbe00e653c0e80595599409e97d0c1af2ce3e",
+ "modeling_mimo_v2.py": "a8c3cb3aae473bcc15f023010547c919f15eba6546e6ed7efb61a8937b12f3ad"
+ },
+ "torch_version": "2.12.1+cu130",
+ "transformers_version": "5.3.0",
+ "engine_env": {
+ "MIMO_DENSE_BITS": "32",
+ "MIMO_IDOT": "0",
+ "COLI_TEMP": "0"
+ },
+ "cases": {
+ "short": {
+ "prompt_ids": [
+ 5,
+ 7,
+ 9,
+ 11,
+ 13,
+ 17,
+ 19,
+ 23
+ ],
+ "teacher_forcing_ids": [
+ 210,
+ 311,
+ 69,
+ 287,
+ 199,
+ 306,
+ 214,
+ 89,
+ 114,
+ 192,
+ 311,
+ 230,
+ 60,
+ 175,
+ 64,
+ 27,
+ 294,
+ 234,
+ 86,
+ 13
+ ],
+ "greedy_full_ids": [
+ 5,
+ 7,
+ 9,
+ 11,
+ 13,
+ 17,
+ 19,
+ 23,
+ 89,
+ 114,
+ 192,
+ 311,
+ 230,
+ 60,
+ 175,
+ 64,
+ 27,
+ 294,
+ 234,
+ 86
+ ],
+ "greedy_new_ids": [
+ 89,
+ 114,
+ 192,
+ 311,
+ 230,
+ 60,
+ 175,
+ 64,
+ 27,
+ 294,
+ 234,
+ 86
+ ],
+ "max_new_tokens": 12,
+ "min_logit_margin": 1.646,
+ "context_free_agreement": 0.0
+ },
+ "window": {
+ "prompt_ids": [
+ 5,
+ 12,
+ 19,
+ 26,
+ 33,
+ 40,
+ 47,
+ 54,
+ 61,
+ 68,
+ 75,
+ 82,
+ 89,
+ 96,
+ 103,
+ 110,
+ 117,
+ 124,
+ 131,
+ 138,
+ 145,
+ 152,
+ 159,
+ 166,
+ 173,
+ 180,
+ 187,
+ 194,
+ 201,
+ 208,
+ 215,
+ 222,
+ 229,
+ 236,
+ 243,
+ 250,
+ 257,
+ 264,
+ 271,
+ 278
+ ],
+ "teacher_forcing_ids": [
+ 210,
+ 238,
+ 192,
+ 121,
+ 6,
+ 161,
+ 311,
+ 250,
+ 134,
+ 220,
+ 56,
+ 120,
+ 0,
+ 32,
+ 21,
+ 64,
+ 286,
+ 115,
+ 145,
+ 145,
+ 77,
+ 93,
+ 35,
+ 150,
+ 255,
+ 151,
+ 255,
+ 113,
+ 146,
+ 148,
+ 254,
+ 317,
+ 79,
+ 46,
+ 266,
+ 95,
+ 159,
+ 291,
+ 105,
+ 220,
+ 64,
+ 238,
+ 150,
+ 146,
+ 317,
+ 279
+ ],
+ "greedy_full_ids": [
+ 5,
+ 12,
+ 19,
+ 26,
+ 33,
+ 40,
+ 47,
+ 54,
+ 61,
+ 68,
+ 75,
+ 82,
+ 89,
+ 96,
+ 103,
+ 110,
+ 117,
+ 124,
+ 131,
+ 138,
+ 145,
+ 152,
+ 159,
+ 166,
+ 173,
+ 180,
+ 187,
+ 194,
+ 201,
+ 208,
+ 215,
+ 222,
+ 229,
+ 236,
+ 243,
+ 250,
+ 257,
+ 264,
+ 271,
+ 278,
+ 220,
+ 64,
+ 238,
+ 150,
+ 146,
+ 317
+ ],
+ "greedy_new_ids": [
+ 220,
+ 64,
+ 238,
+ 150,
+ 146,
+ 317
+ ],
+ "max_new_tokens": 6,
+ "min_logit_margin": 0.896,
+ "context_free_agreement": 0.022
+ },
+ "long": {
+ "prompt_ids": [
+ 5,
+ 16,
+ 27,
+ 38,
+ 49,
+ 60,
+ 71,
+ 82,
+ 93,
+ 104,
+ 115,
+ 126,
+ 137,
+ 148,
+ 159,
+ 170,
+ 181,
+ 192,
+ 203,
+ 214,
+ 225,
+ 236,
+ 247,
+ 258,
+ 269,
+ 280,
+ 291,
+ 302,
+ 313,
+ 4,
+ 15,
+ 26,
+ 37,
+ 48,
+ 59,
+ 70,
+ 81,
+ 92,
+ 103,
+ 114,
+ 125,
+ 136,
+ 147,
+ 158,
+ 169,
+ 180,
+ 191,
+ 202,
+ 213,
+ 224,
+ 235,
+ 246,
+ 257,
+ 268,
+ 279,
+ 290,
+ 301,
+ 312,
+ 3,
+ 14,
+ 25,
+ 36,
+ 47,
+ 58,
+ 69,
+ 80,
+ 91,
+ 102,
+ 113,
+ 124,
+ 135,
+ 146
+ ],
+ "teacher_forcing_ids": [
+ 210,
+ 73,
+ 35,
+ 199,
+ 184,
+ 107,
+ 95,
+ 44,
+ 180,
+ 41,
+ 288,
+ 259,
+ 9,
+ 218,
+ 247,
+ 277,
+ 235,
+ 84,
+ 316,
+ 113,
+ 293,
+ 216,
+ 123,
+ 14,
+ 210,
+ 196,
+ 223,
+ 260,
+ 86,
+ 206,
+ 226,
+ 152,
+ 151,
+ 57,
+ 117,
+ 123,
+ 260,
+ 64,
+ 113,
+ 180,
+ 183,
+ 266,
+ 84,
+ 62,
+ 87,
+ 223,
+ 209,
+ 120,
+ 226,
+ 47,
+ 184,
+ 160,
+ 123,
+ 2,
+ 238,
+ 89,
+ 29,
+ 245,
+ 308,
+ 152,
+ 148,
+ 54,
+ 41,
+ 238,
+ 94,
+ 34,
+ 123,
+ 316,
+ 84,
+ 64,
+ 208,
+ 174,
+ 298,
+ 72,
+ 234,
+ 31,
+ 79,
+ 274
+ ],
+ "greedy_full_ids": [
+ 5,
+ 16,
+ 27,
+ 38,
+ 49,
+ 60,
+ 71,
+ 82,
+ 93,
+ 104,
+ 115,
+ 126,
+ 137,
+ 148,
+ 159,
+ 170,
+ 181,
+ 192,
+ 203,
+ 214,
+ 225,
+ 236,
+ 247,
+ 258,
+ 269,
+ 280,
+ 291,
+ 302,
+ 313,
+ 4,
+ 15,
+ 26,
+ 37,
+ 48,
+ 59,
+ 70,
+ 81,
+ 92,
+ 103,
+ 114,
+ 125,
+ 136,
+ 147,
+ 158,
+ 169,
+ 180,
+ 191,
+ 202,
+ 213,
+ 224,
+ 235,
+ 246,
+ 257,
+ 268,
+ 279,
+ 290,
+ 301,
+ 312,
+ 3,
+ 14,
+ 25,
+ 36,
+ 47,
+ 58,
+ 69,
+ 80,
+ 91,
+ 102,
+ 113,
+ 124,
+ 135,
+ 146,
+ 174,
+ 298,
+ 72,
+ 234,
+ 31,
+ 79
+ ],
+ "greedy_new_ids": [
+ 174,
+ 298,
+ 72,
+ 234,
+ 31,
+ 79
+ ],
+ "max_new_tokens": 6,
+ "min_logit_margin": 0.4713,
+ "context_free_agreement": 0.0
+ }
+ },
+ "image": {
+ "prompt_ids": [
+ 5,
+ 7,
+ 301,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 302,
+ 11,
+ 13,
+ 17,
+ 19
+ ],
+ "grid_h": 8,
+ "grid_w": 12,
+ "patches_sha256": "0c544aef4dec661ce140bcbc3221c43754a29661b5ceef9473755e1b3595d456",
+ "patch_max_diff_vs_official_processor": 0.0,
+ "vision_rows_sum": 60.48851776123047,
+ "vision_rows_abs_sum": 7867.14501953125,
+ "teacher_forcing_ids": [
+ 210,
+ 311,
+ 309,
+ 47,
+ 44,
+ 246,
+ 8,
+ 313,
+ 155,
+ 222,
+ 29,
+ 124,
+ 222,
+ 115,
+ 235,
+ 294,
+ 111,
+ 302,
+ 223,
+ 256,
+ 95,
+ 314,
+ 225,
+ 279,
+ 223,
+ 42,
+ 305,
+ 170,
+ 273,
+ 204,
+ 302,
+ 171,
+ 266,
+ 234,
+ 266,
+ 171,
+ 13,
+ 199,
+ 70,
+ 45
+ ],
+ "greedy_full_ids": [
+ 5,
+ 7,
+ 301,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 300,
+ 302,
+ 11,
+ 13,
+ 17,
+ 19,
+ 171,
+ 266,
+ 234,
+ 266,
+ 171,
+ 13,
+ 199,
+ 70
+ ],
+ "greedy_new_ids": [
+ 171,
+ 266,
+ 234,
+ 266,
+ 171,
+ 13,
+ 199,
+ 70
+ ],
+ "max_new_tokens": 8,
+ "min_logit_margin": 0.5628,
+ "negated_image_new_ids": [
+ 49,
+ 281,
+ 196,
+ 190,
+ 107,
+ 103,
+ 173,
+ 283
+ ]
+ }
+}
diff --git a/c/mimo_vision.h b/c/mimo_vision.h
new file mode 100644
index 00000000..867cf661
--- /dev/null
+++ b/c/mimo_vision.h
@@ -0,0 +1,322 @@
+/* mimo_vision.h -- the MiMo ViT (681M on the release), images only.
+ *
+ * Included by mimo.c after the Model and DW helpers. Mirrors
+ * MiMoVisionTransformer in modeling_mimo_v2.py, with the two points where the
+ * release and that module disagree resolved as the release (and vLLM's
+ * mimo_v2_omni.py) has them:
+ * - the merger stores no biases: its two Linears and its LayerNorm bias are
+ * zero (vLLM builds them with bias=False; a bias present in a container is
+ * still used);
+ * - after a run of column-order blocks that reaches the end of the tower, the
+ * tokens go back to row order before the merger (vLLM does; the release's
+ * last block is a full-attention one, so the two readings never differ on it).
+ *
+ * The tower, per block: x += proj(attn(rms(x))), x += mlp(rms(x))
+ * patches [N, 3*T*16*16] in 2x2-merge-block order (the Qwen2-VL processor)
+ * 2D RoPE head_dim/2 dims: h then w frequencies (theta 1e4), cat(e, e),
+ * rotate-half over the whole head
+ * attention GQA; full over the image in fullatt_block_indexes, else a 1D
+ * window |i - j| <= visual_token_window_size in the CURRENT token
+ * order, plus a sink logit added to key 0 of that order
+ * order vit_window_attn_types: -1 full, 0 row order, 1 column order of
+ * the merge units (entering a type-1 run permutes the tokens and
+ * their rotary angles, leaving it permutes them back)
+ * merger LayerNorm, the 2x2 units concatenated, Linear, exact GELU, Linear
+ */
+
+typedef struct {
+ float *norm1, *norm2, *sinks;
+ DW qkv, proj, gate, up, down;
+ float *qkv_b, *proj_b, *gate_b, *up_b, *down_b;
+ int full, type;
+} VBlock;
+
+struct Vision {
+ int depth, d, inter, heads, kvh, hd, out, patch, temporal, merge, in_ch, window;
+ float eps;
+ DW embed; /* [d, in_ch*T*P*P] */
+ VBlock *b;
+ float *ln_w, *ln_b, *fc1_b, *fc2_b;
+ DW fc1, fc2;
+ int patch_in;
+};
+
+static void vis_mat(DW *d, shards *S, const char *name, int O, int I) {
+ /* the tower stays BF16 (or f32 for the oracle): int8 is a dense-trunk option */
+ int saved = g_dense_bits;
+ g_dense_bits = g_dense_bits == 32 ? 32 : 0;
+ dw_load_bf16(d, S, name, O, I);
+ g_dense_bits = saved;
+}
+
+static float *vis_vec_opt(shards *S, const char *name, int n) {
+ return st_has(S, name) ? load_vec(S, name, n) : NULL;
+}
+
+static int vjson_int(jval *o, const char *key, int fallback) {
+ jval *v = o ? json_get(o, key) : NULL;
+ return (v && v->t == J_NUM) ? (int)v->num : fallback;
+}
+
+static Vision *vision_load(Model *m, const char *dir) {
+ if (!st_has(&m->S, "visual.patch_embed.proj.weight")) return NULL;
+ char path[4096];
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ char *text = read_text(path, NULL);
+ if (!text) return NULL;
+ char *arena = NULL;
+ jval *root = json_parse(text, &arena);
+ jval *vc = root ? json_get(root, "vision_config") : NULL;
+ if (!vc || vc->t != J_OBJ) {
+ fprintf(stderr, "[mimo] visual.* tensors but no vision_config: text only\n");
+ json_free(root); free(arena); free(text);
+ return NULL;
+ }
+ Vision *v = xcalloc(1, sizeof(Vision), "vision");
+ v->depth = vjson_int(vc, "depth", 0);
+ v->d = vjson_int(vc, "hidden_size", 0);
+ v->inter = vjson_int(vc, "intermediate_size", 0);
+ v->heads = vjson_int(vc, "num_heads", 0);
+ v->kvh = vjson_int(vc, "num_key_value_heads", v->heads);
+ v->hd = vjson_int(vc, "qk_channels", 64);
+ v->out = vjson_int(vc, "out_hidden_size", 0);
+ v->patch = vjson_int(vc, "patch_size", 16);
+ v->temporal = vjson_int(vc, "temporal_patch_size", 2);
+ v->merge = vjson_int(vc, "spatial_merge_size", 2);
+ v->in_ch = vjson_int(vc, "in_chans", 3);
+ v->window = vjson_int(vc, "visual_token_window_size", -1);
+ v->eps = (float)jnum(vc, "rms_norm_eps", 1e-6);
+ int use_sink = jbool(vc, "use_sink", 0);
+ if (v->depth < 1 || v->depth > 256 || v->d < 8 || v->d > 16384 || v->inter < 1 || v->inter > 65536 ||
+ v->heads < 1 || v->kvh < 1 || v->heads % v->kvh || v->hd < 4 || v->hd % 4 || v->hd > 1024 ||
+ v->out != m->c.hidden || v->merge != 2 || v->patch < 1 || v->patch > 64 ||
+ v->temporal < 1 || v->temporal > 4 || v->in_ch != 3) {
+ fprintf(stderr, "[mimo] vision_config out of range (or out_hidden_size != hidden_size) -- refusing\n");
+ exit(1);
+ }
+ v->patch_in = v->in_ch * v->temporal * v->patch * v->patch;
+ v->b = xcalloc((size_t)v->depth, sizeof(VBlock), "vision blocks");
+ jval *full = json_get(vc, "fullatt_block_indexes"), *types = json_get(vc, "vit_window_attn_types");
+ for (int i = 0; i < v->depth; i++) v->b[i].type = -1;
+ if (types && types->t == J_ARR) {
+ if (types->len != v->depth) { fprintf(stderr, "[mimo] vit_window_attn_types must list every block\n"); exit(1); }
+ for (int i = 0; i < v->depth; i++) v->b[i].type = (int)types->kids[i]->num;
+ }
+ if (full && full->t == J_ARR)
+ for (int i = 0; i < full->len; i++) {
+ int b = (int)full->kids[i]->num;
+ if (b >= 0 && b < v->depth) v->b[b].full = 1;
+ }
+ json_free(root); free(arena); free(text);
+
+ shards *S = &m->S;
+ st_tensor *t = st_find(S, "visual.patch_embed.proj.weight");
+ if (t->dtype != 0 || t->numel != (int64_t)v->d * v->patch_in || t->nbytes != t->numel * 2) {
+ fprintf(stderr, "[mimo] visual.patch_embed.proj.weight is not BF16 [%d, %d] -- refusing\n", v->d, v->patch_in);
+ exit(1);
+ }
+ /* Conv3d [d, C, T, P, P] with stride = kernel is a Linear over the flattened
+ * patch, in the processor's C, T, row, col order */
+ v->embed.O = v->d; v->embed.I = v->patch_in;
+ if (g_dense_bits == 32) {
+ float *w = xmalloc((size_t)v->d * v->patch_in * sizeof(float), "patch embed");
+ st_read_f32(S, "visual.patch_embed.proj.weight", w, 1);
+ v->embed.fmt = DW_F32; v->embed.w = w;
+ } else {
+ uint16_t *b16 = xmalloc((size_t)v->d * v->patch_in * sizeof(uint16_t), "patch embed");
+ st_read_raw_cap(S, "visual.patch_embed.proj.weight", b16, (int64_t)v->d * v->patch_in * 2, 1);
+ v->embed.fmt = DW_BF16; v->embed.w = b16;
+ }
+ int qkv_rows = (v->heads + 2 * v->kvh) * v->hd;
+ for (int i = 0; i < v->depth; i++) {
+ VBlock *b = &v->b[i];
+ char nm[512];
+#define VN(fmt_) (snprintf(nm, sizeof(nm), "visual.blocks.%d." fmt_, i), nm)
+ b->norm1 = load_vec(S, VN("norm1.weight"), v->d);
+ b->norm2 = load_vec(S, VN("norm2.weight"), v->d);
+ vis_mat(&b->qkv, S, VN("attn.qkv.weight"), qkv_rows, v->d);
+ b->qkv_b = load_vec(S, VN("attn.qkv.bias"), qkv_rows);
+ vis_mat(&b->proj, S, VN("attn.proj.weight"), v->d, v->heads * v->hd);
+ b->proj_b = load_vec(S, VN("attn.proj.bias"), v->d);
+ vis_mat(&b->gate, S, VN("mlp.gate_proj.weight"), v->inter, v->d);
+ b->gate_b = load_vec(S, VN("mlp.gate_proj.bias"), v->inter);
+ vis_mat(&b->up, S, VN("mlp.up_proj.weight"), v->inter, v->d);
+ b->up_b = load_vec(S, VN("mlp.up_proj.bias"), v->inter);
+ vis_mat(&b->down, S, VN("mlp.down_proj.weight"), v->d, v->inter);
+ b->down_b = load_vec(S, VN("mlp.down_proj.bias"), v->d);
+ if (use_sink && !b->full) b->sinks = load_vec(S, VN("attn.sinks"), v->heads);
+#undef VN
+ }
+ int merged = v->d * v->merge * v->merge;
+ v->ln_w = load_vec(S, "visual.merger.ln_q.weight", v->d);
+ v->ln_b = vis_vec_opt(S, "visual.merger.ln_q.bias", v->d);
+ vis_mat(&v->fc1, S, "visual.merger.mlp.0.weight", merged, merged);
+ v->fc1_b = vis_vec_opt(S, "visual.merger.mlp.0.bias", merged);
+ vis_mat(&v->fc2, S, "visual.merger.mlp.2.weight", v->out, merged);
+ v->fc2_b = vis_vec_opt(S, "visual.merger.mlp.2.bias", v->out);
+ return v;
+}
+
+static void add_bias(float *y, const float *b, int n, int dim) {
+ if (!b) return;
+ for (int t = 0; t < n; t++) for (int i = 0; i < dim; i++) y[(size_t)t * dim + i] += b[i];
+}
+
+/* new[unit k] = old[unit idx[k]], units of 4 tokens (the 2x2 merge) */
+static void permute_units(float *x, const int *idx, int units, int width) {
+ size_t unit = (size_t)4 * width;
+ float *tmp = xmalloc((size_t)units * unit * sizeof(float), "vision permute");
+ for (int k = 0; k < units; k++) memcpy(tmp + (size_t)k * unit, x + (size_t)idx[k] * unit, unit * sizeof(float));
+ memcpy(x, tmp, (size_t)units * unit * sizeof(float));
+ free(tmp);
+}
+
+static void vis_rope(float *x, int nh, int hd, const float *ang) {
+ int half = hd / 2;
+ for (int h = 0; h < nh; h++) {
+ float *v = x + (size_t)h * hd;
+ for (int i = 0; i < half; i++) {
+ float cs = cosf(ang[i]), sn = sinf(ang[i]);
+ float a = v[i], b = v[i + half];
+ v[i] = a * cs - b * sn;
+ v[i + half] = b * cs + a * sn;
+ }
+ }
+}
+
+static float *vision_run(Model *m, const float *patches, int gh, int gw, int *n_rows) {
+ Vision *v = g_vision;
+ (void)m;
+ *n_rows = 0;
+ if (!v || gh < 2 || gw < 2 || gh % 2 || gw % 2) return NULL;
+ double t0 = now_s();
+ int N = gh * gw, d = v->d, hd = v->hd, nh = v->heads, kvh = v->kvh, group = nh / kvh;
+ int units = N / 4, uh = gh / 2, uw = gw / 2;
+ float *x = xmalloc((size_t)N * d * sizeof(float), "vision tokens");
+ dw_matmul(x, patches, N, &v->embed);
+ /* rotary angles per token, in the processor's block-major order */
+ int rdim = hd / 2, nf = rdim / 2;
+ float *ang = xmalloc((size_t)N * rdim * sizeof(float), "vision angles");
+ float *inv = xmalloc((size_t)nf * sizeof(float), "vision inv_freq");
+ for (int j = 0; j < nf; j++) inv[j] = 1.0f / powf(10000.0f, (float)(2 * j) / (float)rdim);
+ for (int n = 0; n < N; n++) {
+ int u = n / 4, r = (n % 4) / 2, cc = n % 2;
+ int hp = (u / uw) * 2 + r, wp = (u % uw) * 2 + cc;
+ for (int j = 0; j < nf; j++) {
+ ang[(size_t)n * rdim + j] = (float)hp * inv[j];
+ ang[(size_t)n * rdim + nf + j] = (float)wp * inv[j];
+ }
+ }
+ free(inv);
+ /* column order of the merge units and its inverse */
+ int *col = xmalloc((size_t)units * sizeof(int), "vision col order");
+ int *rev = xmalloc((size_t)units * sizeof(int), "vision row order");
+ for (int k = 0; k < units; k++) col[k] = (k % uh) * uw + k / uh;
+ for (int k = 0; k < units; k++) rev[col[k]] = k;
+ float *ang_col = xmalloc((size_t)N * rdim * sizeof(float), "vision col angles");
+ memcpy(ang_col, ang, (size_t)N * rdim * sizeof(float));
+ permute_units(ang_col, col, units, rdim);
+
+ int qd = nh * hd, kd = kvh * hd, rows = qd + 2 * kd;
+ float *xn = xmalloc((size_t)N * d * sizeof(float), "vision normed");
+ float *qkv = xmalloc((size_t)N * rows * sizeof(float), "vision qkv");
+ float *ctx = xmalloc((size_t)N * qd * sizeof(float), "vision attention");
+ float *tmp = xmalloc((size_t)N * d * sizeof(float), "vision block out");
+ float *g = xmalloc((size_t)N * v->inter * sizeof(float), "vision mlp gate");
+ float *u = xmalloc((size_t)N * v->inter * sizeof(float), "vision mlp up");
+ float scale = 1.0f / sqrtf((float)hd);
+ int prev = -2;
+ for (int bi = 0; bi < v->depth; bi++) {
+ VBlock *b = &v->b[bi];
+ if (b->type == 1 && prev != 1) permute_units(x, col, units, d);
+ if (b->type != 1 && prev == 1) permute_units(x, rev, units, d);
+ prev = b->type;
+ const float *A = b->type == 1 ? ang_col : ang;
+ for (int n = 0; n < N; n++) rmsnorm(xn + (size_t)n * d, x + (size_t)n * d, b->norm1, d, v->eps);
+ dw_matmul(qkv, xn, N, &b->qkv);
+ add_bias(qkv, b->qkv_b, N, rows);
+ for (int n = 0; n < N; n++) {
+ vis_rope(qkv + (size_t)n * rows, nh, hd, A + (size_t)n * rdim);
+ vis_rope(qkv + (size_t)n * rows + qd, kvh, hd, A + (size_t)n * rdim);
+ }
+ int window = b->full ? -1 : v->window;
+ #pragma omp parallel
+ {
+ float *score = xmalloc((size_t)N * sizeof(float), "vision scores");
+ #pragma omp for collapse(2) schedule(static)
+ for (int i = 0; i < N; i++) {
+ for (int h = 0; h < nh; h++) {
+ int kh = h / group;
+ int lo = 0, hi = N - 1;
+ if (window >= 0) { lo = i - window > 0 ? i - window : 0; hi = i + window < N - 1 ? i + window : N - 1; }
+ const float *q = qkv + (size_t)i * rows + (size_t)h * hd;
+ float best = -INFINITY;
+ for (int j = lo; j <= hi; j++) {
+ const float *kr = qkv + (size_t)j * rows + qd + (size_t)kh * hd;
+ float s = 0;
+ for (int e = 0; e < hd; e++) s += q[e] * kr[e];
+ s *= scale;
+ if (j == 0 && b->sinks) s += b->sinks[h];
+ score[j - lo] = s;
+ if (s > best) best = s;
+ }
+ float den = 0;
+ for (int j = lo; j <= hi; j++) { score[j - lo] = expf(score[j - lo] - best); den += score[j - lo]; }
+ float *o = ctx + (size_t)i * qd + (size_t)h * hd;
+ for (int e = 0; e < hd; e++) o[e] = 0;
+ for (int j = lo; j <= hi; j++) {
+ const float *vr = qkv + (size_t)j * rows + qd + kd + (size_t)kh * hd;
+ float w = score[j - lo] / den;
+ for (int e = 0; e < hd; e++) o[e] += w * vr[e];
+ }
+ }
+ }
+ free(score);
+ }
+ dw_matmul(tmp, ctx, N, &b->proj);
+ add_bias(tmp, b->proj_b, N, d);
+ for (size_t i = 0; i < (size_t)N * d; i++) x[i] += tmp[i];
+ for (int n = 0; n < N; n++) rmsnorm(xn + (size_t)n * d, x + (size_t)n * d, b->norm2, d, v->eps);
+ dw_matmul(g, xn, N, &b->gate);
+ add_bias(g, b->gate_b, N, v->inter);
+ dw_matmul(u, xn, N, &b->up);
+ add_bias(u, b->up_b, N, v->inter);
+ for (size_t i = 0; i < (size_t)N * v->inter; i++) g[i] = silu(g[i]) * u[i];
+ dw_matmul(tmp, g, N, &b->down);
+ add_bias(tmp, b->down_b, N, d);
+ for (size_t i = 0; i < (size_t)N * d; i++) x[i] += tmp[i];
+ }
+ if (prev == 1) permute_units(x, rev, units, d);
+ /* merger: LayerNorm per token, 2x2 units concatenated, Linear, GELU, Linear */
+ for (int n = 0; n < N; n++) {
+ float *r = x + (size_t)n * d;
+ float mean = 0, var = 0;
+ for (int i = 0; i < d; i++) mean += r[i];
+ mean /= d;
+ for (int i = 0; i < d; i++) { float z = r[i] - mean; var += z * z; }
+ var /= d;
+ float inv_std = 1.0f / sqrtf(var + 1e-6f);
+ for (int i = 0; i < d; i++) r[i] = (r[i] - mean) * inv_std * v->ln_w[i] + (v->ln_b ? v->ln_b[i] : 0.0f);
+ }
+ int merged = 4 * d;
+ float *h1 = xmalloc((size_t)units * merged * sizeof(float), "vision merger");
+ dw_matmul(h1, x, units, &v->fc1);
+ add_bias(h1, v->fc1_b, units, merged);
+ for (size_t i = 0; i < (size_t)units * merged; i++) h1[i] = 0.5f * h1[i] * (1.0f + erff(h1[i] * 0.70710678118654752f));
+ float *rows_out = xmalloc((size_t)units * v->out * sizeof(float), "image rows");
+ dw_matmul(rows_out, h1, units, &v->fc2);
+ add_bias(rows_out, v->fc2_b, units, v->out);
+ free(h1); free(x); free(ang); free(ang_col); free(col); free(rev);
+ free(xn); free(qkv); free(ctx); free(tmp); free(g); free(u);
+ *n_rows = units;
+ /* MIMO_VISION_DUMP=<file>: the rows the tower produced, f32 [units, hidden],
+ * for tools/mimo_real_check.py to hold against Xiaomi's ViT */
+ const char *dump = getenv("MIMO_VISION_DUMP");
+ if (dump) {
+ FILE *f = fopen(dump, "wb");
+ if (f) { fwrite(rows_out, sizeof(float), (size_t)units * v->out, f); fclose(f); }
+ }
+ if (env_int("MIMO_STATS", 0))
+ fprintf(stderr, "[mimo] image %dx%d patches -> %d rows in %.2fs\n", gh, gw, units, now_s() - t0);
+ return rows_out;
+}
diff --git a/c/tests/mimo_tiny_harness.py b/c/tests/mimo_tiny_harness.py
new file mode 100644
index 00000000..2df3bf8b
--- /dev/null
+++ b/c/tests/mimo_tiny_harness.py
@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+"""Token-exact checks for the MiMo-V2.6 engine against an independent oracle.
+
+Dependency-free (stdlib only). The fixture is generated by
+tools/make_mimo_tiny.py (numpy, deterministic); the expected tokens in
+ref.json come from Xiaomi's own modeling code via tools/make_mimo_ref.py (run
+offline by a maintainer, vendor files pinned by SHA-256), so the engine and its
+oracle never share an author.
+
+What is checked, all token-exact (argmax, no tolerances):
+ - greedy generation for three cases, each longer than the 8-token window
+ - teacher forcing at every position via the MIMO_LOGITS dump
+ - the same tokens from every exact configuration: native FP8/BF16 dense
+ weights, an expert cache that evicts at every layer, prefill in blocks of 3
+ and of 1 (the sliding-window ring across blocks), buffered expert reads
+ - determinism (two identical runs, identical tokens)
+ - with the vision fixture (make_mimo_tiny.py --vision): the picture through
+ the tower, greedy and teacher-forced, f32 and native BF16, and the negated
+ picture giving the reference's own answer for it
+ - the oracle BITES: one corrupted expert scale byte per layer-2 expert, and
+ separately the qkv scale rows read as one continuous grid instead of per
+ chunk, must each change the output (guards against a comparison that
+ silently accepts anything, and pins the pre-sharded qkv layout)
+
+The engine runs in its exact-math configuration (ref.json's engine_env): dense
+weights widened to f32 and float expert matmuls. The fixture ships no
+tokenizer.json on purpose: the engine then prints raw token ids on stdout.
+"""
+
+import argparse
+import array
+import hashlib
+import json
+import os
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def run_engine(binary, fixture, prompt_ids, max_new, env, extra_env=None,
+ image=None):
+ run_env = dict(os.environ, **env)
+ if extra_env:
+ run_env.update(extra_env)
+ command = [binary, str(fixture), "--ids", " ".join(map(str, prompt_ids)),
+ "--ngen", str(max_new)]
+ if image:
+ path, grid_h, grid_w = image
+ command += ["--image", str(path), "--grid", str(grid_h), str(grid_w)]
+ result = subprocess.run(command, env=run_env, capture_output=True,
+ text=True, timeout=600)
+ if result.returncode != 0:
+ raise AssertionError(
+ f"engine exited {result.returncode}:\n{result.stderr[-2000:]}")
+ tokens = []
+ for word in result.stdout.split():
+ if not word.lstrip("-").isdigit():
+ break
+ tokens.append(int(word))
+ return tokens
+
+
+def check_greedy(binary, fixture, reference):
+ for name, case in reference["cases"].items():
+ got = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ want = case["greedy_new_ids"]
+ if got != want:
+ raise AssertionError(f"greedy mismatch in case '{name}': engine "
+ f"{got} vs reference {want}")
+ print(f"ok greedy {name}: {len(want)}/{len(want)} tokens")
+
+
+def check_exact_variants(binary, fixture, reference):
+ """Every configuration that is exact by construction must give the same
+ tokens: the native FP8/BF16 dense path, an expert cache no larger than one
+ routing step (every layer evicts), and prefill in odd-sized blocks and one
+ token at a time (the window ring is filled and read across blocks)."""
+ variants = {
+ "native dense, cache 4, prefill blocks of 3":
+ {"MIMO_DENSE_BITS": "0", "MIMO_CAP": "4", "MIMO_CHUNK": "3"},
+ "prefill one token at a time": {"MIMO_CHUNK": "1"},
+ "buffered expert reads": {"MIMO_DIRECT": "0"},
+ }
+ for label, extra in variants.items():
+ env = dict(reference["engine_env"], **extra)
+ for name, case in reference["cases"].items():
+ got = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], env)
+ if got != case["greedy_new_ids"]:
+ raise AssertionError(f"{label}: greedy mismatch in case '{name}': "
+ f"engine {got} vs reference "
+ f"{case['greedy_new_ids']}")
+ print(f"ok {label}: identical in {len(reference['cases'])} cases")
+
+
+def check_determinism(binary, fixture, reference):
+ case = reference["cases"]["short"]
+ first = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ second = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ if first != second:
+ raise AssertionError(f"non-deterministic greedy: {first} vs {second}")
+ print("ok determinism: two identical runs, identical tokens")
+
+
+def teacher_forced(binary, fixture, ids, env, vocab, image=None):
+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:
+ path = tmp.name
+ try:
+ run_engine(binary, fixture, ids, 0, env, extra_env={"MIMO_LOGITS": path},
+ image=image)
+ raw = array.array("f")
+ with open(path, "rb") as fh:
+ raw.frombytes(fh.read())
+ finally:
+ os.unlink(path)
+ if len(raw) != len(ids) * vocab:
+ raise AssertionError(f"MIMO_LOGITS dumped {len(raw)} floats, expected "
+ f"{len(ids) * vocab}")
+ return [max(range(vocab), key=raw[p * vocab:(p + 1) * vocab].__getitem__)
+ for p in range(len(ids))]
+
+
+def check_teacher_forcing(binary, fixture, reference, vocab):
+ for name, case in reference["cases"].items():
+ got = teacher_forced(binary, fixture, case["greedy_full_ids"],
+ reference["engine_env"], vocab)
+ want = case["teacher_forcing_ids"]
+ if got != want:
+ bad = next(i for i, (a, b) in enumerate(zip(got, want)) if a != b)
+ raise AssertionError(
+ f"teacher forcing mismatch in case '{name}' at position {bad}: "
+ f"engine {got[bad]} vs reference {want[bad]}")
+ print(f"ok teacher forcing {name}: {len(want)}/{len(want)} positions")
+
+
+def patch_fixture(fixture, edit):
+ """A copy of the fixture with edit(raw_bytes_of_shard1, header, base)
+ applied to the first shard; returns the copy's path."""
+ copy = Path(tempfile.mkdtemp(prefix="mimo-tiny-bite-")) / "fixture"
+ shutil.copytree(fixture, copy)
+ shard = copy / "model-00001-of-00002.safetensors"
+ raw = bytearray(shard.read_bytes())
+ header_len = struct.unpack("<Q", bytes(raw[:8]))[0]
+ header = json.loads(bytes(raw[8:8 + header_len]))
+ edit(raw, header, 8 + header_len)
+ shard.write_bytes(bytes(raw))
+ return copy
+
+
+def diverged_cases(binary, fixture, reference):
+ count = 0
+ for case in reference["cases"].values():
+ got = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ if got != case["greedy_new_ids"]:
+ count += 1
+ return count
+
+
+def check_oracle_bites(binary, fixture, reference, experts):
+ def expert_scales(raw, header, base):
+ for expert in range(experts):
+ lo, _ = header[f"model.layers.2.mlp.experts.{expert}"
+ f".gate_proj.weight_scale"]["data_offsets"]
+ raw[base + lo] = (raw[base + lo] + 3) % 256 # x8 on one group
+ copy = patch_fixture(fixture, expert_scales)
+ try:
+ count = diverged_cases(binary, copy, reference)
+ finally:
+ shutil.rmtree(copy.parent)
+ if not count:
+ raise AssertionError("corrupted expert scales produced identical tokens "
+ "in every case: the oracle does not bite")
+ print(f"ok oracle bites: corrupted expert scales diverge in "
+ f"{count}/{len(reference['cases'])} cases")
+
+ # The full layers' qkv scale grid is tiled per chunk (4 rows where one
+ # continuous grid would have 3). Swapping two of its rows is invisible to
+ # a reader that ignores the chunk layout only if it never uses them; the
+ # engine must change its output.
+ def swap_qkv_scale_rows(raw, header, base):
+ meta = header["model.layers.0.self_attn.qkv_proj.weight_scale_inv"]
+ lo, hi = meta["data_offsets"]
+ cols = meta["shape"][1]
+ row = cols * 4
+ grid = raw[base + lo:base + hi]
+ first, second = grid[row:2 * row], grid[2 * row:3 * row]
+ grid[row:2 * row], grid[2 * row:3 * row] = second, first
+ raw[base + lo:base + hi] = grid
+ copy = patch_fixture(fixture, swap_qkv_scale_rows)
+ try:
+ count = diverged_cases(binary, copy, reference)
+ finally:
+ shutil.rmtree(copy.parent)
+ if not count:
+ raise AssertionError("swapping qkv scale rows changed nothing: the "
+ "per-chunk scale grid is not being read")
+ print(f"ok qkv scale grid bites: {count}/{len(reference['cases'])} cases")
+
+
+def check_image(binary, fixture, reference, vocab):
+ """The fixture's picture, preprocessed the gateway's way (patches.f32,
+ pinned by hash to what the official Qwen2-VL processor produced), through
+ the engine's tower: the same tokens as Xiaomi's ViT and language model,
+ greedy and teacher-forced, on the f32 and the native BF16 tower. Negating
+ the patches must give exactly the tokens the reference gave for the
+ negated picture -- the image is read, not just tolerated."""
+ image = reference["image"]
+ patches = fixture / "patches.f32"
+ digest = hashlib.sha256(patches.read_bytes()).hexdigest()
+ if digest != image["patches_sha256"]:
+ raise AssertionError("patches.f32 differs from the patches the reference "
+ "checked against the official processor")
+ spec = (patches, image["grid_h"], image["grid_w"])
+ for label, extra in (("f32", {}), ("native BF16 tower", {"MIMO_DENSE_BITS": "0"})):
+ env = dict(reference["engine_env"], **extra)
+ got = run_engine(binary, fixture, image["prompt_ids"],
+ image["max_new_tokens"], env, image=spec)
+ if got != image["greedy_new_ids"]:
+ raise AssertionError(f"image greedy mismatch ({label}): engine {got} vs "
+ f"reference {image['greedy_new_ids']}")
+ print(f"ok image greedy ({label}): {len(got)}/{len(got)} tokens")
+ got = teacher_forced(binary, fixture, image["greedy_full_ids"],
+ reference["engine_env"], vocab, image=spec)
+ if got != image["teacher_forcing_ids"]:
+ bad = next(i for i, (a, b) in enumerate(zip(got, image["teacher_forcing_ids"]))
+ if a != b)
+ raise AssertionError(f"image teacher forcing mismatch at position {bad}")
+ print(f"ok image teacher forcing: {len(got)}/{len(got)} positions")
+ negated = Path(tempfile.mkdtemp(prefix="mimo-neg-")) / "patches.f32"
+ try:
+ values = array.array("f")
+ values.frombytes(patches.read_bytes())
+ negated.write_bytes(array.array("f", (-v for v in values)).tobytes())
+ got = run_engine(binary, fixture, image["prompt_ids"],
+ image["max_new_tokens"], reference["engine_env"],
+ image=(negated, image["grid_h"], image["grid_w"]))
+ finally:
+ shutil.rmtree(negated.parent)
+ if got != image["negated_image_new_ids"]:
+ raise AssertionError(f"negated image: engine {got} vs reference "
+ f"{image['negated_image_new_ids']}")
+ print("ok image bites: the negated picture gives the reference's other answer")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--binary", required=True)
+ parser.add_argument("--fixture", required=True)
+ args = parser.parse_args()
+
+ fixture = Path(args.fixture)
+ reference = json.loads((fixture / "ref.json").read_text())
+ if reference.get("source") != "xiaomi-vendor":
+ raise AssertionError("ref.json must come from the vendor implementation "
+ f"(source={reference.get('source')!r})")
+ config = json.loads((fixture / "config.json").read_text())
+ if (fixture / "tokenizer.json").exists():
+ raise AssertionError("fixture must not ship tokenizer.json: raw-id "
+ "stdout is the comparison channel")
+
+ check_greedy(args.binary, fixture, reference)
+ check_exact_variants(args.binary, fixture, reference)
+ check_determinism(args.binary, fixture, reference)
+ check_teacher_forcing(args.binary, fixture, reference, config["vocab_size"])
+ check_oracle_bites(args.binary, fixture, reference,
+ config["n_routed_experts"])
+ if "image" in reference:
+ check_image(args.binary, fixture, reference, config["vocab_size"])
+ print("mimo-tiny-check: all checks passed")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/make_mimo_ref.py b/c/tools/make_mimo_ref.py
new file mode 100644
index 00000000..ae8041dd
--- /dev/null
+++ b/c/tools/make_mimo_ref.py
@@ -0,0 +1,503 @@
+#!/usr/bin/env python3
+"""Produce ref.json for the tiny MiMo-V2.6 fixture from Xiaomi's own code.
+
+Maintainer tool, run offline once per fixture change; CI never executes it
+(tests/mimo_tiny_harness.py only reads the committed ref.json). The reference
+tokens come from the vendor's published implementation (modeling_mimo_v2.py on
+the MiMo-V2.6-Flash-MOPD Hugging Face repo, pinned below by SHA-256), so the
+engine and its oracle never share an author.
+
+Parity is by construction:
+- the engine is tested with its dense weights at f32 (ref.json engine_env) and
+ float expert matmuls, and this script dequantizes THE SAME BYTES into the
+ vendor model's f32 Linear weights: BF16 by widening, FP8 e4m3 through torch's
+ own float8_e4m3fn type times the 128x128 block scale, MXFP4 as
+ e2m1 * 2^(scale-127) with the low nibble on the even column;
+- the fused qkv_proj is stored pre-sharded ([Q_0|K_0|V_0|Q_1|...], scale grid
+ tiled per chunk) as on the release, while the vendor module wants [Q|K|V]:
+ this script de-interleaves it with the layout vLLM documents for the release
+ (vllm/model_executor/models/mimo_v2.py, _shard_fp8_qkv_proj), so a reader
+ that gets the layout wrong disagrees with the oracle;
+- token-level argmax must then agree wherever the logit margin exceeds float
+ noise; the generator engineers wide margins and this script VERIFIES them per
+ emitted position (--min-margin, refuse on violation).
+
+Requires: torch (CPU is fine), transformers >= 5.3, numpy. Network only if the
+vendor files are not already in --vendor-dir.
+"""
+
+import argparse
+import hashlib
+import importlib.machinery
+import importlib.util
+import json
+import struct
+import sys
+import urllib.request
+from pathlib import Path
+
+import numpy as np
+
+VENDOR_REPO = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
+VENDOR_REVISION = "2479e2d0029eca9a34cc7e7f55a121925f81908e"
+VENDOR_FILES = {
+ "configuration_mimo_v2.py":
+ "773062ac9850b908eb54751b3e4dbe00e653c0e80595599409e97d0c1af2ce3e",
+ "modeling_mimo_v2.py":
+ "a8c3cb3aae473bcc15f023010547c919f15eba6546e6ed7efb61a8937b12f3ad",
+}
+BLOCK = 128
+
+MX4_LUT = np.array([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
+ -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
+ dtype=np.float32)
+
+
+def read_fixture(fixture):
+ """{name: (dtype string, raw numpy array)} across the indexed shards."""
+ index = json.loads((fixture / "model.safetensors.index.json").read_text())
+ raw_dtypes = {"F32": np.float32, "U8": np.uint8, "BF16": np.uint16,
+ "F8_E4M3": np.uint8}
+ out = {}
+ for shard in sorted(set(index["weight_map"].values())):
+ with open(fixture / shard, "rb") as fh:
+ header_len = struct.unpack("<Q", fh.read(8))[0]
+ header = json.loads(fh.read(header_len))
+ raw = fh.read()
+ for name, meta in header.items():
+ if name == "__metadata__":
+ continue
+ lo, hi = meta["data_offsets"]
+ dtype = raw_dtypes[meta["dtype"]]
+ arr = np.frombuffer(raw, dtype=dtype,
+ count=(hi - lo) // np.dtype(dtype).itemsize,
+ offset=lo).reshape(meta["shape"]).copy()
+ out[name] = (meta["dtype"], arr)
+ return out
+
+
+def bf16_to_f32(bits):
+ return (bits.astype(np.uint32) << 16).view(np.float32)
+
+
+def fp8_to_f32(codes):
+ """e4m3fn decode by torch's own float8 type (not colibri's decoder)."""
+ import torch
+ return torch.from_numpy(codes).view(torch.float8_e4m3fn).float().numpy()
+
+
+def mxfp4_dequant(packed, scale):
+ rows, half = packed.shape
+ cols = half * 2
+ codes = np.empty((rows, cols), dtype=np.uint8)
+ codes[:, 0::2] = packed & 0x0F
+ codes[:, 1::2] = packed >> 4
+ values = MX4_LUT[codes]
+ exponents = np.ldexp(np.float32(1.0), scale.astype(np.int32) - 127)
+ return (values.reshape(rows, cols // 32, 32) *
+ exponents[:, :, None]).reshape(rows, cols).astype(np.float32)
+
+
+def fp8_block_dequant(codes, scale_inv, row_scale_index=None):
+ """w[r, c] = fp8(codes[r, c]) * scale_inv[row_block(r), c // 128]."""
+ w = fp8_to_f32(codes)
+ rows, cols = w.shape
+ if row_scale_index is None:
+ row_scale_index = np.arange(rows) // BLOCK
+ col_block = np.arange(cols) // BLOCK
+ return (w * scale_inv[row_scale_index][:, col_block]).astype(np.float32)
+
+
+def qkv_contiguous(codes, scale_inv, heads, kv_heads, head_dim, v_head_dim,
+ chunks):
+ """Pre-sharded fused qkv (vLLM's documented layout) -> [Q|K|V] f32."""
+ rows = codes.shape[0]
+ per_chunk = rows // chunks
+ q_rows = (heads // chunks) * head_dim
+ k_rows = (kv_heads // chunks) * head_dim
+ v_rows = (kv_heads // chunks) * v_head_dim
+ assert q_rows + k_rows + v_rows == per_chunk, (rows, chunks)
+ chunk_scale_rows = -(-per_chunk // BLOCK)
+ r = np.arange(rows)
+ if scale_inv.shape[0] == chunks * chunk_scale_rows:
+ index = (r // per_chunk) * chunk_scale_rows + (r % per_chunk) // BLOCK
+ elif scale_inv.shape[0] == -(-rows // BLOCK):
+ index = r // BLOCK
+ else:
+ raise SystemExit(f"qkv scale grid {scale_inv.shape} fits neither layout")
+ w = fp8_block_dequant(codes, scale_inv, index)
+ parts = [w[c * per_chunk:(c + 1) * per_chunk] for c in range(chunks)]
+ q = np.concatenate([p[:q_rows] for p in parts])
+ k = np.concatenate([p[q_rows:q_rows + k_rows] for p in parts])
+ v = np.concatenate([p[q_rows + k_rows:] for p in parts])
+ return np.concatenate([q, k, v]).astype(np.float32)
+
+
+def vision_state(tensors, model):
+ """visual.* widened to f32, plus the zeros the release does not store: the
+ merger's two Linear biases and its LayerNorm bias (vLLM builds that merger
+ with bias=False; the HF module declares them and would otherwise keep its
+ random init)."""
+ import torch
+ state = {}
+ for name, (dtype, arr) in tensors.items():
+ if name.startswith("visual."):
+ state[name] = torch.from_numpy(bf16_to_f32(arr) if dtype == "BF16"
+ else arr.astype(np.float32))
+ for name, param in model.named_parameters():
+ if name.startswith("visual.merger.") and name.endswith(".bias") \
+ and name not in state:
+ state[name] = torch.zeros_like(param)
+ return state
+
+
+def to_state_dict(tensors, config):
+ import torch
+ chunks = config["num_key_value_heads"]
+ state = {}
+ consumed = set()
+ for name, (dtype, arr) in tensors.items():
+ if name in consumed:
+ continue
+ if name.endswith(".weight_scale_inv") or name.endswith(".weight_scale"):
+ continue
+ if name.startswith("visual."):
+ continue
+ if dtype == "BF16":
+ value = bf16_to_f32(arr)
+ elif dtype == "F32":
+ value = arr.astype(np.float32)
+ elif name.endswith("qkv_proj.weight"):
+ layer = int(name.split(".")[2])
+ swa = config["hybrid_layer_pattern"][layer] == 1
+ heads = config["swa_num_attention_heads" if swa else "num_attention_heads"]
+ kv = config["swa_num_key_value_heads" if swa else "num_key_value_heads"]
+ value = qkv_contiguous(arr, tensors[name + "_scale_inv"][1], heads,
+ kv, config["head_dim"], config["v_head_dim"],
+ chunks)
+ elif dtype == "F8_E4M3":
+ value = fp8_block_dequant(arr, tensors[name + "_scale_inv"][1])
+ elif dtype == "U8" and ".experts." in name:
+ value = mxfp4_dequant(arr, tensors[name + "_scale"][1])
+ else:
+ raise SystemExit(f"{name}: unexpected {dtype}")
+ state[name] = torch.from_numpy(np.ascontiguousarray(value))
+ return state
+
+
+def fetch_vendor(vendor_dir):
+ vendor_dir.mkdir(parents=True, exist_ok=True)
+ for name, expected in VENDOR_FILES.items():
+ path = vendor_dir / name
+ if not path.exists():
+ url = (f"https://huggingface.co/{VENDOR_REPO}/raw/"
+ f"{VENDOR_REVISION}/{name}")
+ print(f"[ref] downloading {url}", file=sys.stderr)
+ with urllib.request.urlopen(url, timeout=60) as resp:
+ path.write_bytes(resp.read())
+ digest = hashlib.sha256(path.read_bytes()).hexdigest()
+ if digest != expected:
+ raise SystemExit(
+ f"{name}: sha256 {digest} does not match the pinned {expected}; "
+ f"the vendor file changed. Review it, then update VENDOR_FILES.")
+
+
+def load_vendor(vendor_dir):
+ """The vendor files as a synthetic package, so the relative import of the
+ configuration module works whatever the directory is called."""
+ pkg = "mimo_v2_vendor"
+ spec = importlib.machinery.ModuleSpec(pkg, None, is_package=True)
+ package = importlib.util.module_from_spec(spec)
+ package.__path__ = [str(vendor_dir)]
+ sys.modules[pkg] = package
+
+ def load(stem):
+ file_spec = importlib.util.spec_from_file_location(
+ f"{pkg}.{stem}", vendor_dir / f"{stem}.py")
+ module = importlib.util.module_from_spec(file_spec)
+ sys.modules[f"{pkg}.{stem}"] = module
+ file_spec.loader.exec_module(module)
+ return module
+
+ config_module = load("configuration_mimo_v2")
+ modeling = load("modeling_mimo_v2")
+ shim_mask_builders(modeling)
+ return config_module, modeling
+
+
+def shim_mask_builders(modeling):
+ """The vendor file targets transformers 5.3, whose mask builders take
+ `input_embeds` and `cache_position`; later releases renamed the first
+ (`inputs_embeds`, 5.6 dropped the old name) and derive the second
+ themselves. Rename, drop what the installed builder no longer takes, and
+ nothing else: the masks, and every number, stay the library's own. The
+ reference runs without a cache from position 0, where the derived positions
+ are the passed ones; the committed ref.json is identical under 5.3 and 5.12."""
+ import inspect
+ for name in ("create_causal_mask", "create_sliding_window_causal_mask"):
+ builder = getattr(modeling, name)
+ params = inspect.signature(builder).parameters
+ if "input_embeds" in params:
+ continue
+
+ def adapted(*args, _builder=builder, _params=params, **kwargs):
+ if "input_embeds" in kwargs:
+ kwargs["inputs_embeds"] = kwargs.pop("input_embeds")
+ if not any(p.kind == p.VAR_KEYWORD for p in _params.values()):
+ kwargs = {k: v for k, v in kwargs.items() if k in _params}
+ return _builder(*args, **kwargs)
+ setattr(modeling, name, adapted)
+
+
+def build_model(config_module, modeling, config):
+ import torch
+ cfg = config_module.MiMoV2Config(**dict(config, use_cache=False))
+ cfg._attn_implementation = "eager" # the sink path is eager-only
+ torch.manual_seed(0) # every weight is overwritten below
+ model = modeling.MiMoV2ForCausalLM(cfg).float().eval()
+ return model
+
+
+def check_rope(model, config):
+ """The two rotary tables must be what the config says (theta 1e7 on full
+ layers, 1e4 on SWA layers, over int(head_dim * 0.334) dims): refuse a
+ transformers version that silently standardizes them away."""
+ dim = int(config["head_dim"] * config["partial_rotary_factor"])
+ for module, theta in ((model.model.rotary_emb, config["rope_theta"]),
+ (model.model.swa_rotary_emb, config["swa_rope_theta"])):
+ want = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim))
+ got = module.inv_freq.double().numpy()
+ if got.shape != want.shape or not np.allclose(got, want, rtol=1e-6):
+ raise SystemExit(f"rotary table {got} is not theta={theta} over "
+ f"{dim} dims: this transformers version changes "
+ f"the vendor's RoPE")
+
+
+class RouterWatch:
+ """Records, for every token at every MoE layer, the gap between the k-th
+ and (k+1)-th biased router score: a near-tie there lets float noise swap an
+ expert, which no logit margin downstream can protect against."""
+
+ def __init__(self, model):
+ self.gaps = []
+ for layer in model.model.layers:
+ gate = getattr(layer.mlp, "gate", None)
+ if gate is not None:
+ gate.register_forward_hook(self.hook)
+
+ def hook(self, gate, inputs, _output):
+ import torch
+ x = inputs[0].reshape(-1, inputs[0].shape[-1]).float()
+ scores = torch.sigmoid(x @ gate.weight.float().T)
+ biased = scores + gate.e_score_correction_bias.float()
+ top = torch.topk(biased, gate.top_k + 1, dim=-1).values
+ self.gaps.extend((top[:, -2] - top[:, -1]).tolist())
+
+
+def logits_for(model, ids):
+ import torch
+ with torch.no_grad():
+ return model(input_ids=torch.tensor([ids], dtype=torch.long),
+ use_cache=False).logits[0]
+
+
+def case_reference(model, prompt, max_new, min_margin):
+ import torch
+ sequence = list(prompt)
+ margins = []
+ for _ in range(max_new):
+ logits = logits_for(model, sequence)[-1]
+ top2 = torch.topk(logits, 2).values
+ margins.append(float(top2[0] - top2[1]))
+ sequence.append(int(logits.argmax()))
+ full = logits_for(model, sequence)
+ teacher = full.argmax(-1).tolist()
+ # context-free control: the head applied to the embedding alone
+ with torch.no_grad():
+ emb = model.model.embed_tokens(torch.tensor(sequence))
+ normed = model.model.norm(emb)
+ alone = model.lm_head(normed).argmax(-1).tolist()
+ context_free = sum(a == b for a, b in zip(alone, teacher)) / len(teacher)
+ tf_top2 = torch.topk(full, 2, dim=-1).values
+ tf_margins = (tf_top2[:, 0] - tf_top2[:, 1]).tolist()
+ worst = min(margins + tf_margins)
+ if worst < min_margin:
+ raise SystemExit(
+ f"logit margin {worst:.4f} below --min-margin {min_margin}: the "
+ f"fixture rides a near-tie; re-tune the generator instead of "
+ f"lowering the margin.")
+ return {
+ "prompt_ids": list(prompt),
+ "teacher_forcing_ids": teacher,
+ "greedy_full_ids": sequence,
+ "greedy_new_ids": sequence[len(prompt):],
+ "max_new_tokens": max_new,
+ "min_logit_margin": round(worst, 4),
+ "context_free_agreement": round(context_free, 3),
+ }
+
+
+def vision_reference(model, fixture, config, max_new, min_margin):
+ """The official Qwen2-VL processor on the fixture's picture must give the
+ fixture's patches bit for bit (the gateway's preprocessing); then Xiaomi's
+ ViT and language model answer a prompt that carries the picture."""
+ import torch
+ from PIL import Image
+ from transformers import Qwen2VLImageProcessor
+ processor = Qwen2VLImageProcessor.from_pretrained(str(fixture))
+ image = Image.open(fixture / "image.png").convert("RGB")
+ batch = processor(images=[image], return_tensors="pt")
+ official = batch["pixel_values"].float().numpy()
+ thw = batch["image_grid_thw"]
+ grid = json.loads((fixture / "image.json").read_text())
+ ours = np.fromfile(fixture / "patches.f32", dtype=np.float32).reshape(official.shape)
+ if thw.tolist() != [[1, grid["grid_h"], grid["grid_w"]]]:
+ raise SystemExit(f"grid {thw.tolist()} != fixture {grid}")
+ diff = float(np.abs(official - ours).max())
+ if diff > 1e-6:
+ raise SystemExit(f"gateway patches differ from the official processor by {diff}")
+ n_img = grid["tokens"]
+ prompt = ([5, 7, config["vision_start_token_id"]] + [config["image_token_id"]] * n_img
+ + [config["vision_end_token_id"], 11, 13, 17, 19])
+ pixels = torch.from_numpy(official)
+
+ def logits_img(ids, pv):
+ with torch.no_grad():
+ return model(input_ids=torch.tensor([ids]), pixel_values=pv,
+ image_grid_thw=thw, use_cache=False).logits[0]
+
+ def greedy(pv):
+ seq, margins = list(prompt), []
+ for _ in range(max_new):
+ lg = logits_img(seq, pv)[-1]
+ top2 = torch.topk(lg, 2).values
+ margins.append(float(top2[0] - top2[1]))
+ seq.append(int(lg.argmax()))
+ return seq, margins
+
+ seq, margins = greedy(pixels)
+ full = logits_img(seq, pixels)
+ tf_top2 = torch.topk(full, 2, dim=-1).values
+ worst = min(margins + (tf_top2[:, 0] - tf_top2[:, 1]).tolist())
+ if worst < min_margin:
+ raise SystemExit(f"vision case: logit margin {worst:.4f} below {min_margin}")
+ other, _ = greedy(-pixels)
+ if other[len(prompt):] == seq[len(prompt):]:
+ raise SystemExit("the answer does not depend on the picture: re-tune the tower")
+ with torch.no_grad():
+ rows = model.visual(pixel_values=pixels, grid_thw=thw).float().numpy()
+ return {
+ "prompt_ids": prompt,
+ "grid_h": grid["grid_h"], "grid_w": grid["grid_w"],
+ "patches_sha256": hashlib.sha256(ours.tobytes()).hexdigest(),
+ "patch_max_diff_vs_official_processor": diff,
+ "vision_rows_sum": float(rows.sum()),
+ "vision_rows_abs_sum": float(np.abs(rows).sum()),
+ "teacher_forcing_ids": full.argmax(-1).tolist(),
+ "greedy_full_ids": seq,
+ "greedy_new_ids": seq[len(prompt):],
+ "max_new_tokens": max_new,
+ "min_logit_margin": round(worst, 4),
+ "negated_image_new_ids": other[len(prompt):],
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--fixture", default="./mimo_tiny")
+ parser.add_argument("--vendor-dir", default=None,
+ help="default: <fixture>/.vendor_mimo_v2")
+ parser.add_argument("--output", default=None,
+ help="default: <fixture>/ref.json")
+ parser.add_argument("--min-margin", type=float, default=0.05)
+ parser.add_argument(
+ "--min-router-gap", type=float, default=1e-5,
+ help="router scores are f32 dot products over the hidden size; engine "
+ "and torch sum them in different orders, about 1e-6 apart")
+ parser.add_argument("--max-context-free", type=float, default=0.5)
+ args = parser.parse_args()
+
+ fixture = Path(args.fixture)
+ vendor_dir = (Path(args.vendor_dir) if args.vendor_dir
+ else fixture / ".vendor_mimo_v2")
+ output = Path(args.output) if args.output else fixture / "ref.json"
+ config = json.loads((fixture / "config.json").read_text())
+
+ fetch_vendor(vendor_dir)
+ config_module, modeling = load_vendor(vendor_dir)
+ model = build_model(config_module, modeling, config)
+ check_rope(model, config)
+ state = to_state_dict(read_fixture(fixture), config)
+ if "vision_config" in config:
+ state.update(vision_state(read_fixture(fixture), model))
+ missing, unexpected = model.load_state_dict(state, strict=False)
+ if missing or unexpected:
+ raise SystemExit(f"state dict mismatch:\n missing: {missing}\n"
+ f" unexpected: {unexpected}")
+
+ eos = config["eos_token_id"]
+ vocab = config["vocab_size"]
+ cases = {
+ # every case outgrows the 8-token window; "long" crosses a 64-token
+ # prefill chunk too
+ "short": ([5, 7, 9, 11, 13, 17, 19, 23], 12),
+ "window": ([(5 + i * 7) % vocab for i in range(40)], 6),
+ "long": ([(5 + i * 11) % vocab for i in range(72)], 6),
+ }
+ watch = RouterWatch(model)
+ reference = {}
+ for name, (prompt, max_new) in cases.items():
+ prompt = [t if t != eos else t + 1 for t in prompt]
+ reference[name] = case_reference(model, prompt, max_new,
+ args.min_margin)
+ if eos in reference[name]["greedy_new_ids"]:
+ raise SystemExit(f"case {name}: greedy emitted EOS")
+ print(f"[ref] {name}: prompt {len(prompt)} -> +{max_new}, "
+ f"min margin {reference[name]['min_logit_margin']}, "
+ f"context-free agreement "
+ f"{reference[name]['context_free_agreement']}", file=sys.stderr)
+ if reference[name]["context_free_agreement"] > args.max_context_free:
+ raise SystemExit(
+ f"case {name}: the embedding alone predicts "
+ f"{reference[name]['context_free_agreement']:.0%} of the "
+ f"teacher-forced argmax; the layers barely matter and an engine "
+ f"that skipped them would pass. Re-tune the generator.")
+ router_gap = min(watch.gaps)
+ print(f"[ref] router: {len(watch.gaps)} top-k boundaries, min gap "
+ f"{router_gap:.3g}", file=sys.stderr)
+ if router_gap < args.min_router_gap:
+ raise SystemExit(f"router top-k boundary gap {router_gap:.3g} below "
+ f"--min-router-gap {args.min_router_gap}: float noise "
+ f"could swap an expert. Re-tune the generator.")
+
+ image = None
+ if "vision_config" in config:
+ image = vision_reference(model, fixture, config, 8, args.min_margin)
+ print(f"[ref] image: {image['grid_h']}x{image['grid_w']} patches, margin "
+ f"{image['min_logit_margin']}, patches vs official processor "
+ f"{image['patch_max_diff_vs_official_processor']}", file=sys.stderr)
+
+ import torch
+ import transformers
+ output.write_text(json.dumps({
+ "schema_version": 1,
+ "source": "xiaomi-vendor",
+ "vendor_repo": VENDOR_REPO,
+ "vendor_revision": VENDOR_REVISION,
+ "vendor_files": VENDOR_FILES,
+ "torch_version": torch.__version__,
+ "transformers_version": transformers.__version__,
+ "engine_env": {
+ # exact-math configuration: dense weights widened to f32 instead of
+ # the default int8 at load, float expert matmuls instead of int8
+ # activations (a speed approximation, not an oracle target)
+ "MIMO_DENSE_BITS": "32", "MIMO_IDOT": "0", "COLI_TEMP": "0"},
+ "cases": reference,
+ **({"image": image} if image else {}),
+ }, indent=1) + "\n")
+ print(f"[ref] wrote {output}", file=sys.stderr)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/make_mimo_tiny.py b/c/tools/make_mimo_tiny.py
new file mode 100644
index 00000000..657c6e2d
--- /dev/null
+++ b/c/tools/make_mimo_tiny.py
@@ -0,0 +1,436 @@
+#!/usr/bin/env python3
+"""Generate the tiny MiMo-V2.6 checkpoint used by tests/mimo_tiny_harness.py.
+
+numpy-only and deterministic: CI regenerates the shards from SEED and
+byte-identical weights come out on every platform. The reference tokens in
+ref.json are NOT produced here: they come from Xiaomi's own modeling code
+(tools/make_mimo_ref.py, run offline once, vendor files pinned by SHA-256),
+so the engine and its oracle never share an author.
+
+The fixture stores every tensor the way the released checkpoint does, because
+the storage is where a reading of MiMo goes wrong while a float reference
+still agrees:
+
+ - routed experts are MXFP4 g32 (e2m1 nibble pairs, low nibble = even column,
+ one e8m0 byte per 32 columns), the six tensors of an expert back to back
+ in the file order of the real shards: down, down scale, gate, gate scale,
+ up, up scale;
+ - qkv_proj and the layer-0 MLP are FP8 e4m3 with a F32 weight_scale_inv per
+ 128x128 block;
+ - the fused qkv_proj is PRE-SHARDED in num_key_value_heads chunks
+ [Q_0|K_0|V_0|Q_1|K_1|V_1] with the scale grid tiled per chunk. The full
+ layers' chunk (176 rows) is not a whole number of blocks, so the per-chunk
+ grid has 4 rows where a contiguous one would have 3: a reader that assumes
+ [Q|K|V] with one continuous grid fails here, as it would on the real
+ checkpoint (108 scale rows, not 106, on a Flash full-attention layer);
+ - o_proj, router, norms, sinks, embed and lm_head are BF16.
+
+The geometry crosses every path of mimo.c: full-attention and sliding-window
+layers with different KV head counts (2 and 4 heads over 2 chunks), a window
+of 8 that every case outgrows, the SWA sink logit, partial RoPE with the two
+thetas, the value scale, the dense layer 0 and MoE layers.
+
+No tokenizer.json on purpose: without one the engine prints raw generated
+token ids on stdout, which is what the token-exact test parses. Greedy
+determinism is engineered, then verified by make_mimo_ref.py: the head is
+random at a large scale (wide top-2 gaps), the router's top-k boundary is
+checked for near-ties at every token and layer, and the layers are checked to
+matter (the argmax must differ from the embedding-only one at most positions).
+"""
+
+import argparse
+import json
+import struct
+import sys
+from pathlib import Path
+
+import numpy as np
+
+SEED = 20260930
+
+VOCAB = 320
+HIDDEN = 256
+LAYERS = 6
+PATTERN = [0, 1, 1, 0, 1, 1] # 1 = sliding window, 0 = full attention
+MOE_FREQ = [0, 1, 1, 1, 1, 1] # layer 0 dense, like the release
+DENSE_INTER = 256
+HEADS = 4 # full attention
+KV_HEADS = 2 # also the qkv pre-shard count (ckpt_tp)
+SWA_HEADS = 4
+SWA_KV_HEADS = 4 # two per chunk, as on the release
+HEAD_DIM = 48 # rope_dim = int(48 * 0.334) = 16
+V_HEAD_DIM = 32
+EXPERTS = 16
+TOPK = 4
+MOE_INTER = 64 # MXFP4 needs multiples of 32
+WINDOW = 8
+EOS_ID = 1
+BLOCK = 128
+EMBED_S = 0.05 # embedding scale
+LM_S = 4.0 # head scale
+EXP_LO, EXP_HI = 121, 126 # MXFP4 e8m0 exponent range
+
+CONFIG = {
+ "architectures": ["MiMoV2ForCausalLM"],
+ "model_type": "mimo_v2",
+ "vocab_size": VOCAB,
+ "hidden_size": HIDDEN,
+ "intermediate_size": DENSE_INTER,
+ "num_hidden_layers": LAYERS,
+ "hybrid_layer_pattern": PATTERN,
+ "moe_layer_freq": MOE_FREQ,
+ "num_attention_heads": HEADS,
+ "num_key_value_heads": KV_HEADS,
+ "head_dim": HEAD_DIM,
+ "v_head_dim": V_HEAD_DIM,
+ "swa_num_attention_heads": SWA_HEADS,
+ "swa_num_key_value_heads": SWA_KV_HEADS,
+ "swa_head_dim": HEAD_DIM,
+ "swa_v_head_dim": V_HEAD_DIM,
+ "sliding_window": WINDOW,
+ "sliding_window_size": WINDOW,
+ "attention_chunk_size": WINDOW,
+ "add_full_attention_sink_bias": False,
+ "add_swa_attention_sink_bias": True,
+ "attention_value_scale": 0.707,
+ "attention_projection_layout": "fused_qkv",
+ "attention_bias": False,
+ "partial_rotary_factor": 0.334,
+ "rope_theta": 10000000.0,
+ "swa_rope_theta": 10000.0,
+ "rope_parameters": {"partial_rotary_factor": 0.334, "rope_theta": 10000000.0,
+ "rope_type": "default", "type": "default"},
+ "n_routed_experts": EXPERTS,
+ "num_experts_per_tok": TOPK,
+ "moe_intermediate_size": MOE_INTER,
+ "n_shared_experts": None,
+ "n_group": 1,
+ "topk_group": 1,
+ "topk_method": "noaux_tc",
+ "scoring_func": "sigmoid",
+ "norm_topk_prob": True,
+ "routed_scaling_factor": None,
+ "moe_router_dtype": "bfloat16",
+ "hidden_act": "silu",
+ "layernorm_epsilon": 1e-6,
+ "max_position_embeddings": 256,
+ "tie_word_embeddings": False,
+ "bos_token_id": None,
+ "eos_token_id": EOS_ID,
+ "pad_token_id": 0,
+ "dtype": "bfloat16",
+ "quantization_config": {
+ "activation_scheme": "dynamic",
+ "fmt": "e4m3",
+ "quant_method": "fp8",
+ "store_dtype": "mxfp4",
+ "mxfp4_block_size": 32,
+ "weight_block_size": [BLOCK, BLOCK],
+ "ignored_layers": [f"model.layers.{i}.self_attn.o_proj"
+ for i in range(LAYERS)],
+ },
+}
+
+
+# --vision: a tiny MiMo ViT on the same language model. Six blocks cross every
+# path of the tower: full attention at both ends (no sink), row-order and
+# column-order windows, and the permutations entering and leaving a column run
+# (types -1, 0, 1, 1, 0, -1; the release's pattern is the same alphabet).
+VISION = {
+ "depth": 6,
+ "hidden_size": 64,
+ "intermediate_size": 96,
+ "num_heads": 4,
+ "num_key_value_heads": 2,
+ "qk_channels": 16,
+ "out_hidden_size": HIDDEN,
+ "patch_size": 16,
+ "spatial_patch_size": 16,
+ "temporal_patch_size": 2,
+ "spatial_merge_size": 2,
+ "in_chans": 3,
+ "hidden_act": "silu",
+ "fullatt_block_indexes": [0, 5],
+ "vit_window_attn_types": [-1, 0, 1, 1, 0, -1],
+ "visual_token_window_size": 5,
+ "use_sink": True,
+ "window_size": 128,
+ "tokens_per_second": 2,
+}
+IMAGE_TOKEN, VISION_START, VISION_END = 300, 301, 302
+IMAGE_W, IMAGE_H = 192, 128 # already a multiple of 32: no resize
+PREPROCESSOR = {
+ "min_pixels": 3136,
+ "max_pixels": 12845056,
+ "patch_size": 16,
+ "temporal_patch_size": 2,
+ "merge_size": 2,
+ "image_mean": [0.48145466, 0.4578275, 0.40821073],
+ "image_std": [0.26862954, 0.26130258, 0.27577711],
+ "image_processor_type": "Qwen2VLImageProcessor",
+ "processor_class": "Qwen2_5_VLProcessor",
+}
+
+
+def bf16_bits(x):
+ """f32 -> bf16 bit patterns, round to nearest even."""
+ bits = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32)
+ rounding = ((bits >> 16) & 1) + np.uint32(0x7FFF)
+ return ((bits + rounding) >> 16).astype(np.uint16)
+
+
+class BF16(np.ndarray):
+ """uint16 bit patterns tagged for the writer as BF16."""
+
+
+class F8(np.ndarray):
+ """uint8 e4m3fn codes tagged for the writer as F8_E4M3."""
+
+
+def tag(arr, cls):
+ return np.ascontiguousarray(arr).view(cls)
+
+
+def st_dtype(arr):
+ if isinstance(arr, BF16):
+ return "BF16"
+ if isinstance(arr, F8):
+ return "F8_E4M3"
+ return {np.dtype("float32"): "F32", np.dtype("uint8"): "U8"}[arr.dtype]
+
+
+def build_tensors(rng):
+ """Every weight of the tiny model, in emission order, as {name: array}."""
+ tensors = {}
+
+ def put(name, arr):
+ assert name not in tensors, name
+ tensors[name] = arr
+
+ def normal(*shape, s=0.02):
+ return (rng.standard_normal(shape) * s).astype(np.float32)
+
+ def bf16(x):
+ return tag(bf16_bits(x), BF16)
+
+ def norm_weight(n):
+ return bf16(1.0 + normal(n, s=0.1))
+
+ def fp8_codes(rows, cols):
+ """Random finite e4m3fn codes with exponent fields 4..9 (|w| roughly
+ 1/8 .. 7.5 before the block scale); never the 0x7F/0xFF NaN codes."""
+ sign = rng.integers(0, 2, size=(rows, cols), dtype=np.uint8) << 7
+ exp = rng.integers(4, 10, size=(rows, cols), dtype=np.uint8) << 3
+ man = rng.integers(0, 8, size=(rows, cols), dtype=np.uint8)
+ return sign | exp | man
+
+ def fp8_scales(rows, cols):
+ return (rng.uniform(0.004, 0.012, size=(rows, cols))).astype(np.float32)
+
+ def fp8_matrix(name, out, inp):
+ put(name + ".weight", tag(fp8_codes(out, inp), F8))
+ put(name + ".weight_scale_inv",
+ fp8_scales(-(-out // BLOCK), -(-inp // BLOCK)))
+
+ def fp8_qkv(name, heads, kv_heads):
+ """Pre-sharded fused qkv: KV_HEADS chunks of [Q_c|K_c|V_c] rows, the
+ scale grid tiled per chunk (ceil(rows_per_chunk / 128) rows each)."""
+ chunks = KV_HEADS
+ rows_per_chunk = ((heads // chunks) * HEAD_DIM
+ + (kv_heads // chunks) * HEAD_DIM
+ + (kv_heads // chunks) * V_HEAD_DIM)
+ put(name + ".weight",
+ tag(fp8_codes(chunks * rows_per_chunk, HIDDEN), F8))
+ put(name + ".weight_scale_inv",
+ fp8_scales(chunks * -(-rows_per_chunk // BLOCK),
+ -(-HIDDEN // BLOCK)))
+
+ def mxfp4(rows, cols):
+ """One packed MXFP4 matrix. All 16 e2m1 codes are finite; exponents
+ stay near 127 so dequantized magnitudes stay tame, never 0xFF."""
+ packed = rng.integers(0, 256, size=(rows, cols // 2), dtype=np.uint8)
+ scale = rng.integers(EXP_LO, EXP_HI, size=(rows, cols // 32), dtype=np.uint8)
+ return packed, scale
+
+ embed = normal(VOCAB, HIDDEN, s=EMBED_S)
+ put("model.embed_tokens.weight", bf16(embed))
+ # A random head at a large scale: logits spread over ~LM_S * 16, so the
+ # top-2 gap clears float noise by orders of magnitude, while the choice
+ # still depends on the whole residual stream (make_mimo_ref.py checks that
+ # the layers change the argmax, not just the embedding). EOS is a zero
+ # row: never the argmax, so a fixed-length run detects early truncation.
+ lm_head = normal(VOCAB, HIDDEN, s=LM_S)
+ lm_head[EOS_ID] = 0.0
+ put("model.norm.weight", norm_weight(HIDDEN))
+ put("lm_head.weight", bf16(lm_head))
+
+ for i in range(LAYERS):
+ p = f"model.layers.{i}."
+ swa = PATTERN[i] == 1
+ heads = SWA_HEADS if swa else HEADS
+ kv_heads = SWA_KV_HEADS if swa else KV_HEADS
+ put(p + "input_layernorm.weight", norm_weight(HIDDEN))
+ put(p + "post_attention_layernorm.weight", norm_weight(HIDDEN))
+ fp8_qkv(p + "self_attn.qkv_proj", heads, kv_heads)
+ put(p + "self_attn.o_proj.weight",
+ bf16(normal(HIDDEN, heads * V_HEAD_DIM, s=0.05)))
+ if swa:
+ put(p + "self_attn.attention_sink_bias", bf16(normal(heads, s=1.0)))
+ if not MOE_FREQ[i]:
+ fp8_matrix(p + "mlp.gate_proj", DENSE_INTER, HIDDEN)
+ fp8_matrix(p + "mlp.up_proj", DENSE_INTER, HIDDEN)
+ fp8_matrix(p + "mlp.down_proj", HIDDEN, DENSE_INTER)
+ continue
+ # closed-form router: wide, stable expert preferences per hidden
+ # direction, so top-k selection cannot ride a near-tie
+ gate = 0.08 * np.sin(np.arange(EXPERTS * HIDDEN, dtype=np.float32)
+ * 0.013 + 0.7 * i).reshape(EXPERTS, HIDDEN)
+ put(p + "mlp.gate.weight", bf16(gate))
+ put(p + "mlp.gate.e_score_correction_bias",
+ np.linspace(-0.02, 0.02, EXPERTS, dtype=np.float32))
+ for e in range(EXPERTS):
+ q = p + f"mlp.experts.{e}."
+ down, down_s = mxfp4(HIDDEN, MOE_INTER)
+ gate_w, gate_s = mxfp4(MOE_INTER, HIDDEN)
+ up, up_s = mxfp4(MOE_INTER, HIDDEN)
+ # the file order of the release, back to back
+ put(q + "down_proj.weight", down)
+ put(q + "down_proj.weight_scale", down_s)
+ put(q + "gate_proj.weight", gate_w)
+ put(q + "gate_proj.weight_scale", gate_s)
+ put(q + "up_proj.weight", up)
+ put(q + "up_proj.weight_scale", up_s)
+ return tensors
+
+
+def build_vision(rng):
+ """The tower's tensors, named and shaped as in the release (BF16; the merger
+ carries no biases and its LayerNorm no bias, as there)."""
+ v = VISION
+ d, f, heads, kvh = v["hidden_size"], v["intermediate_size"], v["num_heads"], v["num_key_value_heads"]
+ hd = v["qk_channels"]
+ merged = d * v["spatial_merge_size"] ** 2
+ tensors = {}
+
+ def bf16(x):
+ return tag(bf16_bits(x), BF16)
+
+ def normal(*shape, s=0.02):
+ return (rng.standard_normal(shape) * s).astype(np.float32)
+
+ tensors["visual.patch_embed.proj.weight"] = bf16(normal(d, 3, 2, 16, 16, s=0.03))
+ for i in range(v["depth"]):
+ p = f"visual.blocks.{i}."
+ tensors[p + "norm1.weight"] = bf16(1.0 + normal(d, s=0.1))
+ tensors[p + "norm2.weight"] = bf16(1.0 + normal(d, s=0.1))
+ tensors[p + "attn.qkv.weight"] = bf16(normal((heads + 2 * kvh) * hd, d, s=0.15))
+ tensors[p + "attn.qkv.bias"] = bf16(normal((heads + 2 * kvh) * hd, s=0.1))
+ tensors[p + "attn.proj.weight"] = bf16(normal(d, heads * hd, s=0.1))
+ tensors[p + "attn.proj.bias"] = bf16(normal(d, s=0.05))
+ if i not in v["fullatt_block_indexes"]:
+ tensors[p + "attn.sinks"] = bf16(normal(heads, s=1.0))
+ for n in ("gate_proj", "up_proj"):
+ tensors[p + f"mlp.{n}.weight"] = bf16(normal(f, d, s=0.1))
+ tensors[p + f"mlp.{n}.bias"] = bf16(normal(f, s=0.05))
+ tensors[p + "mlp.down_proj.weight"] = bf16(normal(d, f, s=0.1))
+ tensors[p + "mlp.down_proj.bias"] = bf16(normal(d, s=0.05))
+ tensors["visual.merger.ln_q.weight"] = bf16(1.0 + normal(d, s=0.1))
+ tensors["visual.merger.mlp.0.weight"] = bf16(normal(merged, merged, s=0.05))
+ tensors["visual.merger.mlp.2.weight"] = bf16(normal(HIDDEN, merged, s=0.2))
+ return tensors
+
+
+def test_image(rng):
+ """A deterministic picture with structure at several scales, so the
+ windows and the merge see different content everywhere."""
+ y, x = np.mgrid[0:IMAGE_H, 0:IMAGE_W].astype(np.float32)
+ r = 127 + 120 * np.sin(x / 9.0) * np.cos(y / 13.0)
+ g = 127 + 120 * np.sin((x + y) / 17.0)
+ b = (x * 3 + y * 5) % 256
+ img = np.stack([r, g, b], axis=-1) + rng.integers(0, 24, size=(IMAGE_H, IMAGE_W, 3))
+ return np.clip(img, 0, 255).astype(np.uint8)
+
+
+def write_safetensors(path, tensors):
+ """Hand-rolled writer: keeps insertion order, which the expert contiguity
+ the engine exploits depends on (the safetensors library reorders)."""
+ header, offset = {}, 0
+ for name, arr in tensors.items():
+ header[name] = {"dtype": st_dtype(arr), "shape": list(arr.shape),
+ "data_offsets": [offset, offset + arr.nbytes]}
+ offset += arr.nbytes
+ blob = json.dumps(header, separators=(",", ":")).encode()
+ blob += b" " * ((-len(blob)) % 8)
+ with open(path, "wb") as fh:
+ fh.write(struct.pack("<Q", len(blob)))
+ fh.write(blob)
+ for arr in tensors.values():
+ fh.write(np.ascontiguousarray(arr).view(np.uint8).tobytes())
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--output", default="./mimo_tiny")
+ parser.add_argument("--force", action="store_true",
+ help="overwrite existing shards")
+ parser.add_argument("--vision", action="store_true",
+ help="add a tiny ViT, the image tokens, a test picture "
+ "and its patches (the vision fixture)")
+ args = parser.parse_args()
+
+ out = Path(args.output)
+ out.mkdir(parents=True, exist_ok=True)
+ shard_names = ["model-00001-of-00002.safetensors",
+ "model-00002-of-00002.safetensors"]
+ if any((out / s).exists() for s in shard_names) and not args.force:
+ print(f"{out} already has shards; use --force to regenerate",
+ file=sys.stderr)
+ return 1
+
+ rng = np.random.default_rng(SEED)
+ tensors = build_tensors(rng)
+ config = dict(CONFIG)
+ if args.vision:
+ tensors.update(build_vision(np.random.default_rng(SEED + 1)))
+ config.update({"vision_config": VISION, "image_token_id": IMAGE_TOKEN,
+ "vision_start_token_id": VISION_START,
+ "vision_end_token_id": VISION_END,
+ "vision_model_type": "mimovl"})
+ (out / "preprocessor_config.json").write_text(
+ json.dumps(PREPROCESSOR, indent=2) + "\n")
+ from PIL import Image
+ Image.fromarray(test_image(np.random.default_rng(SEED + 2))).save(
+ out / "image.png")
+ # the patches exactly as the gateway builds them (tools/qwen38_image.py,
+ # the Qwen2-VL processor); make_mimo_ref.py checks them bit for bit
+ # against the official processor
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
+ import qwen38_image
+ patches, grid_h, grid_w = qwen38_image.preprocess(
+ (out / "image.png").read_bytes(), model_dir=str(out))
+ patches.tofile(out / "patches.f32")
+ (out / "image.json").write_text(json.dumps(
+ {"grid_h": grid_h, "grid_w": grid_w,
+ "tokens": grid_h * grid_w // 4}) + "\n")
+ # two shards like the release: the experts of layers 3.. in the second,
+ # so the loader has to follow the index across files
+ second = {n: a for n, a in tensors.items()
+ if ".mlp.experts." in n and int(n.split(".")[2]) >= 3}
+ first = {n: a for n, a in tensors.items() if n not in second}
+ weight_map = {}
+ for shard, part in zip(shard_names, (first, second)):
+ write_safetensors(out / shard, part)
+ weight_map.update({n: shard for n in part})
+ total = sum(a.nbytes for a in tensors.values())
+ (out / "model.safetensors.index.json").write_text(json.dumps(
+ {"metadata": {"total_size": total}, "weight_map": weight_map},
+ indent=1) + "\n")
+ (out / "config.json").write_text(json.dumps(config, indent=2) + "\n")
+ print(f"{out}: {len(tensors)} tensors, {total / 1e6:.2f} MB in 2 shards, "
+ f"{LAYERS} layers (pattern {PATTERN}), {EXPERTS} experts top-{TOPK}, "
+ f"seed {SEED}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/mimo_real_check.py b/c/tools/mimo_real_check.py
new file mode 100644
index 00000000..a32b3c5f
--- /dev/null
+++ b/c/tools/mimo_real_check.py
@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+"""mimo.c against Xiaomi's own code on the REAL MiMo-V2.6 checkpoint.
+
+The tiny oracle (tools/make_mimo_tiny.py + make_mimo_ref.py) pins the math on a
+checkpoint stored like the release. This pins the release itself: its MXFP4 nibble
+order, its FP8 block scales, its pre-sharded qkv, its sinks, its ViT. Maintainer tool,
+run by hand on a machine that has the checkpoint; nothing here runs in CI.
+
+ python3 tools/mimo_real_check.py --model DIR --engine ./mimo [--layers N] [--image PNG]
+
+The vendor model is built from the checkpoint's own modeling_mimo_v2.py (its SHA-256
+must match the one the tiny oracle pinned) with the first N layers (default: all),
+dense weights dequantized to f32 from the same bytes the engine reads. Its experts are
+NOT all materialized -- 256 experts of a Flash layer are 26 GB in f32 -- but built one
+at a time when routed: each is the vendor's own MiMoV2MLP, loaded, called and dropped,
+so the math is still theirs and the RAM stays at one expert.
+
+What is compared, at every prompt position: the logits (max absolute difference,
+relative to the logit range), the argmax, and the KL divergence of the engine's
+distribution from the vendor's. With --image, the ViT's output rows too.
+"""
+import argparse
+import hashlib
+import json
+import os
+import struct
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import make_mimo_ref as R # noqa: E402 (reuses the pinned vendor loader and dequantizers)
+
+
+class Shards:
+ """Tensor lookup across the checkpoint's shards, read on demand."""
+
+ def __init__(self, model_dir):
+ self.dir = Path(model_dir)
+ index = json.loads((self.dir / "model.safetensors.index.json").read_text())
+ self.where = index["weight_map"]
+ self.headers = {}
+
+ def header(self, shard):
+ if shard not in self.headers:
+ with open(self.dir / shard, "rb") as fh:
+ size = struct.unpack("<Q", fh.read(8))[0]
+ self.headers[shard] = (json.loads(fh.read(size)), 8 + size)
+ return self.headers[shard]
+
+ def has(self, name):
+ return name in self.where
+
+ def raw(self, name):
+ shard = self.where[name]
+ header, base = self.header(shard)
+ meta = header[name]
+ lo, hi = meta["data_offsets"]
+ dtype = {"F32": np.float32, "U8": np.uint8, "BF16": np.uint16,
+ "F8_E4M3": np.uint8}[meta["dtype"]]
+ with open(self.dir / shard, "rb") as fh:
+ fh.seek(base + lo)
+ data = fh.read(hi - lo)
+ return meta["dtype"], np.frombuffer(data, dtype=dtype).reshape(meta["shape"])
+
+ def f32(self, name):
+ dtype, arr = self.raw(name)
+ if dtype == "BF16":
+ return R.bf16_to_f32(arr)
+ if dtype == "F32":
+ return arr.astype(np.float32)
+ raise SystemExit(f"{name}: {dtype} is not a float tensor")
+
+
+def truncated_config(config, layers, vision):
+ config = dict(config)
+ config["num_hidden_layers"] = layers
+ config["hybrid_layer_pattern"] = config["hybrid_layer_pattern"][:layers]
+ config["moe_layer_freq"] = config["moe_layer_freq"][:layers]
+ config.pop("quantization_config", None)
+ config.pop("audio_config", None)
+ if not vision:
+ config.pop("vision_config", None)
+ return config
+
+
+def load_dense(model, shards, config):
+ """Every non-expert weight of the truncated model, dequantized to f32."""
+ import torch
+ chunks = config["num_key_value_heads"]
+ state = {}
+ for name, param in model.named_parameters():
+ if ".mlp.experts." in name:
+ continue
+ if not shards.has(name):
+ if name.startswith("visual.merger.") and name.endswith(".bias"):
+ state[name] = torch.zeros(tuple(param.shape)) # the release stores none
+ continue
+ raise SystemExit(f"{name}: not in the checkpoint")
+ dtype, arr = shards.raw(name)
+ if name.endswith("qkv_proj.weight"):
+ layer = int(name.split(".")[2])
+ swa = config["hybrid_layer_pattern"][layer] == 1
+ heads = config["swa_num_attention_heads" if swa else "num_attention_heads"]
+ kv = config["swa_num_key_value_heads" if swa else "num_key_value_heads"]
+ value = R.qkv_contiguous(arr, shards.f32(name + "_scale_inv"), heads, kv,
+ config["head_dim"], config["v_head_dim"], chunks)
+ elif dtype == "F8_E4M3":
+ value = R.fp8_block_dequant(arr, shards.f32(name + "_scale_inv"))
+ elif dtype in ("BF16", "F32"):
+ value = shards.f32(name)
+ else:
+ raise SystemExit(f"{name}: unexpected {dtype}")
+ if tuple(value.shape) != tuple(param.shape):
+ if name == "visual.patch_embed.proj.weight":
+ value = value.reshape(param.shape)
+ else:
+ raise SystemExit(f"{name}: {value.shape} vs {tuple(param.shape)}")
+ state[name] = torch.from_numpy(np.ascontiguousarray(value))
+ return state
+
+
+def lazy_experts(model, modeling, shards, config):
+ """Replace each layer's expert list with one that builds the vendor's MiMoV2MLP
+ for an expert only when the router picks it."""
+ import torch
+
+ class LazyExperts(torch.nn.Module):
+ def __init__(self, layer, count):
+ super().__init__()
+ self.layer, self.count = layer, count
+ self.cfg = model.config
+
+ def __len__(self):
+ return self.count
+
+ def __iter__(self):
+ for e in range(self.count):
+ yield self[e]
+
+ def __getitem__(self, e):
+ layer, cfg = self.layer, self.cfg
+
+ class Once:
+ def __call__(self, x):
+ mlp = modeling.MiMoV2MLP(cfg, intermediate_size=cfg.moe_intermediate_size)
+ prefix = f"model.layers.{layer}.mlp.experts.{e}."
+ weights = {}
+ for part in ("gate_proj", "up_proj", "down_proj"):
+ _, packed = shards.raw(prefix + part + ".weight")
+ _, scale = shards.raw(prefix + part + ".weight_scale")
+ weights[part + ".weight"] = torch.from_numpy(
+ R.mxfp4_dequant(packed, scale))
+ mlp.load_state_dict(weights)
+ with torch.no_grad():
+ return mlp.float()(x)
+ return Once()
+
+ for index, layer in enumerate(model.model.layers):
+ if config["moe_layer_freq"][index]:
+ layer.mlp.experts = LazyExperts(index, config["n_routed_experts"])
+
+
+def engine_logits(engine, model_dir, ids, layers, env_extra, vocab):
+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:
+ path = tmp.name
+ env = dict(os.environ, MIMO_LOGITS=path, MIMO_LAYERS=str(layers), **env_extra)
+ try:
+ subprocess.run([engine, str(model_dir), "--ids", " ".join(map(str, ids)), "--ngen", "0"],
+ env=env, check=True, capture_output=True, text=True, timeout=7200)
+ return np.fromfile(path, dtype=np.float32).reshape(len(ids), vocab)
+ finally:
+ os.unlink(path)
+
+
+def image_engine(args, model_dir, config, engine_env, vocab, tokenizer):
+ """The picture through the gateway's preprocessing and the official processor
+ (must agree on the grid), then the engine: its ViT rows and the logits of a prompt
+ that carries the picture. Runs before the vendor model is built, so the two never
+ hold RAM at the same time."""
+ from PIL import Image
+ from transformers import Qwen2VLImageProcessor
+ import qwen38_image
+ data = Path(args.image).read_bytes()
+ ours, grid_h, grid_w = qwen38_image.preprocess(data, model_dir=str(model_dir),
+ max_tokens=args.image_tokens)
+ # the pixel window as the gateway sets it; recent transformers read min_pixels /
+ # max_pixels over `size` when the checkpoint's config names them, so give both
+ raw = json.loads((model_dir / "preprocessor_config.json").read_text())
+ low, high = raw.get("min_pixels", 3136), raw.get("max_pixels", 12845056)
+ if args.image_tokens:
+ high = min(high, args.image_tokens * 32 * 32)
+ low = min(low, high)
+ processor = Qwen2VLImageProcessor.from_pretrained(
+ str(model_dir), min_pixels=low, max_pixels=high,
+ size={"shortest_edge": low, "longest_edge": high})
+ batch = processor(images=[Image.open(args.image).convert("RGB")], return_tensors="pt")
+ official = batch["pixel_values"].float().numpy()
+ thw = batch["image_grid_thw"]
+ if thw.tolist() != [[1, grid_h, grid_w]]:
+ raise SystemExit(f"grid: gateway {grid_h}x{grid_w}, official {thw.tolist()}")
+ print(f"[real] image {grid_h}x{grid_w} patches: gateway vs official processor, max "
+ f"|diff| {np.abs(official - ours).max():.3e} (resampling: Pillow against the "
+ f"processor's own)")
+ n_img = grid_h * grid_w // 4
+ prefix = tokenizer.encode("<|im_start|>user\n<|vision_start|>", add_special_tokens=False).ids
+ suffix = tokenizer.encode("<|vision_end|>Cosa vedi in questa immagine?<|im_end|>"
+ "<|im_start|>assistant\n<think>", add_special_tokens=False).ids
+ ids = prefix + [config["image_token_id"]] * n_img + suffix
+ with tempfile.TemporaryDirectory() as tmp:
+ patches = Path(tmp) / "patches.f32"
+ ours.tofile(patches)
+ dump, logits_path = Path(tmp) / "rows.f32", Path(tmp) / "logits.f32"
+ env = dict(os.environ, MIMO_LOGITS=str(logits_path), MIMO_VISION_DUMP=str(dump),
+ MIMO_LAYERS=str(config["num_hidden_layers"]), **engine_env)
+ subprocess.run([args.engine, str(model_dir), "--ids", " ".join(map(str, ids)),
+ "--ngen", "0", "--image", str(patches), "--grid", str(grid_h),
+ str(grid_w)], env=env, check=True, capture_output=True, text=True,
+ timeout=7200)
+ rows = np.fromfile(dump, dtype=np.float32).reshape(n_img, -1)
+ logits = np.fromfile(logits_path, dtype=np.float32).reshape(len(ids), vocab)
+ return {"patches": ours, "thw": thw, "ids": ids, "rows": rows, "logits": logits}
+
+
+def image_vendor(model, image, config):
+ import torch
+ pixels = torch.from_numpy(image["patches"])
+ with torch.no_grad():
+ rows = model.visual(pixel_values=pixels, grid_thw=image["thw"]).float().numpy()
+ theirs = model(input_ids=torch.tensor([image["ids"]]), pixel_values=pixels,
+ image_grid_thw=image["thw"], use_cache=False).logits[0].float().numpy()
+ diff = np.abs(image["rows"] - rows)
+ print(f"[real] ViT rows ({rows.shape[0]} x {rows.shape[1]}): max |diff| {diff.max():.3e}, "
+ f"relative {diff.max() / np.abs(rows).max():.2e}")
+ return compare(f"text with the picture, {config['num_hidden_layers']} layers",
+ image["logits"], theirs)
+
+
+def compare(name, ours, theirs):
+ diff = np.abs(ours - theirs)
+ span = float(theirs.max() - theirs.min())
+ same = int((ours.argmax(-1) == theirs.argmax(-1)).sum())
+
+ def logsoftmax(x):
+ x = x - x.max(-1, keepdims=True)
+ return x - np.log(np.exp(x).sum(-1, keepdims=True))
+ p = np.exp(logsoftmax(theirs.astype(np.float64)))
+ kl = float((p * (logsoftmax(theirs.astype(np.float64)) - logsoftmax(ours.astype(np.float64)))).sum(-1).max())
+ print(f"[real] {name}: argmax {same}/{len(ours)}, max |diff| {diff.max():.3e} "
+ f"({diff.max() / span:.2e} of the logit range), max KL {kl:.2e}")
+ return same == len(ours)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--model", required=True)
+ parser.add_argument("--engine", required=True)
+ parser.add_argument("--layers", type=int, default=0, help="0: every layer")
+ parser.add_argument("--prompt", default="<|im_start|>user\nCiao! Qual e' la capitale "
+ "d'Italia, e perche'?<|im_end|><|im_start|>assistant\n<think>")
+ parser.add_argument("--image", default=None, help="a picture for the ViT check")
+ parser.add_argument("--image-tokens", type=int, default=256,
+ help="ceiling on the picture's tokens, as the gateway's")
+ parser.add_argument("--dense-bits", default="0", help="the engine's MIMO_DENSE_BITS")
+ parser.add_argument("--ids", default=None, help="prompt token ids instead of --prompt")
+ parser.add_argument("--vendor-dir", default=None,
+ help="where modeling_mimo_v2.py is (default: the model dir)")
+ args = parser.parse_args()
+
+ import torch
+ from tokenizers import Tokenizer
+ model_dir = Path(args.model)
+ vendor_dir = Path(args.vendor_dir) if args.vendor_dir else model_dir
+ for name, expected in R.VENDOR_FILES.items():
+ digest = hashlib.sha256((vendor_dir / name).read_bytes()).hexdigest()
+ if digest != expected:
+ raise SystemExit(f"{name} in the checkpoint is not the pinned vendor file")
+ full = json.loads((model_dir / "config.json").read_text())
+ layers = args.layers or full["num_hidden_layers"]
+ config = truncated_config(full, layers, vision=bool(args.image))
+ tokenizer = None
+ if args.ids:
+ ids = [int(t) for t in args.ids.split()]
+ else:
+ tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json"))
+ ids = tokenizer.encode(args.prompt, add_special_tokens=False).ids
+ print(f"[real] {layers} of {full['num_hidden_layers']} layers, {len(ids)} prompt tokens",
+ file=sys.stderr)
+ engine_env = {"MIMO_DENSE_BITS": args.dense_bits, "MIMO_IDOT": "0"}
+ ours = engine_logits(args.engine, model_dir, ids, layers,
+ engine_env,
+ full["vocab_size"])
+ image = None
+ if args.image:
+ if tokenizer is None:
+ tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json"))
+ image = image_engine(args, model_dir, config, engine_env, full["vocab_size"], tokenizer)
+ config_module, modeling = R.load_vendor(vendor_dir)
+ cfg = config_module.MiMoV2Config(**dict(config, use_cache=False))
+ cfg._attn_implementation = "eager"
+ torch.manual_seed(0)
+ with torch.device("meta"):
+ model = modeling.MiMoV2ForCausalLM(cfg)
+ shards = Shards(model_dir)
+ lazy_experts(model, modeling, shards, config)
+ state = load_dense(model, shards, config)
+ model.load_state_dict(state, strict=False, assign=True)
+ del state
+ # the rotary tables are buffers computed at construction, i.e. on meta: rebuild
+ # them on the CPU from the same config, then refuse anything still on meta
+ model.model.rotary_emb = modeling.MiMoV2RotaryEmbedding(config=cfg, is_swa=False)
+ model.model.swa_rotary_emb = modeling.MiMoV2RotaryEmbedding(config=cfg, is_swa=True)
+ if getattr(model, "visual", None) is not None:
+ head_dim = model.visual.blocks[0].attn.head_dim
+ model.visual.rotary_pos_emb = modeling.MiMoVisionRotaryEmbedding(head_dim // 2)
+ left = [n for n, t in list(model.named_parameters()) + list(model.named_buffers())
+ if t.is_meta]
+ if left:
+ raise SystemExit(f"still on meta after loading: {left[:8]}")
+ R.check_rope(model, config)
+ model = model.float().eval()
+
+ with torch.no_grad():
+ theirs = model(input_ids=torch.tensor([ids]), use_cache=False).logits[0].float().numpy()
+ ok = compare(f"text, {layers} layers", ours, theirs)
+ if image:
+ ok = image_vendor(model, image, config) and ok
+ if tokenizer:
+ top = [tokenizer.decode([int(t)]) for t in theirs.argmax(-1)[-5:]]
+ print(f"[real] last positions, vendor argmax: {top}")
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/qwen38_image.py b/c/tools/qwen38_image.py
index f796d3ec..90505e08 100644
--- a/c/tools/qwen38_image.py
+++ b/c/tools/qwen38_image.py
@@ -129,6 +129,11 @@ def load_config(model_dir):
settings["min_pixels"] = size["shortest_edge"]
if "longest_edge" in size:
settings["max_pixels"] = size["longest_edge"]
+ # The older Qwen2-VL spelling (MiMo-V2.6 ships it): the same window as
+ # top-level keys. `size` wins when both are there, as in the processor.
+ for key in ("min_pixels", "max_pixels"):
+ if key in raw:
+ settings.setdefault(key, raw[key])
config = root / "config.json"
if config.exists():
vision = json.loads(config.read_text()).get("vision_config") or {}
diff --git a/c/tools/requirements-mimo-tiny.txt b/c/tools/requirements-mimo-tiny.txt
new file mode 100644
index 00000000..c1634540
--- /dev/null
+++ b/c/tools/requirements-mimo-tiny.txt
@@ -0,0 +1,4 @@
+# tiny MiMo-V2.6 fixture generation (CI): deterministic numpy weights, and Pillow for
+# the test picture and the gateway's Qwen2-VL preprocessing (tools/qwen38_image.py)
+numpy>=1.24
+pillow>=10