Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
mimo: MiMo-V2.6 engine (Flash, Pro), text and vision, read as released
JustVuggb290d5eb1808
diff --git a/.github/workflows/check.yml b/.github/workflows/check.ymlindex 6c4403f1..6270965a 100644--- a/.github/workflows/check.yml+++ b/.github/workflows/check.yml@@ -83,6 +83,28 @@ jobs:- name: Generate fixture and run vendor oraclerun: make -C c kimi-k3-tiny-check+ mimo-tiny:+ name: Linux (generated tiny MiMo-V2.6 vendor oracle, text + vision)+ runs-on: ubuntu-latest+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@v4+ - uses: actions/setup-python@v5+ with:+ python-version: '3.13'+ cache: pip+ cache-dependency-path: c/tools/requirements-mimo-tiny.txt+ - name: Install pinned tiny-fixture dependencies+ run: python -m pip install -r c/tools/requirements-mimo-tiny.txt+ - name: Generate fixture and run vendor oracle+ run: make -C c mimo-tiny-check+ - name: Same oracle under ASan and UBSan+ run: |+ make -C c clean >/dev/null+ make -C c mimo EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"+ ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \+ python c/tests/mimo_tiny_harness.py --binary c/mimo --fixture c/mimo_tiny+v4-tiny:name: Linux (generated tiny V4 target oracle)runs-on: ubuntu-latestdiff --git a/.gitignore b/.gitignoreindex eeba7b07..ff65ffa1 100644--- a/.gitignore+++ b/.gitignore@@ -158,3 +158,12 @@ c/kimi_k3_tiny/.coli_usagec/qwen38_vision_tiny/c/qwen38_mm_tiny/c/qwen38_tiny_fp8/+# MiMo tiny fixture: config.json and ref.json are committed, the rest is regenerated+c/mimo_tiny/*.safetensors+c/mimo_tiny/model.safetensors.index.json+c/mimo_tiny/.vendor_mimo_v2/+c/mimo_tiny/image.png+c/mimo_tiny/image.json+c/mimo_tiny/patches.f32+c/mimo+c/mimo.exediff --git a/c/Makefile b/c/Makefileindex 1530e1be..d9d97be6 100644--- a/c/Makefile+++ b/c/Makefile@@ -1236,6 +1236,28 @@ deepseek_v41$(EXE): deepseek_v41.c sse41_kernels.h cli_args.h st.h json.h tok.hsparse_attn.h hyper_connections.h serve_codec.h serve_poll.h .build-config$(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)+# Xiaomi MiMo-V2.6 (Flash 309B, Pro 1.02T): one file plus its ViT header, reading the+# release as it is -- MXFP4 experts streamed per layer, FP8/BF16 dense resident.+ifneq ($(EXE),)+.PHONY: mimo+mimo: mimo$(EXE)+endif+mimo$(EXE): mimo.c mimo_vision.h sse41_kernels.h cli_args.h compat.h json.h st.h quant.h idot.h fp8_format.h \+ omp_tune.h kv_prefix.h tok.h tok_unicode.h tok_unicode_o200k.h tok_unicode_deepseek.h serve_codec.h \+ decode_batch.h serve_poll.h stop_ids.h .build-config+ $(CC) $(CFLAGS) mimo.c -o mimo$(EXE) $(LDFLAGS)++# Tiny MiMo fixture + vendor oracle, the Kimi K3 pattern: the generator is numpy (and+# Pillow for the test picture) and deterministic; ref.json is committed and comes from+# Xiaomi's own modeling code (tools/make_mimo_ref.py, maintainer-run, SHA-256-pinned),+# so CI needs neither torch nor network. The fixture carries the vision tower too.+.PHONY: mimo-tiny-generate mimo-tiny-check+mimo-tiny-generate:+ $(PYTHON) tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision++mimo-tiny-check: mimo-tiny-generate mimo$(EXE)+ $(PYTHON) tests/mimo_tiny_harness.py --binary ./mimo$(EXE) --fixture ./mimo_tiny+# Qwen3.8-Flash-Next text-only sibling. qwen38.c owns the direct checkpoint# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);@@ -2453,7 +2475,7 @@ check:$(MAKE) portable$(MAKE) test-install: colibri$(EXE) glm53$(EXE) inkling$(EXE) kimi_k3$(EXE) olmoe$(EXE) qwen36$(EXE) qwen38$(EXE) deepseek_v41$(EXE) qwenimage$(EXE) \+install: colibri$(EXE) glm53$(EXE) inkling$(EXE) kimi_k3$(EXE) olmoe$(EXE) qwen36$(EXE) qwen38$(EXE) deepseek_v41$(EXE) mimo$(EXE) qwenimage$(EXE) \$(if $(filter 1,$(COLI_V4_SUPPORTED)),deepseek-v4,)$(INSTALL) -d $(DESTDIR)$(BINDIR)$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)@@ -2468,6 +2490,7 @@ install: colibri$(EXE) glm53$(EXE) inkling$(EXE) kimi_k3$(EXE) olmoe$(EXE) qwen3$(INSTALL) -m 755 qwen38$(EXE) $(DESTDIR)$(LIBEXECDIR)/qwen38$(EXE)$(INSTALL) -m 755 qwenimage$(EXE) $(DESTDIR)$(LIBEXECDIR)/qwenimage$(EXE)$(INSTALL) -m 755 deepseek_v41$(EXE) $(DESTDIR)$(LIBEXECDIR)/deepseek_v41$(EXE)+ $(INSTALL) -m 755 mimo$(EXE) $(DESTDIR)$(LIBEXECDIR)/mimo$(EXE)@if [ -f deepseek_v4$(EXE) ]; then \$(INSTALL) -m 755 deepseek_v4$(EXE) $(DESTDIR)$(LIBEXECDIR)/deepseek_v4$(EXE); \fidiff --git a/c/mimo.c b/c/mimo.cnew file mode 100644index 00000000..10d569e8--- /dev/null+++ b/c/mimo.c@@ -0,0 +1,1427 @@+/* mimo.c -- Xiaomi MiMo-V2.6 (Flash 309B/15B, Pro 1.02T/42B) on a machine that+ * cannot hold it.+ *+ * The released checkpoint is read as it is, no conversion step:+ *+ * experts MXFP4 g32 (e2m1 nibble pairs, low nibble = even column, one e8m0+ * byte per 32 columns), the six tensors of an expert back to back in+ * the shard, so an expert is ONE read. 256 of them per layer on+ * Flash, 8 routed per token, cached in RAM by a per-layer LRU.+ * dense qkv_proj and the layer-0 MLP in FP8 e4m3 with a F32 scale per+ * 128x128 block; o_proj, router, norms, sinks, embed, lm_head BF16.+ *+ * What makes MiMo MiMo, each mirroring modeling_mimo_v2.py:+ * hybrid attention hybrid_layer_pattern: 1 = sliding window (128 keys,+ * itself included), 0 = full attention; the two kinds have+ * their own KV head count and their own RoPE theta+ * partial RoPE on the first int(head_dim * 0.334) dims of each head,+ * rotate-half style; head_dim 192 for q/k, 128 for v+ * value scale v *= attention_value_scale before attention+ * attention sink SWA layers carry one logit per head that joins the+ * softmax denominator and nothing else+ * router sigmoid scores, e_score_correction_bias picks but does+ * not weigh, top-k renormalised; no shared expert+ *+ * One storage detail the modeling code does not describe: the fused qkv_proj is+ * PRE-SHARDED in num_key_value_heads chunks [Q_0|K_0|V_0|Q_1|K_1|V_1|...] and+ * its FP8 scale grid is tiled per chunk (a Flash full-attention layer has 108+ * scale rows, not the 106 a contiguous grid would have). The layout is vLLM's+ * (vllm/model_executor/models/mimo_v2.py, _shard_fp8_qkv_proj); the loader+ * undoes it and the tiny oracle stores it the same way so a wrong reading fails.+ *+ * Held to Xiaomi's own modeling code through tools/make_mimo_ref.py (vendor+ * files pinned by SHA-256) on the tiny fixture of tools/make_mimo_tiny.py.+ */+#define _GNU_SOURCE+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <stdint.h>+#include <math.h>+#include <errno.h>+#include <time.h>+#include <stdarg.h>+#include <unistd.h>+#include <sys/stat.h>+#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)+#include <sys/resource.h>+#endif+#ifdef _OPENMP+#include <omp.h>+#endif++#include "cli_args.h"+#include "compat.h"+#include "json.h"+#include "st.h"+#include "quant.h"+#include "omp_tune.h"+#include "kv_prefix.h"+#include "tok.h"+#include "serve_codec.h"+#include "serve_poll.h"+#include "stop_ids.h"+#if defined(__AVX2__)+#include <immintrin.h>+#endif++#define MIMO_MAX_LAYERS 128+#define MIMO_MAX_TOPK 16+#define MIMO_MAX_CHUNKS 16++/* ------------------------------------------------------------------ utils ---- */++static double now_s(void) {+ struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t);+ return t.tv_sec + t.tv_nsec * 1e-9;+}++static double rss_gb(void) {+ struct rusage usage;+ getrusage(RUSAGE_SELF, &usage);+#ifdef __APPLE__+ return usage.ru_maxrss / 1e9;+#else+ return usage.ru_maxrss / 1e6;+#endif+}++static void *xmalloc(size_t bytes, const char *what) {+ void *p = malloc(bytes ? bytes : 1);+ if (!p) { fprintf(stderr, "[mimo] out of memory allocating %s (%zu bytes)\n", what, bytes); exit(1); }+ return p;+}++static void *xcalloc(size_t n, size_t size, const char *what) {+ void *p = calloc(n ? n : 1, size ? size : 1);+ if (!p) { fprintf(stderr, "[mimo] out of memory allocating %s\n", what); exit(1); }+ return p;+}++static int env_int(const char *name, int fallback) {+ const char *v = getenv(name);+ return (v && *v) ? atoi(v) : fallback;+}++static inline float silu(float x) { return x / (1.0f + expf(-x)); }+static inline float sigmoid(float x) { return 1.0f / (1.0f + expf(-x)); }++/* ---------------------------------------------------------------- config ---- */++/* kind 0 = full attention, 1 = sliding window */+typedef struct {+ int vocab, hidden, n_layers, dense_inter, moe_inter, n_experts, topk;+ int heads[2], kv_heads[2], head_dim[2], v_dim[2], rope_dim[2];+ float theta[2];+ int sink[2];+ int swa[MIMO_MAX_LAYERS], moe[MIMO_MAX_LAYERS];+ int window, chunks, norm_topk;+ float v_scale, eps, route_scale;+ int max_positions;+ int image_token_id, vision_start_id, vision_end_id;+} Cfg;++static double jnum(jval *o, const char *key, double fallback) {+ jval *v = o ? json_get(o, key) : NULL;+ return (v && v->t == J_NUM) ? v->num : fallback;+}++static int jbool(jval *o, const char *key, int fallback) {+ jval *v = o ? json_get(o, key) : NULL;+ if (!v) return fallback;+ if (v->t == J_BOOL) return v->boolean != 0;+ if (v->t == J_NUM) return v->num != 0;+ return fallback;+}++static char *read_text(const char *path, long *size_out) {+ FILE *f = fopen(path, "rb");+ if (!f) return NULL;+ fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);+ if (size < 0 || size > (64l << 20)) { fclose(f); return NULL; }+ char *text = xmalloc((size_t)size + 1, path);+ if (fread(text, 1, (size_t)size, f) != (size_t)size) { fclose(f); free(text); return NULL; }+ text[size] = 0; fclose(f);+ if (size_out) *size_out = size;+ return text;+}++static void die_cfg(const char *what) {+ fprintf(stderr, "[mimo] config.json: %s -- refusing\n", what);+ exit(1);+}++static void load_cfg(Cfg *c, const char *dir) {+ char path[4096];+ snprintf(path, sizeof(path), "%s/config.json", dir);+ char *text = read_text(path, NULL);+ if (!text) { fprintf(stderr, "[mimo] cannot read %s\n", path); exit(1); }+ char *arena = NULL;+ jval *root = json_parse(text, &arena);+ if (!root) die_cfg("not JSON");+ jval *mt = json_get(root, "model_type");+ if (!mt || mt->t != J_STR || strcmp(mt->str, "mimo_v2"))+ die_cfg("model_type is not mimo_v2");+ memset(c, 0, sizeof(*c));+ c->vocab = (int)jnum(root, "vocab_size", 0);+ c->hidden = (int)jnum(root, "hidden_size", 0);+ c->n_layers = (int)jnum(root, "num_hidden_layers", 0);+ c->dense_inter = (int)jnum(root, "intermediate_size", 0);+ c->moe_inter = (int)jnum(root, "moe_intermediate_size", 0);+ c->n_experts = (int)jnum(root, "n_routed_experts", 0);+ c->topk = (int)jnum(root, "num_experts_per_tok", 0);+ c->heads[0] = (int)jnum(root, "num_attention_heads", 0);+ c->kv_heads[0] = (int)jnum(root, "num_key_value_heads", 0);+ c->head_dim[0] = (int)jnum(root, "head_dim", 0);+ c->v_dim[0] = (int)jnum(root, "v_head_dim", c->head_dim[0]);+ c->heads[1] = (int)jnum(root, "swa_num_attention_heads", c->heads[0]);+ c->kv_heads[1] = (int)jnum(root, "swa_num_key_value_heads", c->kv_heads[0]);+ c->head_dim[1] = (int)jnum(root, "swa_head_dim", c->head_dim[0]);+ c->v_dim[1] = (int)jnum(root, "swa_v_head_dim", c->v_dim[0]);+ double prf = jnum(root, "partial_rotary_factor", 1.0);+ for (int k = 0; k < 2; k++) c->rope_dim[k] = (int)(c->head_dim[k] * prf);+ c->theta[0] = (float)jnum(root, "rope_theta", 10000.0);+ c->theta[1] = (float)jnum(root, "swa_rope_theta", c->theta[0]);+ c->sink[0] = jbool(root, "add_full_attention_sink_bias", 0);+ c->sink[1] = jbool(root, "add_swa_attention_sink_bias", 0);+ c->window = (int)jnum(root, "sliding_window", 0);+ c->v_scale = (float)jnum(root, "attention_value_scale", 1.0);+ c->eps = (float)jnum(root, "layernorm_epsilon", 1e-6);+ c->route_scale = (float)jnum(root, "routed_scaling_factor", 1.0);+ c->norm_topk = jbool(root, "norm_topk_prob", 1);+ c->max_positions = (int)jnum(root, "max_position_embeddings", 32768);+ c->image_token_id = (int)jnum(root, "image_token_id", -1);+ c->vision_start_id = (int)jnum(root, "vision_start_token_id", -1);+ c->vision_end_id = (int)jnum(root, "vision_end_token_id", -1);+ c->chunks = c->kv_heads[0];+ /* MIMO_LAYERS=N: only the first N layers, then the final norm and head. A+ * checking aid (tools/mimo_real_check.py compares it with Xiaomi's code on+ * the real weights), never an answer anyone should read. */+ int keep = env_int("MIMO_LAYERS", 0);++ jval *pat = json_get(root, "hybrid_layer_pattern");+ jval *freq = json_get(root, "moe_layer_freq");+ if (c->n_layers < 1 || c->n_layers > MIMO_MAX_LAYERS) die_cfg("num_hidden_layers out of range");+ if (!pat || pat->t != J_ARR || pat->len != c->n_layers) die_cfg("hybrid_layer_pattern must list every layer");+ if (!freq || freq->t != J_ARR || freq->len != c->n_layers) die_cfg("moe_layer_freq must list every layer");+ for (int i = 0; i < c->n_layers; i++) {+ c->swa[i] = (int)pat->kids[i]->num == 1;+ c->moe[i] = (int)freq->kids[i]->num != 0;+ }+ if (keep > 0 && keep < c->n_layers) {+ fprintf(stderr, "[mimo] MIMO_LAYERS=%d: a truncated model for checking, not for answers\n", keep);+ c->n_layers = keep;+ }+ /* Everything below sizes a buffer or indexes one: refuse a config that+ * would make any of it overflow or divide by zero, before reading a byte. */+ if (c->vocab < 2 || c->vocab > (1 << 22)) die_cfg("vocab_size out of range");+ if (c->hidden < 32 || c->hidden > 65536 || c->hidden % 32) die_cfg("hidden_size must be a positive multiple of 32");+ if (c->moe_inter < 32 || c->moe_inter % 32 || c->moe_inter > 65536) die_cfg("moe_intermediate_size must be a positive multiple of 32");+ if (c->n_experts < 1 || c->n_experts > 4096) die_cfg("n_routed_experts out of range");+ if (c->topk < 1 || c->topk > MIMO_MAX_TOPK || c->topk > c->n_experts) die_cfg("num_experts_per_tok out of range");+ for (int k = 0; k < 2; k++) {+ if (c->heads[k] < 1 || c->kv_heads[k] < 1 || c->heads[k] % c->kv_heads[k]) die_cfg("attention heads must be a multiple of kv heads");+ if (c->heads[k] > 1024 || c->kv_heads[k] > 1024) die_cfg("too many attention heads");+ if (c->head_dim[k] < 2 || c->head_dim[k] > 1024 || c->v_dim[k] < 1 || c->v_dim[k] > 1024) die_cfg("head dims out of range");+ if (c->rope_dim[k] < 0 || c->rope_dim[k] > c->head_dim[k] || c->rope_dim[k] % 2) die_cfg("rotary dims must be even and fit the head");+ }+ if (c->chunks < 1 || c->chunks > MIMO_MAX_CHUNKS) die_cfg("num_key_value_heads (the qkv pre-shard count) out of range");+ for (int k = 0; k < 2; k++)+ if (c->heads[k] % c->chunks || c->kv_heads[k] % c->chunks) die_cfg("head counts do not divide into the qkv pre-shard chunks");+ int any_swa = 0, any_moe = 0;+ for (int i = 0; i < c->n_layers; i++) { any_swa |= c->swa[i]; any_moe |= c->moe[i]; }+ if (any_swa && (c->window < 1 || c->window > (1 << 20))) die_cfg("sliding_window out of range");+ for (int i = 0; i < c->n_layers; i++)+ if (!c->moe[i] && (c->dense_inter < 1 || c->dense_inter > (1 << 20))) die_cfg("intermediate_size out of range");+ if (!any_moe && keep <= 0) die_cfg("no MoE layer");+ if (!(c->eps > 0) || !(c->eps < 1)) die_cfg("layernorm_epsilon out of range");+ if (c->max_positions < 1) c->max_positions = 32768;+ json_free(root); free(arena); free(text);+}++/* ----------------------------------------------------------- dense weights ---- */++/* A dense matrix [O, I] in one of four forms. FP8 keeps the checkpoint bytes and+ * expands the 128x128 block grid to one scale per (row, column block), which is+ * what lets a row move (the qkv de-interleave) without touching its bytes. */+enum { DW_F32 = 0, DW_BF16 = 1, DW_FP8 = 2, DW_I8 = 3 };+typedef struct {+ int fmt, O, I, nblk;+ void *w; /* f32 / bf16 / e4m3 / int8 */+ float *s; /* FP8: [O][nblk]; I8: [O] */+} DW;++/* 0 (default): the checkpoint's own FP8 and BF16 bytes, exact.+ * 8: int8 per row, about 30% less RAM for the dense part, not exact.+ * 32: f32, exact and four times the RAM: the oracle's configuration. */+static int g_dense_bits;++static void matmul_bf16(float *y, const float *x, const uint16_t *W, int S, int I, int O) {+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const uint16_t *w = W + (int64_t)o * I;+ for (int s = 0; s < S; s++) {+ const float *xs = x + (int64_t)s * I;+ float a = 0; int i = 0;+#ifdef __AVX2__+ __m256 acc = _mm256_setzero_ps();+ for (; i + 8 <= I; i += 8)+ acc = _mm256_fmadd_ps(_mm256_loadu_ps(xs + i), bf16_decode8(w + i), acc);+ a = hsum256(acc);+#endif+ for (; i < I; i++) a += xs[i] * bf16_to_f32(w[i]);+ y[(int64_t)s * O + o] = a;+ }+ }+}++/* FP8 with a scale per (row, 128-column block): float inside a block, double+ * across blocks, the convention of quant.h's matmul_fp8. */+static void matmul_fp8_rows(float *y, const float *x, const uint8_t *q8, const float *rs,+ int nblk, int S, int I, int O) {+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const uint8_t *w = q8 + (int64_t)o * I;+ const float *scl = rs + (int64_t)o * nblk;+ for (int s = 0; s < S; s++) {+ const float *xs = x + (int64_t)s * I;+ double a = 0;+ for (int b = 0; b < nblk; b++) {+ int base = b * 128, end = base + 128 < I ? base + 128 : I, i = base;+ float acc = 0;+#ifdef __AVX2__+ __m256 v = _mm256_setzero_ps();+ for (; i + 8 <= end; i += 8)+ v = _mm256_fmadd_ps(_mm256_loadu_ps(xs + i), e4m3_decode8(w + i), v);+ acc = hsum256(v);+#endif+ for (; i < end; i++) acc += e4m3_decode(w[i]) * xs[i];+ a += (double)acc * scl[b];+ }+ y[(int64_t)s * O + o] = (float)a;+ }+ }+}++static void dw_matmul(float *y, const float *x, int S, const DW *d) {+ switch (d->fmt) {+ case DW_F32: matmul(y, x, (const float *)d->w, S, d->I, d->O); break;+ case DW_BF16: matmul_bf16(y, x, (const uint16_t *)d->w, S, d->I, d->O); break;+ case DW_FP8: matmul_fp8_rows(y, x, (const uint8_t *)d->w, d->s, d->nblk, S, d->I, d->O); break;+ case DW_I8: matmul_q(y, x, (const int8_t *)d->w, d->s, S, d->I, d->O); break;+ }+}++static st_tensor *need(shards *S, const char *name, int dtype, int O, int I) {+ st_tensor *t = st_find(S, name);+ if (!t) st_die_missing(S, name);+ int esz = st_dtype_esz(dtype);+ int64_t numel = I > 0 ? (int64_t)O * I : O;+ int rank_ok = I > 0 ? (t->rank == 2 && t->shape[0] == O && t->shape[1] == I)+ : (t->rank == 1 && t->shape[0] == O);+ if (t->dtype != dtype || !rank_ok || t->nbytes != numel * esz) {+ fprintf(stderr, "[mimo] %s: %s %lld bytes, expected %s [%d%s%d] -- refusing (untrusted container)\n",+ name, st_dtype_name(t->dtype), (long long)t->nbytes, st_dtype_name(dtype), O,+ I > 0 ? ", " : "", I > 0 ? I : 0);+ exit(1);+ }+ return t;+}++/* f32 copy of a BF16 or F32 vector */+static float *load_vec(shards *S, const char *name, int n) {+ st_tensor *t = st_find(S, name);+ if (!t) st_die_missing(S, name);+ int dtype = t->dtype == 2 ? 2 : 0;+ need(S, name, dtype, n, 0);+ float *out = xmalloc((size_t)n * sizeof(float), name);+ st_read_f32(S, name, out, 1);+ return out;+}++/* Rows of an f32 matrix [O, I] into the configured dense form. Takes ownership. */+static void dw_from_f32(DW *d, float *w, int O, int I) {+ d->O = O; d->I = I;+ if (g_dense_bits == 8) {+ d->fmt = DW_I8;+ d->w = xmalloc((size_t)O * I, "int8 weights");+ d->s = xmalloc((size_t)O * sizeof(float), "int8 scales");+ quantize_rows(w, (int8_t *)d->w, d->s, O, I, 8);+ free(w);+ } else {+ d->fmt = DW_F32; d->w = w; d->s = NULL;+ }+}++static void dw_load_bf16(DW *d, shards *S, const char *name, int O, int I) {+ st_tensor *t = need(S, name, 0, O, I);+ if (g_dense_bits == 0) {+ d->fmt = DW_BF16; d->O = O; d->I = I; d->s = NULL;+ d->w = xmalloc((size_t)t->nbytes, name);+ st_read_raw_cap(S, name, d->w, t->nbytes, 1);+ return;+ }+ float *w = xmalloc((size_t)O * I * sizeof(float), name);+ st_read_f32(S, name, w, 1);+ dw_from_f32(d, w, O, I);+}++/* An FP8 [O, I] with its block grid; `row_block[r]` names the scale row of+ * stored row r (NULL: r / 128). `order[r]` names which stored row becomes row r+ * of the result (NULL: identity). */+static void dw_load_fp8(DW *d, shards *S, const char *name, int O, int I,+ const int *row_block, int scale_rows, const int *order) {+ char sname[600];+ snprintf(sname, sizeof(sname), "%s_scale_inv", name);+ int nblk = (I + 127) / 128;+ if (scale_rows <= 0) scale_rows = (O + 127) / 128;+ st_tensor *t = need(S, name, 4, O, I);+ need(S, sname, 2, scale_rows, nblk);+ uint8_t *raw = xmalloc((size_t)t->nbytes, name);+ st_read_raw_cap(S, name, raw, t->nbytes, 1);+ float *grid = xmalloc((size_t)scale_rows * nblk * sizeof(float), sname);+ st_read_f32(S, sname, grid, 1);+ for (int64_t i = 0; i < (int64_t)scale_rows * nblk; i++)+ if (!isfinite(grid[i])) { fprintf(stderr, "[mimo] %s: non-finite scale -- refusing\n", sname); exit(1); }+ d->O = O; d->I = I; d->nblk = nblk;+ if (g_dense_bits == 0) {+ d->fmt = DW_FP8;+ uint8_t *w = xmalloc((size_t)O * I, name);+ float *rs = xmalloc((size_t)O * nblk * sizeof(float), sname);+ for (int r = 0; r < O; r++) {+ int src = order ? order[r] : r;+ int sb = row_block ? row_block[src] : src / 128;+ memcpy(w + (int64_t)r * I, raw + (int64_t)src * I, (size_t)I);+ memcpy(rs + (int64_t)r * nblk, grid + (int64_t)sb * nblk, (size_t)nblk * sizeof(float));+ }+ d->w = w; d->s = rs;+ free(raw); free(grid);+ return;+ }+ float *w = xmalloc((size_t)O * I * sizeof(float), name);+ #pragma omp parallel for schedule(static)+ for (int r = 0; r < O; r++) {+ int src = order ? order[r] : r;+ int sb = row_block ? row_block[src] : src / 128;+ const uint8_t *q = raw + (int64_t)src * I;+ float *dst = w + (int64_t)r * I;+ for (int i = 0; i < I; i++) dst[i] = e4m3_decode(q[i]) * grid[(int64_t)sb * nblk + i / 128];+ }+ free(raw); free(grid);+ dw_from_f32(d, w, O, I);+}++/* The fused qkv of layer `li`, read in its pre-sharded order and handed back as+ * [Q | K | V] rows. Two scale grids are legal (vLLM accepts the same two): one+ * tiled per chunk, ceil(rows_per_chunk / 128) rows each, or one continuous grid;+ * they coincide when a chunk is a whole number of blocks. */+static void load_qkv(DW *d, shards *S, const Cfg *c, int li) {+ int k = c->swa[li];+ int T = c->chunks, hd = c->head_dim[k], vd = c->v_dim[k];+ int q_rows = (c->heads[k] / T) * hd, k_rows = (c->kv_heads[k] / T) * hd,+ v_rows = (c->kv_heads[k] / T) * vd;+ int rpc = q_rows + k_rows + v_rows, O = T * rpc;+ int per_chunk = (rpc + 127) / 128, continuous = (O + 127) / 128;+ char name[512], sname[600];+ snprintf(name, sizeof(name), "model.layers.%d.self_attn.qkv_proj.weight", li);+ snprintf(sname, sizeof(sname), "%s_scale_inv", name);+ st_tensor *st = st_find(S, sname);+ if (!st) st_die_missing(S, sname);+ int scale_rows = st->rank == 2 ? (int)st->shape[0] : -1;+ int *row_block = xmalloc((size_t)O * sizeof(int), "qkv scale rows");+ for (int r = 0; r < O; r++) {+ if (scale_rows == T * per_chunk) row_block[r] = (r / rpc) * per_chunk + (r % rpc) / 128;+ else if (scale_rows == continuous) row_block[r] = r / 128;+ else {+ fprintf(stderr, "[mimo] %s: %d scale rows fit neither the per-chunk grid (%d) nor a "+ "continuous one (%d) -- refusing\n", sname, scale_rows, T * per_chunk, continuous);+ exit(1);+ }+ }+ int *order = xmalloc((size_t)O * sizeof(int), "qkv order");+ int w = 0;+ for (int ch = 0; ch < T; ch++) for (int r = 0; r < q_rows; r++) order[w++] = ch * rpc + r;+ for (int ch = 0; ch < T; ch++) for (int r = 0; r < k_rows; r++) order[w++] = ch * rpc + q_rows + r;+ for (int ch = 0; ch < T; ch++) for (int r = 0; r < v_rows; r++) order[w++] = ch * rpc + q_rows + k_rows + r;+ dw_load_fp8(d, S, name, O, c->hidden, row_block, scale_rows, order);+ free(row_block); free(order);+}++/* ------------------------------------------------------------------ model ---- */++typedef struct { int eid; uint64_t used; uint8_t *base, *buf; } Slot;+typedef struct { Slot *s; int n, cap; int *by_expert; } LCache;+typedef struct { int fd[6]; int64_t off[6]; int contig; } ERef;++typedef struct {+ float *ln1, *ln2, *sink;+ DW qkv, o;+ DW gate, up, down; /* dense MLP (layer 0) */+ float *router, *bias; /* [E, H], [E] */+ /* KV: full layers [ctx][kvh][hd]; SWA layers a ring of `window` rows */+ float *K, *V;+ int *ring_pos;+ int rows;+} Layer;++typedef struct {+ Cfg c;+ shards S;+ Layer L[MIMO_MAX_LAYERS];+ uint16_t *embed; /* bf16 [V, H], rows widened on use */+ DW head;+ float *norm;+ ERef *eref;+ int64_t part[6], e_bytes; /* down, down scale, gate, gate scale, up, up scale */+ LCache cache[MIMO_MAX_LAYERS];+ int ctx, pos;+ kv_prefix kvp;+ uint64_t clock, hits, miss, bytes_read, forwards;+ double t_disk, t_expert, t_attn;+ int idot, direct;+ uint8_t **ehit; /* [layer][expert] routed this turn, for HITS */+} Model;++static void expert_table_init(Model *m) {+ Cfg *c = &m->c;+ int H = c->hidden, MI = c->moe_inter;+ m->part[0] = (int64_t)H * (MI / 2); m->part[1] = (int64_t)H * (MI / 32);+ m->part[2] = (int64_t)MI * (H / 2); m->part[3] = (int64_t)MI * (H / 32);+ m->part[4] = m->part[2]; m->part[5] = m->part[3];+ m->e_bytes = 0;+ for (int k = 0; k < 6; k++) m->e_bytes += m->part[k];+ m->eref = xcalloc((size_t)c->n_layers * c->n_experts, sizeof(ERef), "expert table");+ static const char *mat[3] = {"down_proj", "gate_proj", "up_proj"};+ int missing = 0, split = 0;+ for (int li = 0; li < c->n_layers; li++) {+ if (!c->moe[li]) continue;+ for (int e = 0; e < c->n_experts; e++) {+ ERef *er = &m->eref[(int64_t)li * c->n_experts + e];+ for (int k = 0; k < 6; k++) {+ char nm[512];+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.%s.%s", li, e,+ mat[k / 2], (k & 1) ? "weight_scale" : "weight");+ st_tensor *t = st_find(&m->S, nm);+ if (!t) { missing++; er->fd[k] = -1; continue; }+ if (t->dtype != 3 || t->nbytes != m->part[k]) {+ fprintf(stderr, "[mimo] %s: %s %lld bytes, expected U8 %lld -- refusing "+ "(untrusted container)\n", nm, st_dtype_name(t->dtype),+ (long long)t->nbytes, (long long)m->part[k]);+ exit(1);+ }+ er->fd[k] = t->fd; er->off[k] = t->off;+ }+ /* The release stores an expert's six tensors back to back in this+ * order (down, down scale, gate, gate scale, up, up scale): one read. */+ er->contig = er->fd[0] >= 0;+ for (int k = 0; k < 5 && er->contig; k++)+ if (er->fd[k] != er->fd[k + 1] || er->off[k] + m->part[k] != er->off[k + 1]) er->contig = 0;+ split += !er->contig;+ }+ }+ if (missing) fprintf(stderr, "[mimo] WARNING: %d expert tensors missing (incomplete download?) -- touching one aborts\n", missing);+ if (split) fprintf(stderr, "[mimo] %d experts are not contiguous on disk: six reads each\n", split);+}++static void cache_init(Model *m, int cap) {+ Cfg *c = &m->c;+ if (cap < c->topk) cap = c->topk;+ if (cap > c->n_experts) cap = c->n_experts;+ for (int li = 0; li < c->n_layers; li++) {+ if (!c->moe[li]) continue;+ LCache *lc = &m->cache[li];+ lc->cap = cap; lc->n = 0;+ lc->s = xcalloc((size_t)cap, sizeof(Slot), "expert slots");+ lc->by_expert = xmalloc((size_t)c->n_experts * sizeof(int), "expert index");+ for (int e = 0; e < c->n_experts; e++) lc->by_expert[e] = -1;+ for (int k = 0; k < cap; k++) lc->s[k].eid = -1;+ }+}++static void expert_read(Model *m, int li, int eid, Slot *s) {+ if (!s->base && posix_memalign((void **)&s->base, 4096, (size_t)m->e_bytes + 8192)) {+ fprintf(stderr, "[mimo] out of memory for an expert slot\n"); exit(1);+ }+ ERef *er = &m->eref[(int64_t)li * m->c.n_experts + eid];+ if (er->fd[0] < 0) { fprintf(stderr, "[mimo] expert L%d E%d missing on disk\n", li, eid); exit(1); }+ if (er->contig) {+ int dfd = m->direct ? st_direct_fd(&m->S, er->fd[0]) : -1;+ if (dfd >= 0) {+ int64_t a0 = er->off[0] & ~4095LL, pad = er->off[0] - a0, want = pad + m->e_bytes;+ int64_t dlen = (want + 4095) & ~4095LL;+ struct stat sb;+ if (fstat(dfd, &sb) == 0 && a0 + dlen > sb.st_size) dlen = (sb.st_size - a0) & ~4095LL;+ if (dlen > 0) st_pread_full(dfd, s->base, dlen, a0, "pread expert direct");+ if (dlen < want) st_pread_full(er->fd[0], s->base + dlen, want - dlen, a0 + dlen, "pread expert tail");+ s->buf = s->base + pad;+ } else {+ st_pread_full(er->fd[0], s->base, m->e_bytes, er->off[0], "pread expert");+ s->buf = s->base;+ }+ } else {+ uint8_t *dst = s->base;+ for (int k = 0; k < 6; k++) {+ if (er->fd[k] < 0) { fprintf(stderr, "[mimo] expert L%d E%d tensor %d missing\n", li, eid, k); exit(1); }+ st_pread_full(er->fd[k], dst, m->part[k], er->off[k], "pread expert");+ dst += m->part[k];+ }+ s->buf = s->base;+ }+}++/* Make the experts `want[0..n)` of layer li resident, n <= cap, and return their+ * slots in `out`. Hits keep their slot; misses take the least recently used+ * slots not in `want`, and are read in parallel. */+static void experts_ensure(Model *m, int li, const int *want, int n, Slot **out) {+ LCache *lc = &m->cache[li];+ int miss_idx[4096], nmiss = 0;+ uint64_t stamp = ++m->clock;+ for (int j = 0; j < n; j++) {+ int si = lc->by_expert[want[j]];+ if (si >= 0) { out[j] = &lc->s[si]; out[j]->used = stamp; m->hits++; }+ else { out[j] = NULL; miss_idx[nmiss++] = j; }+ }+ for (int k = 0; k < nmiss; k++) {+ int best = -1;+ if (lc->n < lc->cap) best = lc->n++;+ else {+ uint64_t oldest = UINT64_MAX;+ for (int s = 0; s < lc->n; s++)+ if (lc->s[s].used < stamp && lc->s[s].used < oldest) { oldest = lc->s[s].used; best = s; }+ }+ if (best < 0) { fprintf(stderr, "[mimo] expert cache smaller than one routing step\n"); exit(1); }+ Slot *s = &lc->s[best];+ if (s->eid >= 0) lc->by_expert[s->eid] = -1;+ s->eid = want[miss_idx[k]];+ s->used = stamp;+ lc->by_expert[s->eid] = best;+ out[miss_idx[k]] = s;+ }+ if (!nmiss) return;+ double t0 = now_s();+ int threads = env_int("MIMO_READ_THREADS", 8);+ #pragma omp parallel for num_threads(threads) schedule(dynamic, 1)+ for (int k = 0; k < nmiss; k++) {+ Slot *s = out[miss_idx[k]];+ expert_read(m, li, s->eid, s);+ }+ m->miss += (uint64_t)nmiss;+ m->bytes_read += (uint64_t)nmiss * (uint64_t)m->e_bytes;+ m->t_disk += now_s() - t0;+}++static void kv_alloc(Model *m, int ctx) {+ Cfg *c = &m->c;+ m->ctx = ctx;+ for (int li = 0; li < c->n_layers; li++) {+ Layer *l = &m->L[li];+ int k = c->swa[li];+ l->rows = k ? (c->window < ctx ? c->window : ctx) : ctx;+ free(l->K); free(l->V); free(l->ring_pos);+ l->K = xmalloc((size_t)l->rows * c->kv_heads[k] * c->head_dim[k] * sizeof(float), "K cache");+ l->V = xmalloc((size_t)l->rows * c->kv_heads[k] * c->v_dim[k] * sizeof(float), "V cache");+ l->ring_pos = xmalloc((size_t)l->rows * sizeof(int), "ring positions");+ for (int r = 0; r < l->rows; r++) l->ring_pos[r] = -1;+ }+ kv_prefix_alloc(&m->kvp, ctx);+}++static void model_reset(Model *m) {+ Cfg *c = &m->c;+ for (int li = 0; li < c->n_layers; li++)+ for (int r = 0; r < m->L[li].rows; r++) m->L[li].ring_pos[r] = -1;+ m->pos = 0;+ kv_prefix_clear(&m->kvp);+}++static void model_load(Model *m, const char *dir, int cap) {+ Cfg *c = &m->c;+ load_cfg(c, dir);+ st_init_multi(&m->S, dir, getenv("MIMO_DIRS"));+ g_dense_bits = env_int("MIMO_DENSE_BITS", 0);+ if (g_dense_bits != 0 && g_dense_bits != 8 && g_dense_bits != 32) {+ fprintf(stderr, "[mimo] MIMO_DENSE_BITS must be 0 (native FP8/BF16), 8 or 32\n"); exit(1);+ }+ m->idot = env_int("MIMO_IDOT", 0);+ m->direct = env_int("MIMO_DIRECT", 1);+ int H = c->hidden;+ need(&m->S, "model.embed_tokens.weight", 0, c->vocab, H);+ m->embed = xmalloc((size_t)c->vocab * H * sizeof(uint16_t), "embeddings");+ st_read_raw_cap(&m->S, "model.embed_tokens.weight", m->embed, (int64_t)c->vocab * H * 2, 1);+ m->norm = load_vec(&m->S, "model.norm.weight", H);+ dw_load_bf16(&m->head, &m->S, "lm_head.weight", c->vocab, H);+ for (int li = 0; li < c->n_layers; li++) {+ Layer *l = &m->L[li];+ int k = c->swa[li];+ char nm[512];+ snprintf(nm, sizeof(nm), "model.layers.%d.input_layernorm.weight", li);+ l->ln1 = load_vec(&m->S, nm, H);+ snprintf(nm, sizeof(nm), "model.layers.%d.post_attention_layernorm.weight", li);+ l->ln2 = load_vec(&m->S, nm, H);+ load_qkv(&l->qkv, &m->S, c, li);+ snprintf(nm, sizeof(nm), "model.layers.%d.self_attn.o_proj.weight", li);+ dw_load_bf16(&l->o, &m->S, nm, H, c->heads[k] * c->v_dim[k]);+ if (c->sink[k]) {+ snprintf(nm, sizeof(nm), "model.layers.%d.self_attn.attention_sink_bias", li);+ l->sink = load_vec(&m->S, nm, c->heads[k]);+ }+ if (!c->moe[li]) {+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate_proj.weight", li);+ dw_load_fp8(&l->gate, &m->S, nm, c->dense_inter, H, NULL, 0, NULL);+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.up_proj.weight", li);+ dw_load_fp8(&l->up, &m->S, nm, c->dense_inter, H, NULL, 0, NULL);+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.down_proj.weight", li);+ dw_load_fp8(&l->down, &m->S, nm, H, c->dense_inter, NULL, 0, NULL);+ } else {+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate.weight", li);+ need(&m->S, nm, 0, c->n_experts, H);+ l->router = xmalloc((size_t)c->n_experts * H * sizeof(float), nm);+ st_read_f32(&m->S, nm, l->router, 1);+ snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate.e_score_correction_bias", li);+ l->bias = load_vec(&m->S, nm, c->n_experts);+ }+ }+ expert_table_init(m);+ cache_init(m, cap);+ /* coli --ctx arrives as CTX (the family's context_env); MIMO_CTX wins over it */+ const char *ctx_text = getenv("MIMO_CTX");+ if (!ctx_text || !*ctx_text) ctx_text = getenv("CTX");+ int ctx = (ctx_text && *ctx_text) ? atoi(ctx_text) : 8192;+ if (ctx > c->max_positions) ctx = c->max_positions;+ if (ctx < 16) ctx = 16;+ kv_alloc(m, ctx);+ m->ehit = xmalloc((size_t)c->n_layers * sizeof(uint8_t *), "hits");+ for (int li = 0; li < c->n_layers; li++) m->ehit[li] = xcalloc((size_t)c->n_experts, 1, "hits row");+}++/* ------------------------------------------------------------ the forward ---- */++static void rmsnorm(float *out, const float *x, const float *w, int n, float eps) {+ float ss = 0;+ for (int i = 0; i < n; i++) ss += x[i] * x[i];+ float r = 1.0f / sqrtf(ss / n + eps);+ for (int i = 0; i < n; i++) out[i] = w[i] * (x[i] * r);+}++/* rotate-half RoPE on the first `rd` dims of a head, as the reference builds it:+ * inv_freq and the angle in float32, cos/sin of the float angle */+static void rope(float *v, int rd, float theta, int pos) {+ int half = rd / 2;+ for (int i = 0; i < half; i++) {+ float inv = 1.0f / powf(theta, (float)(2 * i) / (float)rd);+ float ang = inv * (float)pos;+ float cs = cosf(ang), sn = sinf(ang);+ float a = v[i], b = v[i + half];+ v[i] = a * cs - b * sn;+ v[i + half] = b * cs + a * sn;+ }+}++/* Attention of `n` rows at positions p0.. of layer li. `xn` is the normed input+ * [n, H], `out` receives o_proj's output [n, H]. */+static void attention(Model *m, int li, const float *xn, int n, int p0, float *out) {+ Cfg *c = &m->c;+ Layer *l = &m->L[li];+ int k = c->swa[li];+ int nh = c->heads[k], kvh = c->kv_heads[k], hd = c->head_dim[k], vd = c->v_dim[k];+ int qd = nh * hd, kd = kvh * hd, vdd = kvh * vd, rows = qd + kd + vdd;+ double t0 = now_s();+ float *qkv = xmalloc((size_t)n * rows * sizeof(float), "qkv");+ dw_matmul(qkv, xn, n, &l->qkv);+ for (int t = 0; t < n; t++) {+ float *q = qkv + (size_t)t * rows, *kk = q + qd, *vv = kk + kd;+ for (int h = 0; h < nh; h++) rope(q + h * hd, c->rope_dim[k], c->theta[k], p0 + t);+ for (int h = 0; h < kvh; h++) rope(kk + h * hd, c->rope_dim[k], c->theta[k], p0 + t);+ for (int i = 0; i < vdd; i++) vv[i] *= c->v_scale;+ }+ /* Keys this block can see: [lo, p0 + n). For a sliding window the ring still+ * holds the ones before p0; for full attention the cache holds everything,+ * and the block is written in first. */+ int window = k ? c->window : 0;+ int lo = k ? (p0 - (window - 1) > 0 ? p0 - (window - 1) : 0) : 0;+ int nkeys = p0 + n - lo;+ float *Kc = NULL, *Vc = NULL;+ if (k) {+ Kc = xmalloc((size_t)nkeys * kd * sizeof(float), "window keys");+ Vc = xmalloc((size_t)nkeys * vdd * sizeof(float), "window values");+ for (int p = lo; p < p0; p++) {+ int slot = p % l->rows;+ if (l->ring_pos[slot] != p) { fprintf(stderr, "[mimo] window ring lost position %d\n", p); exit(1); }+ memcpy(Kc + (size_t)(p - lo) * kd, l->K + (size_t)slot * kd, (size_t)kd * sizeof(float));+ memcpy(Vc + (size_t)(p - lo) * vdd, l->V + (size_t)slot * vdd, (size_t)vdd * sizeof(float));+ }+ for (int t = 0; t < n; t++) {+ const float *src = qkv + (size_t)t * rows + qd;+ memcpy(Kc + (size_t)(p0 + t - lo) * kd, src, (size_t)kd * sizeof(float));+ memcpy(Vc + (size_t)(p0 + t - lo) * vdd, src + kd, (size_t)vdd * sizeof(float));+ }+ } else {+ for (int t = 0; t < n; t++) {+ const float *src = qkv + (size_t)t * rows + qd;+ memcpy(l->K + (size_t)(p0 + t) * kd, src, (size_t)kd * sizeof(float));+ memcpy(l->V + (size_t)(p0 + t) * vdd, src + kd, (size_t)vdd * sizeof(float));+ }+ Kc = l->K; Vc = l->V;+ }+ float *ctxv = xmalloc((size_t)n * nh * vd * sizeof(float), "attention output");+ float scale = 1.0f / sqrtf((float)hd);+ int group = nh / kvh;+ #pragma omp parallel+ {+ float *score = xmalloc((size_t)nkeys * sizeof(float), "scores");+ #pragma omp for collapse(2) schedule(static)+ for (int t = 0; t < n; t++) {+ for (int h = 0; h < nh; h++) {+ int p = p0 + t, kh = h / group;+ int first = window ? (p - (window - 1) > lo ? p - (window - 1) : lo) : 0;+ const float *q = qkv + (size_t)t * rows + (size_t)h * hd;+ float best = -INFINITY;+ for (int j = first; j <= p; j++) {+ const float *kr = Kc + (size_t)(j - lo) * kd + (size_t)kh * hd;+ float s = 0;+ for (int i = 0; i < hd; i++) s += q[i] * kr[i];+ s *= scale;+ score[j - first] = s;+ if (s > best) best = s;+ }+ float sink = l->sink ? l->sink[h] : -INFINITY;+ if (sink > best) best = sink;+ float den = l->sink ? expf(sink - best) : 0.0f;+ for (int j = first; j <= p; j++) { score[j - first] = expf(score[j - first] - best); den += score[j - first]; }+ float *o = ctxv + ((size_t)t * nh + h) * vd;+ for (int i = 0; i < vd; i++) o[i] = 0;+ for (int j = first; j <= p; j++) {+ const float *vr = Vc + (size_t)(j - lo) * vdd + (size_t)kh * vd;+ float w = score[j - first] / den;+ for (int i = 0; i < vd; i++) o[i] += w * vr[i];+ }+ }+ }+ free(score);+ }+ if (k) {+ /* the ring keeps the last `window` positions of the block */+ int from = n > l->rows ? n - l->rows : 0;+ for (int t = from; t < n; t++) {+ int p = p0 + t, slot = p % l->rows;+ const float *src = qkv + (size_t)t * rows + qd;+ memcpy(l->K + (size_t)slot * kd, src, (size_t)kd * sizeof(float));+ memcpy(l->V + (size_t)slot * vdd, src + kd, (size_t)vdd * sizeof(float));+ l->ring_pos[slot] = p;+ }+ free(Kc); free(Vc);+ }+ dw_matmul(out, ctxv, n, &l->o);+ free(ctxv); free(qkv);+ m->t_attn += now_s() - t0;+}++static void dense_mlp(Model *m, Layer *l, const float *xn, int n, float *out) {+ int F = l->gate.O;+ float *g = xmalloc((size_t)n * F * sizeof(float), "mlp gate");+ float *u = xmalloc((size_t)n * F * sizeof(float), "mlp up");+ dw_matmul(g, xn, n, &l->gate);+ dw_matmul(u, xn, n, &l->up);+ for (size_t i = 0; i < (size_t)n * F; i++) g[i] = silu(g[i]) * u[i];+ dw_matmul(out, g, n, &l->down);+ free(g); free(u);+ (void)m;+}++/* Route every row, then run each chosen expert once over the rows that chose it.+ * `out` [n, H] receives the weighted sum. */+static void moe(Model *m, int li, const float *xn, int n, float *out) {+ Cfg *c = &m->c;+ Layer *l = &m->L[li];+ int H = c->hidden, E = c->n_experts, K = c->topk, MI = c->moe_inter;+ int *sel = xmalloc((size_t)n * K * sizeof(int), "routing");+ float *wt = xmalloc((size_t)n * K * sizeof(float), "routing weights");+ float *logits = xmalloc((size_t)n * E * sizeof(float), "router logits");+ matmul(logits, xn, l->router, n, H, E);+ for (int t = 0; t < n; t++) {+ float *score = logits + (size_t)t * E;+ for (int e = 0; e < E; e++) score[e] = sigmoid(score[e]);+ int *s = sel + (size_t)t * K;+ float *w = wt + (size_t)t * K;+ for (int j = 0; j < K; j++) {+ int best = -1; float bv = -INFINITY;+ for (int e = 0; e < E; e++) {+ int taken = 0;+ for (int q = 0; q < j; q++) if (s[q] == e) { taken = 1; break; }+ if (taken) continue;+ float v = score[e] + l->bias[e];+ if (v > bv) { bv = v; best = e; }+ }+ s[j] = best; w[j] = score[best];+ }+ float sum = 0;+ for (int j = 0; j < K; j++) sum += w[j];+ for (int j = 0; j < K; j++) w[j] = (c->norm_topk && K > 1 ? w[j] / (sum + 1e-20f) : w[j]) * c->route_scale;+ for (int j = 0; j < K; j++) m->ehit[li][s[j]] = 1;+ }+ free(logits);+ /* the union of chosen experts, in first-use order */+ int *uni = xmalloc((size_t)E * sizeof(int), "expert union");+ int *seen = xcalloc((size_t)E, sizeof(int), "expert seen");+ int nu = 0;+ for (int i = 0; i < n * K; i++) if (!seen[sel[i]]) { seen[sel[i]] = 1; uni[nu++] = sel[i]; }+ free(seen);+ memset(out, 0, (size_t)n * H * sizeof(float));+ /* Each group of resident experts is computed in two parallel passes -- gate and+ * up of every expert, then down of every expert -- split in row blocks, instead+ * of three OpenMP regions per expert (a thousand a token on Flash, and every+ * barrier costs a scheduling round on a busy machine). The kernel inside a+ * task is the same matmul_mxfp4 on the same rows, so the numbers do not move. */+ int cap = m->cache[li].cap;+ int *row = xmalloc((size_t)n * K * sizeof(int), "expert rows index");+ float *rw = xmalloc((size_t)n * K * sizeof(float), "expert row weights");+ int *first = xmalloc((size_t)(cap + 1) * sizeof(int), "expert row offsets");+ float *xg = xmalloc((size_t)n * K * H * sizeof(float), "expert inputs");+ float *g = xmalloc((size_t)n * K * MI * sizeof(float), "expert gate");+ float *u = xmalloc((size_t)n * K * MI * sizeof(float), "expert up");+ float *y = xmalloc((size_t)n * K * H * sizeof(float), "expert out");+ void (*mm)(float *, const float *, const uint8_t *, const uint8_t *, int, int, int)+ = m->idot ? matmul_mxfp4_i8 : matmul_mxfp4;+ const int BLOCK_ROWS = 256;+ Slot *slots[4096];+ for (int b = 0; b < nu; b += cap) {+ int nb = nu - b < cap ? nu - b : cap;+ experts_ensure(m, li, uni + b, nb, slots);+ double t0 = now_s();+ int total = 0;+ for (int j = 0; j < nb; j++) {+ int e = uni[b + j];+ first[j] = total;+ for (int t = 0; t < n; t++)+ for (int q = 0; q < K; q++)+ if (sel[(size_t)t * K + q] == e) { row[total] = t; rw[total] = wt[(size_t)t * K + q]; total++; }+ }+ first[nb] = total;+ for (int i = 0; i < total; i++) memcpy(xg + (size_t)i * H, xn + (size_t)row[i] * H, (size_t)H * sizeof(float));+ int up_blocks = (MI + BLOCK_ROWS - 1) / BLOCK_ROWS, down_blocks = (H + BLOCK_ROWS - 1) / BLOCK_ROWS;+ int rb_mi = MI / 2, gb_mi = MI / 32, rb_h = H / 2, gb_h = H / 32;+ #pragma omp parallel for schedule(dynamic, 1)+ for (int task = 0; task < nb * 2 * up_blocks; task++) {+ int j = task / (2 * up_blocks), which = (task / up_blocks) % 2, blk = task % up_blocks;+ int r = first[j + 1] - first[j];+ if (!r) continue;+ int o0 = blk * BLOCK_ROWS, rows = MI - o0 < BLOCK_ROWS ? MI - o0 : BLOCK_ROWS;+ const uint8_t *buf = slots[j]->buf;+ const uint8_t *q = buf + m->part[0] + m->part[1] + (which ? m->part[2] + m->part[3] : 0);+ const uint8_t *sc = q + m->part[2];+ float *dst = (which ? u : g) + (size_t)first[j] * MI;+ /* rows [o0, o0+rows) of this expert's gate or up, for its r inputs; the+ * kernel writes [r, rows] contiguously, so it goes through a scratch */+ float tmp[4 * 256];+ float *scratch = r * rows <= (int)(sizeof(tmp) / sizeof(tmp[0])) ? tmp+ : xmalloc((size_t)r * rows * sizeof(float), "expert scratch");+ mm(scratch, xg + (size_t)first[j] * H, q + (size_t)o0 * rb_h, sc + (size_t)o0 * gb_h, r, H, rows);+ for (int i = 0; i < r; i++)+ memcpy(dst + (size_t)i * MI + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));+ if (scratch != tmp) free(scratch);+ }+ for (size_t i = 0; i < (size_t)total * MI; i++) g[i] = silu(g[i]) * u[i];+ #pragma omp parallel for schedule(dynamic, 1)+ for (int task = 0; task < nb * down_blocks; task++) {+ int j = task / down_blocks, blk = task % down_blocks;+ int r = first[j + 1] - first[j];+ if (!r) continue;+ int o0 = blk * BLOCK_ROWS, rows = H - o0 < BLOCK_ROWS ? H - o0 : BLOCK_ROWS;+ const uint8_t *dq = slots[j]->buf, *ds = dq + m->part[0];+ float tmp[4 * 256];+ float *scratch = r * rows <= (int)(sizeof(tmp) / sizeof(tmp[0])) ? tmp+ : xmalloc((size_t)r * rows * sizeof(float), "expert scratch");+ mm(scratch, g + (size_t)first[j] * MI, dq + (size_t)o0 * rb_mi, ds + (size_t)o0 * gb_mi, r, MI, rows);+ float *dst = y + (size_t)first[j] * H;+ for (int i = 0; i < r; i++)+ memcpy(dst + (size_t)i * H + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));+ if (scratch != tmp) free(scratch);+ }+ /* the weighted sum, expert by expert in the order they were chosen */+ for (int j = 0; j < nb; j++)+ for (int i = first[j]; i < first[j + 1]; i++) {+ float *o = out + (size_t)row[i] * H;+ const float *yy = y + (size_t)i * H;+ for (int d = 0; d < H; d++) o[d] += rw[i] * yy[d];+ }+ m->t_expert += now_s() - t0;+ }+ free(first);+ free(xg); free(g); free(u); free(y); free(row); free(rw); free(uni); free(sel); free(wt);+}++static void embed_rows(Model *m, const int *ids, int n, float *h) {+ int H = m->c.hidden;+ for (int t = 0; t < n; t++) {+ int id = ids[t];+ if (id < 0 || id >= m->c.vocab) { fprintf(stderr, "[mimo] token id %d out of range\n", id); exit(1); }+ for (int i = 0; i < H; i++) h[(size_t)t * H + i] = bf16_to_f32(m->embed[(size_t)id * H + i]);+ }+}++/* Image rows to splice in: `rows[r]` replaces the embedding of the r-th+ * image-pad position of the WHOLE prompt (the first at `first_pad`). */+typedef struct { const float *rows; int n_rows; } ImageRows;++/* One block of n tokens at m->pos. logits: NULL, or [n, V] (all rows) when+ * all_rows, else [V] for the last row. */+static void forward(Model *m, const int *ids, int n, float *logits, int all_rows,+ const ImageRows *img, int *img_used) {+ Cfg *c = &m->c;+ int H = c->hidden;+ if (m->pos + n > m->ctx) { fprintf(stderr, "[mimo] context full (%d + %d > %d)\n", m->pos, n, m->ctx); exit(1); }+ float *h = xmalloc((size_t)n * H * sizeof(float), "residual");+ float *xn = xmalloc((size_t)n * H * sizeof(float), "normed");+ float *tmp = xmalloc((size_t)n * H * sizeof(float), "block out");+ embed_rows(m, ids, n, h);+ if (img && img->rows && c->image_token_id >= 0)+ for (int t = 0; t < n; t++)+ if (ids[t] == c->image_token_id && *img_used < img->n_rows) {+ memcpy(h + (size_t)t * H, img->rows + (size_t)(*img_used) * H, (size_t)H * sizeof(float));+ (*img_used)++;+ }+ /* MIMO_TRACE=<file>: the residual after the embedding and after every+ * attention and every MLP of the first block, f32 [2L+1][n][H] -- what+ * tools/make_mimo_ref.py --trace compares against the vendor's hooks. */+ static FILE *trace;+ static int traced;+ if (!traced && getenv("MIMO_TRACE")) { trace = fopen(getenv("MIMO_TRACE"), "wb"); }+ if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);+ for (int li = 0; li < c->n_layers; li++) {+ Layer *l = &m->L[li];+ for (int t = 0; t < n; t++) rmsnorm(xn + (size_t)t * H, h + (size_t)t * H, l->ln1, H, c->eps);+ attention(m, li, xn, n, m->pos, tmp);+ for (size_t i = 0; i < (size_t)n * H; i++) h[i] += tmp[i];+ if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);+ for (int t = 0; t < n; t++) rmsnorm(xn + (size_t)t * H, h + (size_t)t * H, l->ln2, H, c->eps);+ if (c->moe[li]) moe(m, li, xn, n, tmp);+ else dense_mlp(m, l, xn, n, tmp);+ for (size_t i = 0; i < (size_t)n * H; i++) h[i] += tmp[i];+ if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);+ }+ if (trace) { fclose(trace); trace = NULL; }+ traced = 1;+ if (logits) {+ int from = all_rows ? 0 : n - 1, rows = n - from;+ for (int t = from; t < n; t++) rmsnorm(xn + (size_t)(t - from) * H, h + (size_t)t * H, m->norm, H, c->eps);+ dw_matmul(logits, xn, rows, &m->head);+ }+ kv_prefix_record(&m->kvp, ids, m->pos, n);+ m->pos += n;+ m->forwards++;+ free(h); free(xn); free(tmp);+}++/* A prompt in blocks of MIMO_CHUNK rows (default 64): the last block's last+ * row (or every row, for the teacher-forcing dump) comes back in `logits`. */+static void prefill(Model *m, const int *ids, int n, float *logits, float *all_logits,+ const ImageRows *img) {+ int chunk = env_int("MIMO_CHUNK", 64);+ if (chunk < 1) chunk = 1;+ int used = 0;+ for (int b = 0; b < n; b += chunk) {+ int nb = n - b < chunk ? n - b : chunk;+ if (all_logits) forward(m, ids + b, nb, all_logits + (size_t)b * m->c.vocab, 1, img, &used);+ else forward(m, ids + b, nb, b + nb == n ? logits : NULL, 0, img, &used);+ }+ if (all_logits && logits) memcpy(logits, all_logits + (size_t)(n - 1) * m->c.vocab, (size_t)m->c.vocab * sizeof(float));+}++/* ----------------------------------------------------------------- vision ---- */++/* The tower, when the container carries one (visual.*): vision_run turns the+ * gateway's patches into the rows that replace the image-pad embeddings. */+typedef struct Vision Vision;+static Vision *g_vision;+#include "mimo_vision.h"++/* --------------------------------------------------------------- sampling ---- */++static int argmax(const float *x, int n) {+ int best = 0;+ for (int i = 1; i < n; i++) if (x[i] > x[best]) best = i;+ return best;+}++typedef struct { float p; int id; } SampleProb;+static int sample_desc(const void *a, const void *b) {+ float pa = ((const SampleProb *)a)->p, pb = ((const SampleProb *)b)->p;+ return (pb > pa) - (pa > pb);+}++static int sample(const float *logits, int vocab, float temperature, float top_p) {+ if (temperature <= 0.0f) return argmax(logits, vocab);+ SampleProb *rank = xmalloc((size_t)vocab * sizeof(SampleProb), "sampling");+ float top = logits[0];+ for (int i = 1; i < vocab; i++) if (logits[i] > top) top = logits[i];+ double total = 0.0;+ for (int i = 0; i < vocab; i++) {+ float p = expf((logits[i] - top) / temperature);+ total += p; rank[i].p = p; rank[i].id = i;+ }+ qsort(rank, (size_t)vocab, sizeof(SampleProb), sample_desc);+ double cut = (top_p > 0.0f && top_p < 1.0f) ? top_p * total : total, kept = 0.0;+ int n = 0;+ while (n < vocab && kept < cut) kept += rank[n++].p;+ double draw = ((double)rand() / RAND_MAX) * kept, acc = 0.0;+ int pick = rank[0].id;+ for (int i = 0; i < n; i++) { acc += rank[i].p; if (acc >= draw) { pick = rank[i].id; break; } }+ free(rank);+ return pick;+}++/* ------------------------------------------------------------------ serve ---- */++static const ColiServeWireProfile mimo_wire = {+ .max_header_bytes = 511,+ .max_payload_bytes = 1u << 26,+ .max_tokens = 1 << 20,+ .require_exact_lf = 1,+ .require_finite_sampling = 1,+};++static void serve_line(const char *format, ...) {+ va_list args;+ va_start(args, format);+ vfprintf(stdout, format, args);+ va_end(args);+ fflush(stdout);+}++/* the dashboard's grid: rows are the MoE layers, columns the routed experts */+static void serve_emap(Model *m) {+ Cfg *c = &m->c;+ int rows = 0;+ for (int li = 0; li < c->n_layers; li++) rows += c->moe[li];+ int cols = c->n_experts;+ char *hex = xmalloc((size_t)rows * cols * 2 + 1, "emap");+ int w = 0;+ for (int li = 0; li < c->n_layers; li++) {+ if (!c->moe[li]) continue;+ for (int e = 0; e < cols; e++) {+ int byte = (m->cache[li].by_expert[e] >= 0) << 6;+ hex[w++] = "0123456789abcdef"[byte >> 4];+ hex[w++] = "0123456789abcdef"[byte & 15];+ }+ }+ hex[w] = 0;+ serve_line("EMAP %d %d %s\n", rows, cols, hex);+ free(hex);+}++static void serve_hits(Model *m) {+ Cfg *c = &m->c;+ int rows = 0;+ for (int li = 0; li < c->n_layers; li++) rows += c->moe[li];+ int cols = c->n_experts, nb = (rows * cols + 7) / 8, bit = 0;+ uint8_t *bitmap = xcalloc((size_t)nb, 1, "hits bitmap");+ for (int li = 0; li < c->n_layers; li++) {+ if (!c->moe[li]) continue;+ for (int e = 0; e < cols; e++, bit++)+ if (m->ehit[li][e]) { bitmap[bit >> 3] |= (uint8_t)(1 << (bit & 7)); m->ehit[li][e] = 0; }+ }+ char *hex = xmalloc((size_t)nb * 2 + 1, "hits hex");+ for (int b = 0; b < nb; b++) {+ hex[2 * b] = "0123456789abcdef"[bitmap[b] >> 4];+ hex[2 * b + 1] = "0123456789abcdef"[bitmap[b] & 15];+ }+ hex[2 * nb] = 0;+ serve_line("HITS %d %d %s\n", rows, cols, hex);+ free(hex); free(bitmap);+}++/* max_tokens is a ceiling; generation needs at least one free position */+static int serve_budget(int prompt, int requested, int context) {+ if (prompt < 1 || prompt > context) return -1;+ int budget = requested > 0 ? requested : 256;+ int room = context - prompt;+ if (room == 0) return -1;+ return budget < room ? budget : room;+}++static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {+ Cfg *c = &m->c;+ coli_serve_stdio_init();+ int eos[8];+ int n_eos = coli_load_stop_ids(dir, eos, 8, NULL);+ coli_serve_write_ready(stdout, rss_gb());+ serve_emap(m);+ float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");+ int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "prompt ids");+ float *pending = NULL;+ int pending_h = 0, pending_w = 0;+ for (;;) {+ ColiServeCommand command;+ ColiServeReadResult result = coli_serve_read_command(stdin, &mimo_wire, &command);+ if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) break;+ if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); break; }+ if (result == COLI_SERVE_READ_BAD_REQUEST) {+ if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");+ coli_serve_command_dispose(&command);+ continue;+ }+ if (command.kind == COLI_SERVE_COMMAND_IMAGE) {+ /* One image waits for the SUBMIT that names it; a second replaces it. */+ if (!g_vision) {+ coli_serve_write_error(stdout, command.id, "this container has no vision tower");+ coli_serve_command_dispose(&command);+ continue;+ }+ uint64_t expected = (uint64_t)command.grid_h * command.grid_w * g_vision->patch_in * sizeof(float);+ if (command.grid_h < 2 || command.grid_w < 2 || command.grid_h % 2 || command.grid_w % 2 ||+ command.payload_bytes != expected) {+ coli_serve_write_error(stdout, command.id, "BAD_IMAGE payload does not match its grid");+ coli_serve_command_dispose(&command);+ continue;+ }+ free(pending);+ pending = (float *)coli_serve_command_take_payload(&command);+ pending_h = command.grid_h; pending_w = command.grid_w;+ coli_serve_command_dispose(&command);+ continue;+ }+ if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {+ if (command.kind == COLI_SERVE_COMMAND_CANCEL) coli_serve_write_error(stdout, command.id, "NOT_FOUND");+ coli_serve_command_dispose(&command);+ continue;+ }+ double started = now_s();+ double disk0 = m->t_disk, expert0 = m->t_expert, attn0 = m->t_attn;+ uint64_t hits0 = m->hits, miss0 = m->miss, fw0 = m->forwards;+ int n_prompt = tok_encode(tokenizer, (const char *)command.payload, (int)command.payload_bytes,+ ids, m->ctx + 1);+ int budget = serve_budget(n_prompt, command.max_tokens, m->ctx);+ if (budget < 0) {+ char message[160];+ snprintf(message, sizeof(message), "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",+ n_prompt, command.max_tokens, m->ctx);+ coli_serve_write_error(stdout, command.id, n_prompt < 1 ? "EMPTY_PROMPT" : message);+ free(pending); pending = NULL;+ coli_serve_command_dispose(&command);+ continue;+ }+ /* An image: its rows replace the pad ids, which must be exactly as many+ * as the tower produces. Checked before ACCEPT so a mismatch is a clean+ * 400, not a half-answered turn. */+ float *image = NULL;+ int image_rows = 0;+ if (pending) {+ int pads = 0;+ for (int t = 0; t < n_prompt; t++) pads += ids[t] == c->image_token_id;+ int want = (pending_h / 2) * (pending_w / 2);+ if (pads != want) {+ char message[160];+ snprintf(message, sizeof(message), "BAD_IMAGE prompt has %d image pads, the %dx%d grid needs %d",+ pads, pending_h, pending_w, want);+ coli_serve_write_error(stdout, command.id, message);+ free(pending); pending = NULL;+ coli_serve_command_dispose(&command);+ continue;+ }+ }+ /* A chat client resends the whole transcript: if this prompt begins with+ * the ids the state was built from, only the tail is fed. An image+ * refuses it -- the pad ids do not describe the picture. */+ int prefix_on = env_int("COLI_KV_PREFIX", 1) != 0;+ int reuse = (!pending && prefix_on) ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;+ if (getenv("COLI_PREFIX_LOG"))+ fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens\n", reuse ? "reusing" : "no reuse,", reuse, n_prompt);+ if (!reuse) model_reset(m);+ coli_serve_write_accept(stdout, command.id, n_prompt);+ if (pending) {+ image = vision_run(m, pending, pending_h, pending_w, &image_rows);+ free(pending); pending = NULL;+ }+ ImageRows img = { image, image_rows };+ prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL);+ if (image) { kv_prefix_taint(&m->kvp); free(image); }+ int emitted = 0, limited = 1, cancelled = 0, done_early = 0;+ char piece[512];+ while (emitted < budget && !cancelled && !done_early) {+ int token = sample(logits, c->vocab, command.temperature, command.top_p);+ int stop = 0;+ for (int i = 0; i < n_eos; i++) if (token == eos[i]) stop = 1;+ if (stop) { limited = 0; break; }+ int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));+ if (written > 0) coli_serve_write_data(stdout, command.id, piece, (size_t)written);+ emitted++;+ while (coli_serve_stdin_ready()) {+ ColiServeCommand control;+ ColiServeReadResult inner = coli_serve_read_command(stdin, &mimo_wire, &control);+ if (inner == COLI_SERVE_READ_EOF) { done_early = 1; break; }+ if (control.kind == COLI_SERVE_COMMAND_CANCEL && !strcmp(control.id, command.id)) cancelled = 1;+ else if (control.kind == COLI_SERVE_COMMAND_STOP && !strcmp(control.id, command.id)) { limited = 0; done_early = 1; }+ else if (control.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, control.id, "SLOT_BUSY");+ coli_serve_command_dispose(&control);+ }+ if (cancelled || done_early || emitted >= budget) break;+ prefill(m, &token, 1, logits, NULL, NULL);+ }+ if (cancelled) {+ coli_serve_write_error(stdout, command.id, "CANCELLED");+ coli_serve_command_dispose(&command);+ continue;+ }+ double wall = now_s() - started;+ uint64_t th = m->hits - hits0, tm = m->miss - miss0;+ ColiServeDone done = { emitted, wall > 0 ? emitted / wall : 0.0,+ (th + tm) ? 100.0 * th / (double)(th + tm) : 0.0, rss_gb(), n_prompt, limited };+ coli_serve_write_done(stdout, command.id, &done);+ serve_line("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n", wall, n_prompt, emitted,+ m->t_disk - disk0, 0.0, m->t_expert - expert0, m->t_attn - attn0, 0.0,+ (unsigned long long)(m->forwards - fw0));+ serve_hits(m);+ serve_emap(m);+ coli_serve_command_dispose(&command);+ }+ free(ids); free(logits); free(pending);+}++/* ------------------------------------------------------------------- main ---- */++static int parse_ids(const char *text, int *out, int cap) {+ int n = 0;+ const char *p = text;+ while (*p && n < cap) {+ while (*p == ' ' || *p == ',' || *p == '\t' || *p == '\n') p++;+ if (!*p) break;+ char *end;+ long v = strtol(p, &end, 10);+ if (end == p) break;+ out[n++] = (int)v;+ p = end;+ }+ return n;+}++static int is_dir(const char *path) {+ struct stat sb;+ return stat(path, &sb) == 0 && S_ISDIR(sb.st_mode);+}++int main(int argc, char **argv) {+ coli_omp_tune_threads("mimo");+ /* Two ways in. The gateway: SNAP=<dir> mimo <cap>, SERVE=1. By hand and in+ * the tests: mimo <dir> [--ids "..." | --prompt "..."] [--ngen N] [--cap N]. */+ const char *dir = getenv("SNAP");+ int cap = env_int("MIMO_CAP", 0);+ const char *ids_text = NULL, *prompt = NULL, *image_path = NULL;+ int ngen = 32, grid_h = 0, grid_w = 0;+ for (int i = 1; i < argc; i++) {+ if (!strcmp(argv[i], "--ids") && i + 1 < argc) ids_text = argv[++i];+ else if (!strcmp(argv[i], "--prompt") && i + 1 < argc) prompt = argv[++i];+ else if (!strcmp(argv[i], "--ngen") && i + 1 < argc) ngen = coli_arg_int(argv[++i], "--ngen");+ else if (!strcmp(argv[i], "--cap") && i + 1 < argc) cap = coli_arg_int(argv[++i], "--cap");+ else if (!strcmp(argv[i], "--image") && i + 1 < argc) image_path = argv[++i];+ else if (!strcmp(argv[i], "--grid") && i + 2 < argc) {+ grid_h = coli_arg_int(argv[++i], "--grid h"); grid_w = coli_arg_int(argv[++i], "--grid w"); }+ else if (is_dir(argv[i])) dir = argv[i];+ else cap = coli_arg_int(argv[i], "cache/layer");+ }+ if (!dir) {+ fprintf(stderr, "usage: %s <model dir> [--ids \"1 2 3\" | --prompt TEXT] [--ngen N] [--cap N]\n"+ " SNAP=<model dir> SERVE=1 %s <cap>\n", argv[0], argv[0]);+ return 2;+ }+ Model *m = xcalloc(1, sizeof(Model), "model");+ double t0 = now_s();+ if (cap <= 0) cap = 64;+ model_load(m, dir, cap);+ Cfg *c = &m->c;+ g_vision = vision_load(m, dir);+ int swa = 0, shown_cap = 0;+ for (int i = 0; i < c->n_layers; i++) { swa += c->swa[i]; if (c->moe[i] && !shown_cap) shown_cap = m->cache[i].cap; }+ fprintf(stderr, "[mimo] %d layers (%d sliding window %d, %d full), %d experts top-%d, cache %d/layer, "+ "dense %s, ctx %d%s -- loaded in %.2fs\n",+ c->n_layers, swa, c->window, c->n_layers - swa, c->n_experts, c->topk, shown_cap,+ g_dense_bits == 0 ? "native" : g_dense_bits == 8 ? "int8" : "f32", m->ctx,+ g_vision ? ", vision" : "", now_s() - t0);++ char tok_path[4096];+ snprintf(tok_path, sizeof(tok_path), "%s/tokenizer.json", dir);+ int have_tok = access(tok_path, R_OK) == 0;+ Tok tokenizer;+ if (have_tok) tok_load(&tokenizer, tok_path);+ const char *seed = getenv("SEED");+ srand(seed ? (unsigned)strtoul(seed, NULL, 10) : (unsigned)time(NULL));++ if (env_int("SERVE", 0)) {+ if (!have_tok) { fprintf(stderr, "[mimo] SERVE needs %s\n", tok_path); return 1; }+ serve_loop(m, &tokenizer, dir);+ return 0;+ }+ int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "ids");+ int n = 0;+ if (ids_text) n = parse_ids(ids_text, ids, m->ctx);+ else if (prompt && have_tok) n = tok_encode(&tokenizer, prompt, (int)strlen(prompt), ids, m->ctx);+ else { fprintf(stderr, "[mimo] give --ids, or --prompt with a tokenizer.json in the model dir\n"); return 2; }+ if (n < 1) { fprintf(stderr, "[mimo] empty prompt\n"); return 2; }+ if (n + ngen > m->ctx) { fprintf(stderr, "[mimo] prompt + ngen exceed the context %d\n", m->ctx); return 2; }+ float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");+ const char *dump = getenv("MIMO_LOGITS");+ float *all = dump ? xmalloc((size_t)n * c->vocab * sizeof(float), "all logits") : NULL;+ /* --image patches.f32 --grid H W: the gateway's patches for the image-pad+ * ids already in the prompt */+ float *image = NULL;+ int image_rows = 0;+ if (image_path) {+ if (!g_vision) { fprintf(stderr, "[mimo] --image: this container has no vision tower\n"); return 2; }+ size_t want = (size_t)grid_h * grid_w * g_vision->patch_in;+ float *patches = xmalloc(want * sizeof(float), "patches");+ FILE *f = fopen(image_path, "rb");+ if (!f || fread(patches, sizeof(float), want, f) != want || fgetc(f) != EOF) {+ fprintf(stderr, "[mimo] %s does not hold %dx%d patches\n", image_path, grid_h, grid_w); return 2;+ }+ fclose(f);+ int pads = 0;+ for (int t = 0; t < n; t++) pads += ids[t] == c->image_token_id;+ if (pads != (grid_h / 2) * (grid_w / 2)) {+ fprintf(stderr, "[mimo] the prompt has %d image pads, the grid needs %d\n", pads, (grid_h / 2) * (grid_w / 2));+ return 2;+ }+ image = vision_run(m, patches, grid_h, grid_w, &image_rows);+ free(patches);+ }+ ImageRows img = { image, image_rows };+ double tp = now_s();+ prefill(m, ids, n, logits, all, image ? &img : NULL);+ double prefill_s = now_s() - tp;+ if (dump) {+ FILE *f = fopen(dump, "wb");+ if (!f || fwrite(all, sizeof(float), (size_t)n * c->vocab, f) != (size_t)n * c->vocab) {+ fprintf(stderr, "[mimo] cannot write %s\n", dump); return 1;+ }+ fclose(f);+ }+ float temperature = getenv("COLI_TEMP") ? (float)atof(getenv("COLI_TEMP")) : 0.0f;+ int eos[8], n_eos = coli_load_stop_ids(dir, eos, 8, NULL);+ double td = now_s();+ int produced = 0;+ char piece[512];+ for (int g = 0; g < ngen; g++) {+ int token = sample(logits, c->vocab, temperature, 1.0f);+ int stop = 0;+ for (int i = 0; i < n_eos; i++) stop |= token == eos[i];+ if (have_tok && !ids_text) {+ int w = tok_decode(&tokenizer, &token, 1, piece, (int)sizeof(piece));+ fwrite(piece, 1, (size_t)(w > 0 ? w : 0), stdout);+ } else printf("%d ", token);+ fflush(stdout);+ produced++;+ if (stop && have_tok && !ids_text) break;+ if (g + 1 < ngen) prefill(m, &token, 1, logits, NULL, NULL);+ }+ printf("\n");+ fflush(stdout);+ double decode_s = now_s() - td;+ double total = prefill_s + decode_s;+ fprintf(stderr, "[mimo] prefill %d tokens %.2fs, decode %d tokens %.2fs (%.2f tok/s), experts %llu hits "+ "%llu misses %.1f MB read, rss %.2f GB\n"+ "[mimo] time: expert reads %.2fs (%.2f GB/s), expert matmuls %.2fs, attention %.2fs, "+ "the rest (dense MLP, router, head) %.2fs\n",+ n, prefill_s, produced, decode_s, decode_s > 0 ? produced / decode_s : 0.0,+ (unsigned long long)m->hits, (unsigned long long)m->miss, m->bytes_read / 1e6, rss_gb(),+ m->t_disk, m->t_disk > 0 ? m->bytes_read / 1e9 / m->t_disk : 0.0, m->t_expert, m->t_attn,+ total - m->t_disk - m->t_expert - m->t_attn);+ return 0;+}diff --git a/c/mimo_tiny/config.json b/c/mimo_tiny/config.jsonnew file mode 100644index 00000000..4f325d36--- /dev/null+++ b/c/mimo_tiny/config.json@@ -0,0 +1,124 @@+{+ "architectures": [+ "MiMoV2ForCausalLM"+ ],+ "model_type": "mimo_v2",+ "vocab_size": 320,+ "hidden_size": 256,+ "intermediate_size": 256,+ "num_hidden_layers": 6,+ "hybrid_layer_pattern": [+ 0,+ 1,+ 1,+ 0,+ 1,+ 1+ ],+ "moe_layer_freq": [+ 0,+ 1,+ 1,+ 1,+ 1,+ 1+ ],+ "num_attention_heads": 4,+ "num_key_value_heads": 2,+ "head_dim": 48,+ "v_head_dim": 32,+ "swa_num_attention_heads": 4,+ "swa_num_key_value_heads": 4,+ "swa_head_dim": 48,+ "swa_v_head_dim": 32,+ "sliding_window": 8,+ "sliding_window_size": 8,+ "attention_chunk_size": 8,+ "add_full_attention_sink_bias": false,+ "add_swa_attention_sink_bias": true,+ "attention_value_scale": 0.707,+ "attention_projection_layout": "fused_qkv",+ "attention_bias": false,+ "partial_rotary_factor": 0.334,+ "rope_theta": 10000000.0,+ "swa_rope_theta": 10000.0,+ "rope_parameters": {+ "partial_rotary_factor": 0.334,+ "rope_theta": 10000000.0,+ "rope_type": "default",+ "type": "default"+ },+ "n_routed_experts": 16,+ "num_experts_per_tok": 4,+ "moe_intermediate_size": 64,+ "n_shared_experts": null,+ "n_group": 1,+ "topk_group": 1,+ "topk_method": "noaux_tc",+ "scoring_func": "sigmoid",+ "norm_topk_prob": true,+ "routed_scaling_factor": null,+ "moe_router_dtype": "bfloat16",+ "hidden_act": "silu",+ "layernorm_epsilon": 1e-06,+ "max_position_embeddings": 256,+ "tie_word_embeddings": false,+ "bos_token_id": null,+ "eos_token_id": 1,+ "pad_token_id": 0,+ "dtype": "bfloat16",+ "quantization_config": {+ "activation_scheme": "dynamic",+ "fmt": "e4m3",+ "quant_method": "fp8",+ "store_dtype": "mxfp4",+ "mxfp4_block_size": 32,+ "weight_block_size": [+ 128,+ 128+ ],+ "ignored_layers": [+ "model.layers.0.self_attn.o_proj",+ "model.layers.1.self_attn.o_proj",+ "model.layers.2.self_attn.o_proj",+ "model.layers.3.self_attn.o_proj",+ "model.layers.4.self_attn.o_proj",+ "model.layers.5.self_attn.o_proj"+ ]+ },+ "vision_config": {+ "depth": 6,+ "hidden_size": 64,+ "intermediate_size": 96,+ "num_heads": 4,+ "num_key_value_heads": 2,+ "qk_channels": 16,+ "out_hidden_size": 256,+ "patch_size": 16,+ "spatial_patch_size": 16,+ "temporal_patch_size": 2,+ "spatial_merge_size": 2,+ "in_chans": 3,+ "hidden_act": "silu",+ "fullatt_block_indexes": [+ 0,+ 5+ ],+ "vit_window_attn_types": [+ -1,+ 0,+ 1,+ 1,+ 0,+ -1+ ],+ "visual_token_window_size": 5,+ "use_sink": true,+ "window_size": 128,+ "tokens_per_second": 2+ },+ "image_token_id": 300,+ "vision_start_token_id": 301,+ "vision_end_token_id": 302,+ "vision_model_type": "mimovl"+}diff --git a/c/mimo_tiny/preprocessor_config.json b/c/mimo_tiny/preprocessor_config.jsonnew file mode 100644index 00000000..0467ea20--- /dev/null+++ b/c/mimo_tiny/preprocessor_config.json@@ -0,0 +1,19 @@+{+ "min_pixels": 3136,+ "max_pixels": 12845056,+ "patch_size": 16,+ "temporal_patch_size": 2,+ "merge_size": 2,+ "image_mean": [+ 0.48145466,+ 0.4578275,+ 0.40821073+ ],+ "image_std": [+ 0.26862954,+ 0.26130258,+ 0.27577711+ ],+ "image_processor_type": "Qwen2VLImageProcessor",+ "processor_class": "Qwen2_5_VLProcessor"+}diff --git a/c/mimo_tiny/ref.json b/c/mimo_tiny/ref.jsonnew file mode 100644index 00000000..9fb5b0f2--- /dev/null+++ b/c/mimo_tiny/ref.json@@ -0,0 +1,638 @@+{+ "schema_version": 1,+ "source": "xiaomi-vendor",+ "vendor_repo": "XiaomiMiMo/MiMo-V2.6-Flash-MOPD",+ "vendor_revision": "2479e2d0029eca9a34cc7e7f55a121925f81908e",+ "vendor_files": {+ "configuration_mimo_v2.py": "773062ac9850b908eb54751b3e4dbe00e653c0e80595599409e97d0c1af2ce3e",+ "modeling_mimo_v2.py": "a8c3cb3aae473bcc15f023010547c919f15eba6546e6ed7efb61a8937b12f3ad"+ },+ "torch_version": "2.12.1+cu130",+ "transformers_version": "5.3.0",+ "engine_env": {+ "MIMO_DENSE_BITS": "32",+ "MIMO_IDOT": "0",+ "COLI_TEMP": "0"+ },+ "cases": {+ "short": {+ "prompt_ids": [+ 5,+ 7,+ 9,+ 11,+ 13,+ 17,+ 19,+ 23+ ],+ "teacher_forcing_ids": [+ 210,+ 311,+ 69,+ 287,+ 199,+ 306,+ 214,+ 89,+ 114,+ 192,+ 311,+ 230,+ 60,+ 175,+ 64,+ 27,+ 294,+ 234,+ 86,+ 13+ ],+ "greedy_full_ids": [+ 5,+ 7,+ 9,+ 11,+ 13,+ 17,+ 19,+ 23,+ 89,+ 114,+ 192,+ 311,+ 230,+ 60,+ 175,+ 64,+ 27,+ 294,+ 234,+ 86+ ],+ "greedy_new_ids": [+ 89,+ 114,+ 192,+ 311,+ 230,+ 60,+ 175,+ 64,+ 27,+ 294,+ 234,+ 86+ ],+ "max_new_tokens": 12,+ "min_logit_margin": 1.646,+ "context_free_agreement": 0.0+ },+ "window": {+ "prompt_ids": [+ 5,+ 12,+ 19,+ 26,+ 33,+ 40,+ 47,+ 54,+ 61,+ 68,+ 75,+ 82,+ 89,+ 96,+ 103,+ 110,+ 117,+ 124,+ 131,+ 138,+ 145,+ 152,+ 159,+ 166,+ 173,+ 180,+ 187,+ 194,+ 201,+ 208,+ 215,+ 222,+ 229,+ 236,+ 243,+ 250,+ 257,+ 264,+ 271,+ 278+ ],+ "teacher_forcing_ids": [+ 210,+ 238,+ 192,+ 121,+ 6,+ 161,+ 311,+ 250,+ 134,+ 220,+ 56,+ 120,+ 0,+ 32,+ 21,+ 64,+ 286,+ 115,+ 145,+ 145,+ 77,+ 93,+ 35,+ 150,+ 255,+ 151,+ 255,+ 113,+ 146,+ 148,+ 254,+ 317,+ 79,+ 46,+ 266,+ 95,+ 159,+ 291,+ 105,+ 220,+ 64,+ 238,+ 150,+ 146,+ 317,+ 279+ ],+ "greedy_full_ids": [+ 5,+ 12,+ 19,+ 26,+ 33,+ 40,+ 47,+ 54,+ 61,+ 68,+ 75,+ 82,+ 89,+ 96,+ 103,+ 110,+ 117,+ 124,+ 131,+ 138,+ 145,+ 152,+ 159,+ 166,+ 173,+ 180,+ 187,+ 194,+ 201,+ 208,+ 215,+ 222,+ 229,+ 236,+ 243,+ 250,+ 257,+ 264,+ 271,+ 278,+ 220,+ 64,+ 238,+ 150,+ 146,+ 317+ ],+ "greedy_new_ids": [+ 220,+ 64,+ 238,+ 150,+ 146,+ 317+ ],+ "max_new_tokens": 6,+ "min_logit_margin": 0.896,+ "context_free_agreement": 0.022+ },+ "long": {+ "prompt_ids": [+ 5,+ 16,+ 27,+ 38,+ 49,+ 60,+ 71,+ 82,+ 93,+ 104,+ 115,+ 126,+ 137,+ 148,+ 159,+ 170,+ 181,+ 192,+ 203,+ 214,+ 225,+ 236,+ 247,+ 258,+ 269,+ 280,+ 291,+ 302,+ 313,+ 4,+ 15,+ 26,+ 37,+ 48,+ 59,+ 70,+ 81,+ 92,+ 103,+ 114,+ 125,+ 136,+ 147,+ 158,+ 169,+ 180,+ 191,+ 202,+ 213,+ 224,+ 235,+ 246,+ 257,+ 268,+ 279,+ 290,+ 301,+ 312,+ 3,+ 14,+ 25,+ 36,+ 47,+ 58,+ 69,+ 80,+ 91,+ 102,+ 113,+ 124,+ 135,+ 146+ ],+ "teacher_forcing_ids": [+ 210,+ 73,+ 35,+ 199,+ 184,+ 107,+ 95,+ 44,+ 180,+ 41,+ 288,+ 259,+ 9,+ 218,+ 247,+ 277,+ 235,+ 84,+ 316,+ 113,+ 293,+ 216,+ 123,+ 14,+ 210,+ 196,+ 223,+ 260,+ 86,+ 206,+ 226,+ 152,+ 151,+ 57,+ 117,+ 123,+ 260,+ 64,+ 113,+ 180,+ 183,+ 266,+ 84,+ 62,+ 87,+ 223,+ 209,+ 120,+ 226,+ 47,+ 184,+ 160,+ 123,+ 2,+ 238,+ 89,+ 29,+ 245,+ 308,+ 152,+ 148,+ 54,+ 41,+ 238,+ 94,+ 34,+ 123,+ 316,+ 84,+ 64,+ 208,+ 174,+ 298,+ 72,+ 234,+ 31,+ 79,+ 274+ ],+ "greedy_full_ids": [+ 5,+ 16,+ 27,+ 38,+ 49,+ 60,+ 71,+ 82,+ 93,+ 104,+ 115,+ 126,+ 137,+ 148,+ 159,+ 170,+ 181,+ 192,+ 203,+ 214,+ 225,+ 236,+ 247,+ 258,+ 269,+ 280,+ 291,+ 302,+ 313,+ 4,+ 15,+ 26,+ 37,+ 48,+ 59,+ 70,+ 81,+ 92,+ 103,+ 114,+ 125,+ 136,+ 147,+ 158,+ 169,+ 180,+ 191,+ 202,+ 213,+ 224,+ 235,+ 246,+ 257,+ 268,+ 279,+ 290,+ 301,+ 312,+ 3,+ 14,+ 25,+ 36,+ 47,+ 58,+ 69,+ 80,+ 91,+ 102,+ 113,+ 124,+ 135,+ 146,+ 174,+ 298,+ 72,+ 234,+ 31,+ 79+ ],+ "greedy_new_ids": [+ 174,+ 298,+ 72,+ 234,+ 31,+ 79+ ],+ "max_new_tokens": 6,+ "min_logit_margin": 0.4713,+ "context_free_agreement": 0.0+ }+ },+ "image": {+ "prompt_ids": [+ 5,+ 7,+ 301,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 302,+ 11,+ 13,+ 17,+ 19+ ],+ "grid_h": 8,+ "grid_w": 12,+ "patches_sha256": "0c544aef4dec661ce140bcbc3221c43754a29661b5ceef9473755e1b3595d456",+ "patch_max_diff_vs_official_processor": 0.0,+ "vision_rows_sum": 60.48851776123047,+ "vision_rows_abs_sum": 7867.14501953125,+ "teacher_forcing_ids": [+ 210,+ 311,+ 309,+ 47,+ 44,+ 246,+ 8,+ 313,+ 155,+ 222,+ 29,+ 124,+ 222,+ 115,+ 235,+ 294,+ 111,+ 302,+ 223,+ 256,+ 95,+ 314,+ 225,+ 279,+ 223,+ 42,+ 305,+ 170,+ 273,+ 204,+ 302,+ 171,+ 266,+ 234,+ 266,+ 171,+ 13,+ 199,+ 70,+ 45+ ],+ "greedy_full_ids": [+ 5,+ 7,+ 301,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 300,+ 302,+ 11,+ 13,+ 17,+ 19,+ 171,+ 266,+ 234,+ 266,+ 171,+ 13,+ 199,+ 70+ ],+ "greedy_new_ids": [+ 171,+ 266,+ 234,+ 266,+ 171,+ 13,+ 199,+ 70+ ],+ "max_new_tokens": 8,+ "min_logit_margin": 0.5628,+ "negated_image_new_ids": [+ 49,+ 281,+ 196,+ 190,+ 107,+ 103,+ 173,+ 283+ ]+ }+}diff --git a/c/mimo_vision.h b/c/mimo_vision.hnew file mode 100644index 00000000..867cf661--- /dev/null+++ b/c/mimo_vision.h@@ -0,0 +1,322 @@+/* mimo_vision.h -- the MiMo ViT (681M on the release), images only.+ *+ * Included by mimo.c after the Model and DW helpers. Mirrors+ * MiMoVisionTransformer in modeling_mimo_v2.py, with the two points where the+ * release and that module disagree resolved as the release (and vLLM's+ * mimo_v2_omni.py) has them:+ * - the merger stores no biases: its two Linears and its LayerNorm bias are+ * zero (vLLM builds them with bias=False; a bias present in a container is+ * still used);+ * - after a run of column-order blocks that reaches the end of the tower, the+ * tokens go back to row order before the merger (vLLM does; the release's+ * last block is a full-attention one, so the two readings never differ on it).+ *+ * The tower, per block: x += proj(attn(rms(x))), x += mlp(rms(x))+ * patches [N, 3*T*16*16] in 2x2-merge-block order (the Qwen2-VL processor)+ * 2D RoPE head_dim/2 dims: h then w frequencies (theta 1e4), cat(e, e),+ * rotate-half over the whole head+ * attention GQA; full over the image in fullatt_block_indexes, else a 1D+ * window |i - j| <= visual_token_window_size in the CURRENT token+ * order, plus a sink logit added to key 0 of that order+ * order vit_window_attn_types: -1 full, 0 row order, 1 column order of+ * the merge units (entering a type-1 run permutes the tokens and+ * their rotary angles, leaving it permutes them back)+ * merger LayerNorm, the 2x2 units concatenated, Linear, exact GELU, Linear+ */++typedef struct {+ float *norm1, *norm2, *sinks;+ DW qkv, proj, gate, up, down;+ float *qkv_b, *proj_b, *gate_b, *up_b, *down_b;+ int full, type;+} VBlock;++struct Vision {+ int depth, d, inter, heads, kvh, hd, out, patch, temporal, merge, in_ch, window;+ float eps;+ DW embed; /* [d, in_ch*T*P*P] */+ VBlock *b;+ float *ln_w, *ln_b, *fc1_b, *fc2_b;+ DW fc1, fc2;+ int patch_in;+};++static void vis_mat(DW *d, shards *S, const char *name, int O, int I) {+ /* the tower stays BF16 (or f32 for the oracle): int8 is a dense-trunk option */+ int saved = g_dense_bits;+ g_dense_bits = g_dense_bits == 32 ? 32 : 0;+ dw_load_bf16(d, S, name, O, I);+ g_dense_bits = saved;+}++static float *vis_vec_opt(shards *S, const char *name, int n) {+ return st_has(S, name) ? load_vec(S, name, n) : NULL;+}++static int vjson_int(jval *o, const char *key, int fallback) {+ jval *v = o ? json_get(o, key) : NULL;+ return (v && v->t == J_NUM) ? (int)v->num : fallback;+}++static Vision *vision_load(Model *m, const char *dir) {+ if (!st_has(&m->S, "visual.patch_embed.proj.weight")) return NULL;+ char path[4096];+ snprintf(path, sizeof(path), "%s/config.json", dir);+ char *text = read_text(path, NULL);+ if (!text) return NULL;+ char *arena = NULL;+ jval *root = json_parse(text, &arena);+ jval *vc = root ? json_get(root, "vision_config") : NULL;+ if (!vc || vc->t != J_OBJ) {+ fprintf(stderr, "[mimo] visual.* tensors but no vision_config: text only\n");+ json_free(root); free(arena); free(text);+ return NULL;+ }+ Vision *v = xcalloc(1, sizeof(Vision), "vision");+ v->depth = vjson_int(vc, "depth", 0);+ v->d = vjson_int(vc, "hidden_size", 0);+ v->inter = vjson_int(vc, "intermediate_size", 0);+ v->heads = vjson_int(vc, "num_heads", 0);+ v->kvh = vjson_int(vc, "num_key_value_heads", v->heads);+ v->hd = vjson_int(vc, "qk_channels", 64);+ v->out = vjson_int(vc, "out_hidden_size", 0);+ v->patch = vjson_int(vc, "patch_size", 16);+ v->temporal = vjson_int(vc, "temporal_patch_size", 2);+ v->merge = vjson_int(vc, "spatial_merge_size", 2);+ v->in_ch = vjson_int(vc, "in_chans", 3);+ v->window = vjson_int(vc, "visual_token_window_size", -1);+ v->eps = (float)jnum(vc, "rms_norm_eps", 1e-6);+ int use_sink = jbool(vc, "use_sink", 0);+ if (v->depth < 1 || v->depth > 256 || v->d < 8 || v->d > 16384 || v->inter < 1 || v->inter > 65536 ||+ v->heads < 1 || v->kvh < 1 || v->heads % v->kvh || v->hd < 4 || v->hd % 4 || v->hd > 1024 ||+ v->out != m->c.hidden || v->merge != 2 || v->patch < 1 || v->patch > 64 ||+ v->temporal < 1 || v->temporal > 4 || v->in_ch != 3) {+ fprintf(stderr, "[mimo] vision_config out of range (or out_hidden_size != hidden_size) -- refusing\n");+ exit(1);+ }+ v->patch_in = v->in_ch * v->temporal * v->patch * v->patch;+ v->b = xcalloc((size_t)v->depth, sizeof(VBlock), "vision blocks");+ jval *full = json_get(vc, "fullatt_block_indexes"), *types = json_get(vc, "vit_window_attn_types");+ for (int i = 0; i < v->depth; i++) v->b[i].type = -1;+ if (types && types->t == J_ARR) {+ if (types->len != v->depth) { fprintf(stderr, "[mimo] vit_window_attn_types must list every block\n"); exit(1); }+ for (int i = 0; i < v->depth; i++) v->b[i].type = (int)types->kids[i]->num;+ }+ if (full && full->t == J_ARR)+ for (int i = 0; i < full->len; i++) {+ int b = (int)full->kids[i]->num;+ if (b >= 0 && b < v->depth) v->b[b].full = 1;+ }+ json_free(root); free(arena); free(text);++ shards *S = &m->S;+ st_tensor *t = st_find(S, "visual.patch_embed.proj.weight");+ if (t->dtype != 0 || t->numel != (int64_t)v->d * v->patch_in || t->nbytes != t->numel * 2) {+ fprintf(stderr, "[mimo] visual.patch_embed.proj.weight is not BF16 [%d, %d] -- refusing\n", v->d, v->patch_in);+ exit(1);+ }+ /* Conv3d [d, C, T, P, P] with stride = kernel is a Linear over the flattened+ * patch, in the processor's C, T, row, col order */+ v->embed.O = v->d; v->embed.I = v->patch_in;+ if (g_dense_bits == 32) {+ float *w = xmalloc((size_t)v->d * v->patch_in * sizeof(float), "patch embed");+ st_read_f32(S, "visual.patch_embed.proj.weight", w, 1);+ v->embed.fmt = DW_F32; v->embed.w = w;+ } else {+ uint16_t *b16 = xmalloc((size_t)v->d * v->patch_in * sizeof(uint16_t), "patch embed");+ st_read_raw_cap(S, "visual.patch_embed.proj.weight", b16, (int64_t)v->d * v->patch_in * 2, 1);+ v->embed.fmt = DW_BF16; v->embed.w = b16;+ }+ int qkv_rows = (v->heads + 2 * v->kvh) * v->hd;+ for (int i = 0; i < v->depth; i++) {+ VBlock *b = &v->b[i];+ char nm[512];+#define VN(fmt_) (snprintf(nm, sizeof(nm), "visual.blocks.%d." fmt_, i), nm)+ b->norm1 = load_vec(S, VN("norm1.weight"), v->d);+ b->norm2 = load_vec(S, VN("norm2.weight"), v->d);+ vis_mat(&b->qkv, S, VN("attn.qkv.weight"), qkv_rows, v->d);+ b->qkv_b = load_vec(S, VN("attn.qkv.bias"), qkv_rows);+ vis_mat(&b->proj, S, VN("attn.proj.weight"), v->d, v->heads * v->hd);+ b->proj_b = load_vec(S, VN("attn.proj.bias"), v->d);+ vis_mat(&b->gate, S, VN("mlp.gate_proj.weight"), v->inter, v->d);+ b->gate_b = load_vec(S, VN("mlp.gate_proj.bias"), v->inter);+ vis_mat(&b->up, S, VN("mlp.up_proj.weight"), v->inter, v->d);+ b->up_b = load_vec(S, VN("mlp.up_proj.bias"), v->inter);+ vis_mat(&b->down, S, VN("mlp.down_proj.weight"), v->d, v->inter);+ b->down_b = load_vec(S, VN("mlp.down_proj.bias"), v->d);+ if (use_sink && !b->full) b->sinks = load_vec(S, VN("attn.sinks"), v->heads);+#undef VN+ }+ int merged = v->d * v->merge * v->merge;+ v->ln_w = load_vec(S, "visual.merger.ln_q.weight", v->d);+ v->ln_b = vis_vec_opt(S, "visual.merger.ln_q.bias", v->d);+ vis_mat(&v->fc1, S, "visual.merger.mlp.0.weight", merged, merged);+ v->fc1_b = vis_vec_opt(S, "visual.merger.mlp.0.bias", merged);+ vis_mat(&v->fc2, S, "visual.merger.mlp.2.weight", v->out, merged);+ v->fc2_b = vis_vec_opt(S, "visual.merger.mlp.2.bias", v->out);+ return v;+}++static void add_bias(float *y, const float *b, int n, int dim) {+ if (!b) return;+ for (int t = 0; t < n; t++) for (int i = 0; i < dim; i++) y[(size_t)t * dim + i] += b[i];+}++/* new[unit k] = old[unit idx[k]], units of 4 tokens (the 2x2 merge) */+static void permute_units(float *x, const int *idx, int units, int width) {+ size_t unit = (size_t)4 * width;+ float *tmp = xmalloc((size_t)units * unit * sizeof(float), "vision permute");+ for (int k = 0; k < units; k++) memcpy(tmp + (size_t)k * unit, x + (size_t)idx[k] * unit, unit * sizeof(float));+ memcpy(x, tmp, (size_t)units * unit * sizeof(float));+ free(tmp);+}++static void vis_rope(float *x, int nh, int hd, const float *ang) {+ int half = hd / 2;+ for (int h = 0; h < nh; h++) {+ float *v = x + (size_t)h * hd;+ for (int i = 0; i < half; i++) {+ float cs = cosf(ang[i]), sn = sinf(ang[i]);+ float a = v[i], b = v[i + half];+ v[i] = a * cs - b * sn;+ v[i + half] = b * cs + a * sn;+ }+ }+}++static float *vision_run(Model *m, const float *patches, int gh, int gw, int *n_rows) {+ Vision *v = g_vision;+ (void)m;+ *n_rows = 0;+ if (!v || gh < 2 || gw < 2 || gh % 2 || gw % 2) return NULL;+ double t0 = now_s();+ int N = gh * gw, d = v->d, hd = v->hd, nh = v->heads, kvh = v->kvh, group = nh / kvh;+ int units = N / 4, uh = gh / 2, uw = gw / 2;+ float *x = xmalloc((size_t)N * d * sizeof(float), "vision tokens");+ dw_matmul(x, patches, N, &v->embed);+ /* rotary angles per token, in the processor's block-major order */+ int rdim = hd / 2, nf = rdim / 2;+ float *ang = xmalloc((size_t)N * rdim * sizeof(float), "vision angles");+ float *inv = xmalloc((size_t)nf * sizeof(float), "vision inv_freq");+ for (int j = 0; j < nf; j++) inv[j] = 1.0f / powf(10000.0f, (float)(2 * j) / (float)rdim);+ for (int n = 0; n < N; n++) {+ int u = n / 4, r = (n % 4) / 2, cc = n % 2;+ int hp = (u / uw) * 2 + r, wp = (u % uw) * 2 + cc;+ for (int j = 0; j < nf; j++) {+ ang[(size_t)n * rdim + j] = (float)hp * inv[j];+ ang[(size_t)n * rdim + nf + j] = (float)wp * inv[j];+ }+ }+ free(inv);+ /* column order of the merge units and its inverse */+ int *col = xmalloc((size_t)units * sizeof(int), "vision col order");+ int *rev = xmalloc((size_t)units * sizeof(int), "vision row order");+ for (int k = 0; k < units; k++) col[k] = (k % uh) * uw + k / uh;+ for (int k = 0; k < units; k++) rev[col[k]] = k;+ float *ang_col = xmalloc((size_t)N * rdim * sizeof(float), "vision col angles");+ memcpy(ang_col, ang, (size_t)N * rdim * sizeof(float));+ permute_units(ang_col, col, units, rdim);++ int qd = nh * hd, kd = kvh * hd, rows = qd + 2 * kd;+ float *xn = xmalloc((size_t)N * d * sizeof(float), "vision normed");+ float *qkv = xmalloc((size_t)N * rows * sizeof(float), "vision qkv");+ float *ctx = xmalloc((size_t)N * qd * sizeof(float), "vision attention");+ float *tmp = xmalloc((size_t)N * d * sizeof(float), "vision block out");+ float *g = xmalloc((size_t)N * v->inter * sizeof(float), "vision mlp gate");+ float *u = xmalloc((size_t)N * v->inter * sizeof(float), "vision mlp up");+ float scale = 1.0f / sqrtf((float)hd);+ int prev = -2;+ for (int bi = 0; bi < v->depth; bi++) {+ VBlock *b = &v->b[bi];+ if (b->type == 1 && prev != 1) permute_units(x, col, units, d);+ if (b->type != 1 && prev == 1) permute_units(x, rev, units, d);+ prev = b->type;+ const float *A = b->type == 1 ? ang_col : ang;+ for (int n = 0; n < N; n++) rmsnorm(xn + (size_t)n * d, x + (size_t)n * d, b->norm1, d, v->eps);+ dw_matmul(qkv, xn, N, &b->qkv);+ add_bias(qkv, b->qkv_b, N, rows);+ for (int n = 0; n < N; n++) {+ vis_rope(qkv + (size_t)n * rows, nh, hd, A + (size_t)n * rdim);+ vis_rope(qkv + (size_t)n * rows + qd, kvh, hd, A + (size_t)n * rdim);+ }+ int window = b->full ? -1 : v->window;+ #pragma omp parallel+ {+ float *score = xmalloc((size_t)N * sizeof(float), "vision scores");+ #pragma omp for collapse(2) schedule(static)+ for (int i = 0; i < N; i++) {+ for (int h = 0; h < nh; h++) {+ int kh = h / group;+ int lo = 0, hi = N - 1;+ if (window >= 0) { lo = i - window > 0 ? i - window : 0; hi = i + window < N - 1 ? i + window : N - 1; }+ const float *q = qkv + (size_t)i * rows + (size_t)h * hd;+ float best = -INFINITY;+ for (int j = lo; j <= hi; j++) {+ const float *kr = qkv + (size_t)j * rows + qd + (size_t)kh * hd;+ float s = 0;+ for (int e = 0; e < hd; e++) s += q[e] * kr[e];+ s *= scale;+ if (j == 0 && b->sinks) s += b->sinks[h];+ score[j - lo] = s;+ if (s > best) best = s;+ }+ float den = 0;+ for (int j = lo; j <= hi; j++) { score[j - lo] = expf(score[j - lo] - best); den += score[j - lo]; }+ float *o = ctx + (size_t)i * qd + (size_t)h * hd;+ for (int e = 0; e < hd; e++) o[e] = 0;+ for (int j = lo; j <= hi; j++) {+ const float *vr = qkv + (size_t)j * rows + qd + kd + (size_t)kh * hd;+ float w = score[j - lo] / den;+ for (int e = 0; e < hd; e++) o[e] += w * vr[e];+ }+ }+ }+ free(score);+ }+ dw_matmul(tmp, ctx, N, &b->proj);+ add_bias(tmp, b->proj_b, N, d);+ for (size_t i = 0; i < (size_t)N * d; i++) x[i] += tmp[i];+ for (int n = 0; n < N; n++) rmsnorm(xn + (size_t)n * d, x + (size_t)n * d, b->norm2, d, v->eps);+ dw_matmul(g, xn, N, &b->gate);+ add_bias(g, b->gate_b, N, v->inter);+ dw_matmul(u, xn, N, &b->up);+ add_bias(u, b->up_b, N, v->inter);+ for (size_t i = 0; i < (size_t)N * v->inter; i++) g[i] = silu(g[i]) * u[i];+ dw_matmul(tmp, g, N, &b->down);+ add_bias(tmp, b->down_b, N, d);+ for (size_t i = 0; i < (size_t)N * d; i++) x[i] += tmp[i];+ }+ if (prev == 1) permute_units(x, rev, units, d);+ /* merger: LayerNorm per token, 2x2 units concatenated, Linear, GELU, Linear */+ for (int n = 0; n < N; n++) {+ float *r = x + (size_t)n * d;+ float mean = 0, var = 0;+ for (int i = 0; i < d; i++) mean += r[i];+ mean /= d;+ for (int i = 0; i < d; i++) { float z = r[i] - mean; var += z * z; }+ var /= d;+ float inv_std = 1.0f / sqrtf(var + 1e-6f);+ for (int i = 0; i < d; i++) r[i] = (r[i] - mean) * inv_std * v->ln_w[i] + (v->ln_b ? v->ln_b[i] : 0.0f);+ }+ int merged = 4 * d;+ float *h1 = xmalloc((size_t)units * merged * sizeof(float), "vision merger");+ dw_matmul(h1, x, units, &v->fc1);+ add_bias(h1, v->fc1_b, units, merged);+ for (size_t i = 0; i < (size_t)units * merged; i++) h1[i] = 0.5f * h1[i] * (1.0f + erff(h1[i] * 0.70710678118654752f));+ float *rows_out = xmalloc((size_t)units * v->out * sizeof(float), "image rows");+ dw_matmul(rows_out, h1, units, &v->fc2);+ add_bias(rows_out, v->fc2_b, units, v->out);+ free(h1); free(x); free(ang); free(ang_col); free(col); free(rev);+ free(xn); free(qkv); free(ctx); free(tmp); free(g); free(u);+ *n_rows = units;+ /* MIMO_VISION_DUMP=<file>: the rows the tower produced, f32 [units, hidden],+ * for tools/mimo_real_check.py to hold against Xiaomi's ViT */+ const char *dump = getenv("MIMO_VISION_DUMP");+ if (dump) {+ FILE *f = fopen(dump, "wb");+ if (f) { fwrite(rows_out, sizeof(float), (size_t)units * v->out, f); fclose(f); }+ }+ if (env_int("MIMO_STATS", 0))+ fprintf(stderr, "[mimo] image %dx%d patches -> %d rows in %.2fs\n", gh, gw, units, now_s() - t0);+ return rows_out;+}diff --git a/c/tests/mimo_tiny_harness.py b/c/tests/mimo_tiny_harness.pynew file mode 100644index 00000000..2df3bf8b--- /dev/null+++ b/c/tests/mimo_tiny_harness.py@@ -0,0 +1,281 @@+#!/usr/bin/env python3+"""Token-exact checks for the MiMo-V2.6 engine against an independent oracle.++Dependency-free (stdlib only). The fixture is generated by+tools/make_mimo_tiny.py (numpy, deterministic); the expected tokens in+ref.json come from Xiaomi's own modeling code via tools/make_mimo_ref.py (run+offline by a maintainer, vendor files pinned by SHA-256), so the engine and its+oracle never share an author.++What is checked, all token-exact (argmax, no tolerances):+ - greedy generation for three cases, each longer than the 8-token window+ - teacher forcing at every position via the MIMO_LOGITS dump+ - the same tokens from every exact configuration: native FP8/BF16 dense+ weights, an expert cache that evicts at every layer, prefill in blocks of 3+ and of 1 (the sliding-window ring across blocks), buffered expert reads+ - determinism (two identical runs, identical tokens)+ - with the vision fixture (make_mimo_tiny.py --vision): the picture through+ the tower, greedy and teacher-forced, f32 and native BF16, and the negated+ picture giving the reference's own answer for it+ - the oracle BITES: one corrupted expert scale byte per layer-2 expert, and+ separately the qkv scale rows read as one continuous grid instead of per+ chunk, must each change the output (guards against a comparison that+ silently accepts anything, and pins the pre-sharded qkv layout)++The engine runs in its exact-math configuration (ref.json's engine_env): dense+weights widened to f32 and float expert matmuls. The fixture ships no+tokenizer.json on purpose: the engine then prints raw token ids on stdout.+"""++import argparse+import array+import hashlib+import json+import os+import shutil+import struct+import subprocess+import sys+import tempfile+from pathlib import Path+++def run_engine(binary, fixture, prompt_ids, max_new, env, extra_env=None,+ image=None):+ run_env = dict(os.environ, **env)+ if extra_env:+ run_env.update(extra_env)+ command = [binary, str(fixture), "--ids", " ".join(map(str, prompt_ids)),+ "--ngen", str(max_new)]+ if image:+ path, grid_h, grid_w = image+ command += ["--image", str(path), "--grid", str(grid_h), str(grid_w)]+ result = subprocess.run(command, env=run_env, capture_output=True,+ text=True, timeout=600)+ if result.returncode != 0:+ raise AssertionError(+ f"engine exited {result.returncode}:\n{result.stderr[-2000:]}")+ tokens = []+ for word in result.stdout.split():+ if not word.lstrip("-").isdigit():+ break+ tokens.append(int(word))+ return tokens+++def check_greedy(binary, fixture, reference):+ for name, case in reference["cases"].items():+ got = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ want = case["greedy_new_ids"]+ if got != want:+ raise AssertionError(f"greedy mismatch in case '{name}': engine "+ f"{got} vs reference {want}")+ print(f"ok greedy {name}: {len(want)}/{len(want)} tokens")+++def check_exact_variants(binary, fixture, reference):+ """Every configuration that is exact by construction must give the same+ tokens: the native FP8/BF16 dense path, an expert cache no larger than one+ routing step (every layer evicts), and prefill in odd-sized blocks and one+ token at a time (the window ring is filled and read across blocks)."""+ variants = {+ "native dense, cache 4, prefill blocks of 3":+ {"MIMO_DENSE_BITS": "0", "MIMO_CAP": "4", "MIMO_CHUNK": "3"},+ "prefill one token at a time": {"MIMO_CHUNK": "1"},+ "buffered expert reads": {"MIMO_DIRECT": "0"},+ }+ for label, extra in variants.items():+ env = dict(reference["engine_env"], **extra)+ for name, case in reference["cases"].items():+ got = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], env)+ if got != case["greedy_new_ids"]:+ raise AssertionError(f"{label}: greedy mismatch in case '{name}': "+ f"engine {got} vs reference "+ f"{case['greedy_new_ids']}")+ print(f"ok {label}: identical in {len(reference['cases'])} cases")+++def check_determinism(binary, fixture, reference):+ case = reference["cases"]["short"]+ first = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ second = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ if first != second:+ raise AssertionError(f"non-deterministic greedy: {first} vs {second}")+ print("ok determinism: two identical runs, identical tokens")+++def teacher_forced(binary, fixture, ids, env, vocab, image=None):+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:+ path = tmp.name+ try:+ run_engine(binary, fixture, ids, 0, env, extra_env={"MIMO_LOGITS": path},+ image=image)+ raw = array.array("f")+ with open(path, "rb") as fh:+ raw.frombytes(fh.read())+ finally:+ os.unlink(path)+ if len(raw) != len(ids) * vocab:+ raise AssertionError(f"MIMO_LOGITS dumped {len(raw)} floats, expected "+ f"{len(ids) * vocab}")+ return [max(range(vocab), key=raw[p * vocab:(p + 1) * vocab].__getitem__)+ for p in range(len(ids))]+++def check_teacher_forcing(binary, fixture, reference, vocab):+ for name, case in reference["cases"].items():+ got = teacher_forced(binary, fixture, case["greedy_full_ids"],+ reference["engine_env"], vocab)+ want = case["teacher_forcing_ids"]+ if got != want:+ bad = next(i for i, (a, b) in enumerate(zip(got, want)) if a != b)+ raise AssertionError(+ f"teacher forcing mismatch in case '{name}' at position {bad}: "+ f"engine {got[bad]} vs reference {want[bad]}")+ print(f"ok teacher forcing {name}: {len(want)}/{len(want)} positions")+++def patch_fixture(fixture, edit):+ """A copy of the fixture with edit(raw_bytes_of_shard1, header, base)+ applied to the first shard; returns the copy's path."""+ copy = Path(tempfile.mkdtemp(prefix="mimo-tiny-bite-")) / "fixture"+ shutil.copytree(fixture, copy)+ shard = copy / "model-00001-of-00002.safetensors"+ raw = bytearray(shard.read_bytes())+ header_len = struct.unpack("<Q", bytes(raw[:8]))[0]+ header = json.loads(bytes(raw[8:8 + header_len]))+ edit(raw, header, 8 + header_len)+ shard.write_bytes(bytes(raw))+ return copy+++def diverged_cases(binary, fixture, reference):+ count = 0+ for case in reference["cases"].values():+ got = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ if got != case["greedy_new_ids"]:+ count += 1+ return count+++def check_oracle_bites(binary, fixture, reference, experts):+ def expert_scales(raw, header, base):+ for expert in range(experts):+ lo, _ = header[f"model.layers.2.mlp.experts.{expert}"+ f".gate_proj.weight_scale"]["data_offsets"]+ raw[base + lo] = (raw[base + lo] + 3) % 256 # x8 on one group+ copy = patch_fixture(fixture, expert_scales)+ try:+ count = diverged_cases(binary, copy, reference)+ finally:+ shutil.rmtree(copy.parent)+ if not count:+ raise AssertionError("corrupted expert scales produced identical tokens "+ "in every case: the oracle does not bite")+ print(f"ok oracle bites: corrupted expert scales diverge in "+ f"{count}/{len(reference['cases'])} cases")++ # The full layers' qkv scale grid is tiled per chunk (4 rows where one+ # continuous grid would have 3). Swapping two of its rows is invisible to+ # a reader that ignores the chunk layout only if it never uses them; the+ # engine must change its output.+ def swap_qkv_scale_rows(raw, header, base):+ meta = header["model.layers.0.self_attn.qkv_proj.weight_scale_inv"]+ lo, hi = meta["data_offsets"]+ cols = meta["shape"][1]+ row = cols * 4+ grid = raw[base + lo:base + hi]+ first, second = grid[row:2 * row], grid[2 * row:3 * row]+ grid[row:2 * row], grid[2 * row:3 * row] = second, first+ raw[base + lo:base + hi] = grid+ copy = patch_fixture(fixture, swap_qkv_scale_rows)+ try:+ count = diverged_cases(binary, copy, reference)+ finally:+ shutil.rmtree(copy.parent)+ if not count:+ raise AssertionError("swapping qkv scale rows changed nothing: the "+ "per-chunk scale grid is not being read")+ print(f"ok qkv scale grid bites: {count}/{len(reference['cases'])} cases")+++def check_image(binary, fixture, reference, vocab):+ """The fixture's picture, preprocessed the gateway's way (patches.f32,+ pinned by hash to what the official Qwen2-VL processor produced), through+ the engine's tower: the same tokens as Xiaomi's ViT and language model,+ greedy and teacher-forced, on the f32 and the native BF16 tower. Negating+ the patches must give exactly the tokens the reference gave for the+ negated picture -- the image is read, not just tolerated."""+ image = reference["image"]+ patches = fixture / "patches.f32"+ digest = hashlib.sha256(patches.read_bytes()).hexdigest()+ if digest != image["patches_sha256"]:+ raise AssertionError("patches.f32 differs from the patches the reference "+ "checked against the official processor")+ spec = (patches, image["grid_h"], image["grid_w"])+ for label, extra in (("f32", {}), ("native BF16 tower", {"MIMO_DENSE_BITS": "0"})):+ env = dict(reference["engine_env"], **extra)+ got = run_engine(binary, fixture, image["prompt_ids"],+ image["max_new_tokens"], env, image=spec)+ if got != image["greedy_new_ids"]:+ raise AssertionError(f"image greedy mismatch ({label}): engine {got} vs "+ f"reference {image['greedy_new_ids']}")+ print(f"ok image greedy ({label}): {len(got)}/{len(got)} tokens")+ got = teacher_forced(binary, fixture, image["greedy_full_ids"],+ reference["engine_env"], vocab, image=spec)+ if got != image["teacher_forcing_ids"]:+ bad = next(i for i, (a, b) in enumerate(zip(got, image["teacher_forcing_ids"]))+ if a != b)+ raise AssertionError(f"image teacher forcing mismatch at position {bad}")+ print(f"ok image teacher forcing: {len(got)}/{len(got)} positions")+ negated = Path(tempfile.mkdtemp(prefix="mimo-neg-")) / "patches.f32"+ try:+ values = array.array("f")+ values.frombytes(patches.read_bytes())+ negated.write_bytes(array.array("f", (-v for v in values)).tobytes())+ got = run_engine(binary, fixture, image["prompt_ids"],+ image["max_new_tokens"], reference["engine_env"],+ image=(negated, image["grid_h"], image["grid_w"]))+ finally:+ shutil.rmtree(negated.parent)+ if got != image["negated_image_new_ids"]:+ raise AssertionError(f"negated image: engine {got} vs reference "+ f"{image['negated_image_new_ids']}")+ print("ok image bites: the negated picture gives the reference's other answer")+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--binary", required=True)+ parser.add_argument("--fixture", required=True)+ args = parser.parse_args()++ fixture = Path(args.fixture)+ reference = json.loads((fixture / "ref.json").read_text())+ if reference.get("source") != "xiaomi-vendor":+ raise AssertionError("ref.json must come from the vendor implementation "+ f"(source={reference.get('source')!r})")+ config = json.loads((fixture / "config.json").read_text())+ if (fixture / "tokenizer.json").exists():+ raise AssertionError("fixture must not ship tokenizer.json: raw-id "+ "stdout is the comparison channel")++ check_greedy(args.binary, fixture, reference)+ check_exact_variants(args.binary, fixture, reference)+ check_determinism(args.binary, fixture, reference)+ check_teacher_forcing(args.binary, fixture, reference, config["vocab_size"])+ check_oracle_bites(args.binary, fixture, reference,+ config["n_routed_experts"])+ if "image" in reference:+ check_image(args.binary, fixture, reference, config["vocab_size"])+ print("mimo-tiny-check: all checks passed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/make_mimo_ref.py b/c/tools/make_mimo_ref.pynew file mode 100644index 00000000..ae8041dd--- /dev/null+++ b/c/tools/make_mimo_ref.py@@ -0,0 +1,503 @@+#!/usr/bin/env python3+"""Produce ref.json for the tiny MiMo-V2.6 fixture from Xiaomi's own code.++Maintainer tool, run offline once per fixture change; CI never executes it+(tests/mimo_tiny_harness.py only reads the committed ref.json). The reference+tokens come from the vendor's published implementation (modeling_mimo_v2.py on+the MiMo-V2.6-Flash-MOPD Hugging Face repo, pinned below by SHA-256), so the+engine and its oracle never share an author.++Parity is by construction:+- the engine is tested with its dense weights at f32 (ref.json engine_env) and+ float expert matmuls, and this script dequantizes THE SAME BYTES into the+ vendor model's f32 Linear weights: BF16 by widening, FP8 e4m3 through torch's+ own float8_e4m3fn type times the 128x128 block scale, MXFP4 as+ e2m1 * 2^(scale-127) with the low nibble on the even column;+- the fused qkv_proj is stored pre-sharded ([Q_0|K_0|V_0|Q_1|...], scale grid+ tiled per chunk) as on the release, while the vendor module wants [Q|K|V]:+ this script de-interleaves it with the layout vLLM documents for the release+ (vllm/model_executor/models/mimo_v2.py, _shard_fp8_qkv_proj), so a reader+ that gets the layout wrong disagrees with the oracle;+- token-level argmax must then agree wherever the logit margin exceeds float+ noise; the generator engineers wide margins and this script VERIFIES them per+ emitted position (--min-margin, refuse on violation).++Requires: torch (CPU is fine), transformers >= 5.3, numpy. Network only if the+vendor files are not already in --vendor-dir.+"""++import argparse+import hashlib+import importlib.machinery+import importlib.util+import json+import struct+import sys+import urllib.request+from pathlib import Path++import numpy as np++VENDOR_REPO = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"+VENDOR_REVISION = "2479e2d0029eca9a34cc7e7f55a121925f81908e"+VENDOR_FILES = {+ "configuration_mimo_v2.py":+ "773062ac9850b908eb54751b3e4dbe00e653c0e80595599409e97d0c1af2ce3e",+ "modeling_mimo_v2.py":+ "a8c3cb3aae473bcc15f023010547c919f15eba6546e6ed7efb61a8937b12f3ad",+}+BLOCK = 128++MX4_LUT = np.array([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,+ -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],+ dtype=np.float32)+++def read_fixture(fixture):+ """{name: (dtype string, raw numpy array)} across the indexed shards."""+ index = json.loads((fixture / "model.safetensors.index.json").read_text())+ raw_dtypes = {"F32": np.float32, "U8": np.uint8, "BF16": np.uint16,+ "F8_E4M3": np.uint8}+ out = {}+ for shard in sorted(set(index["weight_map"].values())):+ with open(fixture / shard, "rb") as fh:+ header_len = struct.unpack("<Q", fh.read(8))[0]+ header = json.loads(fh.read(header_len))+ raw = fh.read()+ for name, meta in header.items():+ if name == "__metadata__":+ continue+ lo, hi = meta["data_offsets"]+ dtype = raw_dtypes[meta["dtype"]]+ arr = np.frombuffer(raw, dtype=dtype,+ count=(hi - lo) // np.dtype(dtype).itemsize,+ offset=lo).reshape(meta["shape"]).copy()+ out[name] = (meta["dtype"], arr)+ return out+++def bf16_to_f32(bits):+ return (bits.astype(np.uint32) << 16).view(np.float32)+++def fp8_to_f32(codes):+ """e4m3fn decode by torch's own float8 type (not colibri's decoder)."""+ import torch+ return torch.from_numpy(codes).view(torch.float8_e4m3fn).float().numpy()+++def mxfp4_dequant(packed, scale):+ rows, half = packed.shape+ cols = half * 2+ codes = np.empty((rows, cols), dtype=np.uint8)+ codes[:, 0::2] = packed & 0x0F+ codes[:, 1::2] = packed >> 4+ values = MX4_LUT[codes]+ exponents = np.ldexp(np.float32(1.0), scale.astype(np.int32) - 127)+ return (values.reshape(rows, cols // 32, 32) *+ exponents[:, :, None]).reshape(rows, cols).astype(np.float32)+++def fp8_block_dequant(codes, scale_inv, row_scale_index=None):+ """w[r, c] = fp8(codes[r, c]) * scale_inv[row_block(r), c // 128]."""+ w = fp8_to_f32(codes)+ rows, cols = w.shape+ if row_scale_index is None:+ row_scale_index = np.arange(rows) // BLOCK+ col_block = np.arange(cols) // BLOCK+ return (w * scale_inv[row_scale_index][:, col_block]).astype(np.float32)+++def qkv_contiguous(codes, scale_inv, heads, kv_heads, head_dim, v_head_dim,+ chunks):+ """Pre-sharded fused qkv (vLLM's documented layout) -> [Q|K|V] f32."""+ rows = codes.shape[0]+ per_chunk = rows // chunks+ q_rows = (heads // chunks) * head_dim+ k_rows = (kv_heads // chunks) * head_dim+ v_rows = (kv_heads // chunks) * v_head_dim+ assert q_rows + k_rows + v_rows == per_chunk, (rows, chunks)+ chunk_scale_rows = -(-per_chunk // BLOCK)+ r = np.arange(rows)+ if scale_inv.shape[0] == chunks * chunk_scale_rows:+ index = (r // per_chunk) * chunk_scale_rows + (r % per_chunk) // BLOCK+ elif scale_inv.shape[0] == -(-rows // BLOCK):+ index = r // BLOCK+ else:+ raise SystemExit(f"qkv scale grid {scale_inv.shape} fits neither layout")+ w = fp8_block_dequant(codes, scale_inv, index)+ parts = [w[c * per_chunk:(c + 1) * per_chunk] for c in range(chunks)]+ q = np.concatenate([p[:q_rows] for p in parts])+ k = np.concatenate([p[q_rows:q_rows + k_rows] for p in parts])+ v = np.concatenate([p[q_rows + k_rows:] for p in parts])+ return np.concatenate([q, k, v]).astype(np.float32)+++def vision_state(tensors, model):+ """visual.* widened to f32, plus the zeros the release does not store: the+ merger's two Linear biases and its LayerNorm bias (vLLM builds that merger+ with bias=False; the HF module declares them and would otherwise keep its+ random init)."""+ import torch+ state = {}+ for name, (dtype, arr) in tensors.items():+ if name.startswith("visual."):+ state[name] = torch.from_numpy(bf16_to_f32(arr) if dtype == "BF16"+ else arr.astype(np.float32))+ for name, param in model.named_parameters():+ if name.startswith("visual.merger.") and name.endswith(".bias") \+ and name not in state:+ state[name] = torch.zeros_like(param)+ return state+++def to_state_dict(tensors, config):+ import torch+ chunks = config["num_key_value_heads"]+ state = {}+ consumed = set()+ for name, (dtype, arr) in tensors.items():+ if name in consumed:+ continue+ if name.endswith(".weight_scale_inv") or name.endswith(".weight_scale"):+ continue+ if name.startswith("visual."):+ continue+ if dtype == "BF16":+ value = bf16_to_f32(arr)+ elif dtype == "F32":+ value = arr.astype(np.float32)+ elif name.endswith("qkv_proj.weight"):+ layer = int(name.split(".")[2])+ swa = config["hybrid_layer_pattern"][layer] == 1+ heads = config["swa_num_attention_heads" if swa else "num_attention_heads"]+ kv = config["swa_num_key_value_heads" if swa else "num_key_value_heads"]+ value = qkv_contiguous(arr, tensors[name + "_scale_inv"][1], heads,+ kv, config["head_dim"], config["v_head_dim"],+ chunks)+ elif dtype == "F8_E4M3":+ value = fp8_block_dequant(arr, tensors[name + "_scale_inv"][1])+ elif dtype == "U8" and ".experts." in name:+ value = mxfp4_dequant(arr, tensors[name + "_scale"][1])+ else:+ raise SystemExit(f"{name}: unexpected {dtype}")+ state[name] = torch.from_numpy(np.ascontiguousarray(value))+ return state+++def fetch_vendor(vendor_dir):+ vendor_dir.mkdir(parents=True, exist_ok=True)+ for name, expected in VENDOR_FILES.items():+ path = vendor_dir / name+ if not path.exists():+ url = (f"https://huggingface.co/{VENDOR_REPO}/raw/"+ f"{VENDOR_REVISION}/{name}")+ print(f"[ref] downloading {url}", file=sys.stderr)+ with urllib.request.urlopen(url, timeout=60) as resp:+ path.write_bytes(resp.read())+ digest = hashlib.sha256(path.read_bytes()).hexdigest()+ if digest != expected:+ raise SystemExit(+ f"{name}: sha256 {digest} does not match the pinned {expected}; "+ f"the vendor file changed. Review it, then update VENDOR_FILES.")+++def load_vendor(vendor_dir):+ """The vendor files as a synthetic package, so the relative import of the+ configuration module works whatever the directory is called."""+ pkg = "mimo_v2_vendor"+ spec = importlib.machinery.ModuleSpec(pkg, None, is_package=True)+ package = importlib.util.module_from_spec(spec)+ package.__path__ = [str(vendor_dir)]+ sys.modules[pkg] = package++ def load(stem):+ file_spec = importlib.util.spec_from_file_location(+ f"{pkg}.{stem}", vendor_dir / f"{stem}.py")+ module = importlib.util.module_from_spec(file_spec)+ sys.modules[f"{pkg}.{stem}"] = module+ file_spec.loader.exec_module(module)+ return module++ config_module = load("configuration_mimo_v2")+ modeling = load("modeling_mimo_v2")+ shim_mask_builders(modeling)+ return config_module, modeling+++def shim_mask_builders(modeling):+ """The vendor file targets transformers 5.3, whose mask builders take+ `input_embeds` and `cache_position`; later releases renamed the first+ (`inputs_embeds`, 5.6 dropped the old name) and derive the second+ themselves. Rename, drop what the installed builder no longer takes, and+ nothing else: the masks, and every number, stay the library's own. The+ reference runs without a cache from position 0, where the derived positions+ are the passed ones; the committed ref.json is identical under 5.3 and 5.12."""+ import inspect+ for name in ("create_causal_mask", "create_sliding_window_causal_mask"):+ builder = getattr(modeling, name)+ params = inspect.signature(builder).parameters+ if "input_embeds" in params:+ continue++ def adapted(*args, _builder=builder, _params=params, **kwargs):+ if "input_embeds" in kwargs:+ kwargs["inputs_embeds"] = kwargs.pop("input_embeds")+ if not any(p.kind == p.VAR_KEYWORD for p in _params.values()):+ kwargs = {k: v for k, v in kwargs.items() if k in _params}+ return _builder(*args, **kwargs)+ setattr(modeling, name, adapted)+++def build_model(config_module, modeling, config):+ import torch+ cfg = config_module.MiMoV2Config(**dict(config, use_cache=False))+ cfg._attn_implementation = "eager" # the sink path is eager-only+ torch.manual_seed(0) # every weight is overwritten below+ model = modeling.MiMoV2ForCausalLM(cfg).float().eval()+ return model+++def check_rope(model, config):+ """The two rotary tables must be what the config says (theta 1e7 on full+ layers, 1e4 on SWA layers, over int(head_dim * 0.334) dims): refuse a+ transformers version that silently standardizes them away."""+ dim = int(config["head_dim"] * config["partial_rotary_factor"])+ for module, theta in ((model.model.rotary_emb, config["rope_theta"]),+ (model.model.swa_rotary_emb, config["swa_rope_theta"])):+ want = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim))+ got = module.inv_freq.double().numpy()+ if got.shape != want.shape or not np.allclose(got, want, rtol=1e-6):+ raise SystemExit(f"rotary table {got} is not theta={theta} over "+ f"{dim} dims: this transformers version changes "+ f"the vendor's RoPE")+++class RouterWatch:+ """Records, for every token at every MoE layer, the gap between the k-th+ and (k+1)-th biased router score: a near-tie there lets float noise swap an+ expert, which no logit margin downstream can protect against."""++ def __init__(self, model):+ self.gaps = []+ for layer in model.model.layers:+ gate = getattr(layer.mlp, "gate", None)+ if gate is not None:+ gate.register_forward_hook(self.hook)++ def hook(self, gate, inputs, _output):+ import torch+ x = inputs[0].reshape(-1, inputs[0].shape[-1]).float()+ scores = torch.sigmoid(x @ gate.weight.float().T)+ biased = scores + gate.e_score_correction_bias.float()+ top = torch.topk(biased, gate.top_k + 1, dim=-1).values+ self.gaps.extend((top[:, -2] - top[:, -1]).tolist())+++def logits_for(model, ids):+ import torch+ with torch.no_grad():+ return model(input_ids=torch.tensor([ids], dtype=torch.long),+ use_cache=False).logits[0]+++def case_reference(model, prompt, max_new, min_margin):+ import torch+ sequence = list(prompt)+ margins = []+ for _ in range(max_new):+ logits = logits_for(model, sequence)[-1]+ top2 = torch.topk(logits, 2).values+ margins.append(float(top2[0] - top2[1]))+ sequence.append(int(logits.argmax()))+ full = logits_for(model, sequence)+ teacher = full.argmax(-1).tolist()+ # context-free control: the head applied to the embedding alone+ with torch.no_grad():+ emb = model.model.embed_tokens(torch.tensor(sequence))+ normed = model.model.norm(emb)+ alone = model.lm_head(normed).argmax(-1).tolist()+ context_free = sum(a == b for a, b in zip(alone, teacher)) / len(teacher)+ tf_top2 = torch.topk(full, 2, dim=-1).values+ tf_margins = (tf_top2[:, 0] - tf_top2[:, 1]).tolist()+ worst = min(margins + tf_margins)+ if worst < min_margin:+ raise SystemExit(+ f"logit margin {worst:.4f} below --min-margin {min_margin}: the "+ f"fixture rides a near-tie; re-tune the generator instead of "+ f"lowering the margin.")+ return {+ "prompt_ids": list(prompt),+ "teacher_forcing_ids": teacher,+ "greedy_full_ids": sequence,+ "greedy_new_ids": sequence[len(prompt):],+ "max_new_tokens": max_new,+ "min_logit_margin": round(worst, 4),+ "context_free_agreement": round(context_free, 3),+ }+++def vision_reference(model, fixture, config, max_new, min_margin):+ """The official Qwen2-VL processor on the fixture's picture must give the+ fixture's patches bit for bit (the gateway's preprocessing); then Xiaomi's+ ViT and language model answer a prompt that carries the picture."""+ import torch+ from PIL import Image+ from transformers import Qwen2VLImageProcessor+ processor = Qwen2VLImageProcessor.from_pretrained(str(fixture))+ image = Image.open(fixture / "image.png").convert("RGB")+ batch = processor(images=[image], return_tensors="pt")+ official = batch["pixel_values"].float().numpy()+ thw = batch["image_grid_thw"]+ grid = json.loads((fixture / "image.json").read_text())+ ours = np.fromfile(fixture / "patches.f32", dtype=np.float32).reshape(official.shape)+ if thw.tolist() != [[1, grid["grid_h"], grid["grid_w"]]]:+ raise SystemExit(f"grid {thw.tolist()} != fixture {grid}")+ diff = float(np.abs(official - ours).max())+ if diff > 1e-6:+ raise SystemExit(f"gateway patches differ from the official processor by {diff}")+ n_img = grid["tokens"]+ prompt = ([5, 7, config["vision_start_token_id"]] + [config["image_token_id"]] * n_img+ + [config["vision_end_token_id"], 11, 13, 17, 19])+ pixels = torch.from_numpy(official)++ def logits_img(ids, pv):+ with torch.no_grad():+ return model(input_ids=torch.tensor([ids]), pixel_values=pv,+ image_grid_thw=thw, use_cache=False).logits[0]++ def greedy(pv):+ seq, margins = list(prompt), []+ for _ in range(max_new):+ lg = logits_img(seq, pv)[-1]+ top2 = torch.topk(lg, 2).values+ margins.append(float(top2[0] - top2[1]))+ seq.append(int(lg.argmax()))+ return seq, margins++ seq, margins = greedy(pixels)+ full = logits_img(seq, pixels)+ tf_top2 = torch.topk(full, 2, dim=-1).values+ worst = min(margins + (tf_top2[:, 0] - tf_top2[:, 1]).tolist())+ if worst < min_margin:+ raise SystemExit(f"vision case: logit margin {worst:.4f} below {min_margin}")+ other, _ = greedy(-pixels)+ if other[len(prompt):] == seq[len(prompt):]:+ raise SystemExit("the answer does not depend on the picture: re-tune the tower")+ with torch.no_grad():+ rows = model.visual(pixel_values=pixels, grid_thw=thw).float().numpy()+ return {+ "prompt_ids": prompt,+ "grid_h": grid["grid_h"], "grid_w": grid["grid_w"],+ "patches_sha256": hashlib.sha256(ours.tobytes()).hexdigest(),+ "patch_max_diff_vs_official_processor": diff,+ "vision_rows_sum": float(rows.sum()),+ "vision_rows_abs_sum": float(np.abs(rows).sum()),+ "teacher_forcing_ids": full.argmax(-1).tolist(),+ "greedy_full_ids": seq,+ "greedy_new_ids": seq[len(prompt):],+ "max_new_tokens": max_new,+ "min_logit_margin": round(worst, 4),+ "negated_image_new_ids": other[len(prompt):],+ }+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--fixture", default="./mimo_tiny")+ parser.add_argument("--vendor-dir", default=None,+ help="default: <fixture>/.vendor_mimo_v2")+ parser.add_argument("--output", default=None,+ help="default: <fixture>/ref.json")+ parser.add_argument("--min-margin", type=float, default=0.05)+ parser.add_argument(+ "--min-router-gap", type=float, default=1e-5,+ help="router scores are f32 dot products over the hidden size; engine "+ "and torch sum them in different orders, about 1e-6 apart")+ parser.add_argument("--max-context-free", type=float, default=0.5)+ args = parser.parse_args()++ fixture = Path(args.fixture)+ vendor_dir = (Path(args.vendor_dir) if args.vendor_dir+ else fixture / ".vendor_mimo_v2")+ output = Path(args.output) if args.output else fixture / "ref.json"+ config = json.loads((fixture / "config.json").read_text())++ fetch_vendor(vendor_dir)+ config_module, modeling = load_vendor(vendor_dir)+ model = build_model(config_module, modeling, config)+ check_rope(model, config)+ state = to_state_dict(read_fixture(fixture), config)+ if "vision_config" in config:+ state.update(vision_state(read_fixture(fixture), model))+ missing, unexpected = model.load_state_dict(state, strict=False)+ if missing or unexpected:+ raise SystemExit(f"state dict mismatch:\n missing: {missing}\n"+ f" unexpected: {unexpected}")++ eos = config["eos_token_id"]+ vocab = config["vocab_size"]+ cases = {+ # every case outgrows the 8-token window; "long" crosses a 64-token+ # prefill chunk too+ "short": ([5, 7, 9, 11, 13, 17, 19, 23], 12),+ "window": ([(5 + i * 7) % vocab for i in range(40)], 6),+ "long": ([(5 + i * 11) % vocab for i in range(72)], 6),+ }+ watch = RouterWatch(model)+ reference = {}+ for name, (prompt, max_new) in cases.items():+ prompt = [t if t != eos else t + 1 for t in prompt]+ reference[name] = case_reference(model, prompt, max_new,+ args.min_margin)+ if eos in reference[name]["greedy_new_ids"]:+ raise SystemExit(f"case {name}: greedy emitted EOS")+ print(f"[ref] {name}: prompt {len(prompt)} -> +{max_new}, "+ f"min margin {reference[name]['min_logit_margin']}, "+ f"context-free agreement "+ f"{reference[name]['context_free_agreement']}", file=sys.stderr)+ if reference[name]["context_free_agreement"] > args.max_context_free:+ raise SystemExit(+ f"case {name}: the embedding alone predicts "+ f"{reference[name]['context_free_agreement']:.0%} of the "+ f"teacher-forced argmax; the layers barely matter and an engine "+ f"that skipped them would pass. Re-tune the generator.")+ router_gap = min(watch.gaps)+ print(f"[ref] router: {len(watch.gaps)} top-k boundaries, min gap "+ f"{router_gap:.3g}", file=sys.stderr)+ if router_gap < args.min_router_gap:+ raise SystemExit(f"router top-k boundary gap {router_gap:.3g} below "+ f"--min-router-gap {args.min_router_gap}: float noise "+ f"could swap an expert. Re-tune the generator.")++ image = None+ if "vision_config" in config:+ image = vision_reference(model, fixture, config, 8, args.min_margin)+ print(f"[ref] image: {image['grid_h']}x{image['grid_w']} patches, margin "+ f"{image['min_logit_margin']}, patches vs official processor "+ f"{image['patch_max_diff_vs_official_processor']}", file=sys.stderr)++ import torch+ import transformers+ output.write_text(json.dumps({+ "schema_version": 1,+ "source": "xiaomi-vendor",+ "vendor_repo": VENDOR_REPO,+ "vendor_revision": VENDOR_REVISION,+ "vendor_files": VENDOR_FILES,+ "torch_version": torch.__version__,+ "transformers_version": transformers.__version__,+ "engine_env": {+ # exact-math configuration: dense weights widened to f32 instead of+ # the default int8 at load, float expert matmuls instead of int8+ # activations (a speed approximation, not an oracle target)+ "MIMO_DENSE_BITS": "32", "MIMO_IDOT": "0", "COLI_TEMP": "0"},+ "cases": reference,+ **({"image": image} if image else {}),+ }, indent=1) + "\n")+ print(f"[ref] wrote {output}", file=sys.stderr)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/make_mimo_tiny.py b/c/tools/make_mimo_tiny.pynew file mode 100644index 00000000..657c6e2d--- /dev/null+++ b/c/tools/make_mimo_tiny.py@@ -0,0 +1,436 @@+#!/usr/bin/env python3+"""Generate the tiny MiMo-V2.6 checkpoint used by tests/mimo_tiny_harness.py.++numpy-only and deterministic: CI regenerates the shards from SEED and+byte-identical weights come out on every platform. The reference tokens in+ref.json are NOT produced here: they come from Xiaomi's own modeling code+(tools/make_mimo_ref.py, run offline once, vendor files pinned by SHA-256),+so the engine and its oracle never share an author.++The fixture stores every tensor the way the released checkpoint does, because+the storage is where a reading of MiMo goes wrong while a float reference+still agrees:++ - routed experts are MXFP4 g32 (e2m1 nibble pairs, low nibble = even column,+ one e8m0 byte per 32 columns), the six tensors of an expert back to back+ in the file order of the real shards: down, down scale, gate, gate scale,+ up, up scale;+ - qkv_proj and the layer-0 MLP are FP8 e4m3 with a F32 weight_scale_inv per+ 128x128 block;+ - the fused qkv_proj is PRE-SHARDED in num_key_value_heads chunks+ [Q_0|K_0|V_0|Q_1|K_1|V_1] with the scale grid tiled per chunk. The full+ layers' chunk (176 rows) is not a whole number of blocks, so the per-chunk+ grid has 4 rows where a contiguous one would have 3: a reader that assumes+ [Q|K|V] with one continuous grid fails here, as it would on the real+ checkpoint (108 scale rows, not 106, on a Flash full-attention layer);+ - o_proj, router, norms, sinks, embed and lm_head are BF16.++The geometry crosses every path of mimo.c: full-attention and sliding-window+layers with different KV head counts (2 and 4 heads over 2 chunks), a window+of 8 that every case outgrows, the SWA sink logit, partial RoPE with the two+thetas, the value scale, the dense layer 0 and MoE layers.++No tokenizer.json on purpose: without one the engine prints raw generated+token ids on stdout, which is what the token-exact test parses. Greedy+determinism is engineered, then verified by make_mimo_ref.py: the head is+random at a large scale (wide top-2 gaps), the router's top-k boundary is+checked for near-ties at every token and layer, and the layers are checked to+matter (the argmax must differ from the embedding-only one at most positions).+"""++import argparse+import json+import struct+import sys+from pathlib import Path++import numpy as np++SEED = 20260930++VOCAB = 320+HIDDEN = 256+LAYERS = 6+PATTERN = [0, 1, 1, 0, 1, 1] # 1 = sliding window, 0 = full attention+MOE_FREQ = [0, 1, 1, 1, 1, 1] # layer 0 dense, like the release+DENSE_INTER = 256+HEADS = 4 # full attention+KV_HEADS = 2 # also the qkv pre-shard count (ckpt_tp)+SWA_HEADS = 4+SWA_KV_HEADS = 4 # two per chunk, as on the release+HEAD_DIM = 48 # rope_dim = int(48 * 0.334) = 16+V_HEAD_DIM = 32+EXPERTS = 16+TOPK = 4+MOE_INTER = 64 # MXFP4 needs multiples of 32+WINDOW = 8+EOS_ID = 1+BLOCK = 128+EMBED_S = 0.05 # embedding scale+LM_S = 4.0 # head scale+EXP_LO, EXP_HI = 121, 126 # MXFP4 e8m0 exponent range++CONFIG = {+ "architectures": ["MiMoV2ForCausalLM"],+ "model_type": "mimo_v2",+ "vocab_size": VOCAB,+ "hidden_size": HIDDEN,+ "intermediate_size": DENSE_INTER,+ "num_hidden_layers": LAYERS,+ "hybrid_layer_pattern": PATTERN,+ "moe_layer_freq": MOE_FREQ,+ "num_attention_heads": HEADS,+ "num_key_value_heads": KV_HEADS,+ "head_dim": HEAD_DIM,+ "v_head_dim": V_HEAD_DIM,+ "swa_num_attention_heads": SWA_HEADS,+ "swa_num_key_value_heads": SWA_KV_HEADS,+ "swa_head_dim": HEAD_DIM,+ "swa_v_head_dim": V_HEAD_DIM,+ "sliding_window": WINDOW,+ "sliding_window_size": WINDOW,+ "attention_chunk_size": WINDOW,+ "add_full_attention_sink_bias": False,+ "add_swa_attention_sink_bias": True,+ "attention_value_scale": 0.707,+ "attention_projection_layout": "fused_qkv",+ "attention_bias": False,+ "partial_rotary_factor": 0.334,+ "rope_theta": 10000000.0,+ "swa_rope_theta": 10000.0,+ "rope_parameters": {"partial_rotary_factor": 0.334, "rope_theta": 10000000.0,+ "rope_type": "default", "type": "default"},+ "n_routed_experts": EXPERTS,+ "num_experts_per_tok": TOPK,+ "moe_intermediate_size": MOE_INTER,+ "n_shared_experts": None,+ "n_group": 1,+ "topk_group": 1,+ "topk_method": "noaux_tc",+ "scoring_func": "sigmoid",+ "norm_topk_prob": True,+ "routed_scaling_factor": None,+ "moe_router_dtype": "bfloat16",+ "hidden_act": "silu",+ "layernorm_epsilon": 1e-6,+ "max_position_embeddings": 256,+ "tie_word_embeddings": False,+ "bos_token_id": None,+ "eos_token_id": EOS_ID,+ "pad_token_id": 0,+ "dtype": "bfloat16",+ "quantization_config": {+ "activation_scheme": "dynamic",+ "fmt": "e4m3",+ "quant_method": "fp8",+ "store_dtype": "mxfp4",+ "mxfp4_block_size": 32,+ "weight_block_size": [BLOCK, BLOCK],+ "ignored_layers": [f"model.layers.{i}.self_attn.o_proj"+ for i in range(LAYERS)],+ },+}+++# --vision: a tiny MiMo ViT on the same language model. Six blocks cross every+# path of the tower: full attention at both ends (no sink), row-order and+# column-order windows, and the permutations entering and leaving a column run+# (types -1, 0, 1, 1, 0, -1; the release's pattern is the same alphabet).+VISION = {+ "depth": 6,+ "hidden_size": 64,+ "intermediate_size": 96,+ "num_heads": 4,+ "num_key_value_heads": 2,+ "qk_channels": 16,+ "out_hidden_size": HIDDEN,+ "patch_size": 16,+ "spatial_patch_size": 16,+ "temporal_patch_size": 2,+ "spatial_merge_size": 2,+ "in_chans": 3,+ "hidden_act": "silu",+ "fullatt_block_indexes": [0, 5],+ "vit_window_attn_types": [-1, 0, 1, 1, 0, -1],+ "visual_token_window_size": 5,+ "use_sink": True,+ "window_size": 128,+ "tokens_per_second": 2,+}+IMAGE_TOKEN, VISION_START, VISION_END = 300, 301, 302+IMAGE_W, IMAGE_H = 192, 128 # already a multiple of 32: no resize+PREPROCESSOR = {+ "min_pixels": 3136,+ "max_pixels": 12845056,+ "patch_size": 16,+ "temporal_patch_size": 2,+ "merge_size": 2,+ "image_mean": [0.48145466, 0.4578275, 0.40821073],+ "image_std": [0.26862954, 0.26130258, 0.27577711],+ "image_processor_type": "Qwen2VLImageProcessor",+ "processor_class": "Qwen2_5_VLProcessor",+}+++def bf16_bits(x):+ """f32 -> bf16 bit patterns, round to nearest even."""+ bits = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32)+ rounding = ((bits >> 16) & 1) + np.uint32(0x7FFF)+ return ((bits + rounding) >> 16).astype(np.uint16)+++class BF16(np.ndarray):+ """uint16 bit patterns tagged for the writer as BF16."""+++class F8(np.ndarray):+ """uint8 e4m3fn codes tagged for the writer as F8_E4M3."""+++def tag(arr, cls):+ return np.ascontiguousarray(arr).view(cls)+++def st_dtype(arr):+ if isinstance(arr, BF16):+ return "BF16"+ if isinstance(arr, F8):+ return "F8_E4M3"+ return {np.dtype("float32"): "F32", np.dtype("uint8"): "U8"}[arr.dtype]+++def build_tensors(rng):+ """Every weight of the tiny model, in emission order, as {name: array}."""+ tensors = {}++ def put(name, arr):+ assert name not in tensors, name+ tensors[name] = arr++ def normal(*shape, s=0.02):+ return (rng.standard_normal(shape) * s).astype(np.float32)++ def bf16(x):+ return tag(bf16_bits(x), BF16)++ def norm_weight(n):+ return bf16(1.0 + normal(n, s=0.1))++ def fp8_codes(rows, cols):+ """Random finite e4m3fn codes with exponent fields 4..9 (|w| roughly+ 1/8 .. 7.5 before the block scale); never the 0x7F/0xFF NaN codes."""+ sign = rng.integers(0, 2, size=(rows, cols), dtype=np.uint8) << 7+ exp = rng.integers(4, 10, size=(rows, cols), dtype=np.uint8) << 3+ man = rng.integers(0, 8, size=(rows, cols), dtype=np.uint8)+ return sign | exp | man++ def fp8_scales(rows, cols):+ return (rng.uniform(0.004, 0.012, size=(rows, cols))).astype(np.float32)++ def fp8_matrix(name, out, inp):+ put(name + ".weight", tag(fp8_codes(out, inp), F8))+ put(name + ".weight_scale_inv",+ fp8_scales(-(-out // BLOCK), -(-inp // BLOCK)))++ def fp8_qkv(name, heads, kv_heads):+ """Pre-sharded fused qkv: KV_HEADS chunks of [Q_c|K_c|V_c] rows, the+ scale grid tiled per chunk (ceil(rows_per_chunk / 128) rows each)."""+ chunks = KV_HEADS+ rows_per_chunk = ((heads // chunks) * HEAD_DIM+ + (kv_heads // chunks) * HEAD_DIM+ + (kv_heads // chunks) * V_HEAD_DIM)+ put(name + ".weight",+ tag(fp8_codes(chunks * rows_per_chunk, HIDDEN), F8))+ put(name + ".weight_scale_inv",+ fp8_scales(chunks * -(-rows_per_chunk // BLOCK),+ -(-HIDDEN // BLOCK)))++ def mxfp4(rows, cols):+ """One packed MXFP4 matrix. All 16 e2m1 codes are finite; exponents+ stay near 127 so dequantized magnitudes stay tame, never 0xFF."""+ packed = rng.integers(0, 256, size=(rows, cols // 2), dtype=np.uint8)+ scale = rng.integers(EXP_LO, EXP_HI, size=(rows, cols // 32), dtype=np.uint8)+ return packed, scale++ embed = normal(VOCAB, HIDDEN, s=EMBED_S)+ put("model.embed_tokens.weight", bf16(embed))+ # A random head at a large scale: logits spread over ~LM_S * 16, so the+ # top-2 gap clears float noise by orders of magnitude, while the choice+ # still depends on the whole residual stream (make_mimo_ref.py checks that+ # the layers change the argmax, not just the embedding). EOS is a zero+ # row: never the argmax, so a fixed-length run detects early truncation.+ lm_head = normal(VOCAB, HIDDEN, s=LM_S)+ lm_head[EOS_ID] = 0.0+ put("model.norm.weight", norm_weight(HIDDEN))+ put("lm_head.weight", bf16(lm_head))++ for i in range(LAYERS):+ p = f"model.layers.{i}."+ swa = PATTERN[i] == 1+ heads = SWA_HEADS if swa else HEADS+ kv_heads = SWA_KV_HEADS if swa else KV_HEADS+ put(p + "input_layernorm.weight", norm_weight(HIDDEN))+ put(p + "post_attention_layernorm.weight", norm_weight(HIDDEN))+ fp8_qkv(p + "self_attn.qkv_proj", heads, kv_heads)+ put(p + "self_attn.o_proj.weight",+ bf16(normal(HIDDEN, heads * V_HEAD_DIM, s=0.05)))+ if swa:+ put(p + "self_attn.attention_sink_bias", bf16(normal(heads, s=1.0)))+ if not MOE_FREQ[i]:+ fp8_matrix(p + "mlp.gate_proj", DENSE_INTER, HIDDEN)+ fp8_matrix(p + "mlp.up_proj", DENSE_INTER, HIDDEN)+ fp8_matrix(p + "mlp.down_proj", HIDDEN, DENSE_INTER)+ continue+ # closed-form router: wide, stable expert preferences per hidden+ # direction, so top-k selection cannot ride a near-tie+ gate = 0.08 * np.sin(np.arange(EXPERTS * HIDDEN, dtype=np.float32)+ * 0.013 + 0.7 * i).reshape(EXPERTS, HIDDEN)+ put(p + "mlp.gate.weight", bf16(gate))+ put(p + "mlp.gate.e_score_correction_bias",+ np.linspace(-0.02, 0.02, EXPERTS, dtype=np.float32))+ for e in range(EXPERTS):+ q = p + f"mlp.experts.{e}."+ down, down_s = mxfp4(HIDDEN, MOE_INTER)+ gate_w, gate_s = mxfp4(MOE_INTER, HIDDEN)+ up, up_s = mxfp4(MOE_INTER, HIDDEN)+ # the file order of the release, back to back+ put(q + "down_proj.weight", down)+ put(q + "down_proj.weight_scale", down_s)+ put(q + "gate_proj.weight", gate_w)+ put(q + "gate_proj.weight_scale", gate_s)+ put(q + "up_proj.weight", up)+ put(q + "up_proj.weight_scale", up_s)+ return tensors+++def build_vision(rng):+ """The tower's tensors, named and shaped as in the release (BF16; the merger+ carries no biases and its LayerNorm no bias, as there)."""+ v = VISION+ d, f, heads, kvh = v["hidden_size"], v["intermediate_size"], v["num_heads"], v["num_key_value_heads"]+ hd = v["qk_channels"]+ merged = d * v["spatial_merge_size"] ** 2+ tensors = {}++ def bf16(x):+ return tag(bf16_bits(x), BF16)++ def normal(*shape, s=0.02):+ return (rng.standard_normal(shape) * s).astype(np.float32)++ tensors["visual.patch_embed.proj.weight"] = bf16(normal(d, 3, 2, 16, 16, s=0.03))+ for i in range(v["depth"]):+ p = f"visual.blocks.{i}."+ tensors[p + "norm1.weight"] = bf16(1.0 + normal(d, s=0.1))+ tensors[p + "norm2.weight"] = bf16(1.0 + normal(d, s=0.1))+ tensors[p + "attn.qkv.weight"] = bf16(normal((heads + 2 * kvh) * hd, d, s=0.15))+ tensors[p + "attn.qkv.bias"] = bf16(normal((heads + 2 * kvh) * hd, s=0.1))+ tensors[p + "attn.proj.weight"] = bf16(normal(d, heads * hd, s=0.1))+ tensors[p + "attn.proj.bias"] = bf16(normal(d, s=0.05))+ if i not in v["fullatt_block_indexes"]:+ tensors[p + "attn.sinks"] = bf16(normal(heads, s=1.0))+ for n in ("gate_proj", "up_proj"):+ tensors[p + f"mlp.{n}.weight"] = bf16(normal(f, d, s=0.1))+ tensors[p + f"mlp.{n}.bias"] = bf16(normal(f, s=0.05))+ tensors[p + "mlp.down_proj.weight"] = bf16(normal(d, f, s=0.1))+ tensors[p + "mlp.down_proj.bias"] = bf16(normal(d, s=0.05))+ tensors["visual.merger.ln_q.weight"] = bf16(1.0 + normal(d, s=0.1))+ tensors["visual.merger.mlp.0.weight"] = bf16(normal(merged, merged, s=0.05))+ tensors["visual.merger.mlp.2.weight"] = bf16(normal(HIDDEN, merged, s=0.2))+ return tensors+++def test_image(rng):+ """A deterministic picture with structure at several scales, so the+ windows and the merge see different content everywhere."""+ y, x = np.mgrid[0:IMAGE_H, 0:IMAGE_W].astype(np.float32)+ r = 127 + 120 * np.sin(x / 9.0) * np.cos(y / 13.0)+ g = 127 + 120 * np.sin((x + y) / 17.0)+ b = (x * 3 + y * 5) % 256+ img = np.stack([r, g, b], axis=-1) + rng.integers(0, 24, size=(IMAGE_H, IMAGE_W, 3))+ return np.clip(img, 0, 255).astype(np.uint8)+++def write_safetensors(path, tensors):+ """Hand-rolled writer: keeps insertion order, which the expert contiguity+ the engine exploits depends on (the safetensors library reorders)."""+ header, offset = {}, 0+ for name, arr in tensors.items():+ header[name] = {"dtype": st_dtype(arr), "shape": list(arr.shape),+ "data_offsets": [offset, offset + arr.nbytes]}+ offset += arr.nbytes+ blob = json.dumps(header, separators=(",", ":")).encode()+ blob += b" " * ((-len(blob)) % 8)+ with open(path, "wb") as fh:+ fh.write(struct.pack("<Q", len(blob)))+ fh.write(blob)+ for arr in tensors.values():+ fh.write(np.ascontiguousarray(arr).view(np.uint8).tobytes())+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--output", default="./mimo_tiny")+ parser.add_argument("--force", action="store_true",+ help="overwrite existing shards")+ parser.add_argument("--vision", action="store_true",+ help="add a tiny ViT, the image tokens, a test picture "+ "and its patches (the vision fixture)")+ args = parser.parse_args()++ out = Path(args.output)+ out.mkdir(parents=True, exist_ok=True)+ shard_names = ["model-00001-of-00002.safetensors",+ "model-00002-of-00002.safetensors"]+ if any((out / s).exists() for s in shard_names) and not args.force:+ print(f"{out} already has shards; use --force to regenerate",+ file=sys.stderr)+ return 1++ rng = np.random.default_rng(SEED)+ tensors = build_tensors(rng)+ config = dict(CONFIG)+ if args.vision:+ tensors.update(build_vision(np.random.default_rng(SEED + 1)))+ config.update({"vision_config": VISION, "image_token_id": IMAGE_TOKEN,+ "vision_start_token_id": VISION_START,+ "vision_end_token_id": VISION_END,+ "vision_model_type": "mimovl"})+ (out / "preprocessor_config.json").write_text(+ json.dumps(PREPROCESSOR, indent=2) + "\n")+ from PIL import Image+ Image.fromarray(test_image(np.random.default_rng(SEED + 2))).save(+ out / "image.png")+ # the patches exactly as the gateway builds them (tools/qwen38_image.py,+ # the Qwen2-VL processor); make_mimo_ref.py checks them bit for bit+ # against the official processor+ sys.path.insert(0, str(Path(__file__).resolve().parent))+ import qwen38_image+ patches, grid_h, grid_w = qwen38_image.preprocess(+ (out / "image.png").read_bytes(), model_dir=str(out))+ patches.tofile(out / "patches.f32")+ (out / "image.json").write_text(json.dumps(+ {"grid_h": grid_h, "grid_w": grid_w,+ "tokens": grid_h * grid_w // 4}) + "\n")+ # two shards like the release: the experts of layers 3.. in the second,+ # so the loader has to follow the index across files+ second = {n: a for n, a in tensors.items()+ if ".mlp.experts." in n and int(n.split(".")[2]) >= 3}+ first = {n: a for n, a in tensors.items() if n not in second}+ weight_map = {}+ for shard, part in zip(shard_names, (first, second)):+ write_safetensors(out / shard, part)+ weight_map.update({n: shard for n in part})+ total = sum(a.nbytes for a in tensors.values())+ (out / "model.safetensors.index.json").write_text(json.dumps(+ {"metadata": {"total_size": total}, "weight_map": weight_map},+ indent=1) + "\n")+ (out / "config.json").write_text(json.dumps(config, indent=2) + "\n")+ print(f"{out}: {len(tensors)} tensors, {total / 1e6:.2f} MB in 2 shards, "+ f"{LAYERS} layers (pattern {PATTERN}), {EXPERTS} experts top-{TOPK}, "+ f"seed {SEED}")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/mimo_real_check.py b/c/tools/mimo_real_check.pynew file mode 100644index 00000000..a32b3c5f--- /dev/null+++ b/c/tools/mimo_real_check.py@@ -0,0 +1,338 @@+#!/usr/bin/env python3+"""mimo.c against Xiaomi's own code on the REAL MiMo-V2.6 checkpoint.++The tiny oracle (tools/make_mimo_tiny.py + make_mimo_ref.py) pins the math on a+checkpoint stored like the release. This pins the release itself: its MXFP4 nibble+order, its FP8 block scales, its pre-sharded qkv, its sinks, its ViT. Maintainer tool,+run by hand on a machine that has the checkpoint; nothing here runs in CI.++ python3 tools/mimo_real_check.py --model DIR --engine ./mimo [--layers N] [--image PNG]++The vendor model is built from the checkpoint's own modeling_mimo_v2.py (its SHA-256+must match the one the tiny oracle pinned) with the first N layers (default: all),+dense weights dequantized to f32 from the same bytes the engine reads. Its experts are+NOT all materialized -- 256 experts of a Flash layer are 26 GB in f32 -- but built one+at a time when routed: each is the vendor's own MiMoV2MLP, loaded, called and dropped,+so the math is still theirs and the RAM stays at one expert.++What is compared, at every prompt position: the logits (max absolute difference,+relative to the logit range), the argmax, and the KL divergence of the engine's+distribution from the vendor's. With --image, the ViT's output rows too.+"""+import argparse+import hashlib+import json+import os+import struct+import subprocess+import sys+import tempfile+from pathlib import Path++import numpy as np++sys.path.insert(0, str(Path(__file__).resolve().parent))+import make_mimo_ref as R # noqa: E402 (reuses the pinned vendor loader and dequantizers)+++class Shards:+ """Tensor lookup across the checkpoint's shards, read on demand."""++ def __init__(self, model_dir):+ self.dir = Path(model_dir)+ index = json.loads((self.dir / "model.safetensors.index.json").read_text())+ self.where = index["weight_map"]+ self.headers = {}++ def header(self, shard):+ if shard not in self.headers:+ with open(self.dir / shard, "rb") as fh:+ size = struct.unpack("<Q", fh.read(8))[0]+ self.headers[shard] = (json.loads(fh.read(size)), 8 + size)+ return self.headers[shard]++ def has(self, name):+ return name in self.where++ def raw(self, name):+ shard = self.where[name]+ header, base = self.header(shard)+ meta = header[name]+ lo, hi = meta["data_offsets"]+ dtype = {"F32": np.float32, "U8": np.uint8, "BF16": np.uint16,+ "F8_E4M3": np.uint8}[meta["dtype"]]+ with open(self.dir / shard, "rb") as fh:+ fh.seek(base + lo)+ data = fh.read(hi - lo)+ return meta["dtype"], np.frombuffer(data, dtype=dtype).reshape(meta["shape"])++ def f32(self, name):+ dtype, arr = self.raw(name)+ if dtype == "BF16":+ return R.bf16_to_f32(arr)+ if dtype == "F32":+ return arr.astype(np.float32)+ raise SystemExit(f"{name}: {dtype} is not a float tensor")+++def truncated_config(config, layers, vision):+ config = dict(config)+ config["num_hidden_layers"] = layers+ config["hybrid_layer_pattern"] = config["hybrid_layer_pattern"][:layers]+ config["moe_layer_freq"] = config["moe_layer_freq"][:layers]+ config.pop("quantization_config", None)+ config.pop("audio_config", None)+ if not vision:+ config.pop("vision_config", None)+ return config+++def load_dense(model, shards, config):+ """Every non-expert weight of the truncated model, dequantized to f32."""+ import torch+ chunks = config["num_key_value_heads"]+ state = {}+ for name, param in model.named_parameters():+ if ".mlp.experts." in name:+ continue+ if not shards.has(name):+ if name.startswith("visual.merger.") and name.endswith(".bias"):+ state[name] = torch.zeros(tuple(param.shape)) # the release stores none+ continue+ raise SystemExit(f"{name}: not in the checkpoint")+ dtype, arr = shards.raw(name)+ if name.endswith("qkv_proj.weight"):+ layer = int(name.split(".")[2])+ swa = config["hybrid_layer_pattern"][layer] == 1+ heads = config["swa_num_attention_heads" if swa else "num_attention_heads"]+ kv = config["swa_num_key_value_heads" if swa else "num_key_value_heads"]+ value = R.qkv_contiguous(arr, shards.f32(name + "_scale_inv"), heads, kv,+ config["head_dim"], config["v_head_dim"], chunks)+ elif dtype == "F8_E4M3":+ value = R.fp8_block_dequant(arr, shards.f32(name + "_scale_inv"))+ elif dtype in ("BF16", "F32"):+ value = shards.f32(name)+ else:+ raise SystemExit(f"{name}: unexpected {dtype}")+ if tuple(value.shape) != tuple(param.shape):+ if name == "visual.patch_embed.proj.weight":+ value = value.reshape(param.shape)+ else:+ raise SystemExit(f"{name}: {value.shape} vs {tuple(param.shape)}")+ state[name] = torch.from_numpy(np.ascontiguousarray(value))+ return state+++def lazy_experts(model, modeling, shards, config):+ """Replace each layer's expert list with one that builds the vendor's MiMoV2MLP+ for an expert only when the router picks it."""+ import torch++ class LazyExperts(torch.nn.Module):+ def __init__(self, layer, count):+ super().__init__()+ self.layer, self.count = layer, count+ self.cfg = model.config++ def __len__(self):+ return self.count++ def __iter__(self):+ for e in range(self.count):+ yield self[e]++ def __getitem__(self, e):+ layer, cfg = self.layer, self.cfg++ class Once:+ def __call__(self, x):+ mlp = modeling.MiMoV2MLP(cfg, intermediate_size=cfg.moe_intermediate_size)+ prefix = f"model.layers.{layer}.mlp.experts.{e}."+ weights = {}+ for part in ("gate_proj", "up_proj", "down_proj"):+ _, packed = shards.raw(prefix + part + ".weight")+ _, scale = shards.raw(prefix + part + ".weight_scale")+ weights[part + ".weight"] = torch.from_numpy(+ R.mxfp4_dequant(packed, scale))+ mlp.load_state_dict(weights)+ with torch.no_grad():+ return mlp.float()(x)+ return Once()++ for index, layer in enumerate(model.model.layers):+ if config["moe_layer_freq"][index]:+ layer.mlp.experts = LazyExperts(index, config["n_routed_experts"])+++def engine_logits(engine, model_dir, ids, layers, env_extra, vocab):+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:+ path = tmp.name+ env = dict(os.environ, MIMO_LOGITS=path, MIMO_LAYERS=str(layers), **env_extra)+ try:+ subprocess.run([engine, str(model_dir), "--ids", " ".join(map(str, ids)), "--ngen", "0"],+ env=env, check=True, capture_output=True, text=True, timeout=7200)+ return np.fromfile(path, dtype=np.float32).reshape(len(ids), vocab)+ finally:+ os.unlink(path)+++def image_engine(args, model_dir, config, engine_env, vocab, tokenizer):+ """The picture through the gateway's preprocessing and the official processor+ (must agree on the grid), then the engine: its ViT rows and the logits of a prompt+ that carries the picture. Runs before the vendor model is built, so the two never+ hold RAM at the same time."""+ from PIL import Image+ from transformers import Qwen2VLImageProcessor+ import qwen38_image+ data = Path(args.image).read_bytes()+ ours, grid_h, grid_w = qwen38_image.preprocess(data, model_dir=str(model_dir),+ max_tokens=args.image_tokens)+ # the pixel window as the gateway sets it; recent transformers read min_pixels /+ # max_pixels over `size` when the checkpoint's config names them, so give both+ raw = json.loads((model_dir / "preprocessor_config.json").read_text())+ low, high = raw.get("min_pixels", 3136), raw.get("max_pixels", 12845056)+ if args.image_tokens:+ high = min(high, args.image_tokens * 32 * 32)+ low = min(low, high)+ processor = Qwen2VLImageProcessor.from_pretrained(+ str(model_dir), min_pixels=low, max_pixels=high,+ size={"shortest_edge": low, "longest_edge": high})+ batch = processor(images=[Image.open(args.image).convert("RGB")], return_tensors="pt")+ official = batch["pixel_values"].float().numpy()+ thw = batch["image_grid_thw"]+ if thw.tolist() != [[1, grid_h, grid_w]]:+ raise SystemExit(f"grid: gateway {grid_h}x{grid_w}, official {thw.tolist()}")+ print(f"[real] image {grid_h}x{grid_w} patches: gateway vs official processor, max "+ f"|diff| {np.abs(official - ours).max():.3e} (resampling: Pillow against the "+ f"processor's own)")+ n_img = grid_h * grid_w // 4+ prefix = tokenizer.encode("<|im_start|>user\n<|vision_start|>", add_special_tokens=False).ids+ suffix = tokenizer.encode("<|vision_end|>Cosa vedi in questa immagine?<|im_end|>"+ "<|im_start|>assistant\n<think>", add_special_tokens=False).ids+ ids = prefix + [config["image_token_id"]] * n_img + suffix+ with tempfile.TemporaryDirectory() as tmp:+ patches = Path(tmp) / "patches.f32"+ ours.tofile(patches)+ dump, logits_path = Path(tmp) / "rows.f32", Path(tmp) / "logits.f32"+ env = dict(os.environ, MIMO_LOGITS=str(logits_path), MIMO_VISION_DUMP=str(dump),+ MIMO_LAYERS=str(config["num_hidden_layers"]), **engine_env)+ subprocess.run([args.engine, str(model_dir), "--ids", " ".join(map(str, ids)),+ "--ngen", "0", "--image", str(patches), "--grid", str(grid_h),+ str(grid_w)], env=env, check=True, capture_output=True, text=True,+ timeout=7200)+ rows = np.fromfile(dump, dtype=np.float32).reshape(n_img, -1)+ logits = np.fromfile(logits_path, dtype=np.float32).reshape(len(ids), vocab)+ return {"patches": ours, "thw": thw, "ids": ids, "rows": rows, "logits": logits}+++def image_vendor(model, image, config):+ import torch+ pixels = torch.from_numpy(image["patches"])+ with torch.no_grad():+ rows = model.visual(pixel_values=pixels, grid_thw=image["thw"]).float().numpy()+ theirs = model(input_ids=torch.tensor([image["ids"]]), pixel_values=pixels,+ image_grid_thw=image["thw"], use_cache=False).logits[0].float().numpy()+ diff = np.abs(image["rows"] - rows)+ print(f"[real] ViT rows ({rows.shape[0]} x {rows.shape[1]}): max |diff| {diff.max():.3e}, "+ f"relative {diff.max() / np.abs(rows).max():.2e}")+ return compare(f"text with the picture, {config['num_hidden_layers']} layers",+ image["logits"], theirs)+++def compare(name, ours, theirs):+ diff = np.abs(ours - theirs)+ span = float(theirs.max() - theirs.min())+ same = int((ours.argmax(-1) == theirs.argmax(-1)).sum())++ def logsoftmax(x):+ x = x - x.max(-1, keepdims=True)+ return x - np.log(np.exp(x).sum(-1, keepdims=True))+ p = np.exp(logsoftmax(theirs.astype(np.float64)))+ kl = float((p * (logsoftmax(theirs.astype(np.float64)) - logsoftmax(ours.astype(np.float64)))).sum(-1).max())+ print(f"[real] {name}: argmax {same}/{len(ours)}, max |diff| {diff.max():.3e} "+ f"({diff.max() / span:.2e} of the logit range), max KL {kl:.2e}")+ return same == len(ours)+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--model", required=True)+ parser.add_argument("--engine", required=True)+ parser.add_argument("--layers", type=int, default=0, help="0: every layer")+ parser.add_argument("--prompt", default="<|im_start|>user\nCiao! Qual e' la capitale "+ "d'Italia, e perche'?<|im_end|><|im_start|>assistant\n<think>")+ parser.add_argument("--image", default=None, help="a picture for the ViT check")+ parser.add_argument("--image-tokens", type=int, default=256,+ help="ceiling on the picture's tokens, as the gateway's")+ parser.add_argument("--dense-bits", default="0", help="the engine's MIMO_DENSE_BITS")+ parser.add_argument("--ids", default=None, help="prompt token ids instead of --prompt")+ parser.add_argument("--vendor-dir", default=None,+ help="where modeling_mimo_v2.py is (default: the model dir)")+ args = parser.parse_args()++ import torch+ from tokenizers import Tokenizer+ model_dir = Path(args.model)+ vendor_dir = Path(args.vendor_dir) if args.vendor_dir else model_dir+ for name, expected in R.VENDOR_FILES.items():+ digest = hashlib.sha256((vendor_dir / name).read_bytes()).hexdigest()+ if digest != expected:+ raise SystemExit(f"{name} in the checkpoint is not the pinned vendor file")+ full = json.loads((model_dir / "config.json").read_text())+ layers = args.layers or full["num_hidden_layers"]+ config = truncated_config(full, layers, vision=bool(args.image))+ tokenizer = None+ if args.ids:+ ids = [int(t) for t in args.ids.split()]+ else:+ tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json"))+ ids = tokenizer.encode(args.prompt, add_special_tokens=False).ids+ print(f"[real] {layers} of {full['num_hidden_layers']} layers, {len(ids)} prompt tokens",+ file=sys.stderr)+ engine_env = {"MIMO_DENSE_BITS": args.dense_bits, "MIMO_IDOT": "0"}+ ours = engine_logits(args.engine, model_dir, ids, layers,+ engine_env,+ full["vocab_size"])+ image = None+ if args.image:+ if tokenizer is None:+ tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json"))+ image = image_engine(args, model_dir, config, engine_env, full["vocab_size"], tokenizer)+ config_module, modeling = R.load_vendor(vendor_dir)+ cfg = config_module.MiMoV2Config(**dict(config, use_cache=False))+ cfg._attn_implementation = "eager"+ torch.manual_seed(0)+ with torch.device("meta"):+ model = modeling.MiMoV2ForCausalLM(cfg)+ shards = Shards(model_dir)+ lazy_experts(model, modeling, shards, config)+ state = load_dense(model, shards, config)+ model.load_state_dict(state, strict=False, assign=True)+ del state+ # the rotary tables are buffers computed at construction, i.e. on meta: rebuild+ # them on the CPU from the same config, then refuse anything still on meta+ model.model.rotary_emb = modeling.MiMoV2RotaryEmbedding(config=cfg, is_swa=False)+ model.model.swa_rotary_emb = modeling.MiMoV2RotaryEmbedding(config=cfg, is_swa=True)+ if getattr(model, "visual", None) is not None:+ head_dim = model.visual.blocks[0].attn.head_dim+ model.visual.rotary_pos_emb = modeling.MiMoVisionRotaryEmbedding(head_dim // 2)+ left = [n for n, t in list(model.named_parameters()) + list(model.named_buffers())+ if t.is_meta]+ if left:+ raise SystemExit(f"still on meta after loading: {left[:8]}")+ R.check_rope(model, config)+ model = model.float().eval()++ with torch.no_grad():+ theirs = model(input_ids=torch.tensor([ids]), use_cache=False).logits[0].float().numpy()+ ok = compare(f"text, {layers} layers", ours, theirs)+ if image:+ ok = image_vendor(model, image, config) and ok+ if tokenizer:+ top = [tokenizer.decode([int(t)]) for t in theirs.argmax(-1)[-5:]]+ print(f"[real] last positions, vendor argmax: {top}")+ return 0 if ok else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/qwen38_image.py b/c/tools/qwen38_image.pyindex f796d3ec..90505e08 100644--- a/c/tools/qwen38_image.py+++ b/c/tools/qwen38_image.py@@ -129,6 +129,11 @@ def load_config(model_dir):settings["min_pixels"] = size["shortest_edge"]if "longest_edge" in size:settings["max_pixels"] = size["longest_edge"]+ # The older Qwen2-VL spelling (MiMo-V2.6 ships it): the same window as+ # top-level keys. `size` wins when both are there, as in the processor.+ for key in ("min_pixels", "max_pixels"):+ if key in raw:+ settings.setdefault(key, raw[key])config = root / "config.json"if config.exists():vision = json.loads(config.read_text()).get("vision_config") or {}diff --git a/c/tools/requirements-mimo-tiny.txt b/c/tools/requirements-mimo-tiny.txtnew file mode 100644index 00000000..c1634540--- /dev/null+++ b/c/tools/requirements-mimo-tiny.txt@@ -0,0 +1,4 @@+# tiny MiMo-V2.6 fixture generation (CI): deterministic numpy weights, and Pillow for+# the test picture and the gateway's Qwen2-VL preprocessing (tools/qwen38_image.py)+numpy>=1.24+pillow>=10