Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* mimo.c -- Xiaomi MiMo-V2.6 (Flash 309B/15B, Pro 1.02T/42B) on a machine that
* cannot hold it.
*
* The released checkpoint is read as it is, no conversion step:
*
* experts MXFP4 g32 (e2m1 nibble pairs, low nibble = even column, one e8m0
* byte per 32 columns), the six tensors of an expert back to back in
* the shard, so an expert is ONE read. 256 of them per layer on
* Flash, 8 routed per token, cached in RAM by a per-layer LRU.
* dense qkv_proj and the layer-0 MLP in FP8 e4m3 with a F32 scale per
* 128x128 block; o_proj, router, norms, sinks, embed, lm_head BF16.
*
* What makes MiMo MiMo, each mirroring modeling_mimo_v2.py:
* hybrid attention hybrid_layer_pattern: 1 = sliding window (128 keys,
* itself included), 0 = full attention; the two kinds have
* their own KV head count and their own RoPE theta
* partial RoPE on the first int(head_dim * 0.334) dims of each head,
* rotate-half style; head_dim 192 for q/k, 128 for v
* value scale v *= attention_value_scale before attention
* attention sink SWA layers carry one logit per head that joins the
* softmax denominator and nothing else
* router sigmoid scores, e_score_correction_bias picks but does
* not weigh, top-k renormalised; no shared expert
*
* One storage detail the modeling code does not describe: the fused qkv_proj is
* PRE-SHARDED in num_key_value_heads chunks [Q_0|K_0|V_0|Q_1|K_1|V_1|...] and
* its FP8 scale grid is tiled per chunk (a Flash full-attention layer has 108
* scale rows, not the 106 a contiguous grid would have). The layout is vLLM's
* (vllm/model_executor/models/mimo_v2.py, _shard_fp8_qkv_proj); the loader
* undoes it and the tiny oracle stores it the same way so a wrong reading fails.
*
* Held to Xiaomi's own modeling code through tools/make_mimo_ref.py (vendor
* files pinned by SHA-256) on the tiny fixture of tools/make_mimo_tiny.py.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <math.h>
#include <errno.h>
#include <time.h>
#include <stdarg.h>
#include <unistd.h>
#include <sys/stat.h>
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
#include <sys/resource.h>
#endif
#ifdef _OPENMP
#include <omp.h>
#endif
#include "cli_args.h"
#include "compat.h"
#include "json.h"
#include "st.h"
#include "quant.h"
#include "omp_tune.h"
#include "kv_prefix.h"
#include "pin_pool.h"
#include "tok.h"
#include "serve_codec.h"
#include "serve_poll.h"
#include "stop_ids.h"
#if defined(__AVX2__)
#include <immintrin.h>
#endif
#include "vk_tier.h" /* the shared Vulkan routed-expert tier (COLI_VULKAN=1; stubs without VK=1) */
#ifdef COLI_VULKAN
/* VK=1 and COLI_VULKAN=1: the routed experts go to the shared expert tier
* (vk_tier.c) as the release's MXFP4, and the dense matrices of the trunk and of
* the vision tower to the device where coli_vk_dense() puts them, in whatever
* form MIMO_DENSE_BITS gave them (FP8, BF16, int8 or f32). The router, and every
* matrix whose upload fails, stays on the CPU path below. The backend's dense
* matmuls share one command buffer: main thread only. */
#include "backend_vulkan.h"
#include <pthread.h>
static int g_vk_ready;
/* COLI_VK_DENSE_HOST: the dense matrices on the device only (mimo_dho_start); the
* per-matrix path then runs on the device whatever coli_vk_dense() says */
static int g_mimo_dho;
#endif
#define MIMO_MAX_LAYERS 128
#define MIMO_MAX_TOPK 16
#define MIMO_MAX_CHUNKS 16
/* ------------------------------------------------------------------ utils ---- */
static double now_s(void) {
struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec + t.tv_nsec * 1e-9;
}
static double rss_gb(void) {
struct rusage usage;
getrusage(RUSAGE_SELF, &usage);
#ifdef __APPLE__
return usage.ru_maxrss / 1e9;
#else
return usage.ru_maxrss / 1e6;
#endif
}
static void *xmalloc(size_t bytes, const char *what) {
void *p = malloc(bytes ? bytes : 1);
if (!p) { fprintf(stderr, "[mimo] out of memory allocating %s (%zu bytes)\n", what, bytes); exit(1); }
return p;
}
static void *xcalloc(size_t n, size_t size, const char *what) {
void *p = calloc(n ? n : 1, size ? size : 1);
if (!p) { fprintf(stderr, "[mimo] out of memory allocating %s\n", what); exit(1); }
return p;
}
static int env_int(const char *name, int fallback) {
const char *v = getenv(name);
return (v && *v) ? atoi(v) : fallback;
}
static inline float silu(float x) { return x / (1.0f + expf(-x)); }
static inline float sigmoid(float x) { return 1.0f / (1.0f + expf(-x)); }
/* ---------------------------------------------------------------- config ---- */
/* kind 0 = full attention, 1 = sliding window */
typedef struct {
int vocab, hidden, n_layers, dense_inter, moe_inter, n_experts, topk;
int heads[2], kv_heads[2], head_dim[2], v_dim[2], rope_dim[2];
float theta[2];
int sink[2];
int swa[MIMO_MAX_LAYERS], moe[MIMO_MAX_LAYERS];
int window, chunks, norm_topk;
float v_scale, eps, route_scale;
int max_positions;
int image_token_id, vision_start_id, vision_end_id;
} Cfg;
static double jnum(jval *o, const char *key, double fallback) {
jval *v = o ? json_get(o, key) : NULL;
return (v && v->t == J_NUM) ? v->num : fallback;
}
static int jbool(jval *o, const char *key, int fallback) {
jval *v = o ? json_get(o, key) : NULL;
if (!v) return fallback;
if (v->t == J_BOOL) return v->boolean != 0;
if (v->t == J_NUM) return v->num != 0;
return fallback;
}
static char *read_text(const char *path, long *size_out) {
FILE *f = fopen(path, "rb");
if (!f) return NULL;
fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);
if (size < 0 || size > (64l << 20)) { fclose(f); return NULL; }
char *text = xmalloc((size_t)size + 1, path);
if (fread(text, 1, (size_t)size, f) != (size_t)size) { fclose(f); free(text); return NULL; }
text[size] = 0; fclose(f);
if (size_out) *size_out = size;
return text;
}
static void die_cfg(const char *what) {
fprintf(stderr, "[mimo] config.json: %s -- refusing\n", what);
exit(1);
}
static void load_cfg(Cfg *c, const char *dir) {
char path[4096];
snprintf(path, sizeof(path), "%s/config.json", dir);
char *text = read_text(path, NULL);
if (!text) { fprintf(stderr, "[mimo] cannot read %s\n", path); exit(1); }
char *arena = NULL;
jval *root = json_parse(text, &arena);
if (!root) die_cfg("not JSON");
jval *mt = json_get(root, "model_type");
if (!mt || mt->t != J_STR || strcmp(mt->str, "mimo_v2"))
die_cfg("model_type is not mimo_v2");
memset(c, 0, sizeof(*c));
c->vocab = (int)jnum(root, "vocab_size", 0);
c->hidden = (int)jnum(root, "hidden_size", 0);
c->n_layers = (int)jnum(root, "num_hidden_layers", 0);
c->dense_inter = (int)jnum(root, "intermediate_size", 0);
c->moe_inter = (int)jnum(root, "moe_intermediate_size", 0);
c->n_experts = (int)jnum(root, "n_routed_experts", 0);
c->topk = (int)jnum(root, "num_experts_per_tok", 0);
c->heads[0] = (int)jnum(root, "num_attention_heads", 0);
c->kv_heads[0] = (int)jnum(root, "num_key_value_heads", 0);
c->head_dim[0] = (int)jnum(root, "head_dim", 0);
c->v_dim[0] = (int)jnum(root, "v_head_dim", c->head_dim[0]);
c->heads[1] = (int)jnum(root, "swa_num_attention_heads", c->heads[0]);
c->kv_heads[1] = (int)jnum(root, "swa_num_key_value_heads", c->kv_heads[0]);
c->head_dim[1] = (int)jnum(root, "swa_head_dim", c->head_dim[0]);
c->v_dim[1] = (int)jnum(root, "swa_v_head_dim", c->v_dim[0]);
double prf = jnum(root, "partial_rotary_factor", 1.0);
for (int k = 0; k < 2; k++) c->rope_dim[k] = (int)(c->head_dim[k] * prf);
c->theta[0] = (float)jnum(root, "rope_theta", 10000.0);
c->theta[1] = (float)jnum(root, "swa_rope_theta", c->theta[0]);
c->sink[0] = jbool(root, "add_full_attention_sink_bias", 0);
c->sink[1] = jbool(root, "add_swa_attention_sink_bias", 0);
c->window = (int)jnum(root, "sliding_window", 0);
c->v_scale = (float)jnum(root, "attention_value_scale", 1.0);
c->eps = (float)jnum(root, "layernorm_epsilon", 1e-6);
c->route_scale = (float)jnum(root, "routed_scaling_factor", 1.0);
c->norm_topk = jbool(root, "norm_topk_prob", 1);
c->max_positions = (int)jnum(root, "max_position_embeddings", 32768);
c->image_token_id = (int)jnum(root, "image_token_id", -1);
c->vision_start_id = (int)jnum(root, "vision_start_token_id", -1);
c->vision_end_id = (int)jnum(root, "vision_end_token_id", -1);
c->chunks = c->kv_heads[0];
/* MIMO_LAYERS=N: only the first N layers, then the final norm and head. A
* checking aid (tools/mimo_real_check.py compares it with Xiaomi's code on
* the real weights), never an answer anyone should read. */
int keep = env_int("MIMO_LAYERS", 0);
jval *pat = json_get(root, "hybrid_layer_pattern");
jval *freq = json_get(root, "moe_layer_freq");
if (c->n_layers < 1 || c->n_layers > MIMO_MAX_LAYERS) die_cfg("num_hidden_layers out of range");
if (!pat || pat->t != J_ARR || pat->len != c->n_layers) die_cfg("hybrid_layer_pattern must list every layer");
if (!freq || freq->t != J_ARR || freq->len != c->n_layers) die_cfg("moe_layer_freq must list every layer");
for (int i = 0; i < c->n_layers; i++) {
c->swa[i] = (int)pat->kids[i]->num == 1;
c->moe[i] = (int)freq->kids[i]->num != 0;
}
if (keep > 0 && keep < c->n_layers) {
fprintf(stderr, "[mimo] MIMO_LAYERS=%d: a truncated model for checking, not for answers\n", keep);
c->n_layers = keep;
}
/* Everything below sizes a buffer or indexes one: refuse a config that
* would make any of it overflow or divide by zero, before reading a byte. */
if (c->vocab < 2 || c->vocab > (1 << 22)) die_cfg("vocab_size out of range");
if (c->hidden < 32 || c->hidden > 65536 || c->hidden % 32) die_cfg("hidden_size must be a positive multiple of 32");
if (c->moe_inter < 32 || c->moe_inter % 32 || c->moe_inter > 65536) die_cfg("moe_intermediate_size must be a positive multiple of 32");
if (c->n_experts < 1 || c->n_experts > 4096) die_cfg("n_routed_experts out of range");
if (c->topk < 1 || c->topk > MIMO_MAX_TOPK || c->topk > c->n_experts) die_cfg("num_experts_per_tok out of range");
for (int k = 0; k < 2; k++) {
if (c->heads[k] < 1 || c->kv_heads[k] < 1 || c->heads[k] % c->kv_heads[k]) die_cfg("attention heads must be a multiple of kv heads");
if (c->heads[k] > 1024 || c->kv_heads[k] > 1024) die_cfg("too many attention heads");
if (c->head_dim[k] < 2 || c->head_dim[k] > 1024 || c->v_dim[k] < 1 || c->v_dim[k] > 1024) die_cfg("head dims out of range");
if (c->rope_dim[k] < 0 || c->rope_dim[k] > c->head_dim[k] || c->rope_dim[k] % 2) die_cfg("rotary dims must be even and fit the head");
}
if (c->chunks < 1 || c->chunks > MIMO_MAX_CHUNKS) die_cfg("num_key_value_heads (the qkv pre-shard count) out of range");
for (int k = 0; k < 2; k++)
if (c->heads[k] % c->chunks || c->kv_heads[k] % c->chunks) die_cfg("head counts do not divide into the qkv pre-shard chunks");
int any_swa = 0, any_moe = 0;
for (int i = 0; i < c->n_layers; i++) { any_swa |= c->swa[i]; any_moe |= c->moe[i]; }
if (any_swa && (c->window < 1 || c->window > (1 << 20))) die_cfg("sliding_window out of range");
for (int i = 0; i < c->n_layers; i++)
if (!c->moe[i] && (c->dense_inter < 1 || c->dense_inter > (1 << 20))) die_cfg("intermediate_size out of range");
if (!any_moe && keep <= 0) die_cfg("no MoE layer");
if (!(c->eps > 0) || !(c->eps < 1)) die_cfg("layernorm_epsilon out of range");
if (c->max_positions < 1) c->max_positions = 32768;
json_free(root); free(arena); free(text);
}
/* ----------------------------------------------------------- dense weights ---- */
/* A dense matrix [O, I] in one of four forms. FP8 keeps the checkpoint bytes and
* expands the 128x128 block grid to one scale per (row, column block), which is
* what lets a row move (the qkv de-interleave) without touching its bytes. */
enum { DW_F32 = 0, DW_BF16 = 1, DW_FP8 = 2, DW_I8 = 3 };
typedef struct {
int fmt, O, I, nblk;
void *w; /* f32 / bf16 / e4m3 / int8 */
float *s; /* FP8: [O][nblk]; I8: [O] */
void *vk; /* COLI_VULKAN: the device copy, uploaded at start-up */
int vk_off; /* its upload failed: this matrix stays on the CPU */
/* COLI_VULKAN: how to read it back from disk (dw_reload), and 1 while the device
* holds it alone (COLI_VK_DENSE_HOST, the host copy dropped) */
char *vk_name;
int vk_kind, vk_bits, vk_li, vk_gone;
} DW;
/* 0 (default): the checkpoint's own FP8 and BF16 bytes, exact.
* 8: int8 per row, about 30% less RAM for the dense part, not exact.
* 32: f32, exact and four times the RAM: the oracle's configuration. */
static int g_dense_bits;
static void matmul_bf16(float *y, const float *x, const uint16_t *W, int S, int I, int O) {
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint16_t *w = W + (int64_t)o * I;
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s * I;
float a = 0; int i = 0;
#ifdef __AVX2__
__m256 acc = _mm256_setzero_ps();
for (; i + 8 <= I; i += 8)
acc = _mm256_fmadd_ps(_mm256_loadu_ps(xs + i), bf16_decode8(w + i), acc);
a = hsum256(acc);
#endif
for (; i < I; i++) a += xs[i] * bf16_to_f32(w[i]);
y[(int64_t)s * O + o] = a;
}
}
}
/* FP8 with a scale per (row, 128-column block): float inside a block, double
* across blocks, the convention of quant.h's matmul_fp8. */
static void matmul_fp8_rows(float *y, const float *x, const uint8_t *q8, const float *rs,
int nblk, int S, int I, int O) {
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint8_t *w = q8 + (int64_t)o * I;
const float *scl = rs + (int64_t)o * nblk;
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s * I;
double a = 0;
for (int b = 0; b < nblk; b++) {
int base = b * 128, end = base + 128 < I ? base + 128 : I, i = base;
float acc = 0;
#ifdef __AVX2__
__m256 v = _mm256_setzero_ps();
for (; i + 8 <= end; i += 8)
v = _mm256_fmadd_ps(_mm256_loadu_ps(xs + i), e4m3_decode8(w + i), v);
acc = hsum256(v);
#endif
for (; i < end; i++) acc += e4m3_decode(w[i]) * xs[i];
a += (double)acc * scl[b];
}
y[(int64_t)s * O + o] = (float)a;
}
}
}
#ifdef COLI_VULKAN
/* Each form is one of the shader's formats as it is stored, so nothing is
* converted on the way up:
* DW_F32 fmt 10, f32 [O, I], no scales
* DW_BF16 fmt 11, bf16 [O, I] (the low half of a word is the even column)
* DW_FP8 fmt 12, e4m3 [O, I] with the per-(row, 128-column block) scales
* rs[O][nblk] that dw_load_fp8 expanded: a group size of 128
* DW_I8 fmt 1, int8 [O, I] and one f32 scale per row after the sum
* Every one is f32 weights times f32 activations, as on the CPU; what differs
* is the order of the sums (the CPU's FP8 kernel adds its blocks in double). */
static int vk_main_thread(void) {
#ifdef _OPENMP
return !omp_in_parallel();
#else
return 1;
#endif
}
static int dw_vk_fmt(const DW *d, int *gs, const float **sc) {
*gs = 0; *sc = NULL;
switch (d->fmt) {
case DW_F32: return 10;
case DW_BF16: return 11;
case DW_FP8: *gs = 128; *sc = d->s; return 12;
case DW_I8: *sc = d->s; return 1;
}
return -1;
}
static int g_mimo_vk_dev; /* the device dw_upload sends a new matrix to: 1 for the chain's layers on
* COLI_VK_DEV2's (mimo_chain.h) */
static int dw_upload(DW *d) {
int gs; const float *sc;
int fmt = dw_vk_fmt(d, &gs, &sc);
if (d->vk || d->vk_off || !d->w) return d->vk != NULL;
if (fmt < 0 || !(g_mimo_vk_dev ? coli_vk_tensor_ensure2((ColiVkTensor **)&d->vk, d->w, sc, fmt, d->I, d->O, gs)
: coli_vk_tensor_ensure((ColiVkTensor **)&d->vk, d->w, sc, fmt, d->I, d->O, gs))) d->vk_off = 1;
return d->vk != NULL;
}
static int dw_matmul_vk(float *y, const float *x, int S, const DW *d) {
if (!g_vk_ready || !(coli_vk_dense() || g_mimo_dho) || d->vk_off || !vk_main_thread()) return 0;
DW *dev = (DW *)d; /* the device copy is a cache inside a read-only matrix */
if (!dw_upload(dev) || coli_vk_tensor_dev((ColiVkTensor *)dev->vk)) return 0; /* the second device's: only its chain */
int gs; const float *sc;
int fmt = dw_vk_fmt(d, &gs, &sc);
return coli_vk_matmul((ColiVkTensor **)&dev->vk, y, x, d->w, sc, fmt, S, d->I, d->O, gs);
}
static void dw_reload(DW *d); /* below, beside the loaders it re-runs */
#endif
static void dw_matmul(float *y, const float *x, int S, const DW *d) {
#ifdef COLI_VULKAN
if (dw_matmul_vk(y, x, S, d)) return;
if (d->vk_gone) dw_reload((DW *)d); /* the CPU needs a matrix the device held alone (a lost device) */
#endif
switch (d->fmt) {
case DW_F32: matmul(y, x, (const float *)d->w, S, d->I, d->O); break;
case DW_BF16: matmul_bf16(y, x, (const uint16_t *)d->w, S, d->I, d->O); break;
case DW_FP8: matmul_fp8_rows(y, x, (const uint8_t *)d->w, d->s, d->nblk, S, d->I, d->O); break;
case DW_I8: matmul_q(y, x, (const int8_t *)d->w, d->s, S, d->I, d->O); break;
}
}
static st_tensor *need(shards *S, const char *name, int dtype, int O, int I) {
st_tensor *t = st_find(S, name);
if (!t) st_die_missing(S, name);
int esz = st_dtype_esz(dtype);
int64_t numel = I > 0 ? (int64_t)O * I : O;
int rank_ok = I > 0 ? (t->rank == 2 && t->shape[0] == O && t->shape[1] == I)
: (t->rank == 1 && t->shape[0] == O);
if (t->dtype != dtype || !rank_ok || t->nbytes != numel * esz) {
fprintf(stderr, "[mimo] %s: %s %lld bytes, expected %s [%d%s%d] -- refusing (untrusted container)\n",
name, st_dtype_name(t->dtype), (long long)t->nbytes, st_dtype_name(dtype), O,
I > 0 ? ", " : "", I > 0 ? I : 0);
exit(1);
}
return t;
}
/* f32 copy of a BF16 or F32 vector */
static float *load_vec(shards *S, const char *name, int n) {
st_tensor *t = st_find(S, name);
if (!t) st_die_missing(S, name);
int dtype = t->dtype == 2 ? 2 : 0;
need(S, name, dtype, n, 0);
float *out = xmalloc((size_t)n * sizeof(float), name);
st_read_f32(S, name, out, 1);
return out;
}
/* Rows of an f32 matrix [O, I] into the configured dense form. Takes ownership. */
static void dw_from_f32(DW *d, float *w, int O, int I) {
d->O = O; d->I = I;
if (g_dense_bits == 8) {
d->fmt = DW_I8;
d->w = xmalloc((size_t)O * I, "int8 weights");
d->s = xmalloc((size_t)O * sizeof(float), "int8 scales");
quantize_rows(w, (int8_t *)d->w, d->s, O, I, 8);
free(w);
} else {
d->fmt = DW_F32; d->w = w; d->s = NULL;
}
}
#ifdef COLI_VULKAN
enum { DWK_NONE = 0, DWK_BF16, DWK_FP8, DWK_QKV };
/* What dw_reload needs to read the matrix back exactly as it was loaded: its name,
* its loader, the MIMO_DENSE_BITS it was loaded under (the tower forces its own). */
static void dw_note(DW *d, const char *name, int kind) {
free(d->vk_name);
d->vk_name = strdup(name);
if (!d->vk_name) { fprintf(stderr, "OOM weight name\n"); exit(1); }
d->vk_kind = kind; d->vk_bits = g_dense_bits; d->vk_li = -1;
}
#define DW_NOTE(d, name, kind) dw_note(d, name, kind)
#else
#define DW_NOTE(d, name, kind) ((void)0)
#endif
static void dw_load_bf16(DW *d, shards *S, const char *name, int O, int I) {
st_tensor *t = need(S, name, 0, O, I);
DW_NOTE(d, name, DWK_BF16);
if (g_dense_bits == 0) {
d->fmt = DW_BF16; d->O = O; d->I = I; d->s = NULL;
d->w = xmalloc((size_t)t->nbytes, name);
st_read_raw_cap(S, name, d->w, t->nbytes, 1);
return;
}
float *w = xmalloc((size_t)O * I * sizeof(float), name);
st_read_f32(S, name, w, 1);
dw_from_f32(d, w, O, I);
}
/* An FP8 [O, I] with its block grid; `row_block[r]` names the scale row of
* stored row r (NULL: r / 128). `order[r]` names which stored row becomes row r
* of the result (NULL: identity). */
static void dw_load_fp8(DW *d, shards *S, const char *name, int O, int I,
const int *row_block, int scale_rows, const int *order) {
char sname[600];
snprintf(sname, sizeof(sname), "%s_scale_inv", name);
int nblk = (I + 127) / 128;
if (scale_rows <= 0) scale_rows = (O + 127) / 128;
st_tensor *t = need(S, name, 4, O, I);
need(S, sname, 2, scale_rows, nblk);
DW_NOTE(d, name, DWK_FP8);
uint8_t *raw = xmalloc((size_t)t->nbytes, name);
st_read_raw_cap(S, name, raw, t->nbytes, 1);
float *grid = xmalloc((size_t)scale_rows * nblk * sizeof(float), sname);
st_read_f32(S, sname, grid, 1);
for (int64_t i = 0; i < (int64_t)scale_rows * nblk; i++)
if (!isfinite(grid[i])) { fprintf(stderr, "[mimo] %s: non-finite scale -- refusing\n", sname); exit(1); }
d->O = O; d->I = I; d->nblk = nblk;
if (g_dense_bits == 0) {
d->fmt = DW_FP8;
uint8_t *w = xmalloc((size_t)O * I, name);
float *rs = xmalloc((size_t)O * nblk * sizeof(float), sname);
for (int r = 0; r < O; r++) {
int src = order ? order[r] : r;
int sb = row_block ? row_block[src] : src / 128;
memcpy(w + (int64_t)r * I, raw + (int64_t)src * I, (size_t)I);
memcpy(rs + (int64_t)r * nblk, grid + (int64_t)sb * nblk, (size_t)nblk * sizeof(float));
}
d->w = w; d->s = rs;
free(raw); free(grid);
return;
}
float *w = xmalloc((size_t)O * I * sizeof(float), name);
#pragma omp parallel for schedule(static)
for (int r = 0; r < O; r++) {
int src = order ? order[r] : r;
int sb = row_block ? row_block[src] : src / 128;
const uint8_t *q = raw + (int64_t)src * I;
float *dst = w + (int64_t)r * I;
for (int i = 0; i < I; i++) dst[i] = e4m3_decode(q[i]) * grid[(int64_t)sb * nblk + i / 128];
}
free(raw); free(grid);
dw_from_f32(d, w, O, I);
}
/* The fused qkv of layer `li`, read in its pre-sharded order and handed back as
* [Q | K | V] rows. Two scale grids are legal (vLLM accepts the same two): one
* tiled per chunk, ceil(rows_per_chunk / 128) rows each, or one continuous grid;
* they coincide when a chunk is a whole number of blocks. */
static void load_qkv(DW *d, shards *S, const Cfg *c, int li) {
int k = c->swa[li];
int T = c->chunks, hd = c->head_dim[k], vd = c->v_dim[k];
int q_rows = (c->heads[k] / T) * hd, k_rows = (c->kv_heads[k] / T) * hd,
v_rows = (c->kv_heads[k] / T) * vd;
int rpc = q_rows + k_rows + v_rows, O = T * rpc;
int per_chunk = (rpc + 127) / 128, continuous = (O + 127) / 128;
char name[512], sname[600];
snprintf(name, sizeof(name), "model.layers.%d.self_attn.qkv_proj.weight", li);
snprintf(sname, sizeof(sname), "%s_scale_inv", name);
st_tensor *st = st_find(S, sname);
if (!st) st_die_missing(S, sname);
int scale_rows = st->rank == 2 ? (int)st->shape[0] : -1;
int *row_block = xmalloc((size_t)O * sizeof(int), "qkv scale rows");
for (int r = 0; r < O; r++) {
if (scale_rows == T * per_chunk) row_block[r] = (r / rpc) * per_chunk + (r % rpc) / 128;
else if (scale_rows == continuous) row_block[r] = r / 128;
else {
fprintf(stderr, "[mimo] %s: %d scale rows fit neither the per-chunk grid (%d) nor a "
"continuous one (%d) -- refusing\n", sname, scale_rows, T * per_chunk, continuous);
exit(1);
}
}
int *order = xmalloc((size_t)O * sizeof(int), "qkv order");
int w = 0;
for (int ch = 0; ch < T; ch++) for (int r = 0; r < q_rows; r++) order[w++] = ch * rpc + r;
for (int ch = 0; ch < T; ch++) for (int r = 0; r < k_rows; r++) order[w++] = ch * rpc + q_rows + r;
for (int ch = 0; ch < T; ch++) for (int r = 0; r < v_rows; r++) order[w++] = ch * rpc + q_rows + k_rows + r;
dw_load_fp8(d, S, name, O, c->hidden, row_block, scale_rows, order);
free(row_block); free(order);
#ifdef COLI_VULKAN
d->vk_kind = DWK_QKV; d->vk_li = li; /* read back through this function: the de-interleave */
#endif
}
#ifdef COLI_VULKAN
/* A matrix whose host copy was dropped (COLI_VK_DENSE_HOST) and that the CPU needs
* after all: read it back from the checkpoint with the loader and the MIMO_DENSE_BITS
* it was first read with, the same bytes, and keep it from there on. */
static size_t dw_bytes(const DW *d);
static pthread_mutex_t g_dw_dho_mx = PTHREAD_MUTEX_INITIALIZER;
static shards *g_dw_dho_S;
static const Cfg *g_dw_dho_c;
static void dw_reload(DW *d) {
pthread_mutex_lock(&g_dw_dho_mx);
if (d->vk_gone) {
DW t = {0};
int saved = g_dense_bits;
g_dense_bits = d->vk_bits;
if (!g_dw_dho_S || !d->vk_name) { fprintf(stderr, "[VK] mimo: a dense matrix the device held alone cannot be read back\n"); exit(1); }
switch (d->vk_kind) {
case DWK_BF16: dw_load_bf16(&t, g_dw_dho_S, d->vk_name, d->O, d->I); break;
case DWK_FP8: dw_load_fp8(&t, g_dw_dho_S, d->vk_name, d->O, d->I, NULL, 0, NULL); break;
case DWK_QKV: load_qkv(&t, g_dw_dho_S, g_dw_dho_c, d->vk_li); break;
default: fprintf(stderr, "[VK] mimo: %s has no loader to read it back\n", d->vk_name); exit(1);
}
g_dense_bits = saved;
if (t.fmt != d->fmt || t.O != d->O || t.I != d->I) { fprintf(stderr, "[VK] mimo: %s came back in another form\n", d->vk_name); exit(1); }
d->w = t.w; d->s = t.s; d->nblk = t.nblk;
free(t.vk_name);
d->vk_gone = 0;
coli_vk_dense_host_reloaded(dw_bytes(d));
}
pthread_mutex_unlock(&g_dw_dho_mx);
}
#endif
/* ------------------------------------------------------------------ model ---- */
typedef struct { int eid; uint64_t used; uint8_t *base, *buf; } Slot;
typedef struct { Slot *s; int n, cap; int *by_expert; } LCache;
typedef struct { int fd[6]; int64_t off[6]; int contig; } ERef;
typedef struct {
float *ln1, *ln2, *sink;
DW qkv, o;
DW gate, up, down; /* dense MLP (layer 0) */
float *router, *bias; /* [E, H], [E] */
/* KV: full layers [ctx][kvh][hd]; SWA layers a ring of `window` rows */
float *K, *V;
int *ring_pos;
int rows;
} Layer;
/* One conversation's caches for a multiplexed serve (KV_SLOTS>1, serve_mux below):
* every layer's K and V (a full layer's rows, a sliding window's ring and its
* positions), where it stands, and the record of the tokens it holds. The Model's
* layers hold the conversation a prefill runs on (mimo_seq_swap trades it for a parked
* one); a multiplexed decode step parks them all and reads each row's from its MimoRow. */
typedef struct {
float *K[MIMO_MAX_LAYERS], *V[MIMO_MAX_LAYERS];
int *ring_pos[MIMO_MAX_LAYERS];
int pos;
kv_prefix kvp;
} MimoSeq;
typedef struct { MimoSeq *seq; int pos; } MimoRow;
static int g_mimo_mux_slots = 1; /* KV_SLOTS: the conversations a serve decodes at once */
typedef struct {
Cfg c;
shards S;
Layer L[MIMO_MAX_LAYERS];
uint16_t *embed; /* bf16 [V, H], rows widened on use */
DW head;
float *norm;
ERef *eref;
int64_t part[6], e_bytes; /* down, down scale, gate, gate scale, up, up scale */
LCache cache[MIMO_MAX_LAYERS];
int ctx, pos;
kv_prefix kvp;
uint64_t clock, hits, miss, bytes_read, forwards;
double t_disk, t_expert, t_attn;
int idot, direct;
uint8_t **ehit; /* [layer][expert] routed this turn, for HITS */
#ifdef COLI_VULKAN
void *vkchain; /* the dense chain's device state (mimo_chain.h), NULL until it runs */
void *vkchain2; /* its layers on COLI_VK_DEV2's device, after the primary's (mimo_chain.h) */
#endif
/* A multiplexed decode step (forward_rows): row t is the token at mux_rows[t].pos
* of mux_rows[t].seq; NULL in every other forward. */
const MimoRow *mux_rows;
} Model;
static void expert_table_init(Model *m) {
Cfg *c = &m->c;
int H = c->hidden, MI = c->moe_inter;
m->part[0] = (int64_t)H * (MI / 2); m->part[1] = (int64_t)H * (MI / 32);
m->part[2] = (int64_t)MI * (H / 2); m->part[3] = (int64_t)MI * (H / 32);
m->part[4] = m->part[2]; m->part[5] = m->part[3];
m->e_bytes = 0;
for (int k = 0; k < 6; k++) m->e_bytes += m->part[k];
m->eref = xcalloc((size_t)c->n_layers * c->n_experts, sizeof(ERef), "expert table");
static const char *mat[3] = {"down_proj", "gate_proj", "up_proj"};
int missing = 0, split = 0;
for (int li = 0; li < c->n_layers; li++) {
if (!c->moe[li]) continue;
for (int e = 0; e < c->n_experts; e++) {
ERef *er = &m->eref[(int64_t)li * c->n_experts + e];
for (int k = 0; k < 6; k++) {
char nm[512];
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.%s.%s", li, e,
mat[k / 2], (k & 1) ? "weight_scale" : "weight");
st_tensor *t = st_find(&m->S, nm);
if (!t) { missing++; er->fd[k] = -1; continue; }
if (t->dtype != 3 || t->nbytes != m->part[k]) {
fprintf(stderr, "[mimo] %s: %s %lld bytes, expected U8 %lld -- refusing "
"(untrusted container)\n", nm, st_dtype_name(t->dtype),
(long long)t->nbytes, (long long)m->part[k]);
exit(1);
}
er->fd[k] = t->fd; er->off[k] = t->off;
}
/* The release stores an expert's six tensors back to back in this
* order (down, down scale, gate, gate scale, up, up scale): one read. */
er->contig = er->fd[0] >= 0;
for (int k = 0; k < 5 && er->contig; k++)
if (er->fd[k] != er->fd[k + 1] || er->off[k] + m->part[k] != er->off[k + 1]) er->contig = 0;
split += !er->contig;
}
}
if (missing) fprintf(stderr, "[mimo] WARNING: %d expert tensors missing (incomplete download?) -- touching one aborts\n", missing);
if (split) fprintf(stderr, "[mimo] %d experts are not contiguous on disk: six reads each\n", split);
}
static void cache_init(Model *m, int cap) {
Cfg *c = &m->c;
if (cap < c->topk) cap = c->topk;
if (cap > c->n_experts) cap = c->n_experts;
for (int li = 0; li < c->n_layers; li++) {
if (!c->moe[li]) continue;
LCache *lc = &m->cache[li];
lc->cap = cap; lc->n = 0;
lc->s = xcalloc((size_t)cap, sizeof(Slot), "expert slots");
lc->by_expert = xmalloc((size_t)c->n_experts * sizeof(int), "expert index");
for (int e = 0; e < c->n_experts; e++) lc->by_expert[e] = -1;
for (int k = 0; k < cap; k++) lc->s[k].eid = -1;
}
}
static void expert_read(Model *m, int li, int eid, Slot *s) {
if (!s->base && posix_memalign((void **)&s->base, 4096, (size_t)m->e_bytes + 8192)) {
fprintf(stderr, "[mimo] out of memory for an expert slot\n"); exit(1);
}
ERef *er = &m->eref[(int64_t)li * m->c.n_experts + eid];
if (er->fd[0] < 0) { fprintf(stderr, "[mimo] expert L%d E%d missing on disk\n", li, eid); exit(1); }
if (er->contig) {
int dfd = m->direct ? st_direct_fd(&m->S, er->fd[0]) : -1;
if (dfd >= 0) {
int64_t a0 = er->off[0] & ~4095LL, pad = er->off[0] - a0, want = pad + m->e_bytes;
int64_t dlen = (want + 4095) & ~4095LL;
struct stat sb;
if (fstat(dfd, &sb) == 0 && a0 + dlen > sb.st_size) dlen = (sb.st_size - a0) & ~4095LL;
if (dlen > 0) st_pread_full(dfd, s->base, dlen, a0, "pread expert direct");
if (dlen < want) st_pread_full(er->fd[0], s->base + dlen, want - dlen, a0 + dlen, "pread expert tail");
s->buf = s->base + pad;
} else {
st_pread_full(er->fd[0], s->base, m->e_bytes, er->off[0], "pread expert");
s->buf = s->base;
}
} else {
uint8_t *dst = s->base;
for (int k = 0; k < 6; k++) {
if (er->fd[k] < 0) { fprintf(stderr, "[mimo] expert L%d E%d tensor %d missing\n", li, eid, k); exit(1); }
st_pread_full(er->fd[k], dst, m->part[k], er->off[k], "pread expert");
dst += m->part[k];
}
s->buf = s->base;
}
}
/* Make the experts `want[0..n)` of layer li resident, n <= cap, and return their
* slots in `out`. Hits keep their slot; misses take the least recently used
* slots not in `want`, and are read in parallel. */
static void experts_ensure(Model *m, int li, const int *want, int n, Slot **out) {
LCache *lc = &m->cache[li];
int miss_idx[4096], nmiss = 0;
uint64_t stamp = ++m->clock;
for (int j = 0; j < n; j++) {
int si = lc->by_expert[want[j]];
if (si >= 0) { out[j] = &lc->s[si]; out[j]->used = stamp; m->hits++; }
else { out[j] = NULL; miss_idx[nmiss++] = j; }
}
for (int k = 0; k < nmiss; k++) {
int best = -1;
if (lc->n < lc->cap) best = lc->n++;
else { /* the least recently used, and before it one the Vulkan tier holds (vkt_ram_first) */
uint64_t oldest = UINT64_MAX, dev_oldest = UINT64_MAX;
int dev = -1;
for (int s = 0; s < lc->n; s++) {
if (lc->s[s].used >= stamp) continue;
if (lc->s[s].eid >= 0 && vkt_ram_first(li, lc->s[s].eid)) {
if (lc->s[s].used < dev_oldest) { dev_oldest = lc->s[s].used; dev = s; }
} else if (lc->s[s].used < oldest) { oldest = lc->s[s].used; best = s; }
}
if (dev >= 0) { best = dev; vkt_ram_gave(); }
}
if (best < 0) { fprintf(stderr, "[mimo] expert cache smaller than one routing step\n"); exit(1); }
Slot *s = &lc->s[best];
if (s->eid >= 0) lc->by_expert[s->eid] = -1;
s->eid = want[miss_idx[k]];
s->used = stamp;
lc->by_expert[s->eid] = best;
out[miss_idx[k]] = s;
}
if (!nmiss) return;
double t0 = now_s();
int threads = env_int("MIMO_READ_THREADS", 8);
#pragma omp parallel for num_threads(threads) schedule(dynamic, 1)
for (int k = 0; k < nmiss; k++) {
Slot *s = out[miss_idx[k]];
expert_read(m, li, s->eid, s);
}
m->miss += (uint64_t)nmiss;
m->bytes_read += (uint64_t)nmiss * (uint64_t)m->e_bytes;
m->t_disk += now_s() - t0;
}
static void kv_alloc(Model *m, int ctx) {
Cfg *c = &m->c;
m->ctx = ctx;
for (int li = 0; li < c->n_layers; li++) {
Layer *l = &m->L[li];
int k = c->swa[li];
l->rows = k ? (c->window < ctx ? c->window : ctx) : ctx;
free(l->K); free(l->V); free(l->ring_pos);
l->K = xmalloc((size_t)l->rows * c->kv_heads[k] * c->head_dim[k] * sizeof(float), "K cache");
l->V = xmalloc((size_t)l->rows * c->kv_heads[k] * c->v_dim[k] * sizeof(float), "V cache");
l->ring_pos = xmalloc((size_t)l->rows * sizeof(int), "ring positions");
for (int r = 0; r < l->rows; r++) l->ring_pos[r] = -1;
}
kv_prefix_alloc(&m->kvp, ctx);
}
static void model_reset(Model *m) {
Cfg *c = &m->c;
for (int li = 0; li < c->n_layers; li++)
for (int r = 0; r < m->L[li].rows; r++) m->L[li].ring_pos[r] = -1;
m->pos = 0;
kv_prefix_clear(&m->kvp);
}
static void model_load(Model *m, const char *dir, int cap) {
Cfg *c = &m->c;
load_cfg(c, dir);
st_init_multi(&m->S, dir, getenv("MIMO_DIRS"));
g_dense_bits = env_int("MIMO_DENSE_BITS", 0);
if (g_dense_bits != 0 && g_dense_bits != 8 && g_dense_bits != 32) {
fprintf(stderr, "[mimo] MIMO_DENSE_BITS must be 0 (native FP8/BF16), 8 or 32\n"); exit(1);
}
m->idot = env_int("MIMO_IDOT", 0);
m->direct = env_int("MIMO_DIRECT", 1);
int H = c->hidden;
need(&m->S, "model.embed_tokens.weight", 0, c->vocab, H);
m->embed = xmalloc((size_t)c->vocab * H * sizeof(uint16_t), "embeddings");
st_read_raw_cap(&m->S, "model.embed_tokens.weight", m->embed, (int64_t)c->vocab * H * 2, 1);
m->norm = load_vec(&m->S, "model.norm.weight", H);
dw_load_bf16(&m->head, &m->S, "lm_head.weight", c->vocab, H);
for (int li = 0; li < c->n_layers; li++) {
Layer *l = &m->L[li];
int k = c->swa[li];
char nm[512];
snprintf(nm, sizeof(nm), "model.layers.%d.input_layernorm.weight", li);
l->ln1 = load_vec(&m->S, nm, H);
snprintf(nm, sizeof(nm), "model.layers.%d.post_attention_layernorm.weight", li);
l->ln2 = load_vec(&m->S, nm, H);
load_qkv(&l->qkv, &m->S, c, li);
snprintf(nm, sizeof(nm), "model.layers.%d.self_attn.o_proj.weight", li);
dw_load_bf16(&l->o, &m->S, nm, H, c->heads[k] * c->v_dim[k]);
if (c->sink[k]) {
snprintf(nm, sizeof(nm), "model.layers.%d.self_attn.attention_sink_bias", li);
l->sink = load_vec(&m->S, nm, c->heads[k]);
}
if (!c->moe[li]) {
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate_proj.weight", li);
dw_load_fp8(&l->gate, &m->S, nm, c->dense_inter, H, NULL, 0, NULL);
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.up_proj.weight", li);
dw_load_fp8(&l->up, &m->S, nm, c->dense_inter, H, NULL, 0, NULL);
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.down_proj.weight", li);
dw_load_fp8(&l->down, &m->S, nm, H, c->dense_inter, NULL, 0, NULL);
} else {
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate.weight", li);
need(&m->S, nm, 0, c->n_experts, H);
l->router = xmalloc((size_t)c->n_experts * H * sizeof(float), nm);
st_read_f32(&m->S, nm, l->router, 1);
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.gate.e_score_correction_bias", li);
l->bias = load_vec(&m->S, nm, c->n_experts);
}
}
expert_table_init(m);
cache_init(m, cap);
/* coli --ctx arrives as CTX (the family's context_env); MIMO_CTX wins over it */
const char *ctx_text = getenv("MIMO_CTX");
if (!ctx_text || !*ctx_text) ctx_text = getenv("CTX");
int ctx = (ctx_text && *ctx_text) ? atoi(ctx_text) : 8192;
if (ctx > c->max_positions) ctx = c->max_positions;
if (ctx < 16) ctx = 16;
kv_alloc(m, ctx);
m->ehit = xmalloc((size_t)c->n_layers * sizeof(uint8_t *), "hits");
for (int li = 0; li < c->n_layers; li++) m->ehit[li] = xcalloc((size_t)c->n_experts, 1, "hits row");
}
/* ------------------------------------------------------------ the forward ---- */
static void rmsnorm(float *out, const float *x, const float *w, int n, float eps) {
float ss = 0;
for (int i = 0; i < n; i++) ss += x[i] * x[i];
float r = 1.0f / sqrtf(ss / n + eps);
for (int i = 0; i < n; i++) out[i] = w[i] * (x[i] * r);
}
/* rotate-half RoPE on the first `rd` dims of a head, as the reference builds it:
* inv_freq and the angle in float32, cos/sin of the float angle */
static void rope(float *v, int rd, float theta, int pos) {
int half = rd / 2;
for (int i = 0; i < half; i++) {
float inv = 1.0f / powf(theta, (float)(2 * i) / (float)rd);
float ang = inv * (float)pos;
float cs = cosf(ang), sn = sinf(ang);
float a = v[i], b = v[i + half];
v[i] = a * cs - b * sn;
v[i + half] = b * cs + a * sn;
}
}
/* Attention of `n` rows at positions p0.. of layer li. `xn` is the normed input
* [n, H], `out` receives o_proj's output [n, H]. */
/* A multiplexed decode step's attention (KV_SLOTS, m->mux_rows): row t is the next
* token of its own conversation, at mux_rows[t].pos, over that conversation's cache --
* a full layer's rows, or a sliding window's ring. Each row's numbers are attention()'s
* for a block of that one token; the projections run once over the n rows. */
static void attention_rows(Model *m, int li, const float *xn, int n, float *out) {
Cfg *c = &m->c;
Layer *l = &m->L[li];
const MimoRow *mr = m->mux_rows;
int k = c->swa[li];
int nh = c->heads[k], kvh = c->kv_heads[k], hd = c->head_dim[k], vd = c->v_dim[k];
int qd = nh * hd, kd = kvh * hd, vdd = kvh * vd, rows = qd + kd + vdd;
double t0 = now_s();
float *qkv = xmalloc((size_t)n * rows * sizeof(float), "qkv");
dw_matmul(qkv, xn, n, &l->qkv);
for (int t = 0; t < n; t++) {
float *q = qkv + (size_t)t * rows, *kk = q + qd, *vv = kk + kd;
for (int h = 0; h < nh; h++) rope(q + h * hd, c->rope_dim[k], c->theta[k], mr[t].pos);
for (int h = 0; h < kvh; h++) rope(kk + h * hd, c->rope_dim[k], c->theta[k], mr[t].pos);
for (int i = 0; i < vdd; i++) vv[i] *= c->v_scale;
}
int window = k ? c->window : 0, most = 1;
const float **Kc = xmalloc((size_t)n * sizeof *Kc, "row keys"), **Vc = xmalloc((size_t)n * sizeof *Vc, "row values");
int *lo = xmalloc((size_t)n * sizeof(int), "row first keys");
for (int t = 0; t < n; t++) {
MimoSeq *q = mr[t].seq;
int p = mr[t].pos;
const float *src = qkv + (size_t)t * rows + qd;
if (k) { /* the window's keys before p from the ring, then this token's */
int l0 = p - (window - 1) > 0 ? p - (window - 1) : 0;
float *kc = xmalloc((size_t)(p + 1 - l0) * kd * sizeof(float), "window keys");
float *vc = xmalloc((size_t)(p + 1 - l0) * vdd * sizeof(float), "window values");
for (int pp = l0; pp < p; pp++) {
int slot = pp % l->rows;
if (q->ring_pos[li][slot] != pp) { fprintf(stderr, "[mimo] window ring lost position %d\n", pp); exit(1); }
memcpy(kc + (size_t)(pp - l0) * kd, q->K[li] + (size_t)slot * kd, (size_t)kd * sizeof(float));
memcpy(vc + (size_t)(pp - l0) * vdd, q->V[li] + (size_t)slot * vdd, (size_t)vdd * sizeof(float));
}
memcpy(kc + (size_t)(p - l0) * kd, src, (size_t)kd * sizeof(float));
memcpy(vc + (size_t)(p - l0) * vdd, src + kd, (size_t)vdd * sizeof(float));
Kc[t] = kc; Vc[t] = vc; lo[t] = l0;
} else { /* the full cache, this token written in first */
memcpy(q->K[li] + (size_t)p * kd, src, (size_t)kd * sizeof(float));
memcpy(q->V[li] + (size_t)p * vdd, src + kd, (size_t)vdd * sizeof(float));
Kc[t] = q->K[li]; Vc[t] = q->V[li]; lo[t] = 0;
}
if (p + 1 - lo[t] > most) most = p + 1 - lo[t];
}
float *ctxv = xmalloc((size_t)n * nh * vd * sizeof(float), "attention output");
float scale = 1.0f / sqrtf((float)hd);
int group = nh / kvh;
#pragma omp parallel
{
float *score = xmalloc((size_t)most * sizeof(float), "scores");
#pragma omp for collapse(2) schedule(static)
for (int t = 0; t < n; t++) {
for (int h = 0; h < nh; h++) {
int p = mr[t].pos, kh = h / group, first = lo[t];
const float *q = qkv + (size_t)t * rows + (size_t)h * hd;
float best = -INFINITY;
for (int j = first; j <= p; j++) {
const float *kr = Kc[t] + (size_t)(j - lo[t]) * kd + (size_t)kh * hd;
float s = 0;
for (int i = 0; i < hd; i++) s += q[i] * kr[i];
s *= scale;
score[j - first] = s;
if (s > best) best = s;
}
float sink = l->sink ? l->sink[h] : -INFINITY;
if (sink > best) best = sink;
float den = l->sink ? expf(sink - best) : 0.0f;
for (int j = first; j <= p; j++) { score[j - first] = expf(score[j - first] - best); den += score[j - first]; }
float *o = ctxv + ((size_t)t * nh + h) * vd;
for (int i = 0; i < vd; i++) o[i] = 0;
for (int j = first; j <= p; j++) {
const float *vr = Vc[t] + (size_t)(j - lo[t]) * vdd + (size_t)kh * vd;
float w = score[j - first] / den;
for (int i = 0; i < vd; i++) o[i] += w * vr[i];
}
}
}
free(score);
}
if (k) /* the ring keeps each conversation's last `window` positions */
for (int t = 0; t < n; t++) {
MimoSeq *q = mr[t].seq;
int p = mr[t].pos, slot = p % l->rows;
const float *src = qkv + (size_t)t * rows + qd;
memcpy(q->K[li] + (size_t)slot * kd, src, (size_t)kd * sizeof(float));
memcpy(q->V[li] + (size_t)slot * vdd, src + kd, (size_t)vdd * sizeof(float));
q->ring_pos[li][slot] = p;
free((void *)Kc[t]); free((void *)Vc[t]);
}
dw_matmul(out, ctxv, n, &l->o);
free(ctxv); free(qkv); free(Kc); free(Vc); free(lo);
m->t_attn += now_s() - t0;
}
static void attention(Model *m, int li, const float *xn, int n, int p0, float *out) {
if (m->mux_rows) { attention_rows(m, li, xn, n, out); return; }
Cfg *c = &m->c;
Layer *l = &m->L[li];
int k = c->swa[li];
int nh = c->heads[k], kvh = c->kv_heads[k], hd = c->head_dim[k], vd = c->v_dim[k];
int qd = nh * hd, kd = kvh * hd, vdd = kvh * vd, rows = qd + kd + vdd;
double t0 = now_s();
float *qkv = xmalloc((size_t)n * rows * sizeof(float), "qkv");
dw_matmul(qkv, xn, n, &l->qkv);
for (int t = 0; t < n; t++) {
float *q = qkv + (size_t)t * rows, *kk = q + qd, *vv = kk + kd;
for (int h = 0; h < nh; h++) rope(q + h * hd, c->rope_dim[k], c->theta[k], p0 + t);
for (int h = 0; h < kvh; h++) rope(kk + h * hd, c->rope_dim[k], c->theta[k], p0 + t);
for (int i = 0; i < vdd; i++) vv[i] *= c->v_scale;
}
/* Keys this block can see: [lo, p0 + n). For a sliding window the ring still
* holds the ones before p0; for full attention the cache holds everything,
* and the block is written in first. */
int window = k ? c->window : 0;
int lo = k ? (p0 - (window - 1) > 0 ? p0 - (window - 1) : 0) : 0;
int nkeys = p0 + n - lo;
float *Kc = NULL, *Vc = NULL;
if (k) {
Kc = xmalloc((size_t)nkeys * kd * sizeof(float), "window keys");
Vc = xmalloc((size_t)nkeys * vdd * sizeof(float), "window values");
for (int p = lo; p < p0; p++) {
int slot = p % l->rows;
if (l->ring_pos[slot] != p) { fprintf(stderr, "[mimo] window ring lost position %d\n", p); exit(1); }
memcpy(Kc + (size_t)(p - lo) * kd, l->K + (size_t)slot * kd, (size_t)kd * sizeof(float));
memcpy(Vc + (size_t)(p - lo) * vdd, l->V + (size_t)slot * vdd, (size_t)vdd * sizeof(float));
}
for (int t = 0; t < n; t++) {
const float *src = qkv + (size_t)t * rows + qd;
memcpy(Kc + (size_t)(p0 + t - lo) * kd, src, (size_t)kd * sizeof(float));
memcpy(Vc + (size_t)(p0 + t - lo) * vdd, src + kd, (size_t)vdd * sizeof(float));
}
} else {
for (int t = 0; t < n; t++) {
const float *src = qkv + (size_t)t * rows + qd;
memcpy(l->K + (size_t)(p0 + t) * kd, src, (size_t)kd * sizeof(float));
memcpy(l->V + (size_t)(p0 + t) * vdd, src + kd, (size_t)vdd * sizeof(float));
}
Kc = l->K; Vc = l->V;
}
float *ctxv = xmalloc((size_t)n * nh * vd * sizeof(float), "attention output");
float scale = 1.0f / sqrtf((float)hd);
int group = nh / kvh;
#pragma omp parallel
{
float *score = xmalloc((size_t)nkeys * sizeof(float), "scores");
#pragma omp for collapse(2) schedule(static)
for (int t = 0; t < n; t++) {
for (int h = 0; h < nh; h++) {
int p = p0 + t, kh = h / group;
int first = window ? (p - (window - 1) > lo ? p - (window - 1) : lo) : 0;
const float *q = qkv + (size_t)t * rows + (size_t)h * hd;
float best = -INFINITY;
for (int j = first; j <= p; j++) {
const float *kr = Kc + (size_t)(j - lo) * kd + (size_t)kh * hd;
float s = 0;
for (int i = 0; i < hd; i++) s += q[i] * kr[i];
s *= scale;
score[j - first] = s;
if (s > best) best = s;
}
float sink = l->sink ? l->sink[h] : -INFINITY;
if (sink > best) best = sink;
float den = l->sink ? expf(sink - best) : 0.0f;
for (int j = first; j <= p; j++) { score[j - first] = expf(score[j - first] - best); den += score[j - first]; }
float *o = ctxv + ((size_t)t * nh + h) * vd;
for (int i = 0; i < vd; i++) o[i] = 0;
for (int j = first; j <= p; j++) {
const float *vr = Vc + (size_t)(j - lo) * vdd + (size_t)kh * vd;
float w = score[j - first] / den;
for (int i = 0; i < vd; i++) o[i] += w * vr[i];
}
}
}
free(score);
}
if (k) {
/* the ring keeps the last `window` positions of the block */
int from = n > l->rows ? n - l->rows : 0;
for (int t = from; t < n; t++) {
int p = p0 + t, slot = p % l->rows;
const float *src = qkv + (size_t)t * rows + qd;
memcpy(l->K + (size_t)slot * kd, src, (size_t)kd * sizeof(float));
memcpy(l->V + (size_t)slot * vdd, src + kd, (size_t)vdd * sizeof(float));
l->ring_pos[slot] = p;
}
free(Kc); free(Vc);
}
dw_matmul(out, ctxv, n, &l->o);
free(ctxv); free(qkv);
m->t_attn += now_s() - t0;
}
static void dense_mlp(Model *m, Layer *l, const float *xn, int n, float *out) {
int F = l->gate.O;
float *g = xmalloc((size_t)n * F * sizeof(float), "mlp gate");
float *u = xmalloc((size_t)n * F * sizeof(float), "mlp up");
dw_matmul(g, xn, n, &l->gate);
dw_matmul(u, xn, n, &l->up);
for (size_t i = 0; i < (size_t)n * F; i++) g[i] = silu(g[i]) * u[i];
dw_matmul(out, g, n, &l->down);
free(g); free(u);
(void)m;
}
/* An expert's bytes as the shared Vulkan tier reads them (buf: down, its
* scales, gate, its scales, up, its scales; one e8m0 byte per 32 columns). */
static VktExpertSrc mimo_vk_src(const Model *m, const uint8_t *buf) {
const uint8_t *dq = buf, *ds = dq + m->part[0], *gq = ds + m->part[1], *gs = gq + m->part[2],
*uq = gs + m->part[3], *us = uq + m->part[4];
return (VktExpertSrc){gq, uq, dq, gs, us, ds};
}
/* The tier's streaming (a big prompt chunk's cold experts on the device): the experts'
* bytes through the layer cache as moe_cpu gets them (experts_ensure: a group at once,
* read in parallel, up to the cache's capacity), valid until the next ensure. */
static int mimo_vk_load_batch(void *ctx, int li, const int *e, int n, VktExpertSrc *srcs, void **h) {
Model *m = ctx;
int cap = m->cache[li].cap;
if (n > cap) n = cap;
if (n < 1 || n > 64) return 0;
Slot *sl[64];
experts_ensure(m, li, e, n, sl);
for (int i = 0; i < n; i++) { srcs[i] = mimo_vk_src(m, sl[i]->buf); h[i] = sl[i]; }
return n;
}
static int mimo_vk_load(void *ctx, int li, int e, VktExpertSrc *src, void **h) {
return mimo_vk_load_batch(ctx, li, &e, 1, src, h) == 1;
}
static void mimo_vk_release(void *ctx, void *h) { (void)ctx; (void)h; }
/* The CPU's experts of one MoE step: every (row, choice) pair i of sel -- or,
* with a mask, those whose mask[i] is mval -- runs its expert, and its output
* lands in y at row pair[i], numbered on from `total`; returns the new count.
*
* Each group of resident experts is computed in two parallel passes -- gate and
* up of every expert, then down of every expert -- split in row blocks, instead
* of three OpenMP regions per expert (a thousand a token on Flash, and every
* barrier costs a scheduling round on a busy machine). The kernel inside a
* task is the same matmul_mxfp4 on the same rows, so the numbers do not move.
*
* Every (row, choice) pair has its own slot in y, across all the groups of the
* block, so the weighted sum can wait for the last group and run row by row in
* that row's own routing order. With `note`, every expert computed here is
* offered to the Vulkan tier while its bytes are in RAM (vkt_note; nothing
* without the tier). */
static int moe_cpu(Model *m, int li, const float *xn, int n, const int *sel,
const uint8_t *mask, int mval, int *pair, float *y, int total, int note) {
Cfg *c = &m->c;
int H = c->hidden, E = c->n_experts, K = c->topk, MI = c->moe_inter;
#define MIMO_WANT(i) (!mask || (mask[i] != 0) == mval)
/* the union of chosen experts, in first-use order */
int *uni = xmalloc((size_t)E * sizeof(int), "expert union");
int *seen = xcalloc((size_t)E, sizeof(int), "expert seen");
int nu = 0;
for (int i = 0; i < n * K; i++) if (MIMO_WANT(i) && !seen[sel[i]]) { seen[sel[i]] = 1; uni[nu++] = sel[i]; }
free(seen);
int cap = m->cache[li].cap;
int *row = xmalloc((size_t)n * K * sizeof(int), "expert rows index");
int *first = xmalloc((size_t)(cap + 1) * sizeof(int), "expert row offsets");
float *xg = xmalloc((size_t)n * K * H * sizeof(float), "expert inputs");
float *g = xmalloc((size_t)n * K * MI * sizeof(float), "expert gate");
float *u = xmalloc((size_t)n * K * MI * sizeof(float), "expert up");
void (*mm)(float *, const float *, const uint8_t *, const uint8_t *, int, int, int)
= m->idot ? matmul_mxfp4_i8 : matmul_mxfp4;
const int BLOCK_ROWS = 256;
Slot *slots[4096];
int base = total; /* y rows before this call are someone else's */
for (int b = 0; b < nu; b += cap) {
int nb = nu - b < cap ? nu - b : cap;
experts_ensure(m, li, uni + b, nb, slots);
double t0 = now_s();
int from = total;
for (int j = 0; j < nb; j++) {
int e = uni[b + j];
first[j] = total - base;
for (int t = 0; t < n; t++)
for (int q = 0; q < K; q++)
if (sel[(size_t)t * K + q] == e && MIMO_WANT((size_t)t * K + q)) {
row[total - base] = t;
pair[(size_t)t * K + q] = total;
total++;
}
}
first[nb] = total - base;
for (int i = from; i < total; i++) memcpy(xg + (size_t)(i - base) * H, xn + (size_t)row[i - base] * H, (size_t)H * sizeof(float));
int up_blocks = (MI + BLOCK_ROWS - 1) / BLOCK_ROWS, down_blocks = (H + BLOCK_ROWS - 1) / BLOCK_ROWS;
int rb_mi = MI / 2, gb_mi = MI / 32, rb_h = H / 2, gb_h = H / 32;
#pragma omp parallel for schedule(dynamic, 1)
for (int task = 0; task < nb * 2 * up_blocks; task++) {
int j = task / (2 * up_blocks), which = (task / up_blocks) % 2, blk = task % up_blocks;
int r = first[j + 1] - first[j];
if (!r) continue;
int o0 = blk * BLOCK_ROWS, rows = MI - o0 < BLOCK_ROWS ? MI - o0 : BLOCK_ROWS;
const uint8_t *buf = slots[j]->buf;
const uint8_t *q = buf + m->part[0] + m->part[1] + (which ? m->part[2] + m->part[3] : 0);
const uint8_t *sc = q + m->part[2];
float *dst = (which ? u : g) + (size_t)first[j] * MI;
/* rows [o0, o0+rows) of this expert's gate or up, for its r inputs; the
* kernel writes [r, rows] contiguously, so it goes through a scratch */
float tmp[4 * 256];
float *scratch = r * rows <= (int)(sizeof(tmp) / sizeof(tmp[0])) ? tmp
: xmalloc((size_t)r * rows * sizeof(float), "expert scratch");
mm(scratch, xg + (size_t)first[j] * H, q + (size_t)o0 * rb_h, sc + (size_t)o0 * gb_h, r, H, rows);
for (int i = 0; i < r; i++)
memcpy(dst + (size_t)i * MI + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));
if (scratch != tmp) free(scratch);
}
for (size_t i = (size_t)(from - base) * MI; i < (size_t)(total - base) * MI; i++) g[i] = silu(g[i]) * u[i];
#pragma omp parallel for schedule(dynamic, 1)
for (int task = 0; task < nb * down_blocks; task++) {
int j = task / down_blocks, blk = task % down_blocks;
int r = first[j + 1] - first[j];
if (!r) continue;
int o0 = blk * BLOCK_ROWS, rows = H - o0 < BLOCK_ROWS ? H - o0 : BLOCK_ROWS;
const uint8_t *dq = slots[j]->buf, *ds = dq + m->part[0];
float tmp[4 * 256];
float *scratch = r * rows <= (int)(sizeof(tmp) / sizeof(tmp[0])) ? tmp
: xmalloc((size_t)r * rows * sizeof(float), "expert scratch");
mm(scratch, g + (size_t)first[j] * MI, dq + (size_t)o0 * rb_mi, ds + (size_t)o0 * gb_mi, r, MI, rows);
float *dst = y + (size_t)(base + first[j]) * H;
for (int i = 0; i < r; i++)
memcpy(dst + (size_t)i * H + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));
if (scratch != tmp) free(scratch);
}
/* the group's bytes are still in RAM: the tier may promote them */
if (note && vkt_ready())
for (int j = 0; j < nb; j++) { VktExpertSrc vs = mimo_vk_src(m, slots[j]->buf); vkt_note(li, uni[b + j], &vs); }
m->t_expert += now_s() - t0;
}
#undef MIMO_WANT
free(first); free(xg); free(g); free(u); free(row); free(uni);
return total;
}
/* Route every row, then run each chosen expert once over the rows that chose it.
* `out` [n, H] receives the weighted sum.
*
* With the Vulkan tier on, vkt_issue first hands the pairs whose expert is
* resident on the device to one asynchronous batch; moe_cpu reads and computes
* the others meanwhile (an expert the device took entirely is never read from
* disk), and after the join every pair joins the sum in its row's routing order,
* the device's and the CPU's alike: the order never depends on which experts were
* resident. The experts the CPU computed are offered to the tier after the join
* (those still in the RAM cache), not while the batch runs: a promotion that
* displaces a resident then frees it at once, and its upload starts on a pool
* with room instead of waiting on the uploader thread for the join's free. */
#define MIMO_VK_ROWS 64 /* rows whose pairs one device batch carries at most */
static void moe(Model *m, int li, const float *xn, int n, float *out) {
Cfg *c = &m->c;
Layer *l = &m->L[li];
int H = c->hidden, E = c->n_experts, K = c->topk;
int *sel = xmalloc((size_t)n * K * sizeof(int), "routing");
float *wt = xmalloc((size_t)n * K * sizeof(float), "routing weights");
float *logits = xmalloc((size_t)n * E * sizeof(float), "router logits");
matmul(logits, xn, l->router, n, H, E);
for (int t = 0; t < n; t++) {
float *score = logits + (size_t)t * E;
for (int e = 0; e < E; e++) score[e] = sigmoid(score[e]);
int *s = sel + (size_t)t * K;
float *w = wt + (size_t)t * K;
for (int j = 0; j < K; j++) {
int best = -1; float bv = -INFINITY;
for (int e = 0; e < E; e++) {
int taken = 0;
for (int q = 0; q < j; q++) if (s[q] == e) { taken = 1; break; }
if (taken) continue;
float v = score[e] + l->bias[e];
if (v > bv) { bv = v; best = e; }
}
s[j] = best; w[j] = score[best];
}
float sum = 0;
for (int j = 0; j < K; j++) sum += w[j];
for (int j = 0; j < K; j++) w[j] = (c->norm_topk && K > 1 ? w[j] / (sum + 1e-20f) : w[j]) * c->route_scale;
for (int j = 0; j < K; j++) m->ehit[li][s[j]] = 1;
}
free(logits);
memset(out, 0, (size_t)n * H * sizeof(float));
int *pair = xmalloc((size_t)n * K * sizeof(int), "expert pair of a choice");
float *y = xmalloc((size_t)n * K * H * sizeof(float), "expert out");
uint8_t *taken = NULL;
const float **dev = NULL;
int ndev = 0;
if (vkt_ready()) {
taken = xmalloc((size_t)n * K, "device pairs");
dev = xmalloc((size_t)n * K * sizeof(*dev), "device rows");
ndev = vkt_issue(li, xn, n, K, sel, taken);
}
int total = moe_cpu(m, li, xn, n, sel, ndev ? taken : NULL, 0, pair, y, 0, !ndev);
if (ndev && !vkt_join(dev)) { /* the batch failed (the tier stops): those pairs here */
moe_cpu(m, li, xn, n, sel, taken, 1, pair, y, total, 0);
ndev = 0;
} else if (ndev) { /* the CPU's experts still in RAM, now that nothing is in flight */
uint8_t *noted = xcalloc((size_t)E, 1, "noted experts");
int *by = m->cache[li].by_expert;
for (int i = 0; i < n * K; i++) {
int e = sel[i];
if (taken[i] || noted[e] || by[e] < 0) continue;
noted[e] = 1;
VktExpertSrc vs = mimo_vk_src(m, m->cache[li].s[by[e]].buf);
vkt_note(li, e, &vs);
}
free(noted);
}
/* The weighted sum, row by row, each row's experts in the order ITS router
* chose them. A row's sum used to follow the block's first-use order, which
* depends on the rows that happen to share the block: the same position
* came out with different low bits in a 64-row prefill block and in a
* one-row resume, so a prompt resumed from a prefix was not the prompt
* computed cold. Now a row is a function of its own inputs and nothing else,
* whatever the block boundaries. One row (a decode step) adds in the same
* order as before: its first-use order IS its routing order. */
double t1 = now_s();
for (int t = 0; t < n; t++) {
float *o = out + (size_t)t * H;
for (int q = 0; q < K; q++) {
size_t i = (size_t)t * K + q;
const float *yy = ndev && taken[i] ? dev[i] : y + (size_t)pair[i] * H;
float w = wt[i];
for (int d = 0; d < H; d++) o[d] += w * yy[d];
}
}
m->t_expert += now_s() - t1;
free(pair); free(y); free(taken); free(dev); free(sel); free(wt);
}
static void embed_rows(Model *m, const int *ids, int n, float *h) {
int H = m->c.hidden;
for (int t = 0; t < n; t++) {
int id = ids[t];
if (id < 0 || id >= m->c.vocab) { fprintf(stderr, "[mimo] token id %d out of range\n", id); exit(1); }
for (int i = 0; i < H; i++) h[(size_t)t * H + i] = bf16_to_f32(m->embed[(size_t)id * H + i]);
}
}
/* Image rows to splice in: `rows[r]` replaces the embedding of the r-th
* image-pad position of the WHOLE prompt (the first at `first_pad`). */
typedef struct { const float *rows; int n_rows; } ImageRows;
/* Layers [l0, L) on the CPU for the nc rows hc at position pc (in place; xn and tmp hold
* nc rows each). trace: MIMO_TRACE's file, the residual after every block. */
static void layers_cpu(Model *m, float *hc, int nc, int pc, int l0, float *xn, float *tmp, FILE *trace) {
Cfg *c = &m->c;
int H = c->hidden;
for (int li = l0; li < c->n_layers && nc > 0; li++) {
Layer *l = &m->L[li];
for (int t = 0; t < nc; t++) rmsnorm(xn + (size_t)t * H, hc + (size_t)t * H, l->ln1, H, c->eps);
attention(m, li, xn, nc, pc, tmp);
for (size_t i = 0; i < (size_t)nc * H; i++) hc[i] += tmp[i];
if (trace) fwrite(hc, sizeof(float), (size_t)nc * H, trace);
for (int t = 0; t < nc; t++) rmsnorm(xn + (size_t)t * H, hc + (size_t)t * H, l->ln2, H, c->eps);
if (c->moe[li]) moe(m, li, xn, nc, tmp);
else dense_mlp(m, l, xn, nc, tmp);
for (size_t i = 0; i < (size_t)nc * H; i++) hc[i] += tmp[i];
if (trace) fwrite(hc, sizeof(float), (size_t)nc * H, trace);
}
}
/* The final norm and lm_head on the CPU: every row of hc into lc (all_rows), or its last. */
static void head_cpu(Model *m, const float *hc, int nc, float *lc, int all_rows, float *xn) {
Cfg *c = &m->c;
int H = c->hidden, from = all_rows ? 0 : nc - 1, rows = nc - from;
for (int t = from; t < nc; t++) rmsnorm(xn + (size_t)(t - from) * H, hc + (size_t)t * H, m->norm, H, c->eps);
dw_matmul(lc, xn, rows, &m->head);
}
#ifdef COLI_VULKAN
#include "mimo_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */
#endif
/* One block of n tokens at m->pos. logits: NULL, or [n, V] (all rows) when
* all_rows, else [V] for the last row. */
static void forward(Model *m, const int *ids, int n, float *logits, int all_rows,
const ImageRows *img, int *img_used) {
Cfg *c = &m->c;
int H = c->hidden;
if (m->pos + n > m->ctx) { fprintf(stderr, "[mimo] context full (%d + %d > %d)\n", m->pos, n, m->ctx); exit(1); }
float *h = xmalloc((size_t)n * H * sizeof(float), "residual");
float *xn = xmalloc((size_t)n * H * sizeof(float), "normed");
float *tmp = xmalloc((size_t)n * H * sizeof(float), "block out");
embed_rows(m, ids, n, h);
if (img && img->rows && c->image_token_id >= 0)
for (int t = 0; t < n; t++)
if (ids[t] == c->image_token_id && *img_used < img->n_rows) {
memcpy(h + (size_t)t * H, img->rows + (size_t)(*img_used) * H, (size_t)H * sizeof(float));
(*img_used)++;
}
/* MIMO_TRACE=<file>: the residual after the embedding and after every
* attention and every MLP of the first block, f32 [2L+1][n][H] -- what
* tools/make_mimo_ref.py --trace compares against the vendor's hooks. */
static FILE *trace;
static int traced;
if (!traced && getenv("MIMO_TRACE")) { trace = fopen(getenv("MIMO_TRACE"), "wb"); }
if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);
/* The rows the CPU computes: all of them, or with the dense chain on (COLI_VK_CHAIN,
* mimo_chain.h) the ones it did not (a lost device), from where the host's caches end. */
int done = 0;
#ifdef COLI_VULKAN
if (g_vk_chain && !trace) {
done = mc_forward(m, h, n, m->pos, logits, all_rows, xn, tmp);
if (done < n) mc_cpu_step(m, m->pos + done);
}
#endif
float *hc = h + (size_t)done * H, *lc = logits && all_rows ? logits + (size_t)done * c->vocab : logits;
int nc = n - done, pc = m->pos + done;
layers_cpu(m, hc, nc, pc, 0, xn, tmp, trace);
if (trace) { fclose(trace); trace = NULL; }
traced = 1;
if (lc && nc > 0) head_cpu(m, hc, nc, lc, all_rows, xn);
kv_prefix_record(&m->kvp, ids, m->pos, n);
m->pos += n;
m->forwards++;
free(h); free(xn); free(tmp);
}
/* ---- several conversations at once (KV_SLOTS>1, serve_mux) -------------------
* Each conversation owns a MimoSeq. The Model's layers hold the conversation a
* prefill runs on (mimo_seq_swap trades it for a parked one); a decode step parks
* them all and runs one forward over a row of each (forward_rows). */
static void mimo_seq_swap(Model *m, MimoSeq *q) {
for (int li = 0; li < m->c.n_layers; li++) {
Layer *l = &m->L[li];
float *K = l->K, *V = l->V; int *rp = l->ring_pos;
l->K = q->K[li]; l->V = q->V[li]; l->ring_pos = q->ring_pos[li];
q->K[li] = K; q->V[li] = V; q->ring_pos[li] = rp;
}
int pos = m->pos; kv_prefix p = m->kvp;
m->pos = q->pos; m->kvp = q->kvp;
q->pos = pos; q->kvp = p;
}
/* A conversation's caches of its own, the shape kv_alloc gives the Model's. */
static void mimo_seq_alloc(Model *m, MimoSeq *q) {
Cfg *c = &m->c;
memset(q, 0, sizeof *q);
for (int li = 0; li < c->n_layers; li++) {
Layer *l = &m->L[li];
int k = c->swa[li];
q->K[li] = xmalloc((size_t)l->rows * c->kv_heads[k] * c->head_dim[k] * sizeof(float), "K cache");
q->V[li] = xmalloc((size_t)l->rows * c->kv_heads[k] * c->v_dim[k] * sizeof(float), "V cache");
q->ring_pos[li] = xmalloc((size_t)l->rows * sizeof(int), "ring positions");
for (int r = 0; r < l->rows; r++) q->ring_pos[li][r] = -1;
}
kv_prefix_alloc(&q->kvp, m->ctx);
}
/* One decode step of several conversations: row t is the token ids[t] at rows[t].pos
* of the conversation rows[t].seq, every conversation parked. The matrices, the routed
* experts and lm_head run once over the n rows; the attention reads and writes each
* row's own caches (attention_rows). The CPU kernels give a row the same bits whatever
* n is, so each conversation gets the logits it would alone: [n, V] into logits. */
static void forward_rows(Model *m, const MimoRow *rows, const int *ids, int n, float *logits) {
Cfg *c = &m->c;
int H = c->hidden;
float *h = xmalloc((size_t)n * H * sizeof(float), "residual");
float *xn = xmalloc((size_t)n * H * sizeof(float), "normed");
float *tmp = xmalloc((size_t)n * H * sizeof(float), "block out");
embed_rows(m, ids, n, h);
m->mux_rows = rows;
layers_cpu(m, h, n, 0, 0, xn, tmp, NULL);
m->mux_rows = NULL;
head_cpu(m, h, n, logits, 1, xn);
for (int t = 0; t < n; t++) {
kv_prefix_record(&rows[t].seq->kvp, ids + t, rows[t].pos, 1);
rows[t].seq->pos = rows[t].pos + 1;
}
m->forwards++;
free(h); free(xn); free(tmp);
}
/* The read-out of a prefill, for the logprobs channel (SUBMIT logprobs=k): one
* ECHO frame per prompt position this prefill feeds, in position order, before
* any DATA. The logits at position p predict the token at p + 1, so the frame
* of a position carries the log-probability of the token that is actually
* there, over the whole vocabulary, and not only when it is among the top k:
* that is what scores an option the model would never have written.
*
* The first position fed has no predictor among the rows computed here. On a
* cold prompt it is position 0, which has nothing to condition on, and its
* frame carries " nan 0". Resumed from a photo, `first` is the photo's own last
* logits, which predict exactly that token. There is no third case: a prompt
* read out never resumes from a live prefix, because the positions before it
* would have no frame at all (see the serve loop).
*
* This is its own argument and not a mode of something else: it asks for every
* row's logits and changes nothing about how the rows are computed. */
typedef struct {
const char *id;
int k;
Tok *tok;
const float *first;
} Echo;
static void echo_frame(const Echo *e, int pos, int token, const float *lo, int vocab) {
char tail[1024], piece[512];
coli_logprob_tail(tail, sizeof(tail), lo, vocab, token, e->k);
int n = tok_decode(e->tok, &token, 1, piece, (int)sizeof(piece));
if (n < 0) n = 0;
printf("ECHO %s %d %d%s\n", e->id, n, pos, tail);
if (n > 0) fwrite(piece, 1, (size_t)n, stdout);
fputc('\n', stdout);
fflush(stdout);
}
/* A prompt in blocks of MIMO_CHUNK rows (default 64): the last block's last
* row (or every row, for the teacher-forcing dump) comes back in `logits`.
* With `echo`, every row of every block goes through the head and is read out
* as it comes; only one block of logits is ever held, not the prompt's. */
static void prefill(Model *m, const int *ids, int n, float *logits, float *all_logits,
const ImageRows *img, const Echo *echo) {
int chunk = env_int("MIMO_CHUNK", 64);
#ifdef COLI_VULKAN
/* the dense chain with a chunk from the budget (vkc_chunk_auto): blocks of its chunk,
* so a long prompt's MoE steps are big ones the tier can stream (not with a read-out,
* which holds a block's logits) */
if (!getenv("MIMO_CHUNK") && g_vk_chain && vkc_chunk_auto() && !echo) {
MimoChain *ch = mc_setup(m);
if (ch && !ch->failed) chunk = mc_chunk_rows(ch, m);
}
#endif
if (chunk < 1) chunk = 1;
int used = 0, V = m->c.vocab;
float *rows = NULL;
if (echo && !all_logits) {
rows = xmalloc((size_t)(n < chunk ? n : chunk) * V * sizeof(float), "read-out logits");
echo_frame(echo, m->pos, ids[0], echo->first, V);
}
for (int b = 0; b < n; b += chunk) {
int nb = n - b < chunk ? n - b : chunk;
if (all_logits) forward(m, ids + b, nb, all_logits + (size_t)b * V, 1, img, &used);
else if (rows) {
int p0 = m->pos;
forward(m, ids + b, nb, rows, 1, img, &used);
for (int t = 0; t < nb; t++) {
if (b + t + 1 < n) echo_frame(echo, p0 + t + 1, ids[b + t + 1], rows + (size_t)t * V, V);
else if (logits) memcpy(logits, rows + (size_t)t * V, (size_t)V * sizeof(float));
}
}
else forward(m, ids + b, nb, b + nb == n ? logits : NULL, 0, img, &used);
}
free(rows);
if (all_logits && logits) memcpy(logits, all_logits + (size_t)(n - 1) * V, (size_t)V * sizeof(float));
}
/* ----------------------------------------------------------------- vision ---- */
/* The tower, when the container carries one (visual.*): vision_run turns the
* gateway's patches into the rows that replace the image-pad embeddings. */
typedef struct Vision Vision;
static Vision *g_vision;
#include "mimo_vision.h"
#ifdef COLI_VULKAN
/* The dense matrices go up first, at start-up, the trunk before the tower, so
* the experts take only what they leave; one that does not fit stays on the
* CPU. Returns how many went up. The dense chain alone (the per-matrix path off)
* takes the trunk and leaves the tower on the CPU (vision 0). */
static void mc_place(Model *m, int *dropped); /* below: a fit's layers, layer by layer */
static int vk_dense_upload(Model *m, int vision) {
int n = 0;
if (g_mc_fit.L && !g_mc_placed) mc_place(m, NULL); /* the chain's fit: its layers, each whole or none */
for (int li = 0; li < m->c.n_layers; li++) {
Layer *l = &m->L[li];
n += dw_upload(&l->qkv) + dw_upload(&l->o);
if (!m->c.moe[li]) n += dw_upload(&l->gate) + dw_upload(&l->up) + dw_upload(&l->down);
}
n += dw_upload(&m->head);
if (g_vision && vision) {
Vision *v = g_vision;
n += dw_upload(&v->embed) + dw_upload(&v->fc1) + dw_upload(&v->fc2);
for (int i = 0; i < v->depth; i++)
n += dw_upload(&v->b[i].qkv) + dw_upload(&v->b[i].proj) + dw_upload(&v->b[i].gate) +
dw_upload(&v->b[i].up) + dw_upload(&v->b[i].down);
}
return n;
}
/* What vk_dense_upload will place, in bytes (the tier's budget leaves it room). */
static size_t dw_bytes(const DW *d) {
if (!d->w) return 0;
size_t e = (size_t)d->O * d->I;
switch (d->fmt) {
case DW_F32: return e * 4;
case DW_BF16: return e * 2;
case DW_FP8: return e + (size_t)d->O * d->nblk * 4;
case DW_I8: return e + (size_t)d->O * 4;
}
return 0;
}
static size_t vk_dense_bytes(const Model *m, int vision) {
size_t b = dw_bytes(&m->head);
for (int li = 0; li < m->c.n_layers; li++) {
const Layer *l = &m->L[li];
b += dw_bytes(&l->qkv) + dw_bytes(&l->o);
if (!m->c.moe[li]) b += dw_bytes(&l->gate) + dw_bytes(&l->up) + dw_bytes(&l->down);
}
if (g_vision && vision) {
const Vision *v = g_vision;
b += dw_bytes(&v->embed) + dw_bytes(&v->fc1) + dw_bytes(&v->fc2);
for (int i = 0; i < v->depth; i++)
b += dw_bytes(&v->b[i].qkv) + dw_bytes(&v->b[i].proj) + dw_bytes(&v->b[i].gate) +
dw_bytes(&v->b[i].up) + dw_bytes(&v->b[i].down);
}
return b;
}
/* ---- a partial chain (vk_chain.h, vkc_fit): the first n layers on the device --------
* mc_fit_start decides n at start-up, before any upload and before the tier sizes its
* budget. A layer's bytes: its matrices in their dense form (MIMO_DENSE_BITS: the fused
* qkv and o_proj, the dense layer's MLP too), its K/V mirror at the size the chain
* allocates (a sliding layer's ring; a full layer's context, or COLI_VK_KV_DEVICE_ROWS
* when that asks for fewer: the split covers the rest) and its norms and sink logits.
* Fixed: the final norm and the scratch of one block of vkc_fit_rows(MIMO_CHUNK) rows
* (the residual's handoff reuses the normed rows' read-back). Tail: the head, and the
* vision tower when the per-matrix path takes it (COLI_VK_DENSE). The CPU's layers, the
* head and the tower of a partial chain stay on the CPU with their host copies (vk_off:
* the per-matrix path leaves them there). */
static void mc_fit_start(Model *m) {
Cfg *c = &m->c; int L = c->n_layers, H = c->hidden;
for (int k = 0; k < 2; k++) if (c->head_dim[k] > 256 || c->v_dim[k] > 256) return; /* the chain declines */
size_t *per = calloc((size_t)L, sizeof *per), *mat = calloc((size_t)L, sizeof *mat);
int ok = per && mat;
long forced = vkc_kv_env("COLI_VK_KV_DEVICE_ROWS", 0);
#define MC_FITW(d) do { const DW *d_ = (d); int g_; const float *s_; int f_ = d_->w ? dw_vk_fmt(d_, &g_, &s_) : -1; \
if (f_ < 0) ok = 0; else { per[i] += vkc_fit_tensor(f_, d_->I, d_->O, g_); mat[i] += coli_vk_tensor_payload(f_, d_->I, d_->O, g_); } } while (0)
for (int i = 0; i < L && ok; i++) {
const Layer *l = &m->L[i];
MC_FITW(&l->qkv); MC_FITW(&l->o);
if (!c->moe[i]) { MC_FITW(&l->gate); MC_FITW(&l->up); MC_FITW(&l->down); }
McGeo g = mc_geo(m, i);
size_t rows = !g.swa && forced > 0 && forced < g.rows ? (size_t)forced : (size_t)g.rows;
per[i] += vkc_fit_buf(rows * g.kd * sizeof(float)) + vkc_fit_buf(rows * g.vdd * sizeof(float)) +
((size_t)2 * H + (l->sink ? (size_t)c->heads[c->swa[i]] : 0)) * sizeof(float);
}
#undef MC_FITW
int hg; const float *hs;
int hf = m->head.w ? dw_vk_fmt(&m->head, &hg, &hs) : -1;
if (!ok || hf < 0) { free(per); free(mat); return; } /* a matrix the device does not take: the chain declines as before */
size_t tail = vkc_fit_tensor(hf, m->head.I, m->head.O, hg);
if (g_vision && coli_vk_dense()) { /* the tower the per-matrix path puts on the device */
Vision *v = g_vision;
const DW *t[3] = {&v->embed, &v->fc1, &v->fc2};
for (int k = 0; k < 3; k++) if (t[k]->w && dw_vk_fmt(t[k], &hg, &hs) >= 0) tail += vkc_fit_tensor(dw_vk_fmt(t[k], &hg, &hs), t[k]->I, t[k]->O, hg);
for (int b = 0; b < v->depth; b++) {
const DW *d[5] = {&v->b[b].qkv, &v->b[b].proj, &v->b[b].gate, &v->b[b].up, &v->b[b].down};
for (int k = 0; k < 5; k++) if (d[k]->w && dw_vk_fmt(d[k], &hg, &hs) >= 0) tail += vkc_fit_tensor(dw_vk_fmt(d[k], &hg, &hs), d[k]->I, d[k]->O, hg);
}
}
MimoChain g; memset(&g, 0, sizeof g);
g.o_kvd = calloc((size_t)L, sizeof(size_t));
const char *mc = getenv("MIMO_CHUNK");
int R = vkc_fit_rows(mc && *mc && atoi(mc) > 0 ? atoi(mc) : 64);
g_mc_count = 0; g_mc_fitcount = 1;
mc_scratch(&g, m, R, 1);
size_t fixed = (size_t)g_mc_count + vkc_fit_buf((size_t)H * sizeof(float));
g_mc_count = -1; g_mc_fitcount = 0;
free(g.o_kvd);
vkc_fit("mimo", L, per, mat, fixed, tail, &g_mc_fit);
/* the layers the primary leaves, on COLI_VK_DEV2's device: a fit of its own from layer
* n0 with that device's free memory, its pipelines up now (the head goes with it when it
* ends at the last layer with room; the tower stays on the CPU) */
int n0 = g_mc_fit.n;
if (vkc_fit_partial(&g_mc_fit) && n0 > 0 && n0 < L && mc_dev2_wanted() && getenv("COLI_VK_DEV2") &&
coli_vk_dev2_open_env()) {
vkc_device(1);
size_t tail2 = vkc_fit_tensor(hf, m->head.I, m->head.O, hg);
int n2 = vkc_fit("mimo dev2", L - n0, per + n0, mat + n0, fixed, tail2, &g_mc_fit2);
if (n2 > 0 && !vkc_init()) {
fprintf(stderr, "[VK] mimo chain: the second device's pipelines did not come up; its layers stay on the CPU\n");
n2 = 0;
}
g_mc_fit2_on = n2 > 0;
if (!g_mc_fit2_on) g_mc_fit2.tail = 0;
vkc_device(0);
}
free(per); free(mat);
}
/* Layers from..L-1, the head and the tower refused to the device: what of them is there
* is freed, and the per-matrix path leaves them on the CPU. */
static void mc_refuse(DW *d) {
if (d->vk && !d->vk_gone) { coli_vk_tensor_free((ColiVkTensor *)d->vk); d->vk = NULL; }
if (!d->vk_gone) d->vk_off = 1;
}
static void mc_refuse_from(Model *m, int from) {
if (vkc_ready()) vkc_finish();
for (int li = from; li < m->c.n_layers; li++) {
Layer *l = &m->L[li];
mc_refuse(&l->qkv); mc_refuse(&l->o); mc_refuse(&l->gate); mc_refuse(&l->up); mc_refuse(&l->down);
}
if (!(m->head.vk && coli_vk_tensor_dev((ColiVkTensor *)m->head.vk))) /* the second device's chain keeps it */
mc_refuse(&m->head);
if (g_vision) {
Vision *v = g_vision;
mc_refuse(&v->embed); mc_refuse(&v->fc1); mc_refuse(&v->fc2);
for (int b = 0; b < v->depth; b++) {
mc_refuse(&v->b[b].qkv); mc_refuse(&v->b[b].proj); mc_refuse(&v->b[b].gate); mc_refuse(&v->b[b].up); mc_refuse(&v->b[b].down);
}
}
}
static void dw_drop(DW *d, int *n); /* below */
/* The fit's layers up, layer by layer: each whole or not at all (a layer that fails is
* freed, it and the rest refused, vkc_fit_shrink); dropped: with COLI_VK_DENSE_HOST, each
* layer's host copies given back once all of it is on the device (counted there). */
static void mc_place(Model *m, int *dropped) {
int n0 = g_mc_fit.n;
for (int i = 0; i < g_mc_fit.n; i++) {
Layer *l = &m->L[i];
DW *d[5] = {&l->qkv, &l->o, &l->gate, &l->up, &l->down};
int nd = m->c.moe[i] ? 2 : 5, ok = 1;
for (int k = 0; k < nd && ok; k++) ok = dw_upload(d[k]);
if (!ok) {
mc_refuse_from(m, i);
vkc_fit_shrink("mimo", &g_mc_fit, i, "a dense matrix did not go up");
break;
}
if (dropped) for (int k = 0; k < nd; k++) dw_drop(d[k], dropped);
vkc_fit_mark(&g_mc_fit, i);
}
int end = g_mc_fit.n;
if (g_mc_fit2_on && g_mc_fit.n < n0) {
/* the primary placed fewer layers than its fit: the second device's would not follow
* them, so they stay on the CPU too */
fprintf(stderr, "[VK] mimo chain: the primary device stopped before layer %d; layers %d..%d stay on the CPU, "
"not on the second device\n", n0, n0, n0 + g_mc_fit2.n - 1);
g_mc_fit2_on = 0;
vkc_device(1); vkc_shutdown(); vkc_device(0);
} else if (g_mc_fit2_on) { /* the second device's layers (their host copies kept), and the head with the last one */
vkc_device(1); g_mimo_vk_dev = 1;
for (int i = n0; i < n0 + g_mc_fit2.n; i++) {
Layer *l = &m->L[i];
DW *d[5] = {&l->qkv, &l->o, &l->gate, &l->up, &l->down};
int nd = m->c.moe[i] ? 2 : 5, ok = 1;
for (int k = 0; k < nd && ok; k++) ok = dw_upload(d[k]);
if (!ok) {
g_mimo_vk_dev = 0;
mc_refuse_from(m, i);
vkc_fit_shrink("mimo dev2", &g_mc_fit2, i - n0, "a dense matrix did not go up");
g_mimo_vk_dev = 1;
break;
}
vkc_fit_mark(&g_mc_fit2, i - n0);
}
end = n0 + g_mc_fit2.n;
if (end == m->c.n_layers && g_mc_fit2.tail && !dw_upload(&m->head)) g_mc_fit2.tail = 0;
g_mimo_vk_dev = 0;
vkc_fit_placed("mimo dev2", &g_mc_fit2);
if (!g_mc_fit2.n) { g_mc_fit2_on = 0; vkc_shutdown(); }
vkc_device(0);
}
if (vkc_fit_partial(&g_mc_fit)) mc_refuse_from(m, end); /* the CPU's layers, the head (unless the second device's), the tower */
vkc_fit_placed("mimo", &g_mc_fit);
g_mc_placed = 1;
}
/* ---- the dense matrices on the device only (COLI_VK_DENSE_HOST) ----------------
* With the trunk on the device (the chain, or COLI_VK_DENSE), every trunk matrix goes
* up now, before the expert tier sizes its budget, and its host copy is dropped; the
* tower too when the per-matrix path is on (COLI_VK_DENSE), else it stays on the CPU
* with its copy. The per-matrix path then answers on the device whatever
* coli_vk_dense() says (the CPU holds no copy), and the CPU reads a matrix back from
* disk only when it needs it after all (dw_reload: a lost device). Kept on the host:
* the embedding (its rows are gathered on the CPU), the routers (the host computes
* them), norms and vectors, the patch embedding. */
static void dw_drop(DW *d, int *n) {
if (!d->w || d->vk_off || d->vk_gone || !d->vk_name || !dw_upload(d)) return;
size_t b = dw_bytes(d);
free(d->w); free(d->s);
d->w = NULL; d->s = NULL; d->vk_gone = 1;
coli_vk_dense_host_dropped(b);
(*n)++;
}
static void mimo_dho_start(Model *m) {
if (!g_vk_ready) return;
int vision = coli_vk_dense();
/* a partial chain's fit: the first n layers' bytes only (the head and the tower with
* the tail); n = 0 leaves the dense part on the CPU */
size_t bytes = vk_dense_bytes(m, vision);
int fit = g_mc_fit.L > 0, tail = !fit || (g_mc_fit.n == g_mc_fit.L && g_mc_fit.tail);
if (fit) {
bytes = 0;
for (int li = 0; li < g_mc_fit.n; li++) {
const Layer *l = &m->L[li];
bytes += dw_bytes(&l->qkv) + dw_bytes(&l->o);
if (!m->c.moe[li]) bytes += dw_bytes(&l->gate) + dw_bytes(&l->up) + dw_bytes(&l->down);
}
if (tail) bytes += vk_dense_bytes(m, vision) - vk_dense_bytes(m, 0) + dw_bytes(&m->head);
}
if (!coli_vk_dense_host_decide("mimo", (coli_vk_dense() || g_vk_chain) && !(fit && !g_mc_fit.n), bytes)) return;
g_mimo_dho = 1;
g_dw_dho_S = &m->S; g_dw_dho_c = &m->c;
int n = 0;
if (fit) {
coli_vk_dense_host_layers(g_mc_fit.n, g_mc_fit.L);
mc_place(m, &n); /* each layer whole, then its host copies */
tail = g_mc_fit.n == g_mc_fit.L && g_mc_fit.tail;
} else
for (int li = 0; li < m->c.n_layers; li++) {
Layer *l = &m->L[li];
dw_drop(&l->qkv, &n); dw_drop(&l->o, &n);
if (!m->c.moe[li]) { dw_drop(&l->gate, &n); dw_drop(&l->up, &n); dw_drop(&l->down, &n); }
}
if (tail) dw_drop(&m->head, &n);
if (g_vision && tail) {
Vision *v = g_vision;
DW *t[3] = {&v->embed, &v->fc1, &v->fc2};
for (int k = 0; k < 3; k++) { if (vision) dw_drop(t[k], &n); else t[k]->vk_off = 1; }
for (int i = 0; i < v->depth; i++) {
DW *b[5] = {&v->b[i].qkv, &v->b[i].proj, &v->b[i].gate, &v->b[i].up, &v->b[i].down};
for (int k = 0; k < 5; k++) { if (vision) dw_drop(b[k], &n); else b[k]->vk_off = 1; }
}
}
coli_vk_dense_host_placed("mimo", vision || !g_vision
? "the embedding (its rows are gathered on the CPU), the routers, norms, the patch embedding"
: "the embedding (its rows are gathered on the CPU), the routers, norms, the vision tower (on the CPU: COLI_VK_DENSE=1 puts it on the device)");
}
/* Is the expert in this layer's RAM cache now (the tier's balance asks)? */
static int vk_in_ram(void *ctx, int li, int e) {
const Model *m = ctx;
return m->cache[li].by_expert && m->cache[li].by_expert[e] >= 0;
}
/* The routed experts on the shared tier (vk_tier.c): the release's MXFP4 with its
* e8m0 group scales, SwiGLU. This engine keeps no expert history, so the tier
* starts empty and fills as experts pass by, evicting the coldest once its budget
* is full. MIMO_VK_EXPERTS, the switch of the per-engine tier before this one, is
* read as follows: 0 keeps the experts on the CPU (no tier), N > 0 sizes the
* budget at N experts (COLI_VK_TIER_GB, which wins when set); unset, the tier's own
* budget applies. */
static void vk_tier_start(Model *m) {
Cfg *c = &m->c;
VktFmt f = {VKT_SRC_MXFP4_E8M0, 32};
const char *mx = getenv("MIMO_VK_EXPERTS"), *gb = getenv("COLI_VK_TIER_GB");
if (mx && *mx && atoi(mx) > 0 && !(gb && *gb)) {
size_t eb = vkt_expert_bytes(c->hidden, c->moe_inter, f, f);
char buf[64];
snprintf(buf, sizeof(buf), "%.12g", ((double)atoi(mx) * eb + eb / 2) / 1073741824.0);
setenv("COLI_VK_TIER_GB", buf, 1);
fprintf(stderr, "[VK] mimo: MIMO_VK_EXPERTS=%s read as a tier budget of %s experts (COLI_VK_TIER_GB=%s)\n",
mx, mx, buf);
}
int cap = 0, nmoe = 0;
for (int li = 0; li < c->n_layers; li++) if (c->moe[li]) { nmoe++; if (m->cache[li].cap > cap) cap = m->cache[li].cap; }
VktConfig vc = {.engine = "mimo", .layers = c->n_layers, .experts = c->n_experts,
.hidden = c->hidden, .inter = c->moe_inter, .topk = c->topk,
.gate_up = f, .down = f, .act = VKT_ACT_SWIGLU,
.max_rows = MIMO_VK_ROWS * c->topk,
.ram_reserve = (size_t)(m->e_bytes + 8192) * (size_t)cap * (size_t)nmoe,
.dense_bytes = g_mc_fit.L && vkc_fit_partial(&g_mc_fit) ? mc_kv_bytes_n(m, g_mc_fit.n) : /* a partial chain: placed already, its caches to come */
(coli_vk_dense_device_only() ? 0 : /* placed already (mimo_dho_start) */
coli_vk_dense() ? vk_dense_bytes(m, 1) : g_vk_chain ? vk_dense_bytes(m, 0) : 0) +
(g_vk_chain ? mc_kv_bytes(m) : 0), /* the chain's KV caches */
.in_ram = vk_in_ram, .ram_ctx = m,
.load = mimo_vk_load, .release = mimo_vk_release, .load_ctx = m, .load_batch = mimo_vk_load_batch};
atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if (vkt_init(&vc, NULL)) atexit(vkt_shutdown);
}
#endif
/* --------------------------------------------------------------- sampling ---- */
static int argmax(const float *x, int n) {
int best = 0;
for (int i = 1; i < n; i++) if (x[i] > x[best]) best = i;
return best;
}
typedef struct { float p; int id; } SampleProb;
static int sample_desc(const void *a, const void *b) {
float pa = ((const SampleProb *)a)->p, pb = ((const SampleProb *)b)->p;
return (pb > pa) - (pa > pb);
}
static int sample(const float *logits, int vocab, float temperature, float top_p) {
if (temperature <= 0.0f) return argmax(logits, vocab);
SampleProb *rank = xmalloc((size_t)vocab * sizeof(SampleProb), "sampling");
float top = logits[0];
for (int i = 1; i < vocab; i++) if (logits[i] > top) top = logits[i];
double total = 0.0;
for (int i = 0; i < vocab; i++) {
float p = expf((logits[i] - top) / temperature);
total += p; rank[i].p = p; rank[i].id = i;
}
qsort(rank, (size_t)vocab, sizeof(SampleProb), sample_desc);
double cut = (top_p > 0.0f && top_p < 1.0f) ? top_p * total : total, kept = 0.0;
int n = 0;
while (n < vocab && kept < cut) kept += rank[n++].p;
double draw = ((double)rand() / RAND_MAX) * kept, acc = 0.0;
int pick = rank[0].id;
for (int i = 0; i < n; i++) { acc += rank[i].p; if (acc >= draw) { pick = rank[i].id; break; } }
free(rank);
return pick;
}
/* ---------------------------------------------------------------- photos ---- */
/* SUBMIT pin=1: a photo of the state at the end of this prompt, so that the
* prompts that begin with it (the options of a closed question) resume from
* there instead of computing it again. pin_pool.h keeps several, nested, and
* restores the deepest one that is a strict prefix of the new prompt.
*
* WHAT A LAYER CARRIES BETWEEN TOKENS, and so what the photo must hold:
*
* full attention K/V rows indexed by position. Nothing after the photo
* writes below it, so they stay where they are, and the
* photo holds only the token ids; kv_prefix_holds() is what
* says they are still the photo's rows (an unrelated prompt
* in between rewrites them from position 0).
* sliding window a RING of `rows` slots, position p in slot p % rows, and
* the position each slot holds. Every option overwrites the
* ring slots of the photo's last positions -- its own token at
* P lands where P - rows was -- so after one option the ring
* no longer describes the photo. The ring, keys, values and
* positions, is copied into the photo and copied back.
* the position m->pos, which is the photo's length.
*
* Nothing else carries over: the sinks are weights, the router has no state,
* and the expert cache changes the time, never the numbers. On Flash the rings
* of the 39 windowed layers are 128 x 8 x (192 + 128) floats each, about 50 MB
* a photo, so COLI_PIN_SLOTS (default 4) is what bounds the memory. */
typedef struct {
int n_layers;
int rows[MIMO_MAX_LAYERS]; /* 0: a full-attention layer */
size_t kd[MIMO_MAX_LAYERS], vd[MIMO_MAX_LAYERS];
float *K[MIMO_MAX_LAYERS], *V[MIMO_MAX_LAYERS];
int *ring_pos[MIMO_MAX_LAYERS];
} PinState;
static ColiPinPool g_pins;
static void pin_state_free(void *v) {
PinState *st = (PinState *)v;
if (!st) return;
for (int li = 0; li < st->n_layers; li++) { free(st->K[li]); free(st->V[li]); free(st->ring_pos[li]); }
free(st);
}
/* Copies the rings into `st` (allocated on first use). NULL when memory runs
* out: a photo is an optimisation and never the reason a request fails. */
static PinState *pin_state_save(Model *m, PinState *st) {
Cfg *c = &m->c;
if (st && st->n_layers != c->n_layers) { pin_state_free(st); st = NULL; }
if (!st) {
st = calloc(1, sizeof(*st));
if (!st) return NULL;
st->n_layers = c->n_layers;
for (int li = 0; li < c->n_layers; li++) {
if (!c->swa[li]) continue;
Layer *l = &m->L[li];
st->rows[li] = l->rows;
st->kd[li] = (size_t)l->rows * c->kv_heads[1] * c->head_dim[1];
st->vd[li] = (size_t)l->rows * c->kv_heads[1] * c->v_dim[1];
st->K[li] = malloc(st->kd[li] * sizeof(float));
st->V[li] = malloc(st->vd[li] * sizeof(float));
st->ring_pos[li] = malloc((size_t)l->rows * sizeof(int));
if (!st->K[li] || !st->V[li] || !st->ring_pos[li]) { pin_state_free(st); return NULL; }
}
}
for (int li = 0; li < c->n_layers; li++) {
if (!st->rows[li]) continue;
Layer *l = &m->L[li];
memcpy(st->K[li], l->K, st->kd[li] * sizeof(float));
memcpy(st->V[li], l->V, st->vd[li] * sizeof(float));
memcpy(st->ring_pos[li], l->ring_pos, (size_t)l->rows * sizeof(int));
}
return st;
}
static int pin_state_load(Model *m, const PinState *st) {
Cfg *c = &m->c;
if (!st || st->n_layers != c->n_layers) return 0;
for (int li = 0; li < c->n_layers; li++)
if (st->rows[li] != (c->swa[li] ? m->L[li].rows : 0)) return 0;
for (int li = 0; li < c->n_layers; li++) {
if (!st->rows[li]) continue;
Layer *l = &m->L[li];
memcpy(l->K, st->K[li], st->kd[li] * sizeof(float));
memcpy(l->V, st->V[li], st->vd[li] * sizeof(float));
memcpy(l->ring_pos, st->ring_pos[li], (size_t)l->rows * sizeof(int));
}
#ifdef COLI_VULKAN
mc_rings_rewritten(m); /* the dense chain's copies of the rings go up again */
#endif
return 1;
}
/* Only a request that asks for it takes a photo, and it goes into the pool,
* where no other request writes: an ordinary prompt can make a photo stale
* (it rewrites the rows the photo stands on), never replace it. */
static void pin_save(Model *m, const int *ids, int n, const float *logits) {
coli_pin_pool_init(&g_pins, m->c.vocab);
ColiPin *k = coli_pin_store(&g_pins, ids, n, logits);
if (!k) return;
PinState *st = pin_state_save(m, (PinState *)k->state);
if (!st) { k->state = NULL; k->len = 0; return; } /* without its rings it would lie */
k->state = st;
fprintf(stderr, "[PIN] photo of %d tokens\n", n);
fflush(stderr);
}
/* The deepest photo that is a strict prefix of this prompt and whose rows the
* state still holds, put back: the rings, the position and the record. Returns
* its length and its logits (the predictor of the first fresh token), or 0.
* A photo whose rows are gone cannot help any more: it is released, memory and
* all, and the next deepest is tried instead of giving up. */
static int pin_restore(Model *m, const int *ids, int n, const float **logit) {
*logit = NULL;
int s = coli_pin_best(&g_pins, ids, n);
while (s >= 0) {
ColiPin *k = &g_pins.slot[s];
if (k->logit && kv_prefix_holds(&m->kvp, k->ids, k->len) &&
pin_state_load(m, (const PinState *)k->state)) {
m->pos = k->len;
kv_prefix_clear(&m->kvp);
kv_prefix_record(&m->kvp, k->ids, 0, k->len);
*logit = k->logit;
coli_pin_touch(&g_pins, s);
return k->len;
}
fprintf(stderr, "[PIN] released the photo of %d tokens: the state no longer holds it\n", k->len);
k->len = 0;
pin_state_free(k->state);
k->state = NULL;
s = coli_pin_best(&g_pins, ids, n);
}
return 0;
}
/* ------------------------------------------------------------------ serve ---- */
/* The 7th numeric field (the byte count of a payload extension) is accepted, but
* only as 0: the gateway puts it ahead of the logprobs= key on chat requests, and
* this engine takes no grammar, so any other value is still a bad header. */
static const ColiServeWireProfile mimo_wire = {
.max_header_bytes = 511,
.max_payload_bytes = 1u << 26,
.max_extension_bytes = 0,
.max_tokens = 1 << 20,
.require_exact_lf = 1,
.require_finite_sampling = 1,
.allow_extension_bytes = 1,
};
static void serve_line(const char *format, ...) {
va_list args;
va_start(args, format);
vfprintf(stdout, format, args);
va_end(args);
fflush(stdout);
}
/* the dashboard's grid: rows are the MoE layers, columns the routed experts */
static void serve_emap(Model *m) {
Cfg *c = &m->c;
int rows = 0;
for (int li = 0; li < c->n_layers; li++) rows += c->moe[li];
int cols = c->n_experts;
char *hex = xmalloc((size_t)rows * cols * 2 + 1, "emap");
int w = 0;
for (int li = 0; li < c->n_layers; li++) {
if (!c->moe[li]) continue;
for (int e = 0; e < cols; e++) {
int byte = (vkt_resident(li, e) ? 2 : m->cache[li].by_expert[e] >= 0) << 6; /* 2 = on the Vulkan device */
hex[w++] = "0123456789abcdef"[byte >> 4];
hex[w++] = "0123456789abcdef"[byte & 15];
}
}
hex[w] = 0;
serve_line("EMAP %d %d %s\n", rows, cols, hex);
free(hex);
}
static void serve_hits(Model *m) {
Cfg *c = &m->c;
int rows = 0;
for (int li = 0; li < c->n_layers; li++) rows += c->moe[li];
int cols = c->n_experts, nb = (rows * cols + 7) / 8, bit = 0;
uint8_t *bitmap = xcalloc((size_t)nb, 1, "hits bitmap");
for (int li = 0; li < c->n_layers; li++) {
if (!c->moe[li]) continue;
for (int e = 0; e < cols; e++, bit++)
if (m->ehit[li][e]) { bitmap[bit >> 3] |= (uint8_t)(1 << (bit & 7)); m->ehit[li][e] = 0; }
}
char *hex = xmalloc((size_t)nb * 2 + 1, "hits hex");
for (int b = 0; b < nb; b++) {
hex[2 * b] = "0123456789abcdef"[bitmap[b] >> 4];
hex[2 * b + 1] = "0123456789abcdef"[bitmap[b] & 15];
}
hex[2 * nb] = 0;
serve_line("HITS %d %d %s\n", rows, cols, hex);
free(hex); free(bitmap);
}
/* max_tokens is a ceiling; generation needs at least one free position. A
* read-only request (max_tokens=0, legal only with logprobs) generates nothing,
* so its prompt may fill the context. */
static int serve_budget(int prompt, int requested, int context, int logprobs) {
if (prompt < 1 || prompt > context) return -1;
int budget = requested > 0 ? requested : (logprobs > 0 ? 0 : 256);
int room = context - prompt;
if (budget > 0 && room == 0) return -1;
return budget < room ? budget : room;
}
/* When a request was accepted, and the counters then: DONE and PROF report its share. */
typedef struct { double started, disk0, expert0, attn0; uint64_t hits0, miss0, fw0; int n_prompt, budget; } MimoReq;
/* A request's prompt into the caches the Model's layers hold: its tokens, the budget,
* the image, the prefix reuse and the photos, ACCEPT, the prefill and its read-out.
* 1 with the logits after the prompt; 0 when the request ended here, its ERROR written
* and its command disposed. serve_loop and serve_mux start every request here. */
static int mimo_serve_start(Model *m, Tok *tokenizer, ColiServeCommand *cmd, int *ids, float *logits,
float **pending, int *pending_h, int *pending_w, MimoReq *rq) {
Cfg *c = &m->c;
double started = now_s();
double disk0 = m->t_disk, expert0 = m->t_expert, attn0 = m->t_attn;
uint64_t hits0 = m->hits, miss0 = m->miss, fw0 = m->forwards;
int n_prompt = tok_encode(tokenizer, (const char *)cmd->payload, (int)cmd->payload_bytes,
ids, m->ctx + 1);
int budget = serve_budget(n_prompt, cmd->max_tokens, m->ctx, cmd->logprobs);
if (budget < 0) {
char message[160];
snprintf(message, sizeof(message), "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
n_prompt, cmd->max_tokens, m->ctx);
coli_serve_write_error(stdout, cmd->id, n_prompt < 1 ? "EMPTY_PROMPT" : message);
free((*pending)); (*pending) = NULL;
coli_serve_command_dispose(cmd);
return 0;
}
/* An image: its rows replace the pad ids, which must be exactly as many
* as the tower produces. Checked before ACCEPT so a mismatch is a clean
* 400, not a half-answered turn. */
float *image = NULL;
int image_rows = 0;
if ((*pending)) {
int pads = 0;
for (int t = 0; t < n_prompt; t++) pads += ids[t] == c->image_token_id;
int want = ((*pending_h) / 2) * ((*pending_w) / 2);
if (pads != want) {
char message[160];
snprintf(message, sizeof(message), "BAD_IMAGE prompt has %d image pads, the %dx%d grid needs %d",
pads, (*pending_h), (*pending_w), want);
coli_serve_write_error(stdout, cmd->id, message);
free((*pending)); (*pending) = NULL;
coli_serve_command_dispose(cmd);
return 0;
}
}
/* Where this prompt starts. A chat client resends the whole transcript:
* if this prompt begins with the ids the state was built from, only the
* tail is fed (COLI_KV_PREFIX=0 turns that off). A photo (SUBMIT pin=1
* earlier) resumes a prompt that begins with it.
*
* With the read-out on (logprobs=k) only a photo will do. The read-out
* owes a frame to every position it does not inherit a predictor for,
* and a live prefix comes with none: the frames before it would simply
* be missing. So such a prompt resumes from the deepest photo, which
* carries the logits that predict its first fresh token, or is read out
* from position 0. It resumes at the photo even when the live state
* shares more: two options share the text before them, and stopping at
* that would leave the first option token without its predictor.
*
* An image refuses both: the pad ids do not describe the picture. */
int prefix_on = env_int("COLI_KV_PREFIX", 1) != 0;
int reuse = 0;
const float *pin_logit = NULL;
if (!(*pending)) {
if (cmd->logprobs > 0) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
else {
reuse = prefix_on ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;
if (!reuse) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
}
}
if (getenv("COLI_PREFIX_LOG"))
fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens%s\n", reuse ? "reusing" : "no reuse,", reuse,
n_prompt, pin_logit ? " (photo)" : "");
if (pin_logit) {
fprintf(stderr, "[PIN] resumed from the photo of %d tokens, %d to prefill\n", reuse, n_prompt - reuse);
fflush(stderr);
}
if (!reuse) model_reset(m);
coli_serve_write_accept(stdout, cmd->id, n_prompt);
if ((*pending)) {
image = vision_run(m, (*pending), (*pending_h), (*pending_w), &image_rows);
free((*pending)); (*pending) = NULL;
}
ImageRows img = { image, image_rows };
Echo echo = { cmd->id, cmd->logprobs, tokenizer, pin_logit };
prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL,
cmd->logprobs > 0 ? &echo : NULL);
if (image) kv_prefix_taint(&m->kvp);
if (cmd->pin) {
/* a photo is described by its ids, and these do not describe the picture */
if (image) fprintf(stderr, "[PIN] no photo: the prompt carries a picture\n");
else pin_save(m, ids, n_prompt, logits);
}
free(image);
rq->started = started; rq->disk0 = disk0; rq->expert0 = expert0; rq->attn0 = attn0;
rq->hits0 = hits0; rq->miss0 = miss0; rq->fw0 = fw0; rq->n_prompt = n_prompt; rq->budget = budget;
return 1;
}
/* ---- several conversations at once (KV_SLOTS>1) ---------------------------------
* The gateway's cache slots, each a conversation with caches of its own (MimoSeq). A
* SUBMIT on a free slot starts its request at once through mimo_serve_start, on that
* slot's caches: its prefix reuse and photos work as a lone serve's. Then every step
* picks the next token of each active request and runs one forward over a row of each
* (forward_rows): the matrices and the experts are read once for all of them. A
* request's frames are a lone request's; they interleave by id. As alone, STOP ends a
* request with DONE and CANCEL with ERROR CANCELLED. */
static MimoSeq *g_mimo_mux_seq; /* [slots]: the conversations the Model does not hold */
static int g_mimo_mux_cur; /* the slot the Model holds, -1 when every one is parked */
typedef struct {
char id[COLI_SERVE_ID_CAP];
float temperature, top_p;
int logprobs, active, stop, cancel, limited, emitted;
float *logits; /* the logits the next pick reads */
MimoReq rq;
} MimoMuxReq;
static void mimo_mux_bind(Model *m, int slot) {
if (g_mimo_mux_cur == slot) return;
if (g_mimo_mux_cur >= 0) mimo_seq_swap(m, &g_mimo_mux_seq[g_mimo_mux_cur]);
if (slot >= 0) mimo_seq_swap(m, &g_mimo_mux_seq[slot]);
g_mimo_mux_cur = slot;
}
/* A request's end, as serve_loop ends one. */
static void mimo_mux_finish(Model *m, MimoMuxReq *r) {
r->active = 0;
if (r->cancel) { coli_serve_write_error(stdout, r->id, "CANCELLED"); return; }
if (r->stop) r->limited = 0;
double wall = now_s() - r->rq.started;
uint64_t th = m->hits - r->rq.hits0, tm = m->miss - r->rq.miss0;
ColiServeDone done = { r->emitted, wall > 0 ? r->emitted / wall : 0.0,
(th + tm) ? 100.0 * th / (double)(th + tm) : 0.0, rss_gb(), r->rq.n_prompt, r->limited };
coli_serve_write_done(stdout, r->id, &done);
serve_line("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n", wall, r->rq.n_prompt, r->emitted,
m->t_disk - r->rq.disk0, 0.0, m->t_expert - r->rq.expert0, m->t_attn - r->rq.attn0, 0.0,
(unsigned long long)(m->forwards - r->rq.fw0));
serve_hits(m);
}
/* The next token of an active request, as serve_loop picks and sends it: 1 with the
* token when the request goes on, 0 when it ended. */
static int mimo_mux_pick(Model *m, Tok *tokenizer, MimoMuxReq *r, const int *eos, int n_eos, int *tk_out) {
Cfg *c = &m->c;
if (r->cancel || r->stop || r->emitted >= r->rq.budget) { mimo_mux_finish(m, r); return 0; }
int token = sample(r->logits, c->vocab, r->temperature, r->top_p);
for (int i = 0; i < n_eos; i++) if (token == eos[i]) { r->limited = 0; mimo_mux_finish(m, r); return 0; }
char piece[512];
int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
if (written > 0) {
if (r->logprobs > 0) {
char tail[1024];
coli_logprob_tail(tail, sizeof(tail), r->logits, c->vocab, token, r->logprobs);
coli_serve_write_data_lp(stdout, r->id, piece, (size_t)written, tail);
} else coli_serve_write_data(stdout, r->id, piece, (size_t)written);
}
r->emitted++;
if (r->emitted >= r->rq.budget) { mimo_mux_finish(m, r); return 0; }
*tk_out = token; return 1;
}
static void serve_mux(Model *m, Tok *tokenizer, const char *dir) {
Cfg *c = &m->c;
int n = g_mimo_mux_slots, V = c->vocab, input_eof = 0;
int eos[8];
int n_eos = coli_load_stop_ids(dir, eos, 8, NULL);
coli_serve_write_ready_caps(stdout, rss_gb(), g_vision ? "vision=1" : "vision=0");
serve_emap(m);
MimoMuxReq *rq = xcalloc((size_t)n, sizeof *rq, "requests");
MimoRow *rows = xmalloc((size_t)n * sizeof *rows, "rows");
int *tok = xmalloc((size_t)n * sizeof(int), "tokens"), *who = xmalloc((size_t)n * sizeof(int), "slots");
float *lo = xmalloc((size_t)n * V * sizeof(float), "step logits");
int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "prompt ids");
for (int i = 0; i < n; i++) rq[i].logits = xmalloc((size_t)V * sizeof(float), "logits");
float *pending = NULL;
int pending_h = 0, pending_w = 0;
unsigned long long steps = 0, nrows = 0;
fprintf(stderr, "[mimo] serving %d conversations at once (KV_SLOTS)\n", n);
for (;;) {
int active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
/* idle: wait for a command; decoding: take one only when one is there */
if (!input_eof && (!active || coli_serve_stdin_ready())) {
ColiServeCommand command;
ColiServeReadResult result = coli_serve_read_command(stdin, &mimo_wire, &command);
if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) input_eof = 1;
else if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); input_eof = 1; }
else if (result == COLI_SERVE_READ_BAD_REQUEST) {
if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");
coli_serve_command_dispose(&command);
} else if (command.kind == COLI_SERVE_COMMAND_IMAGE) {
/* one image waits for the SUBMIT that names it; a second replaces it */
uint64_t expected = g_vision ? (uint64_t)command.grid_h * command.grid_w * g_vision->patch_in * sizeof(float) : 0;
if (!g_vision) coli_serve_write_error(stdout, command.id, "this container has no vision tower");
else if (command.grid_h < 2 || command.grid_w < 2 || command.grid_h % 2 || command.grid_w % 2 ||
command.payload_bytes != expected)
coli_serve_write_error(stdout, command.id, "BAD_IMAGE payload does not match its grid");
else {
free(pending);
pending = (float *)coli_serve_command_take_payload(&command);
pending_h = command.grid_h; pending_w = command.grid_w;
}
coli_serve_command_dispose(&command);
} else if (command.kind == COLI_SERVE_COMMAND_CANCEL || command.kind == COLI_SERVE_COMMAND_STOP) {
int found = 0;
for (int i = 0; i < n; i++) if (rq[i].active && !strcmp(rq[i].id, command.id)) {
found = 1;
if (command.kind == COLI_SERVE_COMMAND_STOP) rq[i].stop = 1; else rq[i].cancel = 1;
}
if (!found && command.kind == COLI_SERVE_COMMAND_CANCEL) coli_serve_write_error(stdout, command.id, "NOT_FOUND");
coli_serve_command_dispose(&command);
} else if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {
coli_serve_command_dispose(&command);
} else if (command.slot < 0 || command.slot >= n || rq[command.slot].active) {
coli_serve_write_error(stdout, command.id, command.slot < 0 || command.slot >= n ? "invalid cache slot" : "SLOT_BUSY");
coli_serve_command_dispose(&command);
} else {
MimoMuxReq *t = &rq[command.slot];
float *keep = t->logits;
memset(t, 0, sizeof *t); t->logits = keep;
snprintf(t->id, sizeof t->id, "%s", command.id);
t->temperature = command.temperature; t->top_p = command.top_p; t->logprobs = command.logprobs;
mimo_mux_bind(m, command.slot);
if (mimo_serve_start(m, tokenizer, &command, ids, t->logits, &pending, &pending_h, &pending_w, &t->rq)) {
t->active = 1; t->limited = 1;
coli_serve_command_dispose(&command);
}
}
}
active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
if (!active) { if (input_eof) break; continue; }
int S = 0, ended = 0;
for (int i = 0; i < n; i++) if (rq[i].active) {
int tk;
if (!mimo_mux_pick(m, tokenizer, &rq[i], eos, n_eos, &tk)) { ended = 1; continue; }
rows[S] = (MimoRow){&g_mimo_mux_seq[i], g_mimo_mux_cur == i ? m->pos : g_mimo_mux_seq[i].pos};
tok[S] = tk; who[S] = i; S++;
}
if (S) {
mimo_mux_bind(m, -1); /* every conversation parked: the rows read theirs */
for (int s = 0; s < S; s++) rows[s].pos = g_mimo_mux_seq[who[s]].pos;
forward_rows(m, rows, tok, S, lo);
steps++; nrows += (unsigned long long)S;
for (int s = 0; s < S; s++) memcpy(rq[who[s]].logits, lo + (size_t)s * V, (size_t)V * sizeof(float));
}
if (ended) {
serve_emap(m);
vkt_report("turn", m->hits, m->miss);
}
}
fprintf(stderr, "[mimo] KV_SLOTS=%d: %llu decode steps, %llu rows (%.2f a step)\n", n, steps, nrows,
steps ? (double)nrows / (double)steps : 0.0);
mimo_mux_bind(m, 0);
for (int i = 0; i < n; i++) free(rq[i].logits);
free(rq); free(rows); free(tok); free(who); free(lo); free(ids); free(pending);
}
static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
if (g_mimo_mux_slots > 1) { serve_mux(m, tokenizer, dir); return; }
Cfg *c = &m->c;
coli_serve_stdio_init();
int eos[8];
int n_eos = coli_load_stop_ids(dir, eos, 8, NULL);
/* CAPS: the gateway offers pictures only when the tower actually loaded */
coli_serve_write_ready_caps(stdout, rss_gb(), g_vision ? "vision=1" : "vision=0");
serve_emap(m);
float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");
int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "prompt ids");
float *pending = NULL;
int pending_h = 0, pending_w = 0;
for (;;) {
ColiServeCommand command;
ColiServeReadResult result = coli_serve_read_command(stdin, &mimo_wire, &command);
if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) break;
if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); break; }
if (result == COLI_SERVE_READ_BAD_REQUEST) {
if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");
coli_serve_command_dispose(&command);
continue;
}
if (command.kind == COLI_SERVE_COMMAND_IMAGE) {
/* One image waits for the SUBMIT that names it; a second replaces it. */
if (!g_vision) {
coli_serve_write_error(stdout, command.id, "this container has no vision tower");
coli_serve_command_dispose(&command);
continue;
}
uint64_t expected = (uint64_t)command.grid_h * command.grid_w * g_vision->patch_in * sizeof(float);
if (command.grid_h < 2 || command.grid_w < 2 || command.grid_h % 2 || command.grid_w % 2 ||
command.payload_bytes != expected) {
coli_serve_write_error(stdout, command.id, "BAD_IMAGE payload does not match its grid");
coli_serve_command_dispose(&command);
continue;
}
free(pending);
pending = (float *)coli_serve_command_take_payload(&command);
pending_h = command.grid_h; pending_w = command.grid_w;
coli_serve_command_dispose(&command);
continue;
}
if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {
if (command.kind == COLI_SERVE_COMMAND_CANCEL) coli_serve_write_error(stdout, command.id, "NOT_FOUND");
coli_serve_command_dispose(&command);
continue;
}
MimoReq rq;
if (!mimo_serve_start(m, tokenizer, &command, ids, logits, &pending, &pending_h, &pending_w, &rq)) continue;
double started = rq.started;
double disk0 = rq.disk0, expert0 = rq.expert0, attn0 = rq.attn0;
uint64_t hits0 = rq.hits0, miss0 = rq.miss0, fw0 = rq.fw0;
int n_prompt = rq.n_prompt, budget = rq.budget;
int emitted = 0, limited = 1, cancelled = 0, done_early = 0;
char piece[512];
while (emitted < budget && !cancelled && !done_early) {
int token = sample(logits, c->vocab, command.temperature, command.top_p);
int stop = 0;
for (int i = 0; i < n_eos; i++) if (token == eos[i]) stop = 1;
if (stop) { limited = 0; break; }
int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
if (written > 0) {
if (command.logprobs > 0) {
/* the distribution the token was drawn from: sample() only reads it */
char tail[1024];
coli_logprob_tail(tail, sizeof(tail), logits, c->vocab, token, command.logprobs);
coli_serve_write_data_lp(stdout, command.id, piece, (size_t)written, tail);
} else coli_serve_write_data(stdout, command.id, piece, (size_t)written);
}
emitted++;
while (coli_serve_stdin_ready()) {
ColiServeCommand control;
ColiServeReadResult inner = coli_serve_read_command(stdin, &mimo_wire, &control);
if (inner == COLI_SERVE_READ_EOF) { done_early = 1; break; }
if (control.kind == COLI_SERVE_COMMAND_CANCEL && !strcmp(control.id, command.id)) cancelled = 1;
else if (control.kind == COLI_SERVE_COMMAND_STOP && !strcmp(control.id, command.id)) { limited = 0; done_early = 1; }
else if (control.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, control.id, "SLOT_BUSY");
coli_serve_command_dispose(&control);
}
if (cancelled || done_early || emitted >= budget) break;
prefill(m, &token, 1, logits, NULL, NULL, NULL);
}
if (cancelled) {
coli_serve_write_error(stdout, command.id, "CANCELLED");
coli_serve_command_dispose(&command);
continue;
}
double wall = now_s() - started;
uint64_t th = m->hits - hits0, tm = m->miss - miss0;
ColiServeDone done = { emitted, wall > 0 ? emitted / wall : 0.0,
(th + tm) ? 100.0 * th / (double)(th + tm) : 0.0, rss_gb(), n_prompt, limited };
coli_serve_write_done(stdout, command.id, &done);
serve_line("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n", wall, n_prompt, emitted,
m->t_disk - disk0, 0.0, m->t_expert - expert0, m->t_attn - attn0, 0.0,
(unsigned long long)(m->forwards - fw0));
serve_hits(m);
serve_emap(m);
vkt_report("turn", m->hits, m->miss); /* the Vulkan tier's line, when it runs */
#ifdef COLI_VULKAN
mc_report(m, "turn");
#endif
coli_serve_command_dispose(&command);
}
free(ids); free(logits); free(pending);
}
/* ------------------------------------------------------------------- main ---- */
/* Once, at the end: how many products the GPU actually took, so a run that
* asked for Vulkan and quietly stayed on the CPU says so. */
static void vk_report(const Model *m) {
#ifdef COLI_VULKAN
if (!g_vk_ready) return;
fprintf(stderr, "[VK] mimo: %llu matmuls on the GPU\n", coli_vk_matmul_calls());
vkt_report("run", m->hits, m->miss);
mc_report((Model *)m, "run");
#else
(void)m;
#endif
}
static int parse_ids(const char *text, int *out, int cap) {
int n = 0;
const char *p = text;
while (*p && n < cap) {
while (*p == ' ' || *p == ',' || *p == '\t' || *p == '\n') p++;
if (!*p) break;
char *end;
long v = strtol(p, &end, 10);
if (end == p) break;
out[n++] = (int)v;
p = end;
}
return n;
}
static int is_dir(const char *path) {
struct stat sb;
return stat(path, &sb) == 0 && S_ISDIR(sb.st_mode);
}
int main(int argc, char **argv) {
coli_omp_tune_threads("mimo");
/* Two ways in. The gateway: SNAP=<dir> mimo <cap>, SERVE=1. By hand and in
* the tests: mimo <dir> [--ids "..." | --prompt "..."] [--ngen N] [--cap N]. */
const char *dir = getenv("SNAP");
int cap = env_int("MIMO_CAP", 0);
if (env_int("SERVE", 0)) { /* KV_SLOTS: the conversations a serve decodes at once */
const char *ks = getenv("KV_SLOTS");
if (ks && *ks) {
char *end = NULL; long v = strtol(ks, &end, 10);
if (end == ks || *end || v < 1 || v > 16) { fprintf(stderr, "KV_SLOTS must be between 1 and 16\n"); return 2; }
g_mimo_mux_slots = (int)v;
}
}
const char *ids_text = NULL, *prompt = NULL, *image_path = NULL;
int ngen = 32, grid_h = 0, grid_w = 0;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--ids") && i + 1 < argc) ids_text = argv[++i];
else if (!strcmp(argv[i], "--prompt") && i + 1 < argc) prompt = argv[++i];
else if (!strcmp(argv[i], "--ngen") && i + 1 < argc) ngen = coli_arg_int(argv[++i], "--ngen");
else if (!strcmp(argv[i], "--cap") && i + 1 < argc) cap = coli_arg_int(argv[++i], "--cap");
else if (!strcmp(argv[i], "--image") && i + 1 < argc) image_path = argv[++i];
else if (!strcmp(argv[i], "--grid") && i + 2 < argc) {
grid_h = coli_arg_int(argv[++i], "--grid h"); grid_w = coli_arg_int(argv[++i], "--grid w"); }
else if (is_dir(argv[i])) dir = argv[i];
else cap = coli_arg_int(argv[i], "cache/layer");
}
if (!dir) {
fprintf(stderr, "usage: %s <model dir> [--ids \"1 2 3\" | --prompt TEXT] [--ngen N] [--cap N]\n"
" SNAP=<model dir> SERVE=1 %s <cap>\n", argv[0], argv[0]);
return 2;
}
Model *m = xcalloc(1, sizeof(Model), "model");
double t0 = now_s();
if (cap <= 0) cap = 64;
model_load(m, dir, cap);
Cfg *c = &m->c;
g_vision = vision_load(m, dir);
#ifdef COLI_VULKAN
/* after the weights, so a missing device costs one line and nothing else; the
* routed-expert tier first, then the dense matrices where coli_vk_dense() puts
* them (on a device that shares the CPU's RAM they stay on the CPU while the
* tier is on; COLI_VK_DENSE decides when set) */
{
int any_moe = 0;
for (int i = 0; i < c->n_layers; i++) any_moe |= c->moe[i];
const char *mx = getenv("MIMO_VK_EXPERTS");
int tier = vkt_wanted() && any_moe && !(mx && *mx && atoi(mx) <= 0); /* MIMO_VK_EXPERTS=0: experts on the CPU */
g_vk_ready = coli_vk_init_env_tier("mimo", tier);
/* COLI_VK_CHAIN: every layer's dense chain on the device (mimo_chain.h), decided
* before the tier so the tier's budget leaves the trunk room. Not measured on a
* MiMo checkpoint: an integrated GPU keeps it opt-in (docs/vulkan.md). */
if (g_vk_ready) {
g_vk_chain = coli_vk_chain_decide("mimo", tier, COLI_VK_CHAIN_UNMEASURED);
if (g_vk_chain && g_mimo_mux_slots > 1) {
g_vk_chain = 0;
fprintf(stderr, "[VK] mimo: KV_SLOTS=%d: the dense chain is off (it keeps one conversation's caches on the device); "
"the expert tier runs every conversation's experts\n", g_mimo_mux_slots);
}
/* the chain's fit before any upload: its first n layers on the device, the
* others, the head and the tower on the CPU when they do not all fit */
if (g_vk_chain) mc_fit_start(m);
if (g_mc_fit.L && !g_mc_fit.n) { /* its line said the chain stays off: nothing goes up */
g_vk_chain = 0;
mc_refuse_from(m, 0);
vkc_fit_placed("mimo", &g_mc_fit);
g_mc_placed = 1;
}
if (g_vk_chain && !vkc_init()) g_vk_chain = 0;
}
mimo_dho_start(m); /* COLI_VK_DENSE_HOST: the dense matrices on the device only, before the tier sizes its budget */
if (g_vk_chain && vkc_fit_partial(&g_mc_fit) && !g_mc_placed) mc_place(m, NULL); /* before the tier, too */
if (g_vk_ready && tier) vk_tier_start(m);
if (g_vk_ready && !vkt_ready() && !coli_vk_dense()) coli_vk_dense_decide("mimo", 0, 1); /* no tier after all */
if (g_vk_chain) { /* the chain's teardown before the device's (the tier registered the device's) */
if (!tier) atexit(coli_vk_shutdown);
atexit(vkc_shutdown_all);
}
}
if (g_vk_ready && (coli_vk_dense() || g_vk_chain) && !(g_mc_fit.L && !g_mc_fit.n)) { /* COLI_VK_DENSE=0: the trunk stays on the CPU */
int up = vk_dense_upload(m, coli_vk_dense());
size_t used = 0, count = 0;
coli_vk_mem_info(&used, &count);
fprintf(stderr, "[VK] mimo: %d dense matrices on the GPU (%.2f GB)\n", up, used / 1e9);
}
#endif
int swa = 0, shown_cap = 0;
for (int i = 0; i < c->n_layers; i++) { swa += c->swa[i]; if (c->moe[i] && !shown_cap) shown_cap = m->cache[i].cap; }
fprintf(stderr, "[mimo] %d layers (%d sliding window %d, %d full), %d experts top-%d, cache %d/layer, "
"dense %s, ctx %d%s -- loaded in %.2fs\n",
c->n_layers, swa, c->window, c->n_layers - swa, c->n_experts, c->topk, shown_cap,
g_dense_bits == 0 ? "native" : g_dense_bits == 8 ? "int8" : "f32", m->ctx,
g_vision ? ", vision" : "", now_s() - t0);
char tok_path[4096];
snprintf(tok_path, sizeof(tok_path), "%s/tokenizer.json", dir);
int have_tok = access(tok_path, R_OK) == 0;
Tok tokenizer;
if (have_tok) tok_load(&tokenizer, tok_path);
const char *seed = getenv("SEED");
srand(seed ? (unsigned)strtoul(seed, NULL, 10) : (unsigned)time(NULL));
if (env_int("SERVE", 0)) {
if (!have_tok) { fprintf(stderr, "[mimo] SERVE needs %s\n", tok_path); return 1; }
if (g_mimo_mux_slots > 1) { /* slot 0 is the Model's own caches */
g_mimo_mux_seq = xcalloc((size_t)g_mimo_mux_slots, sizeof *g_mimo_mux_seq, "conversations");
for (int i = 1; i < g_mimo_mux_slots; i++) mimo_seq_alloc(m, &g_mimo_mux_seq[i]);
g_mimo_mux_cur = 0;
}
serve_loop(m, &tokenizer, dir);
vk_report(m);
return 0;
}
int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "ids");
int n = 0;
if (ids_text) n = parse_ids(ids_text, ids, m->ctx);
else if (prompt && have_tok) n = tok_encode(&tokenizer, prompt, (int)strlen(prompt), ids, m->ctx);
else { fprintf(stderr, "[mimo] give --ids, or --prompt with a tokenizer.json in the model dir\n"); return 2; }
if (n < 1) { fprintf(stderr, "[mimo] empty prompt\n"); return 2; }
if (n + ngen > m->ctx) { fprintf(stderr, "[mimo] prompt + ngen exceed the context %d\n", m->ctx); return 2; }
float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");
const char *dump = getenv("MIMO_LOGITS");
float *all = dump ? xmalloc((size_t)n * c->vocab * sizeof(float), "all logits") : NULL;
/* --image patches.f32 --grid H W: the gateway's patches for the image-pad
* ids already in the prompt */
float *image = NULL;
int image_rows = 0;
if (image_path) {
if (!g_vision) { fprintf(stderr, "[mimo] --image: this container has no vision tower\n"); return 2; }
size_t want = (size_t)grid_h * grid_w * g_vision->patch_in;
float *patches = xmalloc(want * sizeof(float), "patches");
FILE *f = fopen(image_path, "rb");
if (!f || fread(patches, sizeof(float), want, f) != want || fgetc(f) != EOF) {
fprintf(stderr, "[mimo] %s does not hold %dx%d patches\n", image_path, grid_h, grid_w); return 2;
}
fclose(f);
int pads = 0;
for (int t = 0; t < n; t++) pads += ids[t] == c->image_token_id;
if (pads != (grid_h / 2) * (grid_w / 2)) {
fprintf(stderr, "[mimo] the prompt has %d image pads, the grid needs %d\n", pads, (grid_h / 2) * (grid_w / 2));
return 2;
}
image = vision_run(m, patches, grid_h, grid_w, &image_rows);
free(patches);
}
ImageRows img = { image, image_rows };
double tp = now_s();
prefill(m, ids, n, logits, all, image ? &img : NULL, NULL);
double prefill_s = now_s() - tp;
if (dump) {
FILE *f = fopen(dump, "wb");
if (!f || fwrite(all, sizeof(float), (size_t)n * c->vocab, f) != (size_t)n * c->vocab) {
fprintf(stderr, "[mimo] cannot write %s\n", dump); return 1;
}
fclose(f);
}
float temperature = getenv("COLI_TEMP") ? (float)atof(getenv("COLI_TEMP")) : 0.0f;
int eos[8], n_eos = coli_load_stop_ids(dir, eos, 8, NULL);
double td = now_s();
int produced = 0;
char piece[512];
for (int g = 0; g < ngen; g++) {
int token = sample(logits, c->vocab, temperature, 1.0f);
int stop = 0;
for (int i = 0; i < n_eos; i++) stop |= token == eos[i];
if (have_tok && !ids_text) {
int w = tok_decode(&tokenizer, &token, 1, piece, (int)sizeof(piece));
fwrite(piece, 1, (size_t)(w > 0 ? w : 0), stdout);
} else printf("%d ", token);
fflush(stdout);
produced++;
if (stop && have_tok && !ids_text) break;
if (g + 1 < ngen) prefill(m, &token, 1, logits, NULL, NULL, NULL);
}
printf("\n");
fflush(stdout);
double decode_s = now_s() - td;
double total = prefill_s + decode_s;
fprintf(stderr, "[mimo] prefill %d tokens %.2fs, decode %d tokens %.2fs (%.2f tok/s), experts %llu hits "
"%llu misses %.1f MB read, rss %.2f GB\n"
"[mimo] time: expert reads %.2fs (%.2f GB/s), expert matmuls %.2fs, attention %.2fs, "
"the rest (dense MLP, router, head) %.2fs\n",
n, prefill_s, produced, decode_s, decode_s > 0 ? produced / decode_s : 0.0,
(unsigned long long)m->hits, (unsigned long long)m->miss, m->bytes_read / 1e6, rss_gb(),
m->t_disk, m->t_disk > 0 ? m->bytes_read / 1e9 / m->t_disk : 0.0, m->t_expert, m->t_attn,
total - m->t_disk - m->t_expert - m->t_attn);
vk_report(m);
return 0;
}