Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

mimo: several conversations at once (KV_SLOTS up to 16)

JustVugg8257fca32b95
diff --git a/c/mimo.c b/c/mimo.c
index ba2fd79c..66e753a5 100644
--- a/c/mimo.c
+++ b/c/mimo.c
@@ -593,6 +593,20 @@ typedef struct {
int rows;
} Layer;
+/* One conversation's caches for a multiplexed serve (KV_SLOTS>1, serve_mux below):
+ * every layer's K and V (a full layer's rows, a sliding window's ring and its
+ * positions), where it stands, and the record of the tokens it holds. The Model's
+ * layers hold the conversation a prefill runs on (mimo_seq_swap trades it for a parked
+ * one); a multiplexed decode step parks them all and reads each row's from its MimoRow. */
+typedef struct {
+ float *K[MIMO_MAX_LAYERS], *V[MIMO_MAX_LAYERS];
+ int *ring_pos[MIMO_MAX_LAYERS];
+ int pos;
+ kv_prefix kvp;
+} MimoSeq;
+typedef struct { MimoSeq *seq; int pos; } MimoRow;
+static int g_mimo_mux_slots = 1; /* KV_SLOTS: the conversations a serve decodes at once */
+
typedef struct {
Cfg c;
shards S;
@@ -613,6 +627,9 @@ typedef struct {
void *vkchain; /* the dense chain's device state (mimo_chain.h), NULL until it runs */
void *vkchain2; /* its layers on COLI_VK_DEV2's device, after the primary's (mimo_chain.h) */
#endif
+ /* A multiplexed decode step (forward_rows): row t is the token at mux_rows[t].pos
+ * of mux_rows[t].seq; NULL in every other forward. */
+ const MimoRow *mux_rows;
} Model;
static void expert_table_init(Model *m) {
@@ -858,7 +875,105 @@ static void rope(float *v, int rd, float theta, int pos) {
/* Attention of `n` rows at positions p0.. of layer li. `xn` is the normed input
* [n, H], `out` receives o_proj's output [n, H]. */
+/* A multiplexed decode step's attention (KV_SLOTS, m->mux_rows): row t is the next
+ * token of its own conversation, at mux_rows[t].pos, over that conversation's cache --
+ * a full layer's rows, or a sliding window's ring. Each row's numbers are attention()'s
+ * for a block of that one token; the projections run once over the n rows. */
+static void attention_rows(Model *m, int li, const float *xn, int n, float *out) {
+ Cfg *c = &m->c;
+ Layer *l = &m->L[li];
+ const MimoRow *mr = m->mux_rows;
+ int k = c->swa[li];
+ int nh = c->heads[k], kvh = c->kv_heads[k], hd = c->head_dim[k], vd = c->v_dim[k];
+ int qd = nh * hd, kd = kvh * hd, vdd = kvh * vd, rows = qd + kd + vdd;
+ double t0 = now_s();
+ float *qkv = xmalloc((size_t)n * rows * sizeof(float), "qkv");
+ dw_matmul(qkv, xn, n, &l->qkv);
+ for (int t = 0; t < n; t++) {
+ float *q = qkv + (size_t)t * rows, *kk = q + qd, *vv = kk + kd;
+ for (int h = 0; h < nh; h++) rope(q + h * hd, c->rope_dim[k], c->theta[k], mr[t].pos);
+ for (int h = 0; h < kvh; h++) rope(kk + h * hd, c->rope_dim[k], c->theta[k], mr[t].pos);
+ for (int i = 0; i < vdd; i++) vv[i] *= c->v_scale;
+ }
+ int window = k ? c->window : 0, most = 1;
+ const float **Kc = xmalloc((size_t)n * sizeof *Kc, "row keys"), **Vc = xmalloc((size_t)n * sizeof *Vc, "row values");
+ int *lo = xmalloc((size_t)n * sizeof(int), "row first keys");
+ for (int t = 0; t < n; t++) {
+ MimoSeq *q = mr[t].seq;
+ int p = mr[t].pos;
+ const float *src = qkv + (size_t)t * rows + qd;
+ if (k) { /* the window's keys before p from the ring, then this token's */
+ int l0 = p - (window - 1) > 0 ? p - (window - 1) : 0;
+ float *kc = xmalloc((size_t)(p + 1 - l0) * kd * sizeof(float), "window keys");
+ float *vc = xmalloc((size_t)(p + 1 - l0) * vdd * sizeof(float), "window values");
+ for (int pp = l0; pp < p; pp++) {
+ int slot = pp % l->rows;
+ if (q->ring_pos[li][slot] != pp) { fprintf(stderr, "[mimo] window ring lost position %d\n", pp); exit(1); }
+ memcpy(kc + (size_t)(pp - l0) * kd, q->K[li] + (size_t)slot * kd, (size_t)kd * sizeof(float));
+ memcpy(vc + (size_t)(pp - l0) * vdd, q->V[li] + (size_t)slot * vdd, (size_t)vdd * sizeof(float));
+ }
+ memcpy(kc + (size_t)(p - l0) * kd, src, (size_t)kd * sizeof(float));
+ memcpy(vc + (size_t)(p - l0) * vdd, src + kd, (size_t)vdd * sizeof(float));
+ Kc[t] = kc; Vc[t] = vc; lo[t] = l0;
+ } else { /* the full cache, this token written in first */
+ memcpy(q->K[li] + (size_t)p * kd, src, (size_t)kd * sizeof(float));
+ memcpy(q->V[li] + (size_t)p * vdd, src + kd, (size_t)vdd * sizeof(float));
+ Kc[t] = q->K[li]; Vc[t] = q->V[li]; lo[t] = 0;
+ }
+ if (p + 1 - lo[t] > most) most = p + 1 - lo[t];
+ }
+ float *ctxv = xmalloc((size_t)n * nh * vd * sizeof(float), "attention output");
+ float scale = 1.0f / sqrtf((float)hd);
+ int group = nh / kvh;
+ #pragma omp parallel
+ {
+ float *score = xmalloc((size_t)most * sizeof(float), "scores");
+ #pragma omp for collapse(2) schedule(static)
+ for (int t = 0; t < n; t++) {
+ for (int h = 0; h < nh; h++) {
+ int p = mr[t].pos, kh = h / group, first = lo[t];
+ const float *q = qkv + (size_t)t * rows + (size_t)h * hd;
+ float best = -INFINITY;
+ for (int j = first; j <= p; j++) {
+ const float *kr = Kc[t] + (size_t)(j - lo[t]) * kd + (size_t)kh * hd;
+ float s = 0;
+ for (int i = 0; i < hd; i++) s += q[i] * kr[i];
+ s *= scale;
+ score[j - first] = s;
+ if (s > best) best = s;
+ }
+ float sink = l->sink ? l->sink[h] : -INFINITY;
+ if (sink > best) best = sink;
+ float den = l->sink ? expf(sink - best) : 0.0f;
+ for (int j = first; j <= p; j++) { score[j - first] = expf(score[j - first] - best); den += score[j - first]; }
+ float *o = ctxv + ((size_t)t * nh + h) * vd;
+ for (int i = 0; i < vd; i++) o[i] = 0;
+ for (int j = first; j <= p; j++) {
+ const float *vr = Vc[t] + (size_t)(j - lo[t]) * vdd + (size_t)kh * vd;
+ float w = score[j - first] / den;
+ for (int i = 0; i < vd; i++) o[i] += w * vr[i];
+ }
+ }
+ }
+ free(score);
+ }
+ if (k) /* the ring keeps each conversation's last `window` positions */
+ for (int t = 0; t < n; t++) {
+ MimoSeq *q = mr[t].seq;
+ int p = mr[t].pos, slot = p % l->rows;
+ const float *src = qkv + (size_t)t * rows + qd;
+ memcpy(q->K[li] + (size_t)slot * kd, src, (size_t)kd * sizeof(float));
+ memcpy(q->V[li] + (size_t)slot * vdd, src + kd, (size_t)vdd * sizeof(float));
+ q->ring_pos[li][slot] = p;
+ free((void *)Kc[t]); free((void *)Vc[t]);
+ }
+ dw_matmul(out, ctxv, n, &l->o);
+ free(ctxv); free(qkv); free(Kc); free(Vc); free(lo);
+ m->t_attn += now_s() - t0;
+}
+
static void attention(Model *m, int li, const float *xn, int n, int p0, float *out) {
+ if (m->mux_rows) { attention_rows(m, li, xn, n, out); return; }
Cfg *c = &m->c;
Layer *l = &m->L[li];
int k = c->swa[li];
@@ -1276,6 +1391,60 @@ static void forward(Model *m, const int *ids, int n, float *logits, int all_rows
free(h); free(xn); free(tmp);
}
+/* ---- several conversations at once (KV_SLOTS>1, serve_mux) -------------------
+ * Each conversation owns a MimoSeq. The Model's layers hold the conversation a
+ * prefill runs on (mimo_seq_swap trades it for a parked one); a decode step parks
+ * them all and runs one forward over a row of each (forward_rows). */
+static void mimo_seq_swap(Model *m, MimoSeq *q) {
+ for (int li = 0; li < m->c.n_layers; li++) {
+ Layer *l = &m->L[li];
+ float *K = l->K, *V = l->V; int *rp = l->ring_pos;
+ l->K = q->K[li]; l->V = q->V[li]; l->ring_pos = q->ring_pos[li];
+ q->K[li] = K; q->V[li] = V; q->ring_pos[li] = rp;
+ }
+ int pos = m->pos; kv_prefix p = m->kvp;
+ m->pos = q->pos; m->kvp = q->kvp;
+ q->pos = pos; q->kvp = p;
+}
+/* A conversation's caches of its own, the shape kv_alloc gives the Model's. */
+static void mimo_seq_alloc(Model *m, MimoSeq *q) {
+ Cfg *c = &m->c;
+ memset(q, 0, sizeof *q);
+ for (int li = 0; li < c->n_layers; li++) {
+ Layer *l = &m->L[li];
+ int k = c->swa[li];
+ q->K[li] = xmalloc((size_t)l->rows * c->kv_heads[k] * c->head_dim[k] * sizeof(float), "K cache");
+ q->V[li] = xmalloc((size_t)l->rows * c->kv_heads[k] * c->v_dim[k] * sizeof(float), "V cache");
+ q->ring_pos[li] = xmalloc((size_t)l->rows * sizeof(int), "ring positions");
+ for (int r = 0; r < l->rows; r++) q->ring_pos[li][r] = -1;
+ }
+ kv_prefix_alloc(&q->kvp, m->ctx);
+}
+
+/* One decode step of several conversations: row t is the token ids[t] at rows[t].pos
+ * of the conversation rows[t].seq, every conversation parked. The matrices, the routed
+ * experts and lm_head run once over the n rows; the attention reads and writes each
+ * row's own caches (attention_rows). The CPU kernels give a row the same bits whatever
+ * n is, so each conversation gets the logits it would alone: [n, V] into logits. */
+static void forward_rows(Model *m, const MimoRow *rows, const int *ids, int n, float *logits) {
+ Cfg *c = &m->c;
+ int H = c->hidden;
+ float *h = xmalloc((size_t)n * H * sizeof(float), "residual");
+ float *xn = xmalloc((size_t)n * H * sizeof(float), "normed");
+ float *tmp = xmalloc((size_t)n * H * sizeof(float), "block out");
+ embed_rows(m, ids, n, h);
+ m->mux_rows = rows;
+ layers_cpu(m, h, n, 0, 0, xn, tmp, NULL);
+ m->mux_rows = NULL;
+ head_cpu(m, h, n, logits, 1, xn);
+ for (int t = 0; t < n; t++) {
+ kv_prefix_record(&rows[t].seq->kvp, ids + t, rows[t].pos, 1);
+ rows[t].seq->pos = rows[t].pos + 1;
+ }
+ m->forwards++;
+ free(h); free(xn); free(tmp);
+}
+
/* The read-out of a prefill, for the logprobs channel (SUBMIT logprobs=k): one
* ECHO frame per prompt position this prefill feeds, in position order, before
* any DATA. The logits at position p predict the token at p + 1, so the frame
@@ -1910,7 +2079,264 @@ static int serve_budget(int prompt, int requested, int context, int logprobs) {
return budget < room ? budget : room;
}
+/* When a request was accepted, and the counters then: DONE and PROF report its share. */
+typedef struct { double started, disk0, expert0, attn0; uint64_t hits0, miss0, fw0; int n_prompt, budget; } MimoReq;
+
+/* A request's prompt into the caches the Model's layers hold: its tokens, the budget,
+ * the image, the prefix reuse and the photos, ACCEPT, the prefill and its read-out.
+ * 1 with the logits after the prompt; 0 when the request ended here, its ERROR written
+ * and its command disposed. serve_loop and serve_mux start every request here. */
+static int mimo_serve_start(Model *m, Tok *tokenizer, ColiServeCommand *cmd, int *ids, float *logits,
+ float **pending, int *pending_h, int *pending_w, MimoReq *rq) {
+ Cfg *c = &m->c;
+ double started = now_s();
+ double disk0 = m->t_disk, expert0 = m->t_expert, attn0 = m->t_attn;
+ uint64_t hits0 = m->hits, miss0 = m->miss, fw0 = m->forwards;
+ int n_prompt = tok_encode(tokenizer, (const char *)cmd->payload, (int)cmd->payload_bytes,
+ ids, m->ctx + 1);
+ int budget = serve_budget(n_prompt, cmd->max_tokens, m->ctx, cmd->logprobs);
+ if (budget < 0) {
+ char message[160];
+ snprintf(message, sizeof(message), "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
+ n_prompt, cmd->max_tokens, m->ctx);
+ coli_serve_write_error(stdout, cmd->id, n_prompt < 1 ? "EMPTY_PROMPT" : message);
+ free((*pending)); (*pending) = NULL;
+ coli_serve_command_dispose(cmd);
+ return 0;
+ }
+ /* An image: its rows replace the pad ids, which must be exactly as many
+ * as the tower produces. Checked before ACCEPT so a mismatch is a clean
+ * 400, not a half-answered turn. */
+ float *image = NULL;
+ int image_rows = 0;
+ if ((*pending)) {
+ int pads = 0;
+ for (int t = 0; t < n_prompt; t++) pads += ids[t] == c->image_token_id;
+ int want = ((*pending_h) / 2) * ((*pending_w) / 2);
+ if (pads != want) {
+ char message[160];
+ snprintf(message, sizeof(message), "BAD_IMAGE prompt has %d image pads, the %dx%d grid needs %d",
+ pads, (*pending_h), (*pending_w), want);
+ coli_serve_write_error(stdout, cmd->id, message);
+ free((*pending)); (*pending) = NULL;
+ coli_serve_command_dispose(cmd);
+ return 0;
+ }
+ }
+ /* Where this prompt starts. A chat client resends the whole transcript:
+ * if this prompt begins with the ids the state was built from, only the
+ * tail is fed (COLI_KV_PREFIX=0 turns that off). A photo (SUBMIT pin=1
+ * earlier) resumes a prompt that begins with it.
+ *
+ * With the read-out on (logprobs=k) only a photo will do. The read-out
+ * owes a frame to every position it does not inherit a predictor for,
+ * and a live prefix comes with none: the frames before it would simply
+ * be missing. So such a prompt resumes from the deepest photo, which
+ * carries the logits that predict its first fresh token, or is read out
+ * from position 0. It resumes at the photo even when the live state
+ * shares more: two options share the text before them, and stopping at
+ * that would leave the first option token without its predictor.
+ *
+ * An image refuses both: the pad ids do not describe the picture. */
+ int prefix_on = env_int("COLI_KV_PREFIX", 1) != 0;
+ int reuse = 0;
+ const float *pin_logit = NULL;
+ if (!(*pending)) {
+ if (cmd->logprobs > 0) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
+ else {
+ reuse = prefix_on ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;
+ if (!reuse) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
+ }
+ }
+ if (getenv("COLI_PREFIX_LOG"))
+ fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens%s\n", reuse ? "reusing" : "no reuse,", reuse,
+ n_prompt, pin_logit ? " (photo)" : "");
+ if (pin_logit) {
+ fprintf(stderr, "[PIN] resumed from the photo of %d tokens, %d to prefill\n", reuse, n_prompt - reuse);
+ fflush(stderr);
+ }
+ if (!reuse) model_reset(m);
+ coli_serve_write_accept(stdout, cmd->id, n_prompt);
+ if ((*pending)) {
+ image = vision_run(m, (*pending), (*pending_h), (*pending_w), &image_rows);
+ free((*pending)); (*pending) = NULL;
+ }
+ ImageRows img = { image, image_rows };
+ Echo echo = { cmd->id, cmd->logprobs, tokenizer, pin_logit };
+ prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL,
+ cmd->logprobs > 0 ? &echo : NULL);
+ if (image) kv_prefix_taint(&m->kvp);
+ if (cmd->pin) {
+ /* a photo is described by its ids, and these do not describe the picture */
+ if (image) fprintf(stderr, "[PIN] no photo: the prompt carries a picture\n");
+ else pin_save(m, ids, n_prompt, logits);
+ }
+ free(image);
+ rq->started = started; rq->disk0 = disk0; rq->expert0 = expert0; rq->attn0 = attn0;
+ rq->hits0 = hits0; rq->miss0 = miss0; rq->fw0 = fw0; rq->n_prompt = n_prompt; rq->budget = budget;
+ return 1;
+}
+
+/* ---- several conversations at once (KV_SLOTS>1) ---------------------------------
+ * The gateway's cache slots, each a conversation with caches of its own (MimoSeq). A
+ * SUBMIT on a free slot starts its request at once through mimo_serve_start, on that
+ * slot's caches: its prefix reuse and photos work as a lone serve's. Then every step
+ * picks the next token of each active request and runs one forward over a row of each
+ * (forward_rows): the matrices and the experts are read once for all of them. A
+ * request's frames are a lone request's; they interleave by id. As alone, STOP ends a
+ * request with DONE and CANCEL with ERROR CANCELLED. */
+static MimoSeq *g_mimo_mux_seq; /* [slots]: the conversations the Model does not hold */
+static int g_mimo_mux_cur; /* the slot the Model holds, -1 when every one is parked */
+
+typedef struct {
+ char id[COLI_SERVE_ID_CAP];
+ float temperature, top_p;
+ int logprobs, active, stop, cancel, limited, emitted;
+ float *logits; /* the logits the next pick reads */
+ MimoReq rq;
+} MimoMuxReq;
+
+static void mimo_mux_bind(Model *m, int slot) {
+ if (g_mimo_mux_cur == slot) return;
+ if (g_mimo_mux_cur >= 0) mimo_seq_swap(m, &g_mimo_mux_seq[g_mimo_mux_cur]);
+ if (slot >= 0) mimo_seq_swap(m, &g_mimo_mux_seq[slot]);
+ g_mimo_mux_cur = slot;
+}
+
+/* A request's end, as serve_loop ends one. */
+static void mimo_mux_finish(Model *m, MimoMuxReq *r) {
+ r->active = 0;
+ if (r->cancel) { coli_serve_write_error(stdout, r->id, "CANCELLED"); return; }
+ if (r->stop) r->limited = 0;
+ double wall = now_s() - r->rq.started;
+ uint64_t th = m->hits - r->rq.hits0, tm = m->miss - r->rq.miss0;
+ ColiServeDone done = { r->emitted, wall > 0 ? r->emitted / wall : 0.0,
+ (th + tm) ? 100.0 * th / (double)(th + tm) : 0.0, rss_gb(), r->rq.n_prompt, r->limited };
+ coli_serve_write_done(stdout, r->id, &done);
+ serve_line("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n", wall, r->rq.n_prompt, r->emitted,
+ m->t_disk - r->rq.disk0, 0.0, m->t_expert - r->rq.expert0, m->t_attn - r->rq.attn0, 0.0,
+ (unsigned long long)(m->forwards - r->rq.fw0));
+ serve_hits(m);
+}
+
+/* The next token of an active request, as serve_loop picks and sends it: 1 with the
+ * token when the request goes on, 0 when it ended. */
+static int mimo_mux_pick(Model *m, Tok *tokenizer, MimoMuxReq *r, const int *eos, int n_eos, int *tk_out) {
+ Cfg *c = &m->c;
+ if (r->cancel || r->stop || r->emitted >= r->rq.budget) { mimo_mux_finish(m, r); return 0; }
+ int token = sample(r->logits, c->vocab, r->temperature, r->top_p);
+ for (int i = 0; i < n_eos; i++) if (token == eos[i]) { r->limited = 0; mimo_mux_finish(m, r); return 0; }
+ char piece[512];
+ int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
+ if (written > 0) {
+ if (r->logprobs > 0) {
+ char tail[1024];
+ coli_logprob_tail(tail, sizeof(tail), r->logits, c->vocab, token, r->logprobs);
+ coli_serve_write_data_lp(stdout, r->id, piece, (size_t)written, tail);
+ } else coli_serve_write_data(stdout, r->id, piece, (size_t)written);
+ }
+ r->emitted++;
+ if (r->emitted >= r->rq.budget) { mimo_mux_finish(m, r); return 0; }
+ *tk_out = token; return 1;
+}
+
+static void serve_mux(Model *m, Tok *tokenizer, const char *dir) {
+ Cfg *c = &m->c;
+ int n = g_mimo_mux_slots, V = c->vocab, input_eof = 0;
+ int eos[8];
+ int n_eos = coli_load_stop_ids(dir, eos, 8, NULL);
+ coli_serve_write_ready_caps(stdout, rss_gb(), g_vision ? "vision=1" : "vision=0");
+ serve_emap(m);
+ MimoMuxReq *rq = xcalloc((size_t)n, sizeof *rq, "requests");
+ MimoRow *rows = xmalloc((size_t)n * sizeof *rows, "rows");
+ int *tok = xmalloc((size_t)n * sizeof(int), "tokens"), *who = xmalloc((size_t)n * sizeof(int), "slots");
+ float *lo = xmalloc((size_t)n * V * sizeof(float), "step logits");
+ int *ids = xmalloc(((size_t)m->ctx + 1) * sizeof(int), "prompt ids");
+ for (int i = 0; i < n; i++) rq[i].logits = xmalloc((size_t)V * sizeof(float), "logits");
+ float *pending = NULL;
+ int pending_h = 0, pending_w = 0;
+ unsigned long long steps = 0, nrows = 0;
+ fprintf(stderr, "[mimo] serving %d conversations at once (KV_SLOTS)\n", n);
+ for (;;) {
+ int active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
+ /* idle: wait for a command; decoding: take one only when one is there */
+ if (!input_eof && (!active || coli_serve_stdin_ready())) {
+ ColiServeCommand command;
+ ColiServeReadResult result = coli_serve_read_command(stdin, &mimo_wire, &command);
+ if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) input_eof = 1;
+ else if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); input_eof = 1; }
+ else if (result == COLI_SERVE_READ_BAD_REQUEST) {
+ if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");
+ coli_serve_command_dispose(&command);
+ } else if (command.kind == COLI_SERVE_COMMAND_IMAGE) {
+ /* one image waits for the SUBMIT that names it; a second replaces it */
+ uint64_t expected = g_vision ? (uint64_t)command.grid_h * command.grid_w * g_vision->patch_in * sizeof(float) : 0;
+ if (!g_vision) coli_serve_write_error(stdout, command.id, "this container has no vision tower");
+ else if (command.grid_h < 2 || command.grid_w < 2 || command.grid_h % 2 || command.grid_w % 2 ||
+ command.payload_bytes != expected)
+ coli_serve_write_error(stdout, command.id, "BAD_IMAGE payload does not match its grid");
+ else {
+ free(pending);
+ pending = (float *)coli_serve_command_take_payload(&command);
+ pending_h = command.grid_h; pending_w = command.grid_w;
+ }
+ coli_serve_command_dispose(&command);
+ } else if (command.kind == COLI_SERVE_COMMAND_CANCEL || command.kind == COLI_SERVE_COMMAND_STOP) {
+ int found = 0;
+ for (int i = 0; i < n; i++) if (rq[i].active && !strcmp(rq[i].id, command.id)) {
+ found = 1;
+ if (command.kind == COLI_SERVE_COMMAND_STOP) rq[i].stop = 1; else rq[i].cancel = 1;
+ }
+ if (!found && command.kind == COLI_SERVE_COMMAND_CANCEL) coli_serve_write_error(stdout, command.id, "NOT_FOUND");
+ coli_serve_command_dispose(&command);
+ } else if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {
+ coli_serve_command_dispose(&command);
+ } else if (command.slot < 0 || command.slot >= n || rq[command.slot].active) {
+ coli_serve_write_error(stdout, command.id, command.slot < 0 || command.slot >= n ? "invalid cache slot" : "SLOT_BUSY");
+ coli_serve_command_dispose(&command);
+ } else {
+ MimoMuxReq *t = &rq[command.slot];
+ float *keep = t->logits;
+ memset(t, 0, sizeof *t); t->logits = keep;
+ snprintf(t->id, sizeof t->id, "%s", command.id);
+ t->temperature = command.temperature; t->top_p = command.top_p; t->logprobs = command.logprobs;
+ mimo_mux_bind(m, command.slot);
+ if (mimo_serve_start(m, tokenizer, &command, ids, t->logits, &pending, &pending_h, &pending_w, &t->rq)) {
+ t->active = 1; t->limited = 1;
+ coli_serve_command_dispose(&command);
+ }
+ }
+ }
+ active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
+ if (!active) { if (input_eof) break; continue; }
+ int S = 0, ended = 0;
+ for (int i = 0; i < n; i++) if (rq[i].active) {
+ int tk;
+ if (!mimo_mux_pick(m, tokenizer, &rq[i], eos, n_eos, &tk)) { ended = 1; continue; }
+ rows[S] = (MimoRow){&g_mimo_mux_seq[i], g_mimo_mux_cur == i ? m->pos : g_mimo_mux_seq[i].pos};
+ tok[S] = tk; who[S] = i; S++;
+ }
+ if (S) {
+ mimo_mux_bind(m, -1); /* every conversation parked: the rows read theirs */
+ for (int s = 0; s < S; s++) rows[s].pos = g_mimo_mux_seq[who[s]].pos;
+ forward_rows(m, rows, tok, S, lo);
+ steps++; nrows += (unsigned long long)S;
+ for (int s = 0; s < S; s++) memcpy(rq[who[s]].logits, lo + (size_t)s * V, (size_t)V * sizeof(float));
+ }
+ if (ended) {
+ serve_emap(m);
+ vkt_report("turn", m->hits, m->miss);
+ }
+ }
+ fprintf(stderr, "[mimo] KV_SLOTS=%d: %llu decode steps, %llu rows (%.2f a step)\n", n, steps, nrows,
+ steps ? (double)nrows / (double)steps : 0.0);
+ mimo_mux_bind(m, 0);
+ for (int i = 0; i < n; i++) free(rq[i].logits);
+ free(rq); free(rows); free(tok); free(who); free(lo); free(ids); free(pending);
+}
+
static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
+ if (g_mimo_mux_slots > 1) { serve_mux(m, tokenizer, dir); return; }
Cfg *c = &m->c;
coli_serve_stdio_init();
int eos[8];
@@ -1957,89 +2383,12 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
coli_serve_command_dispose(&command);
continue;
}
- double started = now_s();
- double disk0 = m->t_disk, expert0 = m->t_expert, attn0 = m->t_attn;
- uint64_t hits0 = m->hits, miss0 = m->miss, fw0 = m->forwards;
- int n_prompt = tok_encode(tokenizer, (const char *)command.payload, (int)command.payload_bytes,
- ids, m->ctx + 1);
- int budget = serve_budget(n_prompt, command.max_tokens, m->ctx, command.logprobs);
- if (budget < 0) {
- char message[160];
- snprintf(message, sizeof(message), "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
- n_prompt, command.max_tokens, m->ctx);
- coli_serve_write_error(stdout, command.id, n_prompt < 1 ? "EMPTY_PROMPT" : message);
- free(pending); pending = NULL;
- coli_serve_command_dispose(&command);
- continue;
- }
- /* An image: its rows replace the pad ids, which must be exactly as many
- * as the tower produces. Checked before ACCEPT so a mismatch is a clean
- * 400, not a half-answered turn. */
- float *image = NULL;
- int image_rows = 0;
- if (pending) {
- int pads = 0;
- for (int t = 0; t < n_prompt; t++) pads += ids[t] == c->image_token_id;
- int want = (pending_h / 2) * (pending_w / 2);
- if (pads != want) {
- char message[160];
- snprintf(message, sizeof(message), "BAD_IMAGE prompt has %d image pads, the %dx%d grid needs %d",
- pads, pending_h, pending_w, want);
- coli_serve_write_error(stdout, command.id, message);
- free(pending); pending = NULL;
- coli_serve_command_dispose(&command);
- continue;
- }
- }
- /* Where this prompt starts. A chat client resends the whole transcript:
- * if this prompt begins with the ids the state was built from, only the
- * tail is fed (COLI_KV_PREFIX=0 turns that off). A photo (SUBMIT pin=1
- * earlier) resumes a prompt that begins with it.
- *
- * With the read-out on (logprobs=k) only a photo will do. The read-out
- * owes a frame to every position it does not inherit a predictor for,
- * and a live prefix comes with none: the frames before it would simply
- * be missing. So such a prompt resumes from the deepest photo, which
- * carries the logits that predict its first fresh token, or is read out
- * from position 0. It resumes at the photo even when the live state
- * shares more: two options share the text before them, and stopping at
- * that would leave the first option token without its predictor.
- *
- * An image refuses both: the pad ids do not describe the picture. */
- int prefix_on = env_int("COLI_KV_PREFIX", 1) != 0;
- int reuse = 0;
- const float *pin_logit = NULL;
- if (!pending) {
- if (command.logprobs > 0) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
- else {
- reuse = prefix_on ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;
- if (!reuse) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
- }
- }
- if (getenv("COLI_PREFIX_LOG"))
- fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens%s\n", reuse ? "reusing" : "no reuse,", reuse,
- n_prompt, pin_logit ? " (photo)" : "");
- if (pin_logit) {
- fprintf(stderr, "[PIN] resumed from the photo of %d tokens, %d to prefill\n", reuse, n_prompt - reuse);
- fflush(stderr);
- }
- if (!reuse) model_reset(m);
- coli_serve_write_accept(stdout, command.id, n_prompt);
- if (pending) {
- image = vision_run(m, pending, pending_h, pending_w, &image_rows);
- free(pending); pending = NULL;
- }
- ImageRows img = { image, image_rows };
- Echo echo = { command.id, command.logprobs, tokenizer, pin_logit };
- prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL,
- command.logprobs > 0 ? &echo : NULL);
- if (image) kv_prefix_taint(&m->kvp);
- if (command.pin) {
- /* a photo is described by its ids, and these do not describe the picture */
- if (image) fprintf(stderr, "[PIN] no photo: the prompt carries a picture\n");
- else pin_save(m, ids, n_prompt, logits);
- }
- free(image);
+ MimoReq rq;
+ if (!mimo_serve_start(m, tokenizer, &command, ids, logits, &pending, &pending_h, &pending_w, &rq)) continue;
+ double started = rq.started;
+ double disk0 = rq.disk0, expert0 = rq.expert0, attn0 = rq.attn0;
+ uint64_t hits0 = rq.hits0, miss0 = rq.miss0, fw0 = rq.fw0;
+ int n_prompt = rq.n_prompt, budget = rq.budget;
int emitted = 0, limited = 1, cancelled = 0, done_early = 0;
char piece[512];
while (emitted < budget && !cancelled && !done_early) {
@@ -2134,6 +2483,14 @@ int main(int argc, char **argv) {
* the tests: mimo <dir> [--ids "..." | --prompt "..."] [--ngen N] [--cap N]. */
const char *dir = getenv("SNAP");
int cap = env_int("MIMO_CAP", 0);
+ if (env_int("SERVE", 0)) { /* KV_SLOTS: the conversations a serve decodes at once */
+ const char *ks = getenv("KV_SLOTS");
+ if (ks && *ks) {
+ char *end = NULL; long v = strtol(ks, &end, 10);
+ if (end == ks || *end || v < 1 || v > 16) { fprintf(stderr, "KV_SLOTS must be between 1 and 16\n"); return 2; }
+ g_mimo_mux_slots = (int)v;
+ }
+ }
const char *ids_text = NULL, *prompt = NULL, *image_path = NULL;
int ngen = 32, grid_h = 0, grid_w = 0;
for (int i = 1; i < argc; i++) {
@@ -2174,6 +2531,11 @@ int main(int argc, char **argv) {
* MiMo checkpoint: an integrated GPU keeps it opt-in (docs/vulkan.md). */
if (g_vk_ready) {
g_vk_chain = coli_vk_chain_decide("mimo", tier, COLI_VK_CHAIN_UNMEASURED);
+ if (g_vk_chain && g_mimo_mux_slots > 1) {
+ g_vk_chain = 0;
+ fprintf(stderr, "[VK] mimo: KV_SLOTS=%d: the dense chain is off (it keeps one conversation's caches on the device); "
+ "the expert tier runs every conversation's experts\n", g_mimo_mux_slots);
+ }
/* the chain's fit before any upload: its first n layers on the device, the
* others, the head and the tower on the CPU when they do not all fit */
if (g_vk_chain) mc_fit_start(m);
@@ -2219,6 +2581,11 @@ int main(int argc, char **argv) {
if (env_int("SERVE", 0)) {
if (!have_tok) { fprintf(stderr, "[mimo] SERVE needs %s\n", tok_path); return 1; }
+ if (g_mimo_mux_slots > 1) { /* slot 0 is the Model's own caches */
+ g_mimo_mux_seq = xcalloc((size_t)g_mimo_mux_slots, sizeof *g_mimo_mux_seq, "conversations");
+ for (int i = 1; i < g_mimo_mux_slots; i++) mimo_seq_alloc(m, &g_mimo_mux_seq[i]);
+ g_mimo_mux_cur = 0;
+ }
serve_loop(m, &tokenizer, dir);
vk_report(m);
return 0;