Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(olmoe): implement the SERVE protocol so coli chat/web/serve work
shujaanazhar5684d3f691d1
diff --git a/c/coli b/c/coliindex 55d76dc3..63b724c9 100755--- a/c/coli+++ b/c/coli@@ -850,10 +850,12 @@ def cmd_run(a):except OSError: passsys.exit(result)if arch=="olmoe":- # OLMoE has a persistent stdin chat loop but not the gateway's SERVE- # protocol yet. Feed exactly one turn and close stdin: the engine emits- # the answer and exits cleanly. Keep this scoped to `coli run`; chat/web- # remain unavailable until OLMoE implements READY/SUBMIT/DATA/DONE.+ # olmoe.c now speaks the gateway's SERVE protocol too (`coli chat`/+ # `coli web`/`coli serve` use it), but `coli run` predates that and+ # its one-shot contract is simpler without it: feed exactly one turn+ # over the engine's plain stdin chat loop and close stdin, the engine+ # emits the answer and exits cleanly. No reason to spin up the gateway+ # for a single non-interactive prompt.result=subprocess.run([engine, str(a.cap if a.cap is not None else 16), "8"],input=prompt+"\n", text=True, env=env_for_engine(a,arch), check=False)@@ -995,7 +997,8 @@ def cmd_chat(a):engine=engine_for(a.model)need_model(a.model,engine)model_id=("kimi-k3-colibri" if arch=="kimi" else- "inkling-colibri" if arch=="inkling" else "deepseek-v4-colibri")+ "inkling-colibri" if arch=="inkling" else+ "olmoe-colibri" if arch=="olmoe" else "deepseek-v4-colibri")banner(f"chat · {model_id} · local server", model=a.model)import openai_server# --cap rides along so an explicit `coli chat --cap N` reaches the@@ -1178,6 +1181,7 @@ def cmd_serve(a):model_id=a.model_id or ("kimi-k3-colibri" if arch=="kimi" else"inkling-colibri" if arch=="inkling" else"deepseek-v4-colibri" if arch=="deepseek_v4" else+ "olmoe-colibri" if arch=="olmoe" else"glm-5.2-colibri")try:openai_server.serve(a.model,a.host,a.port,model_id,a.api_key,diff --git a/c/olmoe.c b/c/olmoe.cindex 41429fe2..87a52638 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -1093,6 +1093,183 @@ static void run_chat(Model *m, Tok *T, int ctx_cap) {free(line); free(turn); free(newids); free(gen); free(outbuf); free(hist);}+/* ---------- serve mode: openai_server.py engine protocol ----------+ * stdin: SUBMIT <id> <slot> <len> <max_tokens> <temp> <top_p>\n<payload>\n+ * CANCEL <id>\n+ * stdout: READY sentinel once loaded, then per request a stream of+ * DATA <id> <size>\n<bytes>\n frames and a final+ * DONE <id> STAT <tok> <tps> <hit%> <rss> <prompt_tok> <len_limited>\n+ * Byte-identical to colibri.c's serve protocol (inkling.c documents it in+ * full above its own SUBMIT handling) so the shared openai_server.py gateway+ * drives olmoe unchanged.+ *+ * v1 scope, same as Inkling's first serve mode: one request in flight, full+ * re-prefill every turn, no cross-request KV reuse. The payload arrives+ * already rendered by openai_server.py's render_chat_olmoe (bos/eos turn+ * markers and all) -- this engine tokenizes it as-is, the same way run_chat()+ * feeds fmt_user_turn()'s output to tok_encode() above. Because nothing here+ * persists state across requests, a fresh prefill at pos_base=0 is enough to+ * start clean: attention() only ever reads positions [0, kv_len), so the+ * previous request's leftover K/V contents past the new prompt's length are+ * never touched, the same invariant CHAT mode's /reset already relies on+ * (it clears hist_len, not the K/V buffers themselves). */++typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen; } SReq;+#define SRV_QMAX 16+static SReq g_q[SRV_QMAX]; static int g_qn = 0;++/* read one control line (+ payload for SUBMIT). cur_id: request in flight;+ * returns 1 if that request was cancelled, 0 otherwise, -1 on stdin EOF. */+static int serve_read_cmd(const char *cur_id) {+ char ln[512];+ if (!fgets(ln, sizeof(ln), stdin)) return -1;+ char cmd[16], id[64];+ if (sscanf(ln, "%15s %63s", cmd, id) < 2) return 0;+ if (!strcmp(cmd, "CANCEL")) return cur_id && !strcmp(id, cur_id);+ if (!strcmp(cmd, "SUBMIT")) {+ int slot, plen, max_tok; float temp, top_p;+ int nf = sscanf(ln, "%*s %*s %d %d %d %f %f", &slot, &plen, &max_tok, &temp, &top_p);+ if (nf < 5 || plen < 0 || plen > (1<<22) || max_tok < 1 || max_tok > (1<<20)) {+ printf("ERROR %s bad submit header\n", id); fflush(stdout); return 0; }+ (void)slot;+ char *pl = malloc((size_t)plen + 1);+ if (fread(pl, 1, (size_t)plen, stdin) != (size_t)plen) { free(pl); return -1; }+ pl[plen] = 0;+ int nl = fgetc(stdin); (void)nl;+ if (g_qn < SRV_QMAX) {+ SReq *q = &g_q[g_qn++];+ snprintf(q->id, sizeof(q->id), "%s", id);+ q->max_tok = max_tok; q->temp = temp; q->top_p = top_p;+ q->payload = pl; q->plen = plen;+ } else { printf("ERROR %s queue full\n", id); fflush(stdout); free(pl); }+ }+ return 0;+}++static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {+ Cfg *c = &m->c;+ int cap = q->plen + 16;+ int *ids = malloc((size_t)cap * sizeof(int));+ int np = tok_encode(T, q->payload, q->plen, ids, cap);+ if (np <= 0) { printf("ERROR %s empty prompt\n", q->id); fflush(stdout); free(ids); return; }+ if (np + q->max_tok > ctx_cap) {+ printf("ERROR %s context exceeds CTX (%d + %d > %d)\n", q->id, np, q->max_tok, ctx_cap);+ fflush(stdout); free(ids); return;+ }+ g_temp = q->temp; g_nuc = q->top_p;+ double t0 = now_s();+ uint64_t h0 = m->hits, m0 = m->miss;+ float *logit = step(m, ids, np, 0);+ int hist_len = np, gen = 0, limited = 1, cancelled = 0;+ char buf[512];+ for (int s = 0; s < q->max_tok && !cancelled; s++) {+ int nt = pick_tok(logit, c->vocab, -1);+ free(logit); logit = NULL;+ if (is_stop(nt)) { limited = 0; break; }+ int nb = tok_decode(T, &nt, 1, buf, sizeof(buf)-1);+ printf("DATA %s %d\n", q->id, nb);+ fwrite(buf, 1, (size_t)nb, stdout);+ fputc('\n', stdout); fflush(stdout);+ gen++; hist_len++;+ while (coli_stdin_readable()) {+ int r = serve_read_cmd(q->id);+ if (r < 0) { free(ids); return; }+ if (r > 0) { cancelled = 1; limited = 0; }+ }+ /* Unlike run_chat(), we do not step() the final token just to populate+ * its KV slot: nothing in serve mode reads past this request's own+ * reply, so a discarded logit here costs nothing. */+ if (cancelled || s == q->max_tok - 1 || hist_len >= ctx_cap) break;+ logit = step(m, &nt, 1, hist_len - 1);+ }+ free(logit);+ double dt = now_s() - t0;+ double tot = (double)(m->hits - h0 + m->miss - m0);+ printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n", q->id, gen,+ dt > 0 ? gen/dt : 0.0, tot ? 100.0*(m->hits-h0)/tot : 0.0, rss_gb(), np, limited);+ /* PROF: per-turn phase timings for the dashboard. olmoe.c does not split+ * its wall time into fill/expert/shared/attn phases the way glm.c and+ * inkling.c do, so this reports total time only; a real phase breakdown+ * is future work, not a protocol requirement. */+ printf("PROF %.3f %d %d 0.0 0.0 0.0 0.0 0.0 %d\n", dt, np, gen, gen + 1);+ fflush(stdout);+ free(ids);+}++/* dashboard HWINFO/TIERS/EMAP: same lines the other serve-capable engines+ * emit for the web dashboard's hardware panel and Brain page. olmoe.c is+ * CPU-only (no CUDA/Metal backend), so the GPU fields are always empty, and+ * every layer is a MoE layer (no dense/sparse split like GLM-5.2), so the+ * tier scan below runs over all n_layers unconditionally. */+static void serve_hwinfo(Model *m) {+ (void)m;+ char cpu[256] = ""; int cores = 0; double rt = 0, ra = 0;+ FILE *ci = fopen("/proc/cpuinfo", "r");+ if (ci) { char ln[256];+ while (fgets(ln, sizeof(ln), ci)) if (!strncmp(ln, "model name", 10)) {+ char *p = strchr(ln, ':'); if (p) { p++; while (*p == ' ') p++;+ int n = (int)strlen(p); if (n > 0 && p[n-1] == '\n') p[--n] = 0;+ snprintf(cpu, sizeof(cpu), "%s", p); } break; }+ fclose(ci); }+#ifdef _SC_NPROCESSORS_ONLN+ cores = (int)sysconf(_SC_NPROCESSORS_ONLN);+#endif+ FILE *mi = fopen("/proc/meminfo", "r");+ if (mi) { char ln[256]; double v = 0;+ while (fgets(ln, sizeof(ln), mi)) {+ if (sscanf(ln, "MemTotal: %lf", &v) == 1) rt = v/1e6;+ if (sscanf(ln, "MemAvailable: %lf", &v) == 1) ra = v/1e6;+ } fclose(mi); }+ printf("HWINFO %d %.1f %.1f 0 0.0 %s|\n", cores, rt, ra, cpu[0] ? cpu : "unknown");+ fflush(stdout);+}++static void serve_tiers_emap(Model *m) {+ Cfg *c = &m->c; int E = c->n_experts;+ int filled = 0;+ for (int i = 0; i < c->n_layers; i++) filled += m->cache[i].n;+ int64_t I = c->inter, D = c->hidden;+ /* per-expert resident bytes: int8 gate/up/down + one f32 scale per row */+ int64_t slotb = 3*I*D + (2*I+D)*4;+ printf("TIERS 0 %d %d 0.00 %.2f\n", filled, c->n_layers*E - filled, filled*(double)slotb/1e9);+ /* EMAP: 1 byte/expert hex — tier(2b: 0=disk 1=RAM)<<6 | heat(6b: log2 usage) */+ char *hex = malloc((size_t)c->n_layers*E*2 + 1); int w = 0;+ for (int i = 0; i < c->n_layers; i++) {+ LCache *lc = &m->cache[i];+ for (int e = 0; e < E; e++) {+ int tier = 0;+ for (int z = 0; z < lc->n; z++) if (lc->slots[z].eid == e) { tier = 1; break; }+ uint32_t u = m->freq[i] ? m->freq[i][e] : 0;+ int heat = 0; while (u) { heat++; u >>= 1; } if (heat > 63) heat = 63;+ int b = (tier << 6) | heat;+ hex[w++] = "0123456789abcdef"[b >> 4];+ hex[w++] = "0123456789abcdef"[b & 15];+ }+ }+ hex[w] = 0;+ printf("EMAP %d %d %s\n", c->n_layers, E, hex);+ fflush(stdout); free(hex);+}++static void serve_loop(Model *m, Tok *T, int ctx_cap) {+ coli_serve_binary_mode();+ setvbuf(stdin, NULL, _IONBF, 0);+ int tok_eos = tok_id_of(T, "|||IP_ADDRESS|||");+ stops_arm_tok(&m->c, tok_eos, T);+ fputs("\x01\x01READY\x01\x01\n", stdout);+ printf("STAT 0 0.0 0.0 %.2f 0 0\n", rss_gb());+ fflush(stdout);+ serve_hwinfo(m);+ serve_tiers_emap(m);+ for (;;) {+ while (!g_qn) if (serve_read_cmd(NULL) < 0) return;+ SReq q = g_q[0];+ memmove(g_q, g_q + 1, (size_t)(--g_qn) * sizeof(SReq));+ serve_one(m, T, &q, ctx_cap);+ free(q.payload);+ }+}+/* ---------- lettura ref.json ---------- */static int *read_int_array(jval *o, const char *key, int *n_out) {jval *a = json_get(o, key);@@ -1119,6 +1296,32 @@ int main(int argc, char **argv) {return 1;}+ /* SERVE=1: openai_server.py drives the engine over stdin/stdout (READY+ * handshake, SUBMIT/CANCEL, DATA/DONE/PROF frames, HWINFO/TIERS/EMAP for+ * the dashboard) — same protocol as colibri.c/inkling.c/kimi_k3.c. v1:+ * one request at a time, full re-prefill every turn (see serve_one()). */+ if (getenv("SERVE") && getenv("SERVE")[0] == '1') {+ int ctx_cap = getenv("CTX") ? atoi(getenv("CTX")) : 4096;+ if (ctx_cap < 1 || ctx_cap > 4096) { /* attention()'s sc[4096] score buffer hard-caps this */+ fprintf(stderr, "CTX must be 1..4096 (got %d)\n", ctx_cap);+ return 1;+ }+ Model m; model_init(&m, snap, cap, bits);+ m.max_t = ctx_cap;+ m.K = calloc(m.c.n_layers, sizeof(float*)); m.V = calloc(m.c.n_layers, sizeof(float*));+ for (int i = 0; i < m.c.n_layers; i++) {+ m.K[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);+ m.V[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);+ }+ Tok T;+ char tokpath[2048]; snprintf(tokpath, sizeof(tokpath), "%s/tokenizer.json", snap);+ tok_load(&T, tokpath);+ serve_loop(&m, &T, ctx_cap);+ { const char *up = getenv("COLI_USAGE");+ if (up && *up) rt_save(up, 0); }+ return 0;+ }+if (getenv("CHAT")) { /* interactive mode: bypasses the ref.json harness entirely */g_temp = getenv("TEMP") ? (float)atof(getenv("TEMP")) : g_temp;g_nuc = getenv("NUCLEUS") ? (float)atof(getenv("NUCLEUS")) : g_nuc;diff --git a/c/openai_server.py b/c/openai_server.pyindex 27d0fa6e..c4787e2e 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -705,6 +705,44 @@ def render_chat_v4(messages, enable_thinking=False, reasoning_effort=None, toolsreturn "".join(parts)+def render_chat_olmoe(messages, enable_thinking=False, reasoning_effort=None, tools=None,+ tool_choice=None):+ """OLMoE-Instruct's native chat_template (tokenizer_config.json): one+ bos_token, then per-message <|system|>/<|user|>/<|assistant|> turns each+ closed by a newline, prior assistant turns also closed by eos_token+ (bos_token == eos_token == "|||IP_ADDRESS|||", a PII-scrubbing artifact+ repurposed as this tokenizer's BOS/EOS marker), and a trailing+ "<|assistant|>\\n" generation prompt. No tool-call syntax and no thinking+ mode exist in this template, so both parameters are accepted but unused.+ """+ if not isinstance(messages, list) or not messages:+ raise APIError(400, "`messages` must be a non-empty array.", "messages")+ if tools or tool_choice not in (None, "none"):+ raise APIError(400, "Tool use is not wired up for the OLMoE engine yet.",+ "tools", "unsupported_parameter")+ boundary = "|||IP_ADDRESS|||" # bos_token == eos_token in this tokenizer+ parts = [boundary]+ last = len(messages) - 1+ for index, message in enumerate(messages):+ if not isinstance(message, dict):+ raise APIError(400, "Each message must be an object.", f"messages.{index}")+ role = message.get("role")+ if role not in ("system", "developer", "user", "assistant"):+ raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")+ raw = message.get("content")+ text = content_text(raw, f"messages.{index}.content") if raw is not None else ""+ if role in ("system", "developer"):+ parts.append(f"<|system|>\n{text}\n")+ elif role == "user":+ parts.append(f"<|user|>\n{text}\n")+ else:+ parts.append(f"<|assistant|>\n{text}{boundary}")+ if index != last:+ parts.append("\n")+ parts.append("<|assistant|>\n")+ return "".join(parts)++def render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None, tools=None,tool_choice=None, audio_out=None):"""Text-only subset of Inkling's chat_template.jinja: role tokens with@@ -880,7 +918,8 @@ def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None,return render_chat_inkling(messages, enable_thinking, reasoning_effort, tools,tool_choice, audio_out=audio_out)renderer = (render_chat_kimi if ARCH == "kimi" else- render_chat_v4 if ARCH == "deepseek_v4" else render_chat)+ render_chat_v4 if ARCH == "deepseek_v4" else+ render_chat_olmoe if ARCH == "olmoe" else render_chat)return renderer(messages, enable_thinking, reasoning_effort, tools, tool_choice)@@ -1399,8 +1438,8 @@ def read_engine_turn(stream, sentinel, on_bytes):def model_arch(model):"""The model's engine family from its config.json model_type -- the same- rule as coli's model_arch(): "inkling"/"kimi" substring, everything else- (including an unreadable config) is glm."""+ rule as coli's model_arch(): "inkling"/"kimi"/"olmoe" substring, everything+ else (including an unreadable config) is glm."""try:with open(Path(model) / "config.json", encoding="utf-8") as fh:model_type = (json.load(fh).get("model_type") or "").lower()@@ -1412,6 +1451,8 @@ def model_arch(model):return "kimi"if "deepseek_v4" in model_type or ("deepseek" in model_type and "v4" in model_type):return "deepseek_v4"+ if "olmoe" in model_type:+ return "olmoe"return "glm"@@ -2234,7 +2275,7 @@ class APIHandler(BaseHTTPRequestHandler):sys.stderr.flush()maximum, temperature, top_p, grammar, _requested_stop_sequences = generation_options(body, self.server.max_tokens)- if grammar is not None and ARCH in ("inkling", "kimi"):+ if grammar is not None and ARCH in ("inkling", "kimi", "olmoe"):# sibling engines speak the 6-field SUBMIT header only; sending the# grammar payload extension would desync its stdin framing.raise APIError(400, f"`response_format` grammars are not supported by the {ARCH} "@@ -2817,7 +2858,7 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_keraise ValueError("queue_timeout must be positive")if not 1 <= kv_slots <= 16:raise ValueError("kv_slots must be between 1 and 16")- if ARCH in ("inkling", "kimi", "deepseek_v4") and kv_slots != 1:+ if ARCH in ("inkling", "kimi", "deepseek_v4", "olmoe") and kv_slots != 1:raise ValueError(f"{ARCH} engine currently supports exactly one KV slot")if host not in ("127.0.0.1", "localhost", "::1") and not api_key:# (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes@@ -2854,7 +2895,7 @@ def main():parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--model", default=os.environ.get("COLI_MODEL"), required=not os.environ.get("COLI_MODEL"))parser.add_argument("--engine", default=str(default_engine()))- parser.add_argument("--arch", choices=("auto", "glm", "inkling", "kimi", "deepseek_v4"), default="auto",+ parser.add_argument("--arch", choices=("auto", "glm", "inkling", "kimi", "deepseek_v4", "olmoe"), default="auto",help="chat-template family; auto reads model_type from the model's config.json")parser.add_argument("--host", default="127.0.0.1")parser.add_argument("--port", type=int, default=8000)@@ -2885,6 +2926,7 @@ def main():args.model_id = ("inkling-colibri" if ARCH == "inkling" else"kimi-k3-colibri" if ARCH == "kimi" else"deepseek-v4-colibri" if ARCH == "deepseek_v4" else+ "olmoe-colibri" if ARCH == "olmoe" else"glm-5.2-colibri")serve(args.model, args.host, args.port, args.model_id, args.api_key,args.cap,args.max_tokens,args.engine,cors_origins=args.cors_origin,diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 8e1605da..5e2836b2 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -19,7 +19,7 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,_engine_error, cap_for_arch, conversation_cache_slot, model_arch,generation_options, parse_tool_calls, read_engine_turn,- render_chat, render_chat_kimi, serve,+ render_chat, render_chat_kimi, render_chat_olmoe, serve,split_thinking_reply, stop_policy, tune_child_env)@@ -114,6 +114,37 @@ class TemplateTest(unittest.TestCase):"K3CHAT1\nA 3 6\nwhyanswerG 1\n",)+ def test_olmoe_renders_native_chat_template(self):+ # Matches allenai/OLMoE-1B-7B-0125-Instruct's tokenizer_config.json+ # chat_template exactly: one leading bos_token, per-role turns closed+ # by a trailing newline, a prior (non-final) assistant turn also closed+ # by eos_token before that newline (bos_token == eos_token ==+ # "|||IP_ADDRESS|||" in this tokenizer), and a trailing+ # "<|assistant|>\n" generation prompt.+ prompt = render_chat_olmoe([+ {"role": "system", "content": "Be terse."},+ {"role": "user", "content": "Hi"},+ {"role": "assistant", "content": "Hello"},+ {"role": "user", "content": "Continue"},+ ])+ self.assertEqual(+ prompt,+ "|||IP_ADDRESS|||<|system|>\nBe terse.\n<|user|>\nHi\n"+ "<|assistant|>\nHello|||IP_ADDRESS|||\n<|user|>\nContinue\n"+ "<|assistant|>\n",+ )++ def test_olmoe_rejects_tools_and_unknown_roles(self):+ with self.assertRaisesRegex(APIError, "Tool use"):+ render_chat_olmoe([{"role": "user", "content": "Hi"}],+ tools=[{"type": "function"}])+ with self.assertRaisesRegex(APIError, "Unsupported role"):+ render_chat_olmoe([{"role": "tool", "content": "result"}])++ def test_olmoe_rejects_empty_messages(self):+ with self.assertRaisesRegex(APIError, "non-empty array"):+ render_chat_olmoe([])+def test_validates_generation_limits(self):self.assertEqual(generation_options({"max_tokens": 4, "temperature": 0, "top_p": 1}, 8),(4, 0.0, 1.0, None, ()))@@ -661,6 +692,7 @@ class CapSentinelShimTest(unittest.TestCase):self.assertEqual(cap_for_arch("glm", None), 0)self.assertEqual(cap_for_arch("inkling", None), 8)self.assertEqual(cap_for_arch("kimi", None), 8)+ self.assertEqual(cap_for_arch("olmoe", None), 8)self.assertEqual(cap_for_arch("glm", 3), 3)self.assertEqual(cap_for_arch("inkling", 3), 3)self.assertEqual(cap_for_arch("inkling", 0), 0) # explicit 0 is explicit@@ -671,6 +703,7 @@ class CapSentinelShimTest(unittest.TestCase):self.assertEqual(model_arch(self._model("inkling")), "inkling")self.assertEqual(model_arch(self._model("kimi_k3")), "kimi")self.assertEqual(model_arch(self._model("deepseek_v4")), "deepseek_v4")+ self.assertEqual(model_arch(self._model("olmoe")), "olmoe")self.assertEqual(model_arch("/nonexistent"), "glm")def test_direct_v4_server_gets_bounded_dspark_defaults(self):