Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
refactor(serve): share framing with Inkling
Mario Gutierrezc379f4776089
diff --git a/c/Makefile b/c/Makefileindex 8d5cda4e..46be98b9 100644--- a/c/Makefile+++ b/c/Makefile@@ -955,7 +955,7 @@ qwen36$(EXE): qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(Ctests/test_qwen36_ctx$(EXE): tests/test_qwen36_ctx.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(LDFLAGS)-inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(METAL_OBJ)+inkling$(EXE): inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)# ENGINES WITHOUT A CUDA BACKEND (#783).@@ -1077,7 +1077,7 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.htests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)-tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h $(INK_CUDA_OBJ) $(METAL_OBJ)+tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)tests/test_kimi_serve_framing$(EXE): tests/test_kimi_serve_framing.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.hdiff --git a/c/inkling.c b/c/inkling.cindex 785ea6ff..e9322f99 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -39,6 +39,7 @@#include "omp_tune.h"#include "route_trace.h"#include "kv_prefix.h" /* KV prefix reuse (shared) */ /* shared routing telemetry (#700) */+#include "serve_codec.h"#ifdef COLI_CUDA#include "backend_cuda_ink.h"static int g_cuda = 0;@@ -1858,6 +1859,22 @@ typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; intuint8_t *audio; int alen; } SReq; /* raw DMel bytes after the payload */#define SRV_QMAX 16static SReq g_q[SRV_QMAX]; static int g_qn = 0;+static const ColiServeWireProfile inkling_wire = {+ .max_header_bytes = 511,+ .max_payload_bytes = 1u << 22,+ .max_extension_bytes = 1u << 26,+ .max_tokens = 1 << 20,+ .require_exact_lf = 1,+ .require_finite_sampling = 0,+ .allow_extension_bytes = 1,+ .allow_prefix_hint = 0,+};++static void serve_request_dispose(SReq *request) {+ if (!request) return;+ free(request->payload);+ memset(request, 0, sizeof(*request));+}static int stdin_readable(void) {/* Windows non ha fd_set/select in questa forma: la build falliva del tutto.@@ -1868,14 +1885,15 @@ static int stdin_readable(void) {/* read one control line (+ payload for SUBMIT). cur_id: request in flight;* returns 1 if that request was cancelled, 0 otherwise, -1 on stdin EOF. */static int serve_read_cmd(FILE *input, FILE *output, const char *cur_id) {- char ln[512];- if (!fgets(ln, sizeof(ln), input)) return -1;- char cmd[16], id[64];- if (sscanf(ln, "%15s %63s", cmd, id) < 2) return 0;- if (!strcmp(cmd, "CANCEL")) return cur_id && !strcmp(id, cur_id);- if (!strcmp(cmd, "SUBMIT")) {- int slot, plen, max_tok, alen = 0; float temp, top_p;- int nf = sscanf(ln, "%*s %*s %d %d %d %f %f %d", &slot, &plen, &max_tok, &temp, &top_p, &alen);+ ColiServeCommand command;+ ColiServeReadResult result=coli_serve_read_command(input,&inkling_wire,&command);+ if(result==COLI_SERVE_READ_EOF) return -1;+ if(result==COLI_SERVE_READ_BAD_FRAME) return -2;+ if(result==COLI_SERVE_READ_NOMEM){+ coli_serve_write_error(output,command.id,"out of memory"); return -2;+ }+ if(result==COLI_SERVE_READ_BAD_REQUEST&&+ command.kind==COLI_SERVE_COMMAND_SUBMIT){/* 6th field (optional, backward compatible): DMel byte count appended* verbatim after the text payload β frames x mel_bins u8 levels *//* SEC: max_tok was the one field nobody validated, and it is the one@@ -1891,45 +1909,48 @@ static int serve_read_cmd(FILE *input, FILE *output, const char *cur_id) {* anything bridging it (socat, a custom gateway, a sidecar) exposes it* directly, so the check belongs here, next to the one plen already* has, rather than in one of its callers. */- if (nf < 5 || plen < 0 || plen > (1<<22) || alen < 0 || alen > (1<<26) ||- max_tok < 1 || max_tok > (1<<20)) {- fprintf(output, "ERROR %s bad submit header\n", id); fflush(output); return 0; }- (void)slot;- char *pl = malloc((size_t)plen + 1);- if (fread(pl, 1, (size_t)plen, input) != (size_t)plen) { free(pl); return -1; }- pl[plen] = 0;- uint8_t *au = NULL;- if (alen > 0) {- au = malloc((size_t)alen);- if (fread(au, 1, (size_t)alen, input) != (size_t)alen) { free(pl); free(au); return -1; }- }- int nl = fgetc(input); (void)nl;+ coli_serve_write_error(output,command.id,"bad submit header"); return -2;+ }+ if(result!=COLI_SERVE_READ_OK) return 0;+ if(command.kind==COLI_SERVE_COMMAND_CANCEL){+ int matched=cur_id&&!strcmp(command.id,cur_id);+ coli_serve_command_dispose(&command); return matched;+ }+ if(command.kind==COLI_SERVE_COMMAND_STOP){+ coli_serve_command_dispose(&command); return 0;+ }+ if(command.kind==COLI_SERVE_COMMAND_SUBMIT){if (g_qn < SRV_QMAX) {SReq *q = &g_q[g_qn++];- snprintf(q->id, sizeof(q->id), "%s", id);- q->max_tok = max_tok; q->temp = temp; q->top_p = top_p;- q->payload = pl; q->plen = plen;- q->audio = au; q->alen = alen;- } else { fprintf(output, "ERROR %s queue full\n", id); fflush(output); free(pl); free(au); }- }+ snprintf(q->id, sizeof(q->id), "%s", command.id);+ q->max_tok=command.max_tokens; q->temp=command.temperature;+ q->top_p=command.top_p; q->plen=(int)command.payload_bytes;+ q->alen=(int)command.extension_bytes;+ q->audio=coli_serve_command_extension(&command);+ q->payload=(char*)coli_serve_command_take_payload(&command);+ } else coli_serve_write_error(output,command.id,"queue full");+ }+ coli_serve_command_dispose(&command);return 0;}-static void serve_one(Model *m, Tok *T, SReq *q) {+static int serve_one(Model *m, Tok *T, SReq *q) {Cfg *c = &m->c;int cap = q->plen + 16;int *ids = malloc((size_t)cap * sizeof(int));int np = tok_encode(T, q->payload, q->plen, ids, cap);- if (np <= 0) { printf("ERROR %s empty prompt\n", q->id); fflush(stdout); free(ids); return; }+ if (np <= 0) { coli_serve_write_error(stdout,q->id,"empty prompt"); free(ids); return 0; }const char *bad = prompt_reject(np, q->max_tok);- if (bad) { printf("ERROR %s %s\n", q->id, bad); fflush(stdout); free(ids); return; }+ if (bad) { coli_serve_write_error(stdout,q->id,bad); free(ids); return 0; }/* audio: every <|audio|> placeholder must have exactly one DMel frame */int naud = q->alen / m->c.mel_bins;if (q->alen % m->c.mel_bins != 0 || audio_tok_count(m, ids, np) != naud) {- printf("ERROR %s audio frames (%d) do not match <|audio|> placeholders (%d)%s\n",- q->id, naud, audio_tok_count(m, ids, np),- m->audio_norm ? "" : " β snapshot has no audio tensors");- fflush(stdout); free(ids); return;+ char message[256];+ snprintf(message,sizeof(message),+ "audio frames (%d) do not match <|audio|> placeholders (%d)%s",+ naud,audio_tok_count(m,ids,np),+ m->audio_norm?"":" β snapshot has no audio tensors");+ coli_serve_write_error(stdout,q->id,message); free(ids); return 0;}/* KV PREFIX REUSE (#639 for GLM; this engine re-prefilled every turn).* A chat client resends the whole transcript each turn, so turn N used to@@ -1988,13 +2009,11 @@ static void serve_one(Model *m, Tok *T, SReq *q) {if (nhist < 128) hist[nhist++] = tk;else { memmove(hist, hist+1, 127*sizeof(int)); hist[127] = tk; }int nb = tok_decode(T, &tk, 1, buf, sizeof(buf)-1);- printf("DATA %s %d\n", q->id, nb);- fwrite(buf, 1, (size_t)nb, stdout);- fputc('\n', stdout); fflush(stdout);+ coli_serve_write_data(stdout,q->id,buf,(size_t)nb);gen++; len++;while (stdin_readable()) {int r = serve_read_cmd(stdin, stdout, q->id);- if (r < 0) { free(ids); return; }+ if (r < 0) { free(ids); return -1; }if (r > 0) { cancelled = 1; limited = 0; }}if (cancelled || s == q->max_tok - 1) break;@@ -2003,14 +2022,18 @@ static void serve_one(Model *m, Tok *T, SReq *q) {free(logit);double dt = now_s() - t0;double tot = (double)(m->hits - h0 + m->miss - m0);- printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n", q->id, gen,- dt > 0 ? gen/dt : 0.0, tot ? 100.0*(m->hits-h0)/tot : 0.0, rss_gb(), np, limited);+ ColiServeDone done={gen,dt>0?gen/dt:0.0,+ tot?100.0*(m->hits-h0)/tot:0.0,rss_gb(),np,limited};+ char done_line[256];+ int done_bytes=coli_serve_format_done(done_line,sizeof(done_line),q->id,&done);+ if(done_bytes>0) fwrite(done_line,1,(size_t)done_bytes,stdout);/* PROF: per-turn phase timings for the dashboard (gateway schema β we map* expert_wait -> shared-expert compute, lm_head folded into 0). */printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n", dt, np, gen,m->t_fill - f0, m->t_shared - s0, m->t_expert - e0, m->t_attn - a0, 0.0, gen + 1);fflush(stdout);free(ids);+ return 0;}/* ---------- dashboard protocol (HWINFO / TIERS / EMAP) ----------@@ -2077,25 +2100,23 @@ static void serve_loop(Model *m, Tok *T) {* as \r\n, the gateway never matches it and waits forever (#748). Lives in* compat.h because colibri.c has had it since #195 and this engine was* written without it. */- coli_serve_binary_mode();- setvbuf(stdin, NULL, _IONBF, 0);+ coli_serve_stdio_init();const char *sd = getenv("SEED");if (sd) g_rng ^= (uint64_t)strtoull(sd, NULL, 10);else g_rng ^= (uint64_t)time(NULL) * 2654435761u;/* the gateway reads a STAT line right after the READY sentinel (colibri* reports its load stats there) β match the handshake */- fputs("\x01\x01READY\x01\x01\n", stdout);- printf("STAT 0 0.0 0.0 %.2f 0 0\n", rss_gb());- fflush(stdout);+ coli_serve_write_ready(stdout,rss_gb());serve_hwinfo(m);serve_tiers_emap(m);for (;;) {while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return; /* blocks on stdin */SReq q = g_q[0];memmove(g_q, g_q+1, (size_t)(--g_qn) * sizeof(SReq));- serve_one(m, T, &q);+ int fatal=serve_one(m,T,&q);serve_tiers_emap(m);- free(q.payload); free(q.audio);+ serve_request_dispose(&q);+ if(fatal<0){ while(g_qn) serve_request_dispose(&g_q[--g_qn]); return; }}}diff --git a/c/tests/test_inkling_serve_framing.c b/c/tests/test_inkling_serve_framing.cindex 572e3f47..b78ebf59 100644--- a/c/tests/test_inkling_serve_framing.c+++ b/c/tests/test_inkling_serve_framing.c@@ -28,7 +28,7 @@ static size_t read_output(FILE *stream, unsigned char *output, size_t capacity)static void reset_queue(void){- for(int i=0;i<g_qn;i++){ free(g_q[i].payload); free(g_q[i].audio); }+ for(int i=0;i<g_qn;i++) serve_request_dispose(&g_q[i]);memset(g_q,0,sizeof(g_q)); g_qn=0;}@@ -81,10 +81,30 @@ static void test_controls_and_queue_full_preserve_behavior(void)fclose(input); fclose(output); g_qn=0;}+static void test_malformed_submit_cannot_become_control(void)+{+ static const char frame[]=+ "SUBMIT bad 0 12 0 0.7 0.95 4\nCANCEL liveWXYZ\n";+ FILE *input=bytes_input(frame,sizeof(frame)-1);+ FILE *output=tmpfile(); assert(output); binary_stream(output);+ assert(serve_read_cmd(input,output,"live")==-2);+ assert(fgetc(input)=='C');+ fclose(input); fclose(output);++ static const char bad_term[]=+ "SUBMIT broken 0 1 4 0.7 0.95 4\nxWXYZXSTOP live\n";+ input=bytes_input(bad_term,sizeof(bad_term)-1);+ output=tmpfile(); assert(output); binary_stream(output);+ assert(serve_read_cmd(input,output,"live")==-2);+ assert(fgetc(input)=='S');+ fclose(input); fclose(output);+}+int main(void){test_text_and_audio_submits_are_exact();test_controls_and_queue_full_preserve_behavior();+ test_malformed_submit_cannot_become_control();puts("inkling serve framing baseline: ok");return 0;}