Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(serve): share framing with Inkling

Mario Gutierrezc379f4776089
diff --git a/c/Makefile b/c/Makefile
index 8d5cda4e..46be98b9 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -955,7 +955,7 @@ qwen36$(EXE): qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(C
tests/test_qwen36_ctx$(EXE): tests/test_qwen36_ctx.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(LDFLAGS)
-inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(METAL_OBJ)
+inkling$(EXE): inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)
$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)
# ENGINES WITHOUT A CUDA BACKEND (#783).
@@ -1077,7 +1077,7 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h $(INK_CUDA_OBJ) $(METAL_OBJ)
+tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)
$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)
tests/test_kimi_serve_framing$(EXE): tests/test_kimi_serve_framing.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h
diff --git a/c/inkling.c b/c/inkling.c
index 785ea6ff..e9322f99 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -39,6 +39,7 @@
#include "omp_tune.h"
#include "route_trace.h"
#include "kv_prefix.h" /* KV prefix reuse (shared) */ /* shared routing telemetry (#700) */
+#include "serve_codec.h"
#ifdef COLI_CUDA
#include "backend_cuda_ink.h"
static int g_cuda = 0;
@@ -1858,6 +1859,22 @@ typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int
uint8_t *audio; int alen; } SReq; /* raw DMel bytes after the payload */
#define SRV_QMAX 16
static SReq g_q[SRV_QMAX]; static int g_qn = 0;
+static const ColiServeWireProfile inkling_wire = {
+ .max_header_bytes = 511,
+ .max_payload_bytes = 1u << 22,
+ .max_extension_bytes = 1u << 26,
+ .max_tokens = 1 << 20,
+ .require_exact_lf = 1,
+ .require_finite_sampling = 0,
+ .allow_extension_bytes = 1,
+ .allow_prefix_hint = 0,
+};
+
+static void serve_request_dispose(SReq *request) {
+ if (!request) return;
+ free(request->payload);
+ memset(request, 0, sizeof(*request));
+}
static int stdin_readable(void) {
/* Windows non ha fd_set/select in questa forma: la build falliva del tutto.
@@ -1868,14 +1885,15 @@ static int stdin_readable(void) {
/* read one control line (+ payload for SUBMIT). cur_id: request in flight;
* returns 1 if that request was cancelled, 0 otherwise, -1 on stdin EOF. */
static int serve_read_cmd(FILE *input, FILE *output, const char *cur_id) {
- char ln[512];
- if (!fgets(ln, sizeof(ln), input)) return -1;
- char cmd[16], id[64];
- if (sscanf(ln, "%15s %63s", cmd, id) < 2) return 0;
- if (!strcmp(cmd, "CANCEL")) return cur_id && !strcmp(id, cur_id);
- if (!strcmp(cmd, "SUBMIT")) {
- int slot, plen, max_tok, alen = 0; float temp, top_p;
- int nf = sscanf(ln, "%*s %*s %d %d %d %f %f %d", &slot, &plen, &max_tok, &temp, &top_p, &alen);
+ ColiServeCommand command;
+ ColiServeReadResult result=coli_serve_read_command(input,&inkling_wire,&command);
+ if(result==COLI_SERVE_READ_EOF) return -1;
+ if(result==COLI_SERVE_READ_BAD_FRAME) return -2;
+ if(result==COLI_SERVE_READ_NOMEM){
+ coli_serve_write_error(output,command.id,"out of memory"); return -2;
+ }
+ if(result==COLI_SERVE_READ_BAD_REQUEST&&
+ command.kind==COLI_SERVE_COMMAND_SUBMIT){
/* 6th field (optional, backward compatible): DMel byte count appended
* verbatim after the text payload β€” frames x mel_bins u8 levels */
/* SEC: max_tok was the one field nobody validated, and it is the one
@@ -1891,45 +1909,48 @@ static int serve_read_cmd(FILE *input, FILE *output, const char *cur_id) {
* anything bridging it (socat, a custom gateway, a sidecar) exposes it
* directly, so the check belongs here, next to the one plen already
* has, rather than in one of its callers. */
- if (nf < 5 || plen < 0 || plen > (1<<22) || alen < 0 || alen > (1<<26) ||
- max_tok < 1 || max_tok > (1<<20)) {
- fprintf(output, "ERROR %s bad submit header\n", id); fflush(output); return 0; }
- (void)slot;
- char *pl = malloc((size_t)plen + 1);
- if (fread(pl, 1, (size_t)plen, input) != (size_t)plen) { free(pl); return -1; }
- pl[plen] = 0;
- uint8_t *au = NULL;
- if (alen > 0) {
- au = malloc((size_t)alen);
- if (fread(au, 1, (size_t)alen, input) != (size_t)alen) { free(pl); free(au); return -1; }
- }
- int nl = fgetc(input); (void)nl;
+ coli_serve_write_error(output,command.id,"bad submit header"); return -2;
+ }
+ if(result!=COLI_SERVE_READ_OK) return 0;
+ if(command.kind==COLI_SERVE_COMMAND_CANCEL){
+ int matched=cur_id&&!strcmp(command.id,cur_id);
+ coli_serve_command_dispose(&command); return matched;
+ }
+ if(command.kind==COLI_SERVE_COMMAND_STOP){
+ coli_serve_command_dispose(&command); return 0;
+ }
+ if(command.kind==COLI_SERVE_COMMAND_SUBMIT){
if (g_qn < SRV_QMAX) {
SReq *q = &g_q[g_qn++];
- snprintf(q->id, sizeof(q->id), "%s", id);
- q->max_tok = max_tok; q->temp = temp; q->top_p = top_p;
- q->payload = pl; q->plen = plen;
- q->audio = au; q->alen = alen;
- } else { fprintf(output, "ERROR %s queue full\n", id); fflush(output); free(pl); free(au); }
- }
+ snprintf(q->id, sizeof(q->id), "%s", command.id);
+ q->max_tok=command.max_tokens; q->temp=command.temperature;
+ q->top_p=command.top_p; q->plen=(int)command.payload_bytes;
+ q->alen=(int)command.extension_bytes;
+ q->audio=coli_serve_command_extension(&command);
+ q->payload=(char*)coli_serve_command_take_payload(&command);
+ } else coli_serve_write_error(output,command.id,"queue full");
+ }
+ coli_serve_command_dispose(&command);
return 0;
}
-static void serve_one(Model *m, Tok *T, SReq *q) {
+static int serve_one(Model *m, Tok *T, SReq *q) {
Cfg *c = &m->c;
int cap = q->plen + 16;
int *ids = malloc((size_t)cap * sizeof(int));
int np = tok_encode(T, q->payload, q->plen, ids, cap);
- if (np <= 0) { printf("ERROR %s empty prompt\n", q->id); fflush(stdout); free(ids); return; }
+ if (np <= 0) { coli_serve_write_error(stdout,q->id,"empty prompt"); free(ids); return 0; }
const char *bad = prompt_reject(np, q->max_tok);
- if (bad) { printf("ERROR %s %s\n", q->id, bad); fflush(stdout); free(ids); return; }
+ if (bad) { coli_serve_write_error(stdout,q->id,bad); free(ids); return 0; }
/* audio: every <|audio|> placeholder must have exactly one DMel frame */
int naud = q->alen / m->c.mel_bins;
if (q->alen % m->c.mel_bins != 0 || audio_tok_count(m, ids, np) != naud) {
- printf("ERROR %s audio frames (%d) do not match <|audio|> placeholders (%d)%s\n",
- q->id, naud, audio_tok_count(m, ids, np),
- m->audio_norm ? "" : " β€” snapshot has no audio tensors");
- fflush(stdout); free(ids); return;
+ char message[256];
+ snprintf(message,sizeof(message),
+ "audio frames (%d) do not match <|audio|> placeholders (%d)%s",
+ naud,audio_tok_count(m,ids,np),
+ m->audio_norm?"":" β€” snapshot has no audio tensors");
+ coli_serve_write_error(stdout,q->id,message); free(ids); return 0;
}
/* KV PREFIX REUSE (#639 for GLM; this engine re-prefilled every turn).
* A chat client resends the whole transcript each turn, so turn N used to
@@ -1988,13 +2009,11 @@ static void serve_one(Model *m, Tok *T, SReq *q) {
if (nhist < 128) hist[nhist++] = tk;
else { memmove(hist, hist+1, 127*sizeof(int)); hist[127] = tk; }
int nb = tok_decode(T, &tk, 1, buf, sizeof(buf)-1);
- printf("DATA %s %d\n", q->id, nb);
- fwrite(buf, 1, (size_t)nb, stdout);
- fputc('\n', stdout); fflush(stdout);
+ coli_serve_write_data(stdout,q->id,buf,(size_t)nb);
gen++; len++;
while (stdin_readable()) {
int r = serve_read_cmd(stdin, stdout, q->id);
- if (r < 0) { free(ids); return; }
+ if (r < 0) { free(ids); return -1; }
if (r > 0) { cancelled = 1; limited = 0; }
}
if (cancelled || s == q->max_tok - 1) break;
@@ -2003,14 +2022,18 @@ static void serve_one(Model *m, Tok *T, SReq *q) {
free(logit);
double dt = now_s() - t0;
double tot = (double)(m->hits - h0 + m->miss - m0);
- printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n", q->id, gen,
- dt > 0 ? gen/dt : 0.0, tot ? 100.0*(m->hits-h0)/tot : 0.0, rss_gb(), np, limited);
+ ColiServeDone done={gen,dt>0?gen/dt:0.0,
+ tot?100.0*(m->hits-h0)/tot:0.0,rss_gb(),np,limited};
+ char done_line[256];
+ int done_bytes=coli_serve_format_done(done_line,sizeof(done_line),q->id,&done);
+ if(done_bytes>0) fwrite(done_line,1,(size_t)done_bytes,stdout);
/* PROF: per-turn phase timings for the dashboard (gateway schema β€” we map
* expert_wait -> shared-expert compute, lm_head folded into 0). */
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n", dt, np, gen,
m->t_fill - f0, m->t_shared - s0, m->t_expert - e0, m->t_attn - a0, 0.0, gen + 1);
fflush(stdout);
free(ids);
+ return 0;
}
/* ---------- dashboard protocol (HWINFO / TIERS / EMAP) ----------
@@ -2077,25 +2100,23 @@ static void serve_loop(Model *m, Tok *T) {
* as \r\n, the gateway never matches it and waits forever (#748). Lives in
* compat.h because colibri.c has had it since #195 and this engine was
* written without it. */
- coli_serve_binary_mode();
- setvbuf(stdin, NULL, _IONBF, 0);
+ coli_serve_stdio_init();
const char *sd = getenv("SEED");
if (sd) g_rng ^= (uint64_t)strtoull(sd, NULL, 10);
else g_rng ^= (uint64_t)time(NULL) * 2654435761u;
/* the gateway reads a STAT line right after the READY sentinel (colibri
* reports its load stats there) β€” match the handshake */
- fputs("\x01\x01READY\x01\x01\n", stdout);
- printf("STAT 0 0.0 0.0 %.2f 0 0\n", rss_gb());
- fflush(stdout);
+ coli_serve_write_ready(stdout,rss_gb());
serve_hwinfo(m);
serve_tiers_emap(m);
for (;;) {
while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return; /* blocks on stdin */
SReq q = g_q[0];
memmove(g_q, g_q+1, (size_t)(--g_qn) * sizeof(SReq));
- serve_one(m, T, &q);
+ int fatal=serve_one(m,T,&q);
serve_tiers_emap(m);
- free(q.payload); free(q.audio);
+ serve_request_dispose(&q);
+ if(fatal<0){ while(g_qn) serve_request_dispose(&g_q[--g_qn]); return; }
}
}
diff --git a/c/tests/test_inkling_serve_framing.c b/c/tests/test_inkling_serve_framing.c
index 572e3f47..b78ebf59 100644
--- a/c/tests/test_inkling_serve_framing.c
+++ b/c/tests/test_inkling_serve_framing.c
@@ -28,7 +28,7 @@ static size_t read_output(FILE *stream, unsigned char *output, size_t capacity)
static void reset_queue(void)
{
- for(int i=0;i<g_qn;i++){ free(g_q[i].payload); free(g_q[i].audio); }
+ for(int i=0;i<g_qn;i++) serve_request_dispose(&g_q[i]);
memset(g_q,0,sizeof(g_q)); g_qn=0;
}
@@ -81,10 +81,30 @@ static void test_controls_and_queue_full_preserve_behavior(void)
fclose(input); fclose(output); g_qn=0;
}
+static void test_malformed_submit_cannot_become_control(void)
+{
+ static const char frame[]=
+ "SUBMIT bad 0 12 0 0.7 0.95 4\nCANCEL liveWXYZ\n";
+ FILE *input=bytes_input(frame,sizeof(frame)-1);
+ FILE *output=tmpfile(); assert(output); binary_stream(output);
+ assert(serve_read_cmd(input,output,"live")==-2);
+ assert(fgetc(input)=='C');
+ fclose(input); fclose(output);
+
+ static const char bad_term[]=
+ "SUBMIT broken 0 1 4 0.7 0.95 4\nxWXYZXSTOP live\n";
+ input=bytes_input(bad_term,sizeof(bad_term)-1);
+ output=tmpfile(); assert(output); binary_stream(output);
+ assert(serve_read_cmd(input,output,"live")==-2);
+ assert(fgetc(input)=='S');
+ fclose(input); fclose(output);
+}
+
int main(void)
{
test_text_and_audio_submits_are_exact();
test_controls_and_queue_full_preserve_behavior();
+ test_malformed_submit_cannot_become_control();
puts("inkling serve framing baseline: ok");
return 0;
}