Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(serve): share framing with Kimi K3

Mario Gutierrezcbc72fe12c86
diff --git a/c/Makefile b/c/Makefile
index d7e714cb..46375396 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -891,7 +891,7 @@ NOCUDA_CFLAGS = $(filter-out -DCOLI_CUDA -DCOLI_ANS,$(CFLAGS))
NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \
-Wl$(comma)-rpath$(comma)$(CUDA_HOME)/lib64,$(LDFLAGS))
-kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h backend_cuda.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV)
+kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h backend_cuda.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV)
$(CC) $(CFLAGS) kimi_k3.c $(CUDA_OBJ) $(VK_OBJ) -o kimi_k3$(EXE) $(LDFLAGS)
# Use a baseline that matches the compiler target. macOS already targets a
@@ -931,10 +931,10 @@ tests/test_json$(EXE): tests/test_json.c json.h
tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h
+tests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_k3_mmap$(EXE): tests/test_k3_mmap.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h
+tests/test_k3_mmap$(EXE): tests/test_k3_mmap.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h
$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
tests/test_tok_kimi_tiny$(EXE): tests/test_tok_kimi_tiny.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
@@ -995,7 +995,7 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_kimi_serve_framing$(EXE): tests/test_kimi_serve_framing.c kimi_k3.c kv_prefix.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h
+tests/test_kimi_serve_framing$(EXE): tests/test_kimi_serve_framing.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h
$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
# olmoe's matmul_q, not colibri's: compares the IDOT path against FP32
@@ -1340,5 +1340,5 @@ bench: iobench$(EXE)
tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h
$(CC) $(CFLAGS) tests/test_kv_prefix.c -o tests/test_kv_prefix$(EXE) $(LDFLAGS)
-tests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h
+tests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h
$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index a6453e18..2dea3517 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -92,6 +92,7 @@
#include "omp_tune.h"
#include "route_trace.h"
#include "kv_prefix.h" /* KV prefix reuse (shared) */
+#include "serve_codec.h"
/* ---------- config ---------- */
typedef struct {
@@ -1870,6 +1871,14 @@ typedef struct {
int plen;
} ServeReq;
+static const ColiServeWireProfile kimi_wire={
+ .max_header_bytes=511,
+ .max_payload_bytes=1u<<24,
+ .max_tokens=1<<20,
+ .require_exact_lf=1,
+ .require_finite_sampling=0,
+};
+
static void model_state_reset(Model *m){
Cfg *c=&m->c;
kv_prefix_clear(&m->kvp); /* the record describes the state we are dropping */
@@ -1906,41 +1915,51 @@ static int serve_stdin_readable(void){
}
static int serve_read_req(FILE *in, FILE *out, ServeReq *q, const char *active){
- char line[512], cmd[16], id[64];
- if(!fgets(line,sizeof(line),in)) return -1;
- if(sscanf(line,"%15s %63s",cmd,id)<2) return 0;
- if(!strcmp(cmd,"CANCEL")||!strcmp(cmd,"STOP")) return active&&!strcmp(active,id);
- if(strcmp(cmd,"SUBMIT")) return 0;
- int slot, plen, max_tok; float temp, top_p;
- if(sscanf(line,"%*s %*s %d %d %d %f %f",&slot,&plen,&max_tok,&temp,&top_p)!=5||
- plen<0||plen>(1<<24)||max_tok<1||max_tok>(1<<20)){
+ ColiServeCommand command;
+ ColiServeReadResult result=coli_serve_read_command(in,&kimi_wire,&command);
+ if(result==COLI_SERVE_READ_EOF) return -1;
+ if(result==COLI_SERVE_READ_BAD_FRAME) return -2;
+ if(result==COLI_SERVE_READ_NOMEM){
+ coli_serve_write_error(out,command.id,"out of memory"); return -2;
+ }
+ if(result==COLI_SERVE_READ_BAD_REQUEST&&
+ command.kind==COLI_SERVE_COMMAND_SUBMIT){
/* SEC (GHSA-gf38): max_tok needs an upper bound too. INT_MAX wrapped the
* signed np+max_tok context check below and made the kv_alloc size
* negative, so kv_alloc's early-return kept the previous request's small
* KV buffers and the generation loop wrote past them (heap OOB write). */
- fprintf(out,"ERROR %s bad submit header\n",id); fflush(out); return 0;
+ coli_serve_write_error(out,command.id,"bad submit header"); return -2;
+ }
+ if(result!=COLI_SERVE_READ_OK) return 0;
+ if(command.kind==COLI_SERVE_COMMAND_STOP||
+ command.kind==COLI_SERVE_COMMAND_CANCEL){
+ int matched=active&&!strcmp(active,command.id);
+ coli_serve_command_dispose(&command); return matched;
+ }
+ if(command.kind!=COLI_SERVE_COMMAND_SUBMIT){
+ coli_serve_command_dispose(&command); return 0;
}
- (void)slot;
- char *payload=malloc((size_t)plen+1);
- if(!payload){ fprintf(out,"ERROR %s out of memory\n",id); fflush(out); return 0; }
- if(fread(payload,1,(size_t)plen,in)!=(size_t)plen){ free(payload); return -1; }
- (void)fgetc(in); payload[plen]=0;
- snprintf(q->id,sizeof(q->id),"%s",id);
- q->max_tok=max_tok; q->temp=temp; q->top_p=top_p;
- q->payload=payload; q->plen=plen;
+ snprintf(q->id,sizeof(q->id),"%s",command.id);
+ q->max_tok=command.max_tokens; q->temp=command.temperature; q->top_p=command.top_p;
+ q->payload=(char*)coli_serve_command_take_payload(&command);
+ q->plen=(int)command.payload_bytes;
+ coli_serve_command_dispose(&command);
return 2;
}
/* Drain only commands that can arrive while one request owns the engine. A
* second SUBMIT is refused without stealing the active request's state. */
+typedef struct { const char *active; int fatal; } K3ServePoll;
+
static int k3_serve_poll_cancel(void *context){
- const char *active=context;
+ K3ServePoll *poll=context;
int cancelled=0;
while(serve_stdin_readable()){
- ServeReq queued={0}; int r=serve_read_req(stdin,stdout,&queued,active);
- if(r<0||r==1) cancelled=1;
+ ServeReq queued={0}; int r=serve_read_req(stdin,stdout,&queued,poll->active);
+ if(r<0){ if(r==-2) poll->fatal=1; return 1; }
+ if(r==1) cancelled=1;
if(r==2){
- printf("ERROR %s engine busy\n",queued.id); fflush(stdout);
+ coli_serve_write_error(stdout,queued.id,"engine busy");
free(queued.payload);
}
}
@@ -1955,13 +1974,12 @@ static void k3_cancel_unpublished_state(Model *m){
static void serve_data(const char *id, const char *p, int n){
if(n<=0) return;
- printf("DATA %s %d\n",id,n);
- fwrite(p,1,(size_t)n,stdout); fputc('\n',stdout); fflush(stdout);
+ coli_serve_write_data(stdout,id,p,(size_t)n);
}
-static void serve_one(Model *m, Tok *T, ServeReq *q){
+static int serve_one(Model *m, Tok *T, ServeReq *q){
int cap=65536, *ids=malloc((size_t)cap*sizeof(int)), np=0;
- if(!ids){ printf("ERROR %s out of memory\n",q->id); fflush(stdout); return; }
+ if(!ids){ coli_serve_write_error(stdout,q->id,"out of memory"); return 0; }
int sp[4]={-1,-1,-1,-1}, chat=0, thinking=0;
if(m->c.bos>=0) ids[np++]=m->c.bos;
if(q->plen>=8&&!memcmp(q->payload,"K3CHAT1\n",8)){
@@ -1970,25 +1988,27 @@ static void serve_one(Model *m, Tok *T, ServeReq *q){
/* Not the request's fault: this snapshot's tokenizer.json has no
* <|open|>/<|close|>/<|sep|>/<|end_of_msg|>, so no chat turn can be
* built from it. Say that, and say where a usable one comes from. */
- printf("ERROR %s tokenizer.json has no XTML chat tokens "
- "(<|open|> <|close|> <|sep|> <|end_of_msg|>); "
- "regenerate it with tools/k3_tokenizer.py\n",q->id);
- fflush(stdout);
+ coli_serve_write_error(stdout,q->id,
+ "tokenizer.json has no XTML chat tokens "
+ "(<|open|> <|close|> <|sep|> <|end_of_msg|>); "
+ "regenerate it with tools/k3_tokenizer.py");
fprintf(stderr,"[K3] chat: XTML special tokens not in tokenizer.json — "
"regenerate with tools/k3_tokenizer.py\n");
- free(ids); return; }
- if(n<0){ printf("ERROR %s invalid K3 chat payload\n",q->id); fflush(stdout); free(ids); return; }
+ free(ids); return 0; }
+ if(n<0){ coli_serve_write_error(stdout,q->id,"invalid K3 chat payload"); free(ids); return 0; }
np+=n; chat=1;
} else {
np+=tok_encode(T,q->payload,q->plen,ids+np,cap-np);
}
int max_ctx=getenv("K3_MAXT")?atoi(getenv("K3_MAXT")):8192;
if(np<1||(int64_t)np+q->max_tok>max_ctx){ /* SEC (GHSA-gf38): int64 so np+max_tok can't wrap negative */
- printf("ERROR %s CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d\n",
- q->id,np,q->max_tok,max_ctx);
- fflush(stdout); free(ids); return;
+ char message[160];
+ snprintf(message,sizeof(message),
+ "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
+ np,q->max_tok,max_ctx);
+ coli_serve_write_error(stdout,q->id,message); free(ids); return 0;
}
- printf("ACCEPT %s %d\n",q->id,np); fflush(stdout);
+ coli_serve_write_accept(stdout,q->id,np);
/* KV PREFIX REUSE (#639 for GLM; this engine re-prefilled every turn).
* A chat client resends the whole transcript each turn, so turn N used to
* re-process turns 1..N-1 from scratch — the cost of a message grew with
@@ -2022,15 +2042,17 @@ static void serve_one(Model *m, Tok *T, ServeReq *q){
/* `i` is the ABSOLUTE position: attention and the MLA Lc/Rc slots are
* position-indexed, so the loop starts at `reuse`, not at 0. */
int prefill_cancelled=0;
+ K3ServePoll poll={q->id,0};
for(int i=reuse;i<np;i+=chunk){
int C=np-i<chunk?np-i:chunk;
- free(lo); lo=step_chunk_ex(m,ids+i,i,C,k3_serve_poll_cancel,q->id,
+ free(lo); lo=step_chunk_ex(m,ids+i,i,C,k3_serve_poll_cancel,&poll,
&prefill_cancelled);
if(prefill_cancelled) break;
}
if(prefill_cancelled){
free(lo); k3_cancel_unpublished_state(m); free(ids);
- printf("ERROR %s CANCELLED\n",q->id); fflush(stdout); return;
+ if(!poll.fatal) coli_serve_write_error(stdout,q->id,"CANCELLED");
+ return poll.fatal?-1:0;
}
int gen=0, limited=1, cancelled=0, xsup=0, xopen=0, xtl=0;
char buf[512], xtag[64];
@@ -2064,21 +2086,26 @@ static void serve_one(Model *m, Tok *T, ServeReq *q){
while(serve_stdin_readable()){
ServeReq queued={0};
int r=serve_read_req(stdin,stdout,&queued,q->id);
+ if(r==-2){ poll.fatal=1; cancelled=1; break; }
if(r<0){ cancelled=1; break; }
if(r==1) cancelled=1;
if(r==2){
- printf("ERROR %s engine busy\n",queued.id); fflush(stdout); free(queued.payload);
+ coli_serve_write_error(stdout,queued.id,"engine busy"); free(queued.payload);
}
}
if(cancelled){ limited=0; break; }
if(eos){ limited=0; break; }
if(s+1<q->max_tok) lo=step_chunk(m,&tk,np+s,1);
}
+ if(poll.fatal){ free(lo); free(ids); return -1; }
free(lo); free(ids);
double dt=now_s()-t0, decode=now_s()-tg;
uint64_t hits=m->hits-hit0, misses=m->miss-miss0, total=hits+misses;
- printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n",q->id,gen,
- decode>0?gen/decode:0.0,total?100.0*hits/total:0.0,rss_gb(),np,limited);
+ ColiServeDone done={gen,decode>0?gen/decode:0.0,
+ total?100.0*hits/total:0.0,rss_gb(),np,limited};
+ char done_line[256];
+ int done_bytes=coli_serve_format_done(done_line,sizeof(done_line),q->id,&done);
+ if(done_bytes>0) fwrite(done_line,1,(size_t)done_bytes,stdout);
double moe=m->t_moe-e0, disk=m->t_eload-d0;
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n",
dt,np,gen,disk,0.0,moe>disk?moe-disk:moe,m->t_attn-a0,m->t_head-h0,gen+1);
@@ -2087,6 +2114,7 @@ static void serve_one(Model *m, Tok *T, ServeReq *q){
if(g_k3_vk) fprintf(stderr,"[K3-VK] routed tier: %ld resident, %ld GPU hits so far\n",
g_vk_res,g_vk_hit);
#endif
+ return 0;
}
static void serve_loop(Model *m, Tok *T){
@@ -2094,16 +2122,13 @@ static void serve_loop(Model *m, Tok *T){
* finale in \r\n, il gateway non lo riconosce e resta in attesa per sempre
* (#748). Vive in compat.h perche' colibri.c ce l'ha da #195 e questo motore
* e' nato senza. */
- coli_serve_binary_mode();
- setvbuf(stdin,NULL,_IONBF,0);
- fputs("\x01\x01READY\x01\x01\n",stdout);
- printf("STAT 0 0.0 0.0 %.2f 0 0\n",rss_gb());
- fflush(stdout);
+ coli_serve_stdio_init();
+ coli_serve_write_ready(stdout,rss_gb());
for(;;){
ServeReq q={0}; int r;
do r=serve_read_req(stdin,stdout,&q,NULL); while(r==0);
if(r<0) return;
- if(r==2){ serve_one(m,T,&q); free(q.payload); }
+ if(r==2){ int fatal=serve_one(m,T,&q); free(q.payload); if(fatal<0) return; }
}
}
diff --git a/c/serve_codec.h b/c/serve_codec.h
index f0e03c8e..d1324a8f 100644
--- a/c/serve_codec.h
+++ b/c/serve_codec.h
@@ -291,4 +291,15 @@ static inline int coli_serve_write_done(
return fflush(output) == 0;
}
+static inline int coli_serve_format_done(
+ char *output, size_t capacity, const char *id, const ColiServeDone *done)
+{
+ int count = snprintf(output, capacity,
+ "DONE %s STAT %d %.3f %.1f %.2f %d %d\n", id,
+ done->completion_tokens, done->tokens_per_second,
+ done->cache_hit_percent, done->rss_gb,
+ done->prompt_tokens, done->length_limited);
+ return count >= 0 && (size_t)count < capacity ? count : -1;
+}
+
#endif
diff --git a/c/tests/test_kimi_serve_framing.c b/c/tests/test_kimi_serve_framing.c
index 4f521fd0..27fa5b80 100644
--- a/c/tests/test_kimi_serve_framing.c
+++ b/c/tests/test_kimi_serve_framing.c
@@ -74,7 +74,7 @@ static void test_errors_and_busy_payload_are_byte_exact(void)
FILE *in=bytes_input(bad,sizeof(bad)-1);
FILE *out=tmpfile(); assert(out); binary_stream(out);
ServeReq request={0};
- assert(serve_read_req(in,out,&request,NULL)==0);
+ assert(serve_read_req(in,out,&request,NULL)==-2);
char output[128]; size_t count=read_output(out,output,sizeof(output));
static const char want[]="ERROR bad bad submit header\n";
assert(count==sizeof(want)-1&&memcmp(output,want,count)==0);
@@ -93,11 +93,40 @@ static void test_errors_and_busy_payload_are_byte_exact(void)
fclose(in); fclose(out);
}
+static void test_malformed_submit_cannot_become_a_control(void)
+{
+ static const char *frames[]={
+ "SUBMIT bad 0 12 0 0.7 0.95\nCANCEL live\n",
+ "SUBMIT bad 0 12 4 0.7 0.95 extra extra\nCANCEL live\n",
+ };
+ for(size_t i=0;i<sizeof(frames)/sizeof(frames[0]);i++){
+ FILE *in=bytes_input(frames[i],strlen(frames[i]));
+ FILE *out=tmpfile(); assert(out); binary_stream(out);
+ ServeReq request={0};
+ assert(serve_read_req(in,out,&request,"live")==-2);
+ assert(fgetc(in)=='C');
+ char output[64]; size_t count=read_output(out,output,sizeof(output));
+ static const char want[]="ERROR bad bad submit header\n";
+ assert(count==sizeof(want)-1&&memcmp(output,want,count)==0);
+ fclose(in); fclose(out);
+ }
+
+ static const char bad_term[]=
+ "SUBMIT broken 0 1 4 0.7 0.95\nxXSTOP live\n";
+ FILE *in=bytes_input(bad_term,sizeof(bad_term)-1);
+ FILE *out=tmpfile(); assert(out); binary_stream(out);
+ ServeReq request={0};
+ assert(serve_read_req(in,out,&request,"live")==-2);
+ assert(fgetc(in)=='S');
+ fclose(in); fclose(out);
+}
+
int main(void)
{
test_submit_preserves_k3_wire_bytes();
test_controls_match_only_active_request();
test_errors_and_busy_payload_are_byte_exact();
+ test_malformed_submit_cannot_become_a_control();
puts("kimi serve framing baseline: ok");
return 0;
}
diff --git a/c/tests/test_serve_codec.c b/c/tests/test_serve_codec.c
index 42646713..55aaaeeb 100644
--- a/c/tests/test_serve_codec.c
+++ b/c/tests/test_serve_codec.c
@@ -157,6 +157,10 @@ static void test_writer_golden_bytes(void)
assert(coli_serve_write_error(output, "req-7", "bad\r\nframe"));
ColiServeDone done = {3, 1.25, 50.0, 1.25, 7, 1};
assert(coli_serve_write_done(output, "req-7", &done));
+ char done_line[128];
+ assert(coli_serve_format_done(done_line, sizeof(done_line), "req-7", &done) ==
+ (int)strlen("DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n"));
+ assert(strcmp(done_line, "DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n") == 0);
static const unsigned char expected[] =
"\x01\x01READY\x01\x01\nSTAT 0 0.0 0.0 1.25 0 0\n"