Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
test(inkling): freeze serve wire framing
Mario Gutierrezbc46e96fa37b
diff --git a/c/.gitignore b/c/.gitignoreindex c843d62a..ed6d0910 100644--- a/c/.gitignore+++ b/c/.gitignore@@ -39,6 +39,7 @@ tests/test_pipe_blocktests/test_kimi_request_statetests/test_kimi_serve_framingtests/test_v4_serve_framing+tests/test_inkling_serve_framing*.o*.dllcublasLt64*.dlldiff --git a/c/Makefile b/c/Makefileindex fb8c6722..8d5cda4e 100644--- a/c/Makefile+++ b/c/Makefile@@ -1077,6 +1077,9 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.htests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h $(INK_CUDA_OBJ) $(METAL_OBJ)+ $(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)+tests/test_kimi_serve_framing$(EXE): tests/test_kimi_serve_framing.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)diff --git a/c/inkling.c b/c/inkling.cindex 45c3f60d..785ea6ff 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -1867,9 +1867,9 @@ static int stdin_readable(void) {/* read one control line (+ payload for SUBMIT). cur_id: request in flight;* returns 1 if that request was cancelled, 0 otherwise, -1 on stdin EOF. */-static int serve_read_cmd(const char *cur_id) {+static int serve_read_cmd(FILE *input, FILE *output, const char *cur_id) {char ln[512];- if (!fgets(ln, sizeof(ln), stdin)) return -1;+ if (!fgets(ln, sizeof(ln), input)) return -1;char cmd[16], id[64];if (sscanf(ln, "%15s %63s", cmd, id) < 2) return 0;if (!strcmp(cmd, "CANCEL")) return cur_id && !strcmp(id, cur_id);@@ -1893,24 +1893,24 @@ static int serve_read_cmd(const char *cur_id) {* has, rather than in one of its callers. */if (nf < 5 || plen < 0 || plen > (1<<22) || alen < 0 || alen > (1<<26) ||max_tok < 1 || max_tok > (1<<20)) {- printf("ERROR %s bad submit header\n", id); fflush(stdout); return 0; }+ fprintf(output, "ERROR %s bad submit header\n", id); fflush(output); return 0; }(void)slot;char *pl = malloc((size_t)plen + 1);- if (fread(pl, 1, (size_t)plen, stdin) != (size_t)plen) { free(pl); return -1; }+ if (fread(pl, 1, (size_t)plen, input) != (size_t)plen) { free(pl); return -1; }pl[plen] = 0;uint8_t *au = NULL;if (alen > 0) {au = malloc((size_t)alen);- if (fread(au, 1, (size_t)alen, stdin) != (size_t)alen) { free(pl); free(au); return -1; }+ if (fread(au, 1, (size_t)alen, input) != (size_t)alen) { free(pl); free(au); return -1; }}- int nl = fgetc(stdin); (void)nl;+ int nl = fgetc(input); (void)nl;if (g_qn < SRV_QMAX) {SReq *q = &g_q[g_qn++];snprintf(q->id, sizeof(q->id), "%s", id);q->max_tok = max_tok; q->temp = temp; q->top_p = top_p;q->payload = pl; q->plen = plen;q->audio = au; q->alen = alen;- } else { printf("ERROR %s queue full\n", id); fflush(stdout); free(pl); free(au); }+ } else { fprintf(output, "ERROR %s queue full\n", id); fflush(output); free(pl); free(au); }}return 0;}@@ -1993,7 +1993,7 @@ static void serve_one(Model *m, Tok *T, SReq *q) {fputc('\n', stdout); fflush(stdout);gen++; len++;while (stdin_readable()) {- int r = serve_read_cmd(q->id);+ int r = serve_read_cmd(stdin, stdout, q->id);if (r < 0) { free(ids); return; }if (r > 0) { cancelled = 1; limited = 0; }}@@ -2090,7 +2090,7 @@ static void serve_loop(Model *m, Tok *T) {serve_hwinfo(m);serve_tiers_emap(m);for (;;) {- while (!g_qn) if (serve_read_cmd(NULL) < 0) return; /* blocks on stdin */+ while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return; /* blocks on stdin */SReq q = g_q[0];memmove(g_q, g_q+1, (size_t)(--g_qn) * sizeof(SReq));serve_one(m, T, &q);diff --git a/c/tests/test_inkling_serve_framing.c b/c/tests/test_inkling_serve_framing.cnew file mode 100644index 00000000..572e3f47--- /dev/null+++ b/c/tests/test_inkling_serve_framing.c@@ -0,0 +1,90 @@+/* Freeze Inkling's text + binary DMel framing before adopting serve_codec.h. */+#define main inkling_main_unused+#include "../inkling.c"+#undef main++#include <assert.h>++static void binary_stream(FILE *stream)+{+#ifdef _WIN32+ assert(_setmode(_fileno(stream), _O_BINARY) != -1);+#else+ (void)stream;+#endif+}++static FILE *bytes_input(const void *data, size_t size)+{+ FILE *stream=tmpfile(); assert(stream); binary_stream(stream);+ assert(fwrite(data,1,size,stream)==size); rewind(stream); return stream;+}++static size_t read_output(FILE *stream, unsigned char *output, size_t capacity)+{+ assert(fflush(stream)==0); rewind(stream);+ return fread(output,1,capacity,stream);+}++static void reset_queue(void)+{+ for(int i=0;i<g_qn;i++){ free(g_q[i].payload); free(g_q[i].audio); }+ memset(g_q,0,sizeof(g_q)); g_qn=0;+}++static void test_text_and_audio_submits_are_exact(void)+{+ static const unsigned char audio[]={0,1,2,3,'\n',5,6,7,'\r',9,10,11,12,13,14,15};+ static const char prompt[]="Hé\nx";+ unsigned char frame[256];+ int header=snprintf((char*)frame,sizeof(frame),"SUBMIT audio 0 %zu 4 0.25 0.9 %zu\n",+ sizeof(prompt)-1,sizeof(audio));+ assert(header>0);+ memcpy(frame+header,prompt,sizeof(prompt)-1);+ memcpy(frame+header+sizeof(prompt)-1,audio,sizeof(audio));+ frame[header+sizeof(prompt)-1+sizeof(audio)]='\n';+ FILE *input=bytes_input(frame,(size_t)header+sizeof(prompt)+sizeof(audio));+ FILE *output=tmpfile(); assert(output); binary_stream(output);+ assert(serve_read_cmd(input,output,NULL)==0);+ assert(g_qn==1&&strcmp(g_q[0].id,"audio")==0);+ assert(g_q[0].plen==(int)sizeof(prompt)-1&&memcmp(g_q[0].payload,prompt,sizeof(prompt)-1)==0);+ assert(g_q[0].alen==(int)sizeof(audio)&&memcmp(g_q[0].audio,audio,sizeof(audio))==0);+ assert(fgetc(input)==EOF);+ unsigned char bytes[16]; assert(read_output(output,bytes,sizeof(bytes))==0);+ fclose(input); fclose(output); reset_queue();++ static const char text[]="SUBMIT text 7 3 2 0 1\na\nb\n";+ input=bytes_input(text,sizeof(text)-1); output=tmpfile(); assert(output); binary_stream(output);+ assert(serve_read_cmd(input,output,NULL)==0);+ assert(g_qn==1&&g_q[0].alen==0&&memcmp(g_q[0].payload,"a\nb",3)==0);+ fclose(input); fclose(output); reset_queue();+}++static void test_controls_and_queue_full_preserve_behavior(void)+{+ FILE *output=tmpfile(); assert(output); binary_stream(output);+ static const char cancel[]="CANCEL live\n";+ FILE *input=bytes_input(cancel,sizeof(cancel)-1);+ assert(serve_read_cmd(input,output,"live")==1); fclose(input);+ static const char stop[]="STOP live\n";+ input=bytes_input(stop,sizeof(stop)-1);+ assert(serve_read_cmd(input,output,"live")==0); fclose(input);++ g_qn=SRV_QMAX;+ static const char full[]="SUBMIT full 0 1 1 0 1 4\nxWXYZ\n";+ input=bytes_input(full,sizeof(full)-1);+ assert(serve_read_cmd(input,output,NULL)==0);+ assert(fgetc(input)==EOF);+ unsigned char bytes[64]; size_t count=read_output(output,bytes,sizeof(bytes));+ static const char want[]="ERROR full queue full\n";+ assert(count==sizeof(want)-1&&memcmp(bytes,want,count)==0);+ fclose(input); fclose(output); g_qn=0;+}++int main(void)+{+ test_text_and_audio_submits_are_exact();+ test_controls_and_queue_full_preserve_behavior();+ puts("inkling serve framing baseline: ok");+ return 0;+}diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex c3caa8d1..5cc6f1b4 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -472,6 +472,33 @@ class FakeProcess:class DispatcherTest(unittest.TestCase):+ def test_inkling_audio_request_and_response_transcript_is_byte_exact(self):+ prompt = "<|message_user|><|content_audio_input|><|audio|><|end_message|>"+ payload = prompt.encode("utf-8")+ audio = bytes(range(16)) * 5+ expected = (f"SUBMIT 1 0 {len(payload)} 4 0.25 0.9 {len(audio)}\n".encode() ++ payload + audio + b"\n")++ def respond(process, frame):+ self.assertEqual(frame, expected)+ process.stdout.feed(+ b"DATA 1 4\nA\n\xc3\xa9\n"+ b"DONE 1 STAT 1 2.500 50.0 1.25 7 0\n"+ )++ process = FakeProcess(respond)+ with patch("openai_server.ARCH", "inkling"), \+ patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("inkling", "model")+ chunks = []+ stats = engine.generate(prompt, 4, 0.25, 0.9, chunks.append,+ audio=audio)+ engine.close()++ self.assertEqual(process.writes, [expected])+ self.assertEqual(chunks, ["A\né"])+ self.assertEqual(stats["prompt_tokens"], 7)+def test_v4_request_and_response_transcript_is_byte_exact(self):prompt = "<|begin▁of▁sentence|>System<|User|>Hello<|Assistant|>"payload = prompt.encode("utf-8")