Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

test(inkling): freeze serve wire framing

Mario Gutierrezbc46e96fa37b
diff --git a/c/.gitignore b/c/.gitignore
index c843d62a..ed6d0910 100644
--- a/c/.gitignore
+++ b/c/.gitignore
@@ -39,6 +39,7 @@ tests/test_pipe_block
tests/test_kimi_request_state
tests/test_kimi_serve_framing
tests/test_v4_serve_framing
+tests/test_inkling_serve_framing
*.o
*.dll
cublasLt64*.dll
diff --git a/c/Makefile b/c/Makefile
index fb8c6722..8d5cda4e 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1077,6 +1077,9 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h $(INK_CUDA_OBJ) $(METAL_OBJ)
+ $(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)
+
tests/test_kimi_serve_framing$(EXE): tests/test_kimi_serve_framing.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h
$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
diff --git a/c/inkling.c b/c/inkling.c
index 45c3f60d..785ea6ff 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -1867,9 +1867,9 @@ static int stdin_readable(void) {
/* read one control line (+ payload for SUBMIT). cur_id: request in flight;
* returns 1 if that request was cancelled, 0 otherwise, -1 on stdin EOF. */
-static int serve_read_cmd(const char *cur_id) {
+static int serve_read_cmd(FILE *input, FILE *output, const char *cur_id) {
char ln[512];
- if (!fgets(ln, sizeof(ln), stdin)) return -1;
+ if (!fgets(ln, sizeof(ln), input)) return -1;
char cmd[16], id[64];
if (sscanf(ln, "%15s %63s", cmd, id) < 2) return 0;
if (!strcmp(cmd, "CANCEL")) return cur_id && !strcmp(id, cur_id);
@@ -1893,24 +1893,24 @@ static int serve_read_cmd(const char *cur_id) {
* has, rather than in one of its callers. */
if (nf < 5 || plen < 0 || plen > (1<<22) || alen < 0 || alen > (1<<26) ||
max_tok < 1 || max_tok > (1<<20)) {
- printf("ERROR %s bad submit header\n", id); fflush(stdout); return 0; }
+ fprintf(output, "ERROR %s bad submit header\n", id); fflush(output); return 0; }
(void)slot;
char *pl = malloc((size_t)plen + 1);
- if (fread(pl, 1, (size_t)plen, stdin) != (size_t)plen) { free(pl); return -1; }
+ if (fread(pl, 1, (size_t)plen, input) != (size_t)plen) { free(pl); return -1; }
pl[plen] = 0;
uint8_t *au = NULL;
if (alen > 0) {
au = malloc((size_t)alen);
- if (fread(au, 1, (size_t)alen, stdin) != (size_t)alen) { free(pl); free(au); return -1; }
+ if (fread(au, 1, (size_t)alen, input) != (size_t)alen) { free(pl); free(au); return -1; }
}
- int nl = fgetc(stdin); (void)nl;
+ int nl = fgetc(input); (void)nl;
if (g_qn < SRV_QMAX) {
SReq *q = &g_q[g_qn++];
snprintf(q->id, sizeof(q->id), "%s", id);
q->max_tok = max_tok; q->temp = temp; q->top_p = top_p;
q->payload = pl; q->plen = plen;
q->audio = au; q->alen = alen;
- } else { printf("ERROR %s queue full\n", id); fflush(stdout); free(pl); free(au); }
+ } else { fprintf(output, "ERROR %s queue full\n", id); fflush(output); free(pl); free(au); }
}
return 0;
}
@@ -1993,7 +1993,7 @@ static void serve_one(Model *m, Tok *T, SReq *q) {
fputc('\n', stdout); fflush(stdout);
gen++; len++;
while (stdin_readable()) {
- int r = serve_read_cmd(q->id);
+ int r = serve_read_cmd(stdin, stdout, q->id);
if (r < 0) { free(ids); return; }
if (r > 0) { cancelled = 1; limited = 0; }
}
@@ -2090,7 +2090,7 @@ static void serve_loop(Model *m, Tok *T) {
serve_hwinfo(m);
serve_tiers_emap(m);
for (;;) {
- while (!g_qn) if (serve_read_cmd(NULL) < 0) return; /* blocks on stdin */
+ while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return; /* blocks on stdin */
SReq q = g_q[0];
memmove(g_q, g_q+1, (size_t)(--g_qn) * sizeof(SReq));
serve_one(m, T, &q);
diff --git a/c/tests/test_inkling_serve_framing.c b/c/tests/test_inkling_serve_framing.c
new file mode 100644
index 00000000..572e3f47
--- /dev/null
+++ b/c/tests/test_inkling_serve_framing.c
@@ -0,0 +1,90 @@
+/* Freeze Inkling's text + binary DMel framing before adopting serve_codec.h. */
+#define main inkling_main_unused
+#include "../inkling.c"
+#undef main
+
+#include <assert.h>
+
+static void binary_stream(FILE *stream)
+{
+#ifdef _WIN32
+ assert(_setmode(_fileno(stream), _O_BINARY) != -1);
+#else
+ (void)stream;
+#endif
+}
+
+static FILE *bytes_input(const void *data, size_t size)
+{
+ FILE *stream=tmpfile(); assert(stream); binary_stream(stream);
+ assert(fwrite(data,1,size,stream)==size); rewind(stream); return stream;
+}
+
+static size_t read_output(FILE *stream, unsigned char *output, size_t capacity)
+{
+ assert(fflush(stream)==0); rewind(stream);
+ return fread(output,1,capacity,stream);
+}
+
+static void reset_queue(void)
+{
+ for(int i=0;i<g_qn;i++){ free(g_q[i].payload); free(g_q[i].audio); }
+ memset(g_q,0,sizeof(g_q)); g_qn=0;
+}
+
+static void test_text_and_audio_submits_are_exact(void)
+{
+ static const unsigned char audio[]={0,1,2,3,'\n',5,6,7,'\r',9,10,11,12,13,14,15};
+ static const char prompt[]="Hé\nx";
+ unsigned char frame[256];
+ int header=snprintf((char*)frame,sizeof(frame),"SUBMIT audio 0 %zu 4 0.25 0.9 %zu\n",
+ sizeof(prompt)-1,sizeof(audio));
+ assert(header>0);
+ memcpy(frame+header,prompt,sizeof(prompt)-1);
+ memcpy(frame+header+sizeof(prompt)-1,audio,sizeof(audio));
+ frame[header+sizeof(prompt)-1+sizeof(audio)]='\n';
+ FILE *input=bytes_input(frame,(size_t)header+sizeof(prompt)+sizeof(audio));
+ FILE *output=tmpfile(); assert(output); binary_stream(output);
+ assert(serve_read_cmd(input,output,NULL)==0);
+ assert(g_qn==1&&strcmp(g_q[0].id,"audio")==0);
+ assert(g_q[0].plen==(int)sizeof(prompt)-1&&memcmp(g_q[0].payload,prompt,sizeof(prompt)-1)==0);
+ assert(g_q[0].alen==(int)sizeof(audio)&&memcmp(g_q[0].audio,audio,sizeof(audio))==0);
+ assert(fgetc(input)==EOF);
+ unsigned char bytes[16]; assert(read_output(output,bytes,sizeof(bytes))==0);
+ fclose(input); fclose(output); reset_queue();
+
+ static const char text[]="SUBMIT text 7 3 2 0 1\na\nb\n";
+ input=bytes_input(text,sizeof(text)-1); output=tmpfile(); assert(output); binary_stream(output);
+ assert(serve_read_cmd(input,output,NULL)==0);
+ assert(g_qn==1&&g_q[0].alen==0&&memcmp(g_q[0].payload,"a\nb",3)==0);
+ fclose(input); fclose(output); reset_queue();
+}
+
+static void test_controls_and_queue_full_preserve_behavior(void)
+{
+ FILE *output=tmpfile(); assert(output); binary_stream(output);
+ static const char cancel[]="CANCEL live\n";
+ FILE *input=bytes_input(cancel,sizeof(cancel)-1);
+ assert(serve_read_cmd(input,output,"live")==1); fclose(input);
+ static const char stop[]="STOP live\n";
+ input=bytes_input(stop,sizeof(stop)-1);
+ assert(serve_read_cmd(input,output,"live")==0); fclose(input);
+
+ g_qn=SRV_QMAX;
+ static const char full[]="SUBMIT full 0 1 1 0 1 4\nxWXYZ\n";
+ input=bytes_input(full,sizeof(full)-1);
+ assert(serve_read_cmd(input,output,NULL)==0);
+ assert(fgetc(input)==EOF);
+ unsigned char bytes[64]; size_t count=read_output(output,bytes,sizeof(bytes));
+ static const char want[]="ERROR full queue full\n";
+ assert(count==sizeof(want)-1&&memcmp(bytes,want,count)==0);
+ fclose(input); fclose(output); g_qn=0;
+}
+
+int main(void)
+{
+ test_text_and_audio_submits_are_exact();
+ test_controls_and_queue_full_preserve_behavior();
+ puts("inkling serve framing baseline: ok");
+ return 0;
+}
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index c3caa8d1..5cc6f1b4 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -472,6 +472,33 @@ class FakeProcess:
class DispatcherTest(unittest.TestCase):
+ def test_inkling_audio_request_and_response_transcript_is_byte_exact(self):
+ prompt = "<|message_user|><|content_audio_input|><|audio|><|end_message|>"
+ payload = prompt.encode("utf-8")
+ audio = bytes(range(16)) * 5
+ expected = (f"SUBMIT 1 0 {len(payload)} 4 0.25 0.9 {len(audio)}\n".encode() +
+ payload + audio + b"\n")
+
+ def respond(process, frame):
+ self.assertEqual(frame, expected)
+ process.stdout.feed(
+ b"DATA 1 4\nA\n\xc3\xa9\n"
+ b"DONE 1 STAT 1 2.500 50.0 1.25 7 0\n"
+ )
+
+ process = FakeProcess(respond)
+ with patch("openai_server.ARCH", "inkling"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("inkling", "model")
+ chunks = []
+ stats = engine.generate(prompt, 4, 0.25, 0.9, chunks.append,
+ audio=audio)
+ engine.close()
+
+ self.assertEqual(process.writes, [expected])
+ self.assertEqual(chunks, ["A\né"])
+ self.assertEqual(stats["prompt_tokens"], 7)
+
def test_v4_request_and_response_transcript_is_byte_exact(self):
prompt = "<|begin▁of▁sentence|>System<|User|>Hello<|Assistant|>"
payload = prompt.encode("utf-8")