Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
refactor(serve): share framing with OLMoE
Mario Gutierrez6104bbb3b0b9
diff --git a/c/Makefile b/c/Makefileindex 5d0660c0..ee941025 100644--- a/c/Makefile+++ b/c/Makefile@@ -873,7 +873,7 @@ fp8-bench: fp8_bench$(EXE)# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py# asserts this shape.-olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h+olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h$(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(METAL_OBJ)@@ -916,7 +916,7 @@ portable:iobench$(EXE): iobench.c compat.h$(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS)-tests/test_serve_sentinel$(EXE): tests/test_serve_sentinel.c compat.h+tests/test_serve_sentinel$(EXE): tests/test_serve_sentinel.c compat.h serve_codec.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_cluster_protocol$(EXE): tests/test_cluster_protocol.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h@@ -992,13 +992,16 @@ tests/test_spec_decode_state$(EXE): tests/test_spec_decode_state.c colibri.c st.tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# olmoe's matmul_q, not colibri's: compares the IDOT path against FP32# ACTIVATIONS rather than an integer reference. NOCUDA_* for the same reason# the olmoe target uses it -- olmoe.c contains no COLI_CUDA code.-tests/test_olmoe_matmul_q$(EXE): tests/test_olmoe_matmul_q.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h+tests/test_olmoe_matmul_q$(EXE): tests/test_olmoe_matmul_q.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)-tests/test_olmoe_serve_framing$(EXE): tests/test_olmoe_serve_framing.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h+tests/test_olmoe_serve_framing$(EXE): tests/test_olmoe_serve_framing.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.hdiff --git a/c/olmoe.c b/c/olmoe.cindex dadd1fb0..599a9571 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -37,6 +37,7 @@#endif#include "omp_tune.h"#include "route_trace.h" /* shared routing telemetry (#700) */+#include "serve_codec.h"#ifdef _WIN32#include <windows.h>@@ -1168,44 +1169,63 @@ static void run_chat(Model *m, Tok *T, int ctx_cap) {typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen; } SReq;#define SRV_QMAX 16static SReq g_q[SRV_QMAX]; static int g_qn = 0;+static const ColiServeWireProfile olmoe_wire = {+ .max_header_bytes = 511,+ .max_payload_bytes = 1u << 22,+ .max_tokens = 1 << 20,+ .require_exact_lf = 1,+ .require_finite_sampling = 0,+};/* read one control line (+ payload for SUBMIT). cur_id: request in flight;* returns 1 if that request was cancelled, 0 otherwise, -1 on input EOF. */static int serve_read_cmd(FILE *in, FILE *out, const char *cur_id) {- char ln[512];- if (!fgets(ln, sizeof(ln), in)) return -1;- char cmd[16], id[64];- if (sscanf(ln, "%15s %63s", cmd, id) < 2) return 0;- if (!strcmp(cmd, "CANCEL")) return cur_id && !strcmp(id, cur_id);- if (!strcmp(cmd, "SUBMIT")) {- int slot, plen, max_tok; float temp, top_p;- int nf = sscanf(ln, "%*s %*s %d %d %d %f %f", &slot, &plen, &max_tok, &temp, &top_p);- if (nf < 5 || plen < 0 || plen > (1<<22) || max_tok < 1 || max_tok > (1<<20)) {- fprintf(out, "ERROR %s bad submit header\n", id); fflush(out); return 0; }- (void)slot;- char *pl = malloc((size_t)plen + 1);- if (fread(pl, 1, (size_t)plen, in) != (size_t)plen) { free(pl); return -1; }- pl[plen] = 0;- int nl = fgetc(in); (void)nl;+ ColiServeCommand command;+ ColiServeReadResult result = coli_serve_read_command(in, &olmoe_wire, &command);+ if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) return -1;+ if (result == COLI_SERVE_READ_NOMEM) {+ coli_serve_write_error(out, command.id, "out of memory");+ return -1;+ }+ if (result == COLI_SERVE_READ_BAD_REQUEST &&+ command.kind == COLI_SERVE_COMMAND_SUBMIT) {+ coli_serve_write_error(out, command.id, "bad submit header");+ return -1;+ }+ if (result != COLI_SERVE_READ_OK) return 0;+ if (command.kind == COLI_SERVE_COMMAND_CANCEL) {+ int cancelled = cur_id && !strcmp(command.id, cur_id);+ coli_serve_command_dispose(&command);+ return cancelled;+ }+ if (command.kind == COLI_SERVE_COMMAND_SUBMIT) {if (g_qn < SRV_QMAX) {SReq *q = &g_q[g_qn++];- snprintf(q->id, sizeof(q->id), "%s", id);- q->max_tok = max_tok; q->temp = temp; q->top_p = top_p;- q->payload = pl; q->plen = plen;- } else { fprintf(out, "ERROR %s queue full\n", id); fflush(out); free(pl); }+ snprintf(q->id, sizeof(q->id), "%s", command.id);+ q->max_tok = command.max_tokens;+ q->temp = command.temperature;+ q->top_p = command.top_p;+ q->payload = (char *)coli_serve_command_take_payload(&command);+ q->plen = (int)command.payload_bytes;+ } else {+ coli_serve_write_error(out, command.id, "queue full");+ }}+ coli_serve_command_dispose(&command);return 0;}-static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {+static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {Cfg *c = &m->c;int cap = q->plen + 16;int *ids = malloc((size_t)cap * sizeof(int));int np = tok_encode(T, q->payload, q->plen, ids, cap);- if (np <= 0) { printf("ERROR %s empty prompt\n", q->id); fflush(stdout); free(ids); return; }+ if (np <= 0) { coli_serve_write_error(stdout, q->id, "empty prompt"); free(ids); return 0; }if (np + q->max_tok > ctx_cap) {- printf("ERROR %s context exceeds CTX (%d + %d > %d)\n", q->id, np, q->max_tok, ctx_cap);- fflush(stdout); free(ids); return;+ char message[128];+ snprintf(message, sizeof(message), "context exceeds CTX (%d + %d > %d)",+ np, q->max_tok, ctx_cap);+ coli_serve_write_error(stdout, q->id, message); free(ids); return 0;}g_temp = q->temp; g_nuc = q->top_p;double t0 = now_s();@@ -1218,13 +1238,11 @@ static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {free(logit); logit = NULL;if (is_stop(nt)) { limited = 0; break; }int nb = tok_decode(T, &nt, 1, buf, sizeof(buf)-1);- printf("DATA %s %d\n", q->id, nb);- fwrite(buf, 1, (size_t)nb, stdout);- fputc('\n', stdout); fflush(stdout);+ coli_serve_write_data(stdout, q->id, buf, (size_t)nb);gen++; hist_len++;while (coli_stdin_readable()) {int r = serve_read_cmd(stdin, stdout, q->id);- if (r < 0) { free(ids); return; }+ if (r < 0) { free(ids); return -1; }if (r > 0) { cancelled = 1; limited = 0; }}/* Unlike run_chat(), we do not step() the final token just to populate@@ -1236,8 +1254,15 @@ static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {free(logit);double dt = now_s() - t0;double tot = (double)(m->hits - h0 + m->miss - m0);- printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n", q->id, gen,- dt > 0 ? gen/dt : 0.0, tot ? 100.0*(m->hits-h0)/tot : 0.0, rss_gb(), np, limited);+ ColiServeDone done = {+ .completion_tokens = gen,+ .tokens_per_second = dt > 0 ? gen/dt : 0.0,+ .cache_hit_percent = tot ? 100.0*(m->hits-h0)/tot : 0.0,+ .rss_gb = rss_gb(),+ .prompt_tokens = np,+ .length_limited = limited,+ };+ coli_serve_write_done(stdout, q->id, &done);/* PROF: per-turn phase timings for the dashboard. olmoe.c does not split* its wall time into fill/expert/shared/attn phases the way glm.c and* inkling.c do, so this reports total time only; a real phase breakdown@@ -1245,6 +1270,7 @@ static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {printf("PROF %.3f %d %d 0.0 0.0 0.0 0.0 0.0 %d\n", dt, np, gen, gen + 1);fflush(stdout);free(ids);+ return 0;}/* dashboard HWINFO/TIERS/EMAP: same lines the other serve-capable engines@@ -1303,21 +1329,19 @@ static void serve_tiers_emap(Model *m) {}static void serve_loop(Model *m, Tok *T, int ctx_cap) {- coli_serve_binary_mode();- setvbuf(stdin, NULL, _IONBF, 0);+ coli_serve_stdio_init();int tok_eos = tok_id_of(T, "|||IP_ADDRESS|||");stops_arm_tok(&m->c, tok_eos, T);- fputs("\x01\x01READY\x01\x01\n", stdout);- printf("STAT 0 0.0 0.0 %.2f 0 0\n", rss_gb());- fflush(stdout);+ coli_serve_write_ready(stdout, rss_gb());serve_hwinfo(m);serve_tiers_emap(m);for (;;) {while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return;SReq q = g_q[0];memmove(g_q, g_q + 1, (size_t)(--g_qn) * sizeof(SReq));- serve_one(m, T, &q, ctx_cap);+ int fatal = serve_one(m, T, &q, ctx_cap);free(q.payload);+ if (fatal < 0) return;}}diff --git a/c/serve_codec.h b/c/serve_codec.hnew file mode 100644index 00000000..f0e03c8e--- /dev/null+++ b/c/serve_codec.h@@ -0,0 +1,294 @@+#ifndef COLI_SERVE_CODEC_H+#define COLI_SERVE_CODEC_H++#include <errno.h>+#include <math.h>+#include <stdint.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#include "compat.h"++/* Transport only: parse and emit frames. Admission, queueing, cancellation,+ * KV ownership, scheduling, and generation remain with each family engine. */++#define COLI_SERVE_ID_CAP 64++typedef enum {+ COLI_SERVE_COMMAND_UNKNOWN = 0,+ COLI_SERVE_COMMAND_SUBMIT,+ COLI_SERVE_COMMAND_STOP,+ COLI_SERVE_COMMAND_CANCEL,+} ColiServeCommandKind;++typedef enum {+ COLI_SERVE_READ_EOF = -1,+ COLI_SERVE_READ_BAD_FRAME = -2,+ COLI_SERVE_READ_NOMEM = -3,+ COLI_SERVE_READ_BAD_REQUEST = -4,+ COLI_SERVE_READ_IGNORED = 0,+ COLI_SERVE_READ_OK = 1,+} ColiServeReadResult;++typedef struct {+ size_t max_header_bytes;+ uint64_t max_payload_bytes;+ int max_tokens;+ int require_exact_lf;+ int require_finite_sampling;+} ColiServeWireProfile;++typedef struct {+ ColiServeCommandKind kind;+ char id[COLI_SERVE_ID_CAP];+ int slot;+ uint64_t payload_bytes;+ int max_tokens;+ float temperature;+ float top_p;+ unsigned char *payload;+} ColiServeCommand;++typedef struct {+ int completion_tokens;+ double tokens_per_second;+ double cache_hit_percent;+ double rss_gb;+ int prompt_tokens;+ int length_limited;+} ColiServeDone;++static inline void coli_serve_command_dispose(ColiServeCommand *command)+{+ if (!command) return;+ free(command->payload);+ memset(command, 0, sizeof(*command));+}++static inline void coli_serve_stdio_init(void)+{+ coli_serve_binary_mode();+ setvbuf(stdin, NULL, _IONBF, 0);+}++static inline unsigned char *coli_serve_command_take_payload(ColiServeCommand *command)+{+ unsigned char *payload = command ? command->payload : NULL;+ if (command) command->payload = NULL;+ return payload;+}++static inline int coli_serve_parse_i32(const char *text, int *value)+{+ char *end = NULL;+ errno = 0;+ long parsed = strtol(text, &end, 10);+ if (errno || !end || *end || parsed < INT32_MIN || parsed > INT32_MAX) return 0;+ *value = (int)parsed;+ return 1;+}++static inline int coli_serve_parse_u64(const char *text, uint64_t *value)+{+ char *end = NULL;+ if (!text[0] || text[0] == '-') return 0;+ errno = 0;+ unsigned long long parsed = strtoull(text, &end, 10);+ if (errno || !end || *end) return 0;+ *value = (uint64_t)parsed;+ return 1;+}++static inline int coli_serve_parse_f32(const char *text, float *value)+{+ char *end = NULL;+ float parsed = strtof(text, &end);+ if (!end || end == text || *end) return 0;+ *value = parsed;+ return 1;+}++static inline void coli_serve_classify_line(+ const char *line, ColiServeCommand *command)+{+ const char *cursor = line;+ while (*cursor == ' ' || *cursor == '\t') cursor++;+ const char *name = cursor;+ while (*cursor && *cursor != ' ' && *cursor != '\t') cursor++;+ size_t name_size = (size_t)(cursor - name);+ if (name_size == 6 && !memcmp(name, "SUBMIT", 6))+ command->kind = COLI_SERVE_COMMAND_SUBMIT;+ else if (name_size == 4 && !memcmp(name, "STOP", 4))+ command->kind = COLI_SERVE_COMMAND_STOP;+ else if (name_size == 6 && !memcmp(name, "CANCEL", 6))+ command->kind = COLI_SERVE_COMMAND_CANCEL;+ else+ return;+ while (*cursor == ' ' || *cursor == '\t') cursor++;+ const char *id = cursor;+ while (*cursor && *cursor != ' ' && *cursor != '\t') cursor++;+ size_t id_size = (size_t)(cursor - id);+ if (id_size && id_size < sizeof(command->id)) {+ memcpy(command->id, id, id_size);+ command->id[id_size] = 0;+ }+}++static inline ColiServeReadResult coli_serve_read_command_alloc(+ FILE *input, const ColiServeWireProfile *profile, ColiServeCommand *command,+ void *(*allocate)(size_t))+{+ char *line;+ size_t count = 0;+ char *fields[8];+ int nfields = 0;++ if (!input || !profile || !command) return COLI_SERVE_READ_BAD_FRAME;+ memset(command, 0, sizeof(*command));+ size_t maximum = profile->max_header_bytes ? profile->max_header_bytes : 4096;+ if (maximum == SIZE_MAX) return COLI_SERVE_READ_NOMEM;+ line = allocate(maximum + 1);+ if (!line) return COLI_SERVE_READ_NOMEM;+ for (;;) {+ int ch = fgetc(input);+ if (ch == EOF) {+ free(line);+ return count ? COLI_SERVE_READ_BAD_FRAME : COLI_SERVE_READ_EOF;+ }+ if (ch == '\n') break;+ if (count == maximum) {+ do ch = fgetc(input); while (ch != EOF && ch != '\n');+ line[count] = 0;+ coli_serve_classify_line(line, command);+ free(line);+ return COLI_SERVE_READ_BAD_REQUEST;+ }+ line[count++] = (char)ch;+ }+ line[count] = 0;+ if (count && line[count - 1] == '\r') line[--count] = 0;+ coli_serve_classify_line(line, command);++ char *cursor = line;+ while (*cursor) {+ while (*cursor == ' ' || *cursor == '\t') cursor++;+ if (!*cursor) break;+ if (nfields == (int)(sizeof(fields) / sizeof(fields[0]))) {+ free(line);+ return COLI_SERVE_READ_BAD_REQUEST;+ }+ fields[nfields++] = cursor;+ while (*cursor && *cursor != ' ' && *cursor != '\t') cursor++;+ if (*cursor) *cursor++ = 0;+ }+ if (!nfields) {+ free(line);+ return COLI_SERVE_READ_IGNORED;+ }+ if (!strcmp(fields[0], "SUBMIT")) command->kind = COLI_SERVE_COMMAND_SUBMIT;+ else if (!strcmp(fields[0], "STOP")) command->kind = COLI_SERVE_COMMAND_STOP;+ else if (!strcmp(fields[0], "CANCEL")) command->kind = COLI_SERVE_COMMAND_CANCEL;+ else {+ free(line);+ return COLI_SERVE_READ_IGNORED;+ }+ if (nfields < 2 || strlen(fields[1]) >= sizeof(command->id)) {+ free(line);+ return COLI_SERVE_READ_BAD_REQUEST;+ }+ memcpy(command->id, fields[1], strlen(fields[1]) + 1);+ if (command->kind == COLI_SERVE_COMMAND_STOP ||+ command->kind == COLI_SERVE_COMMAND_CANCEL) {+ if (nfields != 2) { free(line); return COLI_SERVE_READ_BAD_REQUEST; }+ free(line);+ return COLI_SERVE_READ_OK;+ }+ if (nfields != 7) {+ free(line);+ return COLI_SERVE_READ_BAD_REQUEST;+ }+ if (!coli_serve_parse_i32(fields[2], &command->slot) ||+ !coli_serve_parse_u64(fields[3], &command->payload_bytes) ||+ !coli_serve_parse_i32(fields[4], &command->max_tokens) ||+ !coli_serve_parse_f32(fields[5], &command->temperature) ||+ !coli_serve_parse_f32(fields[6], &command->top_p) ||+ command->payload_bytes > profile->max_payload_bytes ||+ command->max_tokens < 1 ||+ (profile->max_tokens && command->max_tokens > profile->max_tokens) ||+ (profile->require_finite_sampling &&+ (!isfinite(command->temperature) || !isfinite(command->top_p)))) {+ free(line);+ return COLI_SERVE_READ_BAD_REQUEST;+ }+ free(line);++ if (command->payload_bytes > SIZE_MAX - 1) return COLI_SERVE_READ_NOMEM;+ size_t payload_bytes = (size_t)command->payload_bytes;+ command->payload = allocate(payload_bytes + 1);+ if (!command->payload) return COLI_SERVE_READ_NOMEM;+ if (fread(command->payload, 1, payload_bytes, input) != payload_bytes) {+ coli_serve_command_dispose(command);+ return COLI_SERVE_READ_BAD_FRAME;+ }+ command->payload[payload_bytes] = 0;+ int terminator = fgetc(input);+ if (terminator == EOF || (profile->require_exact_lf && terminator != '\n')) {+ coli_serve_command_dispose(command);+ return COLI_SERVE_READ_BAD_FRAME;+ }+ return COLI_SERVE_READ_OK;+}++static inline ColiServeReadResult coli_serve_read_command(+ FILE *input, const ColiServeWireProfile *profile, ColiServeCommand *command)+{+ return coli_serve_read_command_alloc(input, profile, command, malloc);+}++static inline int coli_serve_write_ready(FILE *output, double rss_gb)+{+ if (fputs("\x01\x01READY\x01\x01\n", output) == EOF ||+ fprintf(output, "STAT 0 0.0 0.0 %.2f 0 0\n", rss_gb) < 0)+ return 0;+ return fflush(output) == 0;+}++static inline int coli_serve_write_accept(FILE *output, const char *id, int prompt_tokens)+{+ return fprintf(output, "ACCEPT %s %d\n", id, prompt_tokens) >= 0 &&+ fflush(output) == 0;+}++static inline int coli_serve_write_data(+ FILE *output, const char *id, const void *data, size_t bytes)+{+ if (fprintf(output, "DATA %s %zu\n", id, bytes) < 0 ||+ (bytes && fwrite(data, 1, bytes, output) != bytes) ||+ fputc('\n', output) == EOF)+ return 0;+ return fflush(output) == 0;+}++static inline int coli_serve_write_error(FILE *output, const char *id, const char *message)+{+ if (fprintf(output, "ERROR %s ", id && *id ? id : "0") < 0) return 0;+ for (const unsigned char *p = (const unsigned char *)(message ? message : ""); *p; p++) {+ unsigned char ch = (*p == '\r' || *p == '\n') ? ' ' : *p;+ if (fputc(ch, output) == EOF) return 0;+ }+ return fputc('\n', output) != EOF && fflush(output) == 0;+}++static inline int coli_serve_write_done(+ FILE *output, const char *id, const ColiServeDone *done)+{+ if (fprintf(output, "DONE %s STAT %d %.3f %.1f %.2f %d %d\n", id,+ done->completion_tokens, done->tokens_per_second,+ done->cache_hit_percent, done->rss_gb, done->prompt_tokens,+ done->length_limited) < 0)+ return 0;+ return fflush(output) == 0;+}++#endifdiff --git a/c/tests/test_olmoe_serve_framing.c b/c/tests/test_olmoe_serve_framing.cindex bcf2a4f6..41858edd 100644--- a/c/tests/test_olmoe_serve_framing.c+++ b/c/tests/test_olmoe_serve_framing.c@@ -7,10 +7,20 @@#include <assert.h>+static void binary_stream(FILE *stream)+{+#ifdef _WIN32+ assert(_setmode(_fileno(stream), _O_BINARY) != -1);+#else+ (void)stream;+#endif+}+static FILE *bytes_input(const void *data, size_t size){FILE *stream = tmpfile();assert(stream);+ binary_stream(stream);assert(fwrite(data, 1, size, stream) == size);rewind(stream);return stream;@@ -37,6 +47,7 @@ static void test_submit_moves_exact_payload_into_queue(void)FILE *in = bytes_input(frame, sizeof(frame) - 1);FILE *out = tmpfile();assert(out);+ binary_stream(out);assert(serve_read_cmd(in, out, NULL) == 0);assert(g_qn == 1);@@ -59,6 +70,7 @@ static void test_cancel_only_matches_active_request(void)static const char cancel[] = "CANCEL req-7\n";FILE *out = tmpfile();assert(out);+ binary_stream(out);FILE *match = bytes_input(cancel, sizeof(cancel) - 1);assert(serve_read_cmd(match, out, "req-7") == 1);@@ -76,7 +88,8 @@ static void test_errors_are_byte_exact_and_frames_are_consumed(void)FILE *bad_in = bytes_input(bad, sizeof(bad) - 1);FILE *bad_out = tmpfile();assert(bad_out);- assert(serve_read_cmd(bad_in, bad_out, NULL) == 0);+ binary_stream(bad_out);+ assert(serve_read_cmd(bad_in, bad_out, NULL) == -1);char output[128];size_t count = read_output(bad_out, output, sizeof(output));static const char bad_want[] = "ERROR bad bad submit header\n";@@ -90,6 +103,7 @@ static void test_errors_are_byte_exact_and_frames_are_consumed(void)FILE *full_in = bytes_input(full, sizeof(full) - 1);FILE *full_out = tmpfile();assert(full_out);+ binary_stream(full_out);assert(serve_read_cmd(full_in, full_out, NULL) == 0);count = read_output(full_out, output, sizeof(output));static const char full_want[] = "ERROR full queue full\n";@@ -101,11 +115,73 @@ static void test_errors_are_byte_exact_and_frames_are_consumed(void)g_qn = 0;}+static void test_invalid_submit_does_not_parse_its_payload_as_control(void)+{+ static const char *frames[] = {+ "SUBMIT bad 0 13 0 0.7 0.95\nCANCEL live\n\n",+ "SUBMIT bad 0 13 4 0.7 0.95 extra extra\nCANCEL live\n\n",+ };+ for (size_t i = 0; i < sizeof(frames) / sizeof(frames[0]); i++) {+ FILE *input = bytes_input(frames[i], strlen(frames[i]));+ FILE *output = tmpfile();+ assert(output);+ binary_stream(output);++ assert(serve_read_cmd(input, output, "live") == -1);+ assert(fgetc(input) == 'C');+ char bytes[64];+ size_t count = read_output(output, bytes, sizeof(bytes));+ static const char want[] = "ERROR bad bad submit header\n";+ assert(count == sizeof(want) - 1);+ assert(memcmp(bytes, want, count) == 0);+ fclose(input);+ fclose(output);+ }++ char overlong[640];+ int prefix = snprintf(overlong, sizeof(overlong),+ "SUBMIT bad 0 13 4 0.7 0.95 ");+ assert(prefix > 0);+ memset(overlong + prefix, 'x', 520);+ size_t offset = (size_t)prefix + 520;+ memcpy(overlong + offset, "\nCANCEL live\n\n", 15);+ offset += 15;+ FILE *input = bytes_input(overlong, offset);+ FILE *output = tmpfile();+ assert(output);+ binary_stream(output);+ assert(serve_read_cmd(input, output, "live") == -1);+ assert(fgetc(input) == 'C');+ fclose(input);+ fclose(output);+}++static void test_bad_frame_stops_before_the_next_command(void)+{+ static const char frame[] =+ "SUBMIT broken 0 1 4 0.7 0.95\nxX"+ "SUBMIT next 0 1 4 0.7 0.95\ny\n";+ FILE *input = bytes_input(frame, sizeof(frame) - 1);+ FILE *output = tmpfile();+ assert(output);+ binary_stream(output);++ assert(serve_read_cmd(input, output, NULL) == -1);+ assert(g_qn == 0);+ assert(fgetc(input) == 'S');+ char bytes[16];+ assert(read_output(output, bytes, sizeof(bytes)) == 0);+ fclose(input);+ fclose(output);+}+int main(void){test_submit_moves_exact_payload_into_queue();test_cancel_only_matches_active_request();test_errors_are_byte_exact_and_frames_are_consumed();+ test_invalid_submit_does_not_parse_its_payload_as_control();+ test_bad_frame_stops_before_the_next_command();puts("olmoe serve framing baseline: ok");return 0;}diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 09b807ff..03a45acd 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -464,7 +464,7 @@ class DispatcherTest(unittest.TestCase):self.assertEqual(frame, expected)process.stdout.feed(b"DATA 1 4\nA\n\xc3\xa9\n"- b"DONE 1 STAT 1 2.5 50.0 1.25 5 0\n"+ b"DONE 1 STAT 1 2.500 50.0 1.25 5 0\n")process = FakeProcess(respond)diff --git a/c/tests/test_serve_codec.c b/c/tests/test_serve_codec.cnew file mode 100644index 00000000..42646713--- /dev/null+++ b/c/tests/test_serve_codec.c@@ -0,0 +1,199 @@+#include <assert.h>+#include <stdio.h>+#include <string.h>++#include "../compat.h"+#include "../serve_codec.h"++static void binary_stream(FILE *stream)+{+#ifdef _WIN32+ assert(_setmode(_fileno(stream), _O_BINARY) != -1);+#else+ (void)stream;+#endif+}++static FILE *bytes_input(const void *data, size_t size)+{+ FILE *stream = tmpfile();+ assert(stream);+ binary_stream(stream);+ assert(fwrite(data, 1, size, stream) == size);+ rewind(stream);+ return stream;+}++static size_t read_output(FILE *stream, unsigned char *output, size_t capacity)+{+ assert(fflush(stream) == 0);+ rewind(stream);+ return fread(output, 1, capacity, stream);+}++static const ColiServeWireProfile profile = {+ .max_header_bytes = 128,+ .max_payload_bytes = 5,+ .max_tokens = 4,+ .require_exact_lf = 1,+ .require_finite_sampling = 1,+};++static int allocation_count;+static int fail_allocation;++static void *fault_alloc(size_t size)+{+ allocation_count++;+ return allocation_count == fail_allocation ? NULL : malloc(size);+}++static void test_submit_and_controls(void)+{+ static const char frame[] = "SUBMIT req-7 0 5 4 0.7 0.95\nab\ncd\n";+ FILE *input = bytes_input(frame, sizeof(frame) - 1);+ ColiServeCommand command;+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);+ assert(command.kind == COLI_SERVE_COMMAND_SUBMIT);+ assert(strcmp(command.id, "req-7") == 0);+ assert(command.slot == 0 && command.payload_bytes == 5 && command.max_tokens == 4);+ assert(memcmp(command.payload, "ab\ncd", 5) == 0);+ assert(fgetc(input) == EOF);+ coli_serve_command_dispose(&command);+ fclose(input);++ static const char subnormal[] = "SUBMIT req-8 0 1 1 1e-40 0.95\nx\n";+ input = bytes_input(subnormal, sizeof(subnormal) - 1);+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);+ assert(command.temperature > 0.0f && isfinite(command.temperature));+ assert(command.payload_bytes == 1 && command.payload[0] == 'x');+ coli_serve_command_dispose(&command);+ fclose(input);++ static const char stop[] = "STOP req-7\n";+ input = bytes_input(stop, sizeof(stop) - 1);+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);+ assert(command.kind == COLI_SERVE_COMMAND_STOP);+ coli_serve_command_dispose(&command);+ fclose(input);++ static const char cancel[] = "CANCEL req-7\n";+ input = bytes_input(cancel, sizeof(cancel) - 1);+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);+ assert(command.kind == COLI_SERVE_COMMAND_CANCEL);+ coli_serve_command_dispose(&command);+ fclose(input);+}++static void test_invalid_headers_and_bodies(void)+{+ static const char *invalid[] = {+ "SUBMIT req 0 6 4 0.7 0.95\n",+ "SUBMIT req 0 5 0 0.7 0.95\n",+ "SUBMIT req 0 5 5 0.7 0.95\n",+ "SUBMIT req 0 5 4 nan 0.95\n",+ "SUBMIT req 0 5 4 0.7 inf\n",+ "SUBMIT req 0 5 4 0.7 0.95 extra\n",+ };+ for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++) {+ FILE *input = bytes_input(invalid[i], strlen(invalid[i]));+ ColiServeCommand command;+ assert(coli_serve_read_command(input, &profile, &command) ==+ COLI_SERVE_READ_BAD_REQUEST);+ fclose(input);+ }++ static const char short_body[] = "SUBMIT req 0 5 4 0.7 0.95\nabc";+ FILE *input = bytes_input(short_body, sizeof(short_body) - 1);+ ColiServeCommand command;+ assert(coli_serve_read_command(input, &profile, &command) ==+ COLI_SERVE_READ_BAD_FRAME);+ fclose(input);++ static const char bad_lf[] = "SUBMIT req 0 5 4 0.7 0.95\nabcdeX";+ input = bytes_input(bad_lf, sizeof(bad_lf) - 1);+ assert(coli_serve_read_command(input, &profile, &command) ==+ COLI_SERVE_READ_BAD_FRAME);+ fclose(input);++ char long_id[COLI_SERVE_ID_CAP + 1];+ memset(long_id, 'x', sizeof(long_id) - 1);+ long_id[sizeof(long_id) - 1] = 0;+ char long_header[256];+ int length = snprintf(long_header, sizeof(long_header),+ "SUBMIT %s 0 0 1 0.7 0.95\n\n", long_id);+ assert(length > 0 && (size_t)length < sizeof(long_header));+ input = bytes_input(long_header, (size_t)length);+ assert(coli_serve_read_command(input, &profile, &command) ==+ COLI_SERVE_READ_BAD_REQUEST);+ fclose(input);++ static const char unknown[] = "PING req\n";+ input = bytes_input(unknown, sizeof(unknown) - 1);+ assert(coli_serve_read_command(input, &profile, &command) ==+ COLI_SERVE_READ_IGNORED);+ fclose(input);++ char long_line[160];+ memset(long_line, 'x', sizeof(long_line));+ long_line[sizeof(long_line) - 2] = '\n';+ long_line[sizeof(long_line) - 1] = 0;+ input = bytes_input(long_line, sizeof(long_line) - 1);+ assert(coli_serve_read_command(input, &profile, &command) ==+ COLI_SERVE_READ_BAD_REQUEST);+ assert(fgetc(input) == EOF);+ fclose(input);+}++static void test_writer_golden_bytes(void)+{+ FILE *output = tmpfile();+ assert(output);+ binary_stream(output);+ assert(coli_serve_write_ready(output, 1.25));+ assert(coli_serve_write_accept(output, "req-7", 12));+ static const unsigned char data[] = {'A', '\n', 0, 'B'};+ assert(coli_serve_write_data(output, "req-7", data, sizeof(data)));+ assert(coli_serve_write_error(output, "req-7", "bad\r\nframe"));+ ColiServeDone done = {3, 1.25, 50.0, 1.25, 7, 1};+ assert(coli_serve_write_done(output, "req-7", &done));++ static const unsigned char expected[] =+ "\x01\x01READY\x01\x01\nSTAT 0 0.0 0.0 1.25 0 0\n"+ "ACCEPT req-7 12\n"+ "DATA req-7 4\nA\n\0B\n"+ "ERROR req-7 bad frame\n"+ "DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n";+ unsigned char actual[sizeof(expected) + 16];+ size_t count = read_output(output, actual, sizeof(actual));+ assert(count == sizeof(expected) - 1);+ assert(memcmp(actual, expected, count) == 0);+ fclose(output);+}++static void test_allocation_failures_are_fatal_to_the_caller(void)+{+ static const char frame[] = "SUBMIT req 0 1 1 0.7 0.95\nx\n";+ for (int failure = 1; failure <= 2; failure++) {+ FILE *input = bytes_input(frame, sizeof(frame) - 1);+ ColiServeCommand command;+ allocation_count = 0;+ fail_allocation = failure;+ assert(coli_serve_read_command_alloc(input, &profile, &command, fault_alloc) ==+ COLI_SERVE_READ_NOMEM);+ assert(command.payload == NULL);+ if (failure == 1) assert(fgetc(input) == 'S');+ else assert(fgetc(input) == 'x');+ fclose(input);+ }+}++int main(void)+{+ test_submit_and_controls();+ test_invalid_headers_and_bodies();+ test_writer_golden_bytes();+ test_allocation_failures_are_fatal_to_the_caller();+ puts("serve codec tests: ok");+ return 0;+}diff --git a/c/tests/test_serve_sentinel.c b/c/tests/test_serve_sentinel.cindex 50328f62..b3c83dbf 100644--- a/c/tests/test_serve_sentinel.c+++ b/c/tests/test_serve_sentinel.c@@ -24,6 +24,7 @@#include <unistd.h> /* close() dopo mkstemp */#endif#include "../compat.h"+#include "../serve_codec.h"#define SENTINEL "\x01\x01READY\x01\x01\n"@@ -55,8 +56,7 @@ int main(void)_setmode(_fileno(f), _O_BINARY);#endif- fputs(SENTINEL, f);- fprintf(f, "STAT 0 0.0 0.0 %.2f 0 0\n", 1.0);+ check(coli_serve_write_ready(f, 1.0), "shared READY writer succeeds");fclose(f);FILE *r = fopen(path, "rb"); /* rb: read the bytes as they landed */