Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(serve): share framing with OLMoE

Mario Gutierrez6104bbb3b0b9
diff --git a/c/Makefile b/c/Makefile
index 5d0660c0..ee941025 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -873,7 +873,7 @@ fp8-bench: fp8_bench$(EXE)
# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an
# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py
# asserts this shape.
-olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h
+olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h
$(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)
inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(METAL_OBJ)
@@ -916,7 +916,7 @@ portable:
iobench$(EXE): iobench.c compat.h
$(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS)
-tests/test_serve_sentinel$(EXE): tests/test_serve_sentinel.c compat.h
+tests/test_serve_sentinel$(EXE): tests/test_serve_sentinel.c compat.h serve_codec.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_cluster_protocol$(EXE): tests/test_cluster_protocol.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
@@ -992,13 +992,16 @@ tests/test_spec_decode_state$(EXE): tests/test_spec_decode_state.c colibri.c st.
tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
# olmoe's matmul_q, not colibri's: compares the IDOT path against FP32
# ACTIVATIONS rather than an integer reference. NOCUDA_* for the same reason
# the olmoe target uses it -- olmoe.c contains no COLI_CUDA code.
-tests/test_olmoe_matmul_q$(EXE): tests/test_olmoe_matmul_q.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h
+tests/test_olmoe_matmul_q$(EXE): tests/test_olmoe_matmul_q.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h
$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
-tests/test_olmoe_serve_framing$(EXE): tests/test_olmoe_serve_framing.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h
+tests/test_olmoe_serve_framing$(EXE): tests/test_olmoe_serve_framing.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h serve_codec.h
$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
diff --git a/c/olmoe.c b/c/olmoe.c
index dadd1fb0..599a9571 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -37,6 +37,7 @@
#endif
#include "omp_tune.h"
#include "route_trace.h" /* shared routing telemetry (#700) */
+#include "serve_codec.h"
#ifdef _WIN32
#include <windows.h>
@@ -1168,44 +1169,63 @@ static void run_chat(Model *m, Tok *T, int ctx_cap) {
typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen; } SReq;
#define SRV_QMAX 16
static SReq g_q[SRV_QMAX]; static int g_qn = 0;
+static const ColiServeWireProfile olmoe_wire = {
+ .max_header_bytes = 511,
+ .max_payload_bytes = 1u << 22,
+ .max_tokens = 1 << 20,
+ .require_exact_lf = 1,
+ .require_finite_sampling = 0,
+};
/* read one control line (+ payload for SUBMIT). cur_id: request in flight;
* returns 1 if that request was cancelled, 0 otherwise, -1 on input EOF. */
static int serve_read_cmd(FILE *in, FILE *out, const char *cur_id) {
- char ln[512];
- if (!fgets(ln, sizeof(ln), in)) return -1;
- char cmd[16], id[64];
- if (sscanf(ln, "%15s %63s", cmd, id) < 2) return 0;
- if (!strcmp(cmd, "CANCEL")) return cur_id && !strcmp(id, cur_id);
- if (!strcmp(cmd, "SUBMIT")) {
- int slot, plen, max_tok; float temp, top_p;
- int nf = sscanf(ln, "%*s %*s %d %d %d %f %f", &slot, &plen, &max_tok, &temp, &top_p);
- if (nf < 5 || plen < 0 || plen > (1<<22) || max_tok < 1 || max_tok > (1<<20)) {
- fprintf(out, "ERROR %s bad submit header\n", id); fflush(out); return 0; }
- (void)slot;
- char *pl = malloc((size_t)plen + 1);
- if (fread(pl, 1, (size_t)plen, in) != (size_t)plen) { free(pl); return -1; }
- pl[plen] = 0;
- int nl = fgetc(in); (void)nl;
+ ColiServeCommand command;
+ ColiServeReadResult result = coli_serve_read_command(in, &olmoe_wire, &command);
+ if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) return -1;
+ if (result == COLI_SERVE_READ_NOMEM) {
+ coli_serve_write_error(out, command.id, "out of memory");
+ return -1;
+ }
+ if (result == COLI_SERVE_READ_BAD_REQUEST &&
+ command.kind == COLI_SERVE_COMMAND_SUBMIT) {
+ coli_serve_write_error(out, command.id, "bad submit header");
+ return -1;
+ }
+ if (result != COLI_SERVE_READ_OK) return 0;
+ if (command.kind == COLI_SERVE_COMMAND_CANCEL) {
+ int cancelled = cur_id && !strcmp(command.id, cur_id);
+ coli_serve_command_dispose(&command);
+ return cancelled;
+ }
+ if (command.kind == COLI_SERVE_COMMAND_SUBMIT) {
if (g_qn < SRV_QMAX) {
SReq *q = &g_q[g_qn++];
- snprintf(q->id, sizeof(q->id), "%s", id);
- q->max_tok = max_tok; q->temp = temp; q->top_p = top_p;
- q->payload = pl; q->plen = plen;
- } else { fprintf(out, "ERROR %s queue full\n", id); fflush(out); free(pl); }
+ snprintf(q->id, sizeof(q->id), "%s", command.id);
+ q->max_tok = command.max_tokens;
+ q->temp = command.temperature;
+ q->top_p = command.top_p;
+ q->payload = (char *)coli_serve_command_take_payload(&command);
+ q->plen = (int)command.payload_bytes;
+ } else {
+ coli_serve_write_error(out, command.id, "queue full");
+ }
}
+ coli_serve_command_dispose(&command);
return 0;
}
-static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
+static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
Cfg *c = &m->c;
int cap = q->plen + 16;
int *ids = malloc((size_t)cap * sizeof(int));
int np = tok_encode(T, q->payload, q->plen, ids, cap);
- if (np <= 0) { printf("ERROR %s empty prompt\n", q->id); fflush(stdout); free(ids); return; }
+ if (np <= 0) { coli_serve_write_error(stdout, q->id, "empty prompt"); free(ids); return 0; }
if (np + q->max_tok > ctx_cap) {
- printf("ERROR %s context exceeds CTX (%d + %d > %d)\n", q->id, np, q->max_tok, ctx_cap);
- fflush(stdout); free(ids); return;
+ char message[128];
+ snprintf(message, sizeof(message), "context exceeds CTX (%d + %d > %d)",
+ np, q->max_tok, ctx_cap);
+ coli_serve_write_error(stdout, q->id, message); free(ids); return 0;
}
g_temp = q->temp; g_nuc = q->top_p;
double t0 = now_s();
@@ -1218,13 +1238,11 @@ static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
free(logit); logit = NULL;
if (is_stop(nt)) { limited = 0; break; }
int nb = tok_decode(T, &nt, 1, buf, sizeof(buf)-1);
- printf("DATA %s %d\n", q->id, nb);
- fwrite(buf, 1, (size_t)nb, stdout);
- fputc('\n', stdout); fflush(stdout);
+ coli_serve_write_data(stdout, q->id, buf, (size_t)nb);
gen++; hist_len++;
while (coli_stdin_readable()) {
int r = serve_read_cmd(stdin, stdout, q->id);
- if (r < 0) { free(ids); return; }
+ if (r < 0) { free(ids); return -1; }
if (r > 0) { cancelled = 1; limited = 0; }
}
/* Unlike run_chat(), we do not step() the final token just to populate
@@ -1236,8 +1254,15 @@ static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
free(logit);
double dt = now_s() - t0;
double tot = (double)(m->hits - h0 + m->miss - m0);
- printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n", q->id, gen,
- dt > 0 ? gen/dt : 0.0, tot ? 100.0*(m->hits-h0)/tot : 0.0, rss_gb(), np, limited);
+ ColiServeDone done = {
+ .completion_tokens = gen,
+ .tokens_per_second = dt > 0 ? gen/dt : 0.0,
+ .cache_hit_percent = tot ? 100.0*(m->hits-h0)/tot : 0.0,
+ .rss_gb = rss_gb(),
+ .prompt_tokens = np,
+ .length_limited = limited,
+ };
+ coli_serve_write_done(stdout, q->id, &done);
/* PROF: per-turn phase timings for the dashboard. olmoe.c does not split
* its wall time into fill/expert/shared/attn phases the way glm.c and
* inkling.c do, so this reports total time only; a real phase breakdown
@@ -1245,6 +1270,7 @@ static void serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
printf("PROF %.3f %d %d 0.0 0.0 0.0 0.0 0.0 %d\n", dt, np, gen, gen + 1);
fflush(stdout);
free(ids);
+ return 0;
}
/* dashboard HWINFO/TIERS/EMAP: same lines the other serve-capable engines
@@ -1303,21 +1329,19 @@ static void serve_tiers_emap(Model *m) {
}
static void serve_loop(Model *m, Tok *T, int ctx_cap) {
- coli_serve_binary_mode();
- setvbuf(stdin, NULL, _IONBF, 0);
+ coli_serve_stdio_init();
int tok_eos = tok_id_of(T, "|||IP_ADDRESS|||");
stops_arm_tok(&m->c, tok_eos, T);
- fputs("\x01\x01READY\x01\x01\n", stdout);
- printf("STAT 0 0.0 0.0 %.2f 0 0\n", rss_gb());
- fflush(stdout);
+ coli_serve_write_ready(stdout, rss_gb());
serve_hwinfo(m);
serve_tiers_emap(m);
for (;;) {
while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return;
SReq q = g_q[0];
memmove(g_q, g_q + 1, (size_t)(--g_qn) * sizeof(SReq));
- serve_one(m, T, &q, ctx_cap);
+ int fatal = serve_one(m, T, &q, ctx_cap);
free(q.payload);
+ if (fatal < 0) return;
}
}
diff --git a/c/serve_codec.h b/c/serve_codec.h
new file mode 100644
index 00000000..f0e03c8e
--- /dev/null
+++ b/c/serve_codec.h
@@ -0,0 +1,294 @@
+#ifndef COLI_SERVE_CODEC_H
+#define COLI_SERVE_CODEC_H
+
+#include <errno.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "compat.h"
+
+/* Transport only: parse and emit frames. Admission, queueing, cancellation,
+ * KV ownership, scheduling, and generation remain with each family engine. */
+
+#define COLI_SERVE_ID_CAP 64
+
+typedef enum {
+ COLI_SERVE_COMMAND_UNKNOWN = 0,
+ COLI_SERVE_COMMAND_SUBMIT,
+ COLI_SERVE_COMMAND_STOP,
+ COLI_SERVE_COMMAND_CANCEL,
+} ColiServeCommandKind;
+
+typedef enum {
+ COLI_SERVE_READ_EOF = -1,
+ COLI_SERVE_READ_BAD_FRAME = -2,
+ COLI_SERVE_READ_NOMEM = -3,
+ COLI_SERVE_READ_BAD_REQUEST = -4,
+ COLI_SERVE_READ_IGNORED = 0,
+ COLI_SERVE_READ_OK = 1,
+} ColiServeReadResult;
+
+typedef struct {
+ size_t max_header_bytes;
+ uint64_t max_payload_bytes;
+ int max_tokens;
+ int require_exact_lf;
+ int require_finite_sampling;
+} ColiServeWireProfile;
+
+typedef struct {
+ ColiServeCommandKind kind;
+ char id[COLI_SERVE_ID_CAP];
+ int slot;
+ uint64_t payload_bytes;
+ int max_tokens;
+ float temperature;
+ float top_p;
+ unsigned char *payload;
+} ColiServeCommand;
+
+typedef struct {
+ int completion_tokens;
+ double tokens_per_second;
+ double cache_hit_percent;
+ double rss_gb;
+ int prompt_tokens;
+ int length_limited;
+} ColiServeDone;
+
+static inline void coli_serve_command_dispose(ColiServeCommand *command)
+{
+ if (!command) return;
+ free(command->payload);
+ memset(command, 0, sizeof(*command));
+}
+
+static inline void coli_serve_stdio_init(void)
+{
+ coli_serve_binary_mode();
+ setvbuf(stdin, NULL, _IONBF, 0);
+}
+
+static inline unsigned char *coli_serve_command_take_payload(ColiServeCommand *command)
+{
+ unsigned char *payload = command ? command->payload : NULL;
+ if (command) command->payload = NULL;
+ return payload;
+}
+
+static inline int coli_serve_parse_i32(const char *text, int *value)
+{
+ char *end = NULL;
+ errno = 0;
+ long parsed = strtol(text, &end, 10);
+ if (errno || !end || *end || parsed < INT32_MIN || parsed > INT32_MAX) return 0;
+ *value = (int)parsed;
+ return 1;
+}
+
+static inline int coli_serve_parse_u64(const char *text, uint64_t *value)
+{
+ char *end = NULL;
+ if (!text[0] || text[0] == '-') return 0;
+ errno = 0;
+ unsigned long long parsed = strtoull(text, &end, 10);
+ if (errno || !end || *end) return 0;
+ *value = (uint64_t)parsed;
+ return 1;
+}
+
+static inline int coli_serve_parse_f32(const char *text, float *value)
+{
+ char *end = NULL;
+ float parsed = strtof(text, &end);
+ if (!end || end == text || *end) return 0;
+ *value = parsed;
+ return 1;
+}
+
+static inline void coli_serve_classify_line(
+ const char *line, ColiServeCommand *command)
+{
+ const char *cursor = line;
+ while (*cursor == ' ' || *cursor == '\t') cursor++;
+ const char *name = cursor;
+ while (*cursor && *cursor != ' ' && *cursor != '\t') cursor++;
+ size_t name_size = (size_t)(cursor - name);
+ if (name_size == 6 && !memcmp(name, "SUBMIT", 6))
+ command->kind = COLI_SERVE_COMMAND_SUBMIT;
+ else if (name_size == 4 && !memcmp(name, "STOP", 4))
+ command->kind = COLI_SERVE_COMMAND_STOP;
+ else if (name_size == 6 && !memcmp(name, "CANCEL", 6))
+ command->kind = COLI_SERVE_COMMAND_CANCEL;
+ else
+ return;
+ while (*cursor == ' ' || *cursor == '\t') cursor++;
+ const char *id = cursor;
+ while (*cursor && *cursor != ' ' && *cursor != '\t') cursor++;
+ size_t id_size = (size_t)(cursor - id);
+ if (id_size && id_size < sizeof(command->id)) {
+ memcpy(command->id, id, id_size);
+ command->id[id_size] = 0;
+ }
+}
+
+static inline ColiServeReadResult coli_serve_read_command_alloc(
+ FILE *input, const ColiServeWireProfile *profile, ColiServeCommand *command,
+ void *(*allocate)(size_t))
+{
+ char *line;
+ size_t count = 0;
+ char *fields[8];
+ int nfields = 0;
+
+ if (!input || !profile || !command) return COLI_SERVE_READ_BAD_FRAME;
+ memset(command, 0, sizeof(*command));
+ size_t maximum = profile->max_header_bytes ? profile->max_header_bytes : 4096;
+ if (maximum == SIZE_MAX) return COLI_SERVE_READ_NOMEM;
+ line = allocate(maximum + 1);
+ if (!line) return COLI_SERVE_READ_NOMEM;
+ for (;;) {
+ int ch = fgetc(input);
+ if (ch == EOF) {
+ free(line);
+ return count ? COLI_SERVE_READ_BAD_FRAME : COLI_SERVE_READ_EOF;
+ }
+ if (ch == '\n') break;
+ if (count == maximum) {
+ do ch = fgetc(input); while (ch != EOF && ch != '\n');
+ line[count] = 0;
+ coli_serve_classify_line(line, command);
+ free(line);
+ return COLI_SERVE_READ_BAD_REQUEST;
+ }
+ line[count++] = (char)ch;
+ }
+ line[count] = 0;
+ if (count && line[count - 1] == '\r') line[--count] = 0;
+ coli_serve_classify_line(line, command);
+
+ char *cursor = line;
+ while (*cursor) {
+ while (*cursor == ' ' || *cursor == '\t') cursor++;
+ if (!*cursor) break;
+ if (nfields == (int)(sizeof(fields) / sizeof(fields[0]))) {
+ free(line);
+ return COLI_SERVE_READ_BAD_REQUEST;
+ }
+ fields[nfields++] = cursor;
+ while (*cursor && *cursor != ' ' && *cursor != '\t') cursor++;
+ if (*cursor) *cursor++ = 0;
+ }
+ if (!nfields) {
+ free(line);
+ return COLI_SERVE_READ_IGNORED;
+ }
+ if (!strcmp(fields[0], "SUBMIT")) command->kind = COLI_SERVE_COMMAND_SUBMIT;
+ else if (!strcmp(fields[0], "STOP")) command->kind = COLI_SERVE_COMMAND_STOP;
+ else if (!strcmp(fields[0], "CANCEL")) command->kind = COLI_SERVE_COMMAND_CANCEL;
+ else {
+ free(line);
+ return COLI_SERVE_READ_IGNORED;
+ }
+ if (nfields < 2 || strlen(fields[1]) >= sizeof(command->id)) {
+ free(line);
+ return COLI_SERVE_READ_BAD_REQUEST;
+ }
+ memcpy(command->id, fields[1], strlen(fields[1]) + 1);
+ if (command->kind == COLI_SERVE_COMMAND_STOP ||
+ command->kind == COLI_SERVE_COMMAND_CANCEL) {
+ if (nfields != 2) { free(line); return COLI_SERVE_READ_BAD_REQUEST; }
+ free(line);
+ return COLI_SERVE_READ_OK;
+ }
+ if (nfields != 7) {
+ free(line);
+ return COLI_SERVE_READ_BAD_REQUEST;
+ }
+ if (!coli_serve_parse_i32(fields[2], &command->slot) ||
+ !coli_serve_parse_u64(fields[3], &command->payload_bytes) ||
+ !coli_serve_parse_i32(fields[4], &command->max_tokens) ||
+ !coli_serve_parse_f32(fields[5], &command->temperature) ||
+ !coli_serve_parse_f32(fields[6], &command->top_p) ||
+ command->payload_bytes > profile->max_payload_bytes ||
+ command->max_tokens < 1 ||
+ (profile->max_tokens && command->max_tokens > profile->max_tokens) ||
+ (profile->require_finite_sampling &&
+ (!isfinite(command->temperature) || !isfinite(command->top_p)))) {
+ free(line);
+ return COLI_SERVE_READ_BAD_REQUEST;
+ }
+ free(line);
+
+ if (command->payload_bytes > SIZE_MAX - 1) return COLI_SERVE_READ_NOMEM;
+ size_t payload_bytes = (size_t)command->payload_bytes;
+ command->payload = allocate(payload_bytes + 1);
+ if (!command->payload) return COLI_SERVE_READ_NOMEM;
+ if (fread(command->payload, 1, payload_bytes, input) != payload_bytes) {
+ coli_serve_command_dispose(command);
+ return COLI_SERVE_READ_BAD_FRAME;
+ }
+ command->payload[payload_bytes] = 0;
+ int terminator = fgetc(input);
+ if (terminator == EOF || (profile->require_exact_lf && terminator != '\n')) {
+ coli_serve_command_dispose(command);
+ return COLI_SERVE_READ_BAD_FRAME;
+ }
+ return COLI_SERVE_READ_OK;
+}
+
+static inline ColiServeReadResult coli_serve_read_command(
+ FILE *input, const ColiServeWireProfile *profile, ColiServeCommand *command)
+{
+ return coli_serve_read_command_alloc(input, profile, command, malloc);
+}
+
+static inline int coli_serve_write_ready(FILE *output, double rss_gb)
+{
+ if (fputs("\x01\x01READY\x01\x01\n", output) == EOF ||
+ fprintf(output, "STAT 0 0.0 0.0 %.2f 0 0\n", rss_gb) < 0)
+ return 0;
+ return fflush(output) == 0;
+}
+
+static inline int coli_serve_write_accept(FILE *output, const char *id, int prompt_tokens)
+{
+ return fprintf(output, "ACCEPT %s %d\n", id, prompt_tokens) >= 0 &&
+ fflush(output) == 0;
+}
+
+static inline int coli_serve_write_data(
+ FILE *output, const char *id, const void *data, size_t bytes)
+{
+ if (fprintf(output, "DATA %s %zu\n", id, bytes) < 0 ||
+ (bytes && fwrite(data, 1, bytes, output) != bytes) ||
+ fputc('\n', output) == EOF)
+ return 0;
+ return fflush(output) == 0;
+}
+
+static inline int coli_serve_write_error(FILE *output, const char *id, const char *message)
+{
+ if (fprintf(output, "ERROR %s ", id && *id ? id : "0") < 0) return 0;
+ for (const unsigned char *p = (const unsigned char *)(message ? message : ""); *p; p++) {
+ unsigned char ch = (*p == '\r' || *p == '\n') ? ' ' : *p;
+ if (fputc(ch, output) == EOF) return 0;
+ }
+ return fputc('\n', output) != EOF && fflush(output) == 0;
+}
+
+static inline int coli_serve_write_done(
+ FILE *output, const char *id, const ColiServeDone *done)
+{
+ if (fprintf(output, "DONE %s STAT %d %.3f %.1f %.2f %d %d\n", id,
+ done->completion_tokens, done->tokens_per_second,
+ done->cache_hit_percent, done->rss_gb, done->prompt_tokens,
+ done->length_limited) < 0)
+ return 0;
+ return fflush(output) == 0;
+}
+
+#endif
diff --git a/c/tests/test_olmoe_serve_framing.c b/c/tests/test_olmoe_serve_framing.c
index bcf2a4f6..41858edd 100644
--- a/c/tests/test_olmoe_serve_framing.c
+++ b/c/tests/test_olmoe_serve_framing.c
@@ -7,10 +7,20 @@
#include <assert.h>
+static void binary_stream(FILE *stream)
+{
+#ifdef _WIN32
+ assert(_setmode(_fileno(stream), _O_BINARY) != -1);
+#else
+ (void)stream;
+#endif
+}
+
static FILE *bytes_input(const void *data, size_t size)
{
FILE *stream = tmpfile();
assert(stream);
+ binary_stream(stream);
assert(fwrite(data, 1, size, stream) == size);
rewind(stream);
return stream;
@@ -37,6 +47,7 @@ static void test_submit_moves_exact_payload_into_queue(void)
FILE *in = bytes_input(frame, sizeof(frame) - 1);
FILE *out = tmpfile();
assert(out);
+ binary_stream(out);
assert(serve_read_cmd(in, out, NULL) == 0);
assert(g_qn == 1);
@@ -59,6 +70,7 @@ static void test_cancel_only_matches_active_request(void)
static const char cancel[] = "CANCEL req-7\n";
FILE *out = tmpfile();
assert(out);
+ binary_stream(out);
FILE *match = bytes_input(cancel, sizeof(cancel) - 1);
assert(serve_read_cmd(match, out, "req-7") == 1);
@@ -76,7 +88,8 @@ static void test_errors_are_byte_exact_and_frames_are_consumed(void)
FILE *bad_in = bytes_input(bad, sizeof(bad) - 1);
FILE *bad_out = tmpfile();
assert(bad_out);
- assert(serve_read_cmd(bad_in, bad_out, NULL) == 0);
+ binary_stream(bad_out);
+ assert(serve_read_cmd(bad_in, bad_out, NULL) == -1);
char output[128];
size_t count = read_output(bad_out, output, sizeof(output));
static const char bad_want[] = "ERROR bad bad submit header\n";
@@ -90,6 +103,7 @@ static void test_errors_are_byte_exact_and_frames_are_consumed(void)
FILE *full_in = bytes_input(full, sizeof(full) - 1);
FILE *full_out = tmpfile();
assert(full_out);
+ binary_stream(full_out);
assert(serve_read_cmd(full_in, full_out, NULL) == 0);
count = read_output(full_out, output, sizeof(output));
static const char full_want[] = "ERROR full queue full\n";
@@ -101,11 +115,73 @@ static void test_errors_are_byte_exact_and_frames_are_consumed(void)
g_qn = 0;
}
+static void test_invalid_submit_does_not_parse_its_payload_as_control(void)
+{
+ static const char *frames[] = {
+ "SUBMIT bad 0 13 0 0.7 0.95\nCANCEL live\n\n",
+ "SUBMIT bad 0 13 4 0.7 0.95 extra extra\nCANCEL live\n\n",
+ };
+ for (size_t i = 0; i < sizeof(frames) / sizeof(frames[0]); i++) {
+ FILE *input = bytes_input(frames[i], strlen(frames[i]));
+ FILE *output = tmpfile();
+ assert(output);
+ binary_stream(output);
+
+ assert(serve_read_cmd(input, output, "live") == -1);
+ assert(fgetc(input) == 'C');
+ char bytes[64];
+ size_t count = read_output(output, bytes, sizeof(bytes));
+ static const char want[] = "ERROR bad bad submit header\n";
+ assert(count == sizeof(want) - 1);
+ assert(memcmp(bytes, want, count) == 0);
+ fclose(input);
+ fclose(output);
+ }
+
+ char overlong[640];
+ int prefix = snprintf(overlong, sizeof(overlong),
+ "SUBMIT bad 0 13 4 0.7 0.95 ");
+ assert(prefix > 0);
+ memset(overlong + prefix, 'x', 520);
+ size_t offset = (size_t)prefix + 520;
+ memcpy(overlong + offset, "\nCANCEL live\n\n", 15);
+ offset += 15;
+ FILE *input = bytes_input(overlong, offset);
+ FILE *output = tmpfile();
+ assert(output);
+ binary_stream(output);
+ assert(serve_read_cmd(input, output, "live") == -1);
+ assert(fgetc(input) == 'C');
+ fclose(input);
+ fclose(output);
+}
+
+static void test_bad_frame_stops_before_the_next_command(void)
+{
+ static const char frame[] =
+ "SUBMIT broken 0 1 4 0.7 0.95\nxX"
+ "SUBMIT next 0 1 4 0.7 0.95\ny\n";
+ FILE *input = bytes_input(frame, sizeof(frame) - 1);
+ FILE *output = tmpfile();
+ assert(output);
+ binary_stream(output);
+
+ assert(serve_read_cmd(input, output, NULL) == -1);
+ assert(g_qn == 0);
+ assert(fgetc(input) == 'S');
+ char bytes[16];
+ assert(read_output(output, bytes, sizeof(bytes)) == 0);
+ fclose(input);
+ fclose(output);
+}
+
int main(void)
{
test_submit_moves_exact_payload_into_queue();
test_cancel_only_matches_active_request();
test_errors_are_byte_exact_and_frames_are_consumed();
+ test_invalid_submit_does_not_parse_its_payload_as_control();
+ test_bad_frame_stops_before_the_next_command();
puts("olmoe serve framing baseline: ok");
return 0;
}
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 09b807ff..03a45acd 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -464,7 +464,7 @@ class DispatcherTest(unittest.TestCase):
self.assertEqual(frame, expected)
process.stdout.feed(
b"DATA 1 4\nA\n\xc3\xa9\n"
- b"DONE 1 STAT 1 2.5 50.0 1.25 5 0\n"
+ b"DONE 1 STAT 1 2.500 50.0 1.25 5 0\n"
)
process = FakeProcess(respond)
diff --git a/c/tests/test_serve_codec.c b/c/tests/test_serve_codec.c
new file mode 100644
index 00000000..42646713
--- /dev/null
+++ b/c/tests/test_serve_codec.c
@@ -0,0 +1,199 @@
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "../compat.h"
+#include "../serve_codec.h"
+
+static void binary_stream(FILE *stream)
+{
+#ifdef _WIN32
+ assert(_setmode(_fileno(stream), _O_BINARY) != -1);
+#else
+ (void)stream;
+#endif
+}
+
+static FILE *bytes_input(const void *data, size_t size)
+{
+ FILE *stream = tmpfile();
+ assert(stream);
+ binary_stream(stream);
+ assert(fwrite(data, 1, size, stream) == size);
+ rewind(stream);
+ return stream;
+}
+
+static size_t read_output(FILE *stream, unsigned char *output, size_t capacity)
+{
+ assert(fflush(stream) == 0);
+ rewind(stream);
+ return fread(output, 1, capacity, stream);
+}
+
+static const ColiServeWireProfile profile = {
+ .max_header_bytes = 128,
+ .max_payload_bytes = 5,
+ .max_tokens = 4,
+ .require_exact_lf = 1,
+ .require_finite_sampling = 1,
+};
+
+static int allocation_count;
+static int fail_allocation;
+
+static void *fault_alloc(size_t size)
+{
+ allocation_count++;
+ return allocation_count == fail_allocation ? NULL : malloc(size);
+}
+
+static void test_submit_and_controls(void)
+{
+ static const char frame[] = "SUBMIT req-7 0 5 4 0.7 0.95\nab\ncd\n";
+ FILE *input = bytes_input(frame, sizeof(frame) - 1);
+ ColiServeCommand command;
+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);
+ assert(command.kind == COLI_SERVE_COMMAND_SUBMIT);
+ assert(strcmp(command.id, "req-7") == 0);
+ assert(command.slot == 0 && command.payload_bytes == 5 && command.max_tokens == 4);
+ assert(memcmp(command.payload, "ab\ncd", 5) == 0);
+ assert(fgetc(input) == EOF);
+ coli_serve_command_dispose(&command);
+ fclose(input);
+
+ static const char subnormal[] = "SUBMIT req-8 0 1 1 1e-40 0.95\nx\n";
+ input = bytes_input(subnormal, sizeof(subnormal) - 1);
+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);
+ assert(command.temperature > 0.0f && isfinite(command.temperature));
+ assert(command.payload_bytes == 1 && command.payload[0] == 'x');
+ coli_serve_command_dispose(&command);
+ fclose(input);
+
+ static const char stop[] = "STOP req-7\n";
+ input = bytes_input(stop, sizeof(stop) - 1);
+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);
+ assert(command.kind == COLI_SERVE_COMMAND_STOP);
+ coli_serve_command_dispose(&command);
+ fclose(input);
+
+ static const char cancel[] = "CANCEL req-7\n";
+ input = bytes_input(cancel, sizeof(cancel) - 1);
+ assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);
+ assert(command.kind == COLI_SERVE_COMMAND_CANCEL);
+ coli_serve_command_dispose(&command);
+ fclose(input);
+}
+
+static void test_invalid_headers_and_bodies(void)
+{
+ static const char *invalid[] = {
+ "SUBMIT req 0 6 4 0.7 0.95\n",
+ "SUBMIT req 0 5 0 0.7 0.95\n",
+ "SUBMIT req 0 5 5 0.7 0.95\n",
+ "SUBMIT req 0 5 4 nan 0.95\n",
+ "SUBMIT req 0 5 4 0.7 inf\n",
+ "SUBMIT req 0 5 4 0.7 0.95 extra\n",
+ };
+ for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++) {
+ FILE *input = bytes_input(invalid[i], strlen(invalid[i]));
+ ColiServeCommand command;
+ assert(coli_serve_read_command(input, &profile, &command) ==
+ COLI_SERVE_READ_BAD_REQUEST);
+ fclose(input);
+ }
+
+ static const char short_body[] = "SUBMIT req 0 5 4 0.7 0.95\nabc";
+ FILE *input = bytes_input(short_body, sizeof(short_body) - 1);
+ ColiServeCommand command;
+ assert(coli_serve_read_command(input, &profile, &command) ==
+ COLI_SERVE_READ_BAD_FRAME);
+ fclose(input);
+
+ static const char bad_lf[] = "SUBMIT req 0 5 4 0.7 0.95\nabcdeX";
+ input = bytes_input(bad_lf, sizeof(bad_lf) - 1);
+ assert(coli_serve_read_command(input, &profile, &command) ==
+ COLI_SERVE_READ_BAD_FRAME);
+ fclose(input);
+
+ char long_id[COLI_SERVE_ID_CAP + 1];
+ memset(long_id, 'x', sizeof(long_id) - 1);
+ long_id[sizeof(long_id) - 1] = 0;
+ char long_header[256];
+ int length = snprintf(long_header, sizeof(long_header),
+ "SUBMIT %s 0 0 1 0.7 0.95\n\n", long_id);
+ assert(length > 0 && (size_t)length < sizeof(long_header));
+ input = bytes_input(long_header, (size_t)length);
+ assert(coli_serve_read_command(input, &profile, &command) ==
+ COLI_SERVE_READ_BAD_REQUEST);
+ fclose(input);
+
+ static const char unknown[] = "PING req\n";
+ input = bytes_input(unknown, sizeof(unknown) - 1);
+ assert(coli_serve_read_command(input, &profile, &command) ==
+ COLI_SERVE_READ_IGNORED);
+ fclose(input);
+
+ char long_line[160];
+ memset(long_line, 'x', sizeof(long_line));
+ long_line[sizeof(long_line) - 2] = '\n';
+ long_line[sizeof(long_line) - 1] = 0;
+ input = bytes_input(long_line, sizeof(long_line) - 1);
+ assert(coli_serve_read_command(input, &profile, &command) ==
+ COLI_SERVE_READ_BAD_REQUEST);
+ assert(fgetc(input) == EOF);
+ fclose(input);
+}
+
+static void test_writer_golden_bytes(void)
+{
+ FILE *output = tmpfile();
+ assert(output);
+ binary_stream(output);
+ assert(coli_serve_write_ready(output, 1.25));
+ assert(coli_serve_write_accept(output, "req-7", 12));
+ static const unsigned char data[] = {'A', '\n', 0, 'B'};
+ assert(coli_serve_write_data(output, "req-7", data, sizeof(data)));
+ assert(coli_serve_write_error(output, "req-7", "bad\r\nframe"));
+ ColiServeDone done = {3, 1.25, 50.0, 1.25, 7, 1};
+ assert(coli_serve_write_done(output, "req-7", &done));
+
+ static const unsigned char expected[] =
+ "\x01\x01READY\x01\x01\nSTAT 0 0.0 0.0 1.25 0 0\n"
+ "ACCEPT req-7 12\n"
+ "DATA req-7 4\nA\n\0B\n"
+ "ERROR req-7 bad frame\n"
+ "DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n";
+ unsigned char actual[sizeof(expected) + 16];
+ size_t count = read_output(output, actual, sizeof(actual));
+ assert(count == sizeof(expected) - 1);
+ assert(memcmp(actual, expected, count) == 0);
+ fclose(output);
+}
+
+static void test_allocation_failures_are_fatal_to_the_caller(void)
+{
+ static const char frame[] = "SUBMIT req 0 1 1 0.7 0.95\nx\n";
+ for (int failure = 1; failure <= 2; failure++) {
+ FILE *input = bytes_input(frame, sizeof(frame) - 1);
+ ColiServeCommand command;
+ allocation_count = 0;
+ fail_allocation = failure;
+ assert(coli_serve_read_command_alloc(input, &profile, &command, fault_alloc) ==
+ COLI_SERVE_READ_NOMEM);
+ assert(command.payload == NULL);
+ if (failure == 1) assert(fgetc(input) == 'S');
+ else assert(fgetc(input) == 'x');
+ fclose(input);
+ }
+}
+
+int main(void)
+{
+ test_submit_and_controls();
+ test_invalid_headers_and_bodies();
+ test_writer_golden_bytes();
+ test_allocation_failures_are_fatal_to_the_caller();
+ puts("serve codec tests: ok");
+ return 0;
+}
diff --git a/c/tests/test_serve_sentinel.c b/c/tests/test_serve_sentinel.c
index 50328f62..b3c83dbf 100644
--- a/c/tests/test_serve_sentinel.c
+++ b/c/tests/test_serve_sentinel.c
@@ -24,6 +24,7 @@
#include <unistd.h> /* close() dopo mkstemp */
#endif
#include "../compat.h"
+#include "../serve_codec.h"
#define SENTINEL "\x01\x01READY\x01\x01\n"
@@ -55,8 +56,7 @@ int main(void)
_setmode(_fileno(f), _O_BINARY);
#endif
- fputs(SENTINEL, f);
- fprintf(f, "STAT 0 0.0 0.0 %.2f 0 0\n", 1.0);
+ check(coli_serve_write_ready(f, 1.0), "shared READY writer succeeds");
fclose(f);
FILE *r = fopen(path, "rb"); /* rb: read the bytes as they landed */