Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
refactor(serve): share framing with DeepSeek V4
Mario Gutierrez8cbebcef90fd
diff --git a/c/Makefile b/c/Makefileindex a3c24abf..52777d40 100644--- a/c/Makefile+++ b/c/Makefile@@ -1202,7 +1202,7 @@ tests/test_v4_ownership$(EXE): tests/test_v4_ownership.c $(V4_OWNERSHIP_OBJS)$(CC) $(V4_OWN_CFLAGS) $< $(V4_OWNERSHIP_OBJS) -o $@ -pthread $(LDFLAGS)tests/test_v4_serve_framing$(EXE): tests/test_v4_serve_framing.c deepseek_v4.c \- deepseek_v4.h deepseek_v4_internal.h Makefile.deepseek-v4 Makefile.deepseek-v4.units+ deepseek_v4.h deepseek_v4_internal.h serve_codec.h Makefile.deepseek-v4 Makefile.deepseek-v4.units$(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) $@endifdiff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4index 6b856cda..57fa291a 100644--- a/c/Makefile.deepseek-v4+++ b/c/Makefile.deepseek-v4@@ -199,8 +199,10 @@ $(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \native_quant_fp4_rows16.h expert_store_registry.h$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@+COLI_V4_UNIT_GENERATE_STATS.o: serve_codec.h+$(V4_SERVE_TEST): tests/test_v4_serve_framing.c deepseek_v4.c deepseek_v4.h \- deepseek_v4_internal.h $(V4_SERVE_TEST_OBJS)+ deepseek_v4_internal.h serve_codec.h $(V4_SERVE_TEST_OBJS)$(CC) $(CFLAGS) $< $(V4_SERVE_TEST_OBJS) -o $@ $(LDFLAGS)# The CUDA tier loader object. Compiled only on Windows, where it is appendeddiff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 3ec7b009..af8fae21 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -10436,6 +10436,7 @@ int coli_v4_prompt_build(char **output, size_t *output_length,#include "deepseek_v4_internal.h"#include "json.h"#include "native_quant.h"+#include "serve_codec.h"#include "tok.h"static int load_embedding(float *state, const ColiSafetensorsIndex *index,@@ -12521,8 +12522,20 @@ typedef struct {ColiV4Session *session;const char *request_id;int cancelled;+ int fatal;} V4ServeStream;+static const ColiServeWireProfile v4_wire = {+ .max_header_bytes = 511,+ .max_payload_bytes = 1u << 24,+ .max_extension_bytes = 1u << 24,+ .max_tokens = 0,+ .require_exact_lf = 1,+ .require_finite_sampling = 0,+ .allow_extension_bytes = 1,+ .allow_prefix_hint = 1,+};+static double v4_serve_rss_gb(void) {#ifdef _WIN32return 0.0;@@ -12601,79 +12614,74 @@ static void v4_prof_emit(double wall_s, int prompt_tokens, int completion,static int v4_serve_read_request(FILE *input, FILE *output,V4ServeRequest *request,const char *active_id) {- char line[512], command[16], id[64];- if (!fgets(line, sizeof(line), input)) return -1;- if (sscanf(line, "%15s %63s", command, id) < 2) return 0;- if (!strcmp(command, "CANCEL") || !strcmp(command, "STOP"))- return active_id && !strcmp(active_id, id);- if (strcmp(command, "SUBMIT")) return 0;-- int slot = 0, prompt_bytes = 0, max_tokens = 0, extension_bytes = 0;- int prefix_bytes = 0;- float temperature = 0.0f, top_p = 1.0f;- /* Optional 8th field: byte length of the prompt's stable system prefix- * (gateway hint for the prefix checkpoint). Older gateways omit it. */- int fields = sscanf(line, "%*s %*s %d %d %d %f %f %d %d",- &slot, &prompt_bytes, &max_tokens,- &temperature, &top_p, &extension_bytes, &prefix_bytes);- if (fields < 7) prefix_bytes = 0;- if (prefix_bytes < 0 || prefix_bytes > prompt_bytes) prefix_bytes = 0;- if (fields < 5 || slot != 0 || prompt_bytes < 0 ||- prompt_bytes > (1 << 24) || max_tokens < 1 ||- extension_bytes < 0 || extension_bytes > (1 << 24)) {- fprintf(output, "ERROR %s bad submit header\n", id);- fflush(output);- return 0;- }- size_t total = (size_t)prompt_bytes + (size_t)extension_bytes;- char *payload = malloc(total + 1);- if (!payload) {- fprintf(output, "ERROR %s out of memory\n", id);- fflush(output);+ ColiServeCommand command;+ ColiServeReadResult result = coli_serve_read_command(input, &v4_wire, &command);+ if (result == COLI_SERVE_READ_EOF) return -1;+ if (result == COLI_SERVE_READ_BAD_FRAME) return -2;+ if (result == COLI_SERVE_READ_NOMEM) {+ coli_serve_write_error(output, command.id, "out of memory");+ return -2;+ }+ if (result == COLI_SERVE_READ_BAD_REQUEST &&+ command.kind == COLI_SERVE_COMMAND_SUBMIT) {+ coli_serve_write_error(output, command.id, "bad submit header");+ return -2;+ }+ if (result != COLI_SERVE_READ_OK) return 0;+ if (command.kind == COLI_SERVE_COMMAND_STOP ||+ command.kind == COLI_SERVE_COMMAND_CANCEL) {+ int matched = active_id && !strcmp(active_id, command.id);+ coli_serve_command_dispose(&command);+ return matched;+ }+ if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {+ coli_serve_command_dispose(&command);return 0;}- if (fread(payload, 1, total, input) != total) {- free(payload);- return -1;+ if (command.slot != 0) {+ coli_serve_write_error(output, command.id, "bad submit header");+ coli_serve_command_dispose(&command);+ return -2;}- (void)fgetc(input);- payload[prompt_bytes] = 0;+ int prefix_bytes = command.prefix_bytes;+ if (prefix_bytes < 0 || (uint64_t)prefix_bytes > command.payload_bytes)+ prefix_bytes = 0;memset(request, 0, sizeof(*request));- snprintf(request->id, sizeof(request->id), "%s", id);- request->prompt = payload;- request->prompt_bytes = prompt_bytes;- request->max_tokens = max_tokens;- request->temperature = temperature;- request->top_p = top_p;- request->extension_bytes = extension_bytes;+ snprintf(request->id, sizeof(request->id), "%s", command.id);+ request->prompt = (char *)coli_serve_command_take_payload(&command);+ request->prompt_bytes = (int)command.payload_bytes;+ request->max_tokens = command.max_tokens;+ request->temperature = command.temperature;+ request->top_p = command.top_p;+ request->extension_bytes = (int)command.extension_bytes;request->prefix_bytes = prefix_bytes;+ coli_serve_command_dispose(&command);return 2;}static void v4_serve_data(FILE *output, const char *id,const char *data, int bytes) {if (bytes <= 0) return;- fprintf(output, "DATA %s %d\n", id, bytes);- fwrite(data, 1, (size_t)bytes, output);- fputc('\n', output);- fflush(output);+ coli_serve_write_data(output, id, data, (size_t)bytes);}/* Drain gateway commands queued behind an active request: CANCEL/STOP for the* active id (or stdin EOF) reports "stop generating"; a SUBMIT that arrives* while the engine is busy is answered with an engine-busy ERROR so its* gateway thread fails fast instead of waiting on a pipe nobody is reading. */-static int v4_serve_drain_commands(const char *active_id) {+static int v4_serve_drain_commands(V4ServeStream *stream) {while (coli_stdin_readable()) {V4ServeRequest queued = {0};- int result = v4_serve_read_request(stdin, stdout, &queued, active_id);- if (result < 0 || result == 1) {+ int result = v4_serve_read_request(stdin, stdout, &queued,+ stream->request_id);+ if (result < 0) {+ if (result == -2) stream->fatal = 1;free(queued.prompt);return 1;}+ if (result == 1) { free(queued.prompt); return 1; }if (result == 2) {- printf("ERROR %s engine busy\n", queued.id);- fflush(stdout);+ coli_serve_write_error(stdout, queued.id, "engine busy");free(queued.prompt);}}@@ -12692,7 +12700,7 @@ static int v4_serve_token(void *user_data, int token, float logit,piece, (int)sizeof(piece) - 1);v4_serve_data(stdout, stream->request_id, piece, bytes);}- if (v4_serve_drain_commands(stream->request_id)) {+ if (v4_serve_drain_commands(stream)) {stream->cancelled = 1;return 1;}@@ -12706,7 +12714,7 @@ static int v4_serve_token(void *user_data, int token, float logit,static int v4_serve_abort(void *user_data) {V4ServeStream *stream = user_data;if (stream->cancelled) return 1;- if (v4_serve_drain_commands(stream->request_id)) {+ if (v4_serve_drain_commands(stream)) {stream->cancelled = 1;return 1;}@@ -12714,29 +12722,24 @@ static int v4_serve_abort(void *user_data) {}static void v4_serve_error(FILE *output, const char *id, const char *message) {- char clean[512];- snprintf(clean, sizeof(clean), "%s", message && *message ? message : "engine request failed");- for (char *p = clean; *p; p++)- if (*p == '\r' || *p == '\n') *p = ' ';- fprintf(output, "ERROR %s %s\n", id, clean);- fflush(output);+ coli_serve_write_error(output, id,+ message && *message ? message : "engine request failed");}static void v4_serve_done(FILE *output, const char *id, int completion,double tokens_per_second, double hit_rate,double rss, int prompt_tokens, int length_limited,int prefix_reused) {- fprintf(output, "DONE %s STAT %d %.3f %.1f %.2f %d %d %d\n",- id, completion, tokens_per_second, hit_rate, rss, prompt_tokens,- length_limited, prefix_reused);- fflush(output);+ ColiServeDone done = {completion, tokens_per_second, hit_rate, rss,+ prompt_tokens, length_limited};+ coli_serve_write_done_i32_suffix(output, id, &done, &prefix_reused, 1);}-static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,- V4ServeRequest *request) {+static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,+ V4ServeRequest *request) {if (request->extension_bytes) {v4_serve_error(stdout, request->id, "unsupported request extension");- return;+ return 0;}if (request->temperature != 0.0f)fprintf(stderr, "[V4] temperature %.3g ignored; target engine is greedy\n",@@ -12767,7 +12770,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,snprintf(message, sizeof(message), "CONTEXT_EXCEEDED %d %d",prompt_count, prompt_capacity);v4_serve_error(stdout, request->id, message);- return;+ return 0;}/* max_tokens is a CEILING, not a target (#260/#382): generation ends at* EOS either way, so an oversized budget is clamped to what the context@@ -12782,8 +12785,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,prompt_count);request->max_tokens = context - prompt_count;}- printf("ACCEPT %s %d\n", request->id, prompt_count);- fflush(stdout);+ coli_serve_write_accept(stdout, request->id, prompt_count);ColiExpertStoreStats before = {0}, after = {0};if (engine->experts && engine->experts->ops && engine->experts->ops->stats)@@ -12792,7 +12794,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,engine->experts ? coli_v4_expert_store_disk_sec(engine->experts) : 0.0;double matmul_before =engine->experts ? coli_v4_expert_store_matmul_sec(engine->experts) : 0.0;- V4ServeStream stream = {session, request->id, 0};+ V4ServeStream stream = {session, request->id, 0, 0};ColiV4SessionGenerateStats stats = {0};char error[512] = {0};double started = spec_now();@@ -12808,9 +12810,10 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,},v4_serve_token, &stream, &stats, error, sizeof(error));double elapsed = spec_now() - started;+ if (stream.fatal) return -1;if (result) {v4_serve_error(stdout, request->id, error);- return;+ return 0;}if (engine->experts && engine->experts->ops && engine->experts->ops->stats)engine->experts->ops->stats(engine->experts, &after);@@ -12859,6 +12862,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,&g_v4_mir_nread[r], __ATOMIC_RELAXED));fprintf(stderr, "%s\n", line);}+ return 0;}static int v4_serve_main(void) {@@ -12915,11 +12919,8 @@ static int v4_serve_main(void) {}}}- coli_serve_binary_mode();- setvbuf(stdin, NULL, _IONBF, 0);- fputs("\x01\x01READY\x01\x01\n", stdout);- printf("STAT 0 0.0 0.0 %.2f 0 0\n", v4_serve_rss_gb());- fflush(stdout);+ coli_serve_stdio_init();+ coli_serve_write_ready(stdout, v4_serve_rss_gb());v4_hwinfo_emit();coli_v4_expert_store_emit_tiers(engine->experts);coli_v4_expert_store_emit_emap(engine->experts);@@ -12930,8 +12931,9 @@ static int v4_serve_main(void) {while (result == 0);if (result < 0) break;if (result == 2) {- v4_serve_one(engine, session, &request);+ int fatal = v4_serve_one(engine, session, &request);free(request.prompt);+ if (fatal < 0) break;}}coli_v4_session_destroy(session);diff --git a/c/serve_codec.h b/c/serve_codec.hindex d1324a8f..67c1bfd7 100644--- a/c/serve_codec.h+++ b/c/serve_codec.h@@ -34,9 +34,12 @@ typedef enum {typedef struct {size_t max_header_bytes;uint64_t max_payload_bytes;+ uint64_t max_extension_bytes;int max_tokens;int require_exact_lf;int require_finite_sampling;+ int allow_extension_bytes;+ int allow_prefix_hint;} ColiServeWireProfile;typedef struct {@@ -47,6 +50,8 @@ typedef struct {int max_tokens;float temperature;float top_p;+ uint64_t extension_bytes;+ int prefix_bytes;unsigned char *payload;} ColiServeCommand;@@ -79,6 +84,12 @@ static inline unsigned char *coli_serve_command_take_payload(ColiServeCommand *creturn payload;}+static inline unsigned char *coli_serve_command_extension(ColiServeCommand *command)+{+ if (!command || !command->payload || !command->extension_bytes) return NULL;+ return command->payload + (size_t)command->payload_bytes + 1;+}+static inline int coli_serve_parse_i32(const char *text, int *value){char *end = NULL;@@ -141,7 +152,7 @@ static inline ColiServeReadResult coli_serve_read_command_alloc({char *line;size_t count = 0;- char *fields[8];+ char *fields[10];int nfields = 0;if (!input || !profile || !command) return COLI_SERVE_READ_BAD_FRAME;@@ -204,16 +215,28 @@ static inline ColiServeReadResult coli_serve_read_command_alloc(free(line);return COLI_SERVE_READ_OK;}- if (nfields != 7) {+ int minimum_fields = 7;+ int maximum_fields = minimum_fields + !!profile->allow_extension_bytes ++ !!profile->allow_prefix_hint;+ if (nfields < minimum_fields || nfields > maximum_fields ||+ (nfields >= 8 && !profile->allow_extension_bytes) ||+ (nfields >= 9 && (!profile->allow_prefix_hint ||+ !profile->allow_extension_bytes))) {free(line);return COLI_SERVE_READ_BAD_REQUEST;}+ command->extension_bytes = 0;+ command->prefix_bytes = 0;if (!coli_serve_parse_i32(fields[2], &command->slot) ||!coli_serve_parse_u64(fields[3], &command->payload_bytes) ||!coli_serve_parse_i32(fields[4], &command->max_tokens) ||!coli_serve_parse_f32(fields[5], &command->temperature) ||!coli_serve_parse_f32(fields[6], &command->top_p) ||+ (nfields >= 8 &&+ !coli_serve_parse_u64(fields[7], &command->extension_bytes)) ||+ (nfields >= 9 && !coli_serve_parse_i32(fields[8], &command->prefix_bytes)) ||command->payload_bytes > profile->max_payload_bytes ||+ command->extension_bytes > profile->max_extension_bytes ||command->max_tokens < 1 ||(profile->max_tokens && command->max_tokens > profile->max_tokens) ||(profile->require_finite_sampling &&@@ -223,11 +246,17 @@ static inline ColiServeReadResult coli_serve_read_command_alloc(}free(line);- if (command->payload_bytes > SIZE_MAX - 1) return COLI_SERVE_READ_NOMEM;+ if (command->payload_bytes > SIZE_MAX - 1 ||+ command->extension_bytes > SIZE_MAX - 1 - (size_t)command->payload_bytes)+ return COLI_SERVE_READ_NOMEM;size_t payload_bytes = (size_t)command->payload_bytes;- command->payload = allocate(payload_bytes + 1);+ size_t extension_bytes = (size_t)command->extension_bytes;+ command->payload = allocate(payload_bytes + 1 + extension_bytes);if (!command->payload) return COLI_SERVE_READ_NOMEM;- if (fread(command->payload, 1, payload_bytes, input) != payload_bytes) {+ if (fread(command->payload, 1, payload_bytes, input) != payload_bytes ||+ (extension_bytes &&+ fread(command->payload + payload_bytes + 1, 1, extension_bytes, input) !=+ extension_bytes)) {coli_serve_command_dispose(command);return COLI_SERVE_READ_BAD_FRAME;}@@ -291,6 +320,20 @@ static inline int coli_serve_write_done(return fflush(output) == 0;}+static inline int coli_serve_write_done_i32_suffix(+ FILE *output, const char *id, const ColiServeDone *done,+ const int *suffix, size_t suffix_count)+{+ if (fprintf(output, "DONE %s STAT %d %.3f %.1f %.2f %d %d", id,+ done->completion_tokens, done->tokens_per_second,+ done->cache_hit_percent, done->rss_gb, done->prompt_tokens,+ done->length_limited) < 0)+ return 0;+ for (size_t i = 0; i < suffix_count; i++)+ if (fprintf(output, " %d", suffix[i]) < 0) return 0;+ return fputc('\n', output) != EOF && fflush(output) == 0;+}+static inline int coli_serve_format_done(char *output, size_t capacity, const char *id, const ColiServeDone *done){diff --git a/c/tests/test_serve_codec.c b/c/tests/test_serve_codec.cindex 55aaaeeb..dd0082ac 100644--- a/c/tests/test_serve_codec.c+++ b/c/tests/test_serve_codec.c@@ -62,6 +62,21 @@ static void test_submit_and_controls(void)coli_serve_command_dispose(&command);fclose(input);+ ColiServeWireProfile extended_profile = profile;+ extended_profile.max_extension_bytes = 4;+ extended_profile.allow_extension_bytes = 1;+ extended_profile.allow_prefix_hint = 1;+ static const char extended[] = "SUBMIT req-v4 0 3 4 0 1 4 2\nabcWXYZ\n";+ input = bytes_input(extended, sizeof(extended) - 1);+ assert(coli_serve_read_command(input, &extended_profile, &command) ==+ COLI_SERVE_READ_OK);+ assert(command.payload_bytes == 3 && command.extension_bytes == 4);+ assert(command.prefix_bytes == 2);+ assert(memcmp(command.payload, "abc", 3) == 0 && command.payload[3] == 0);+ assert(memcmp(coli_serve_command_extension(&command), "WXYZ", 4) == 0);+ coli_serve_command_dispose(&command);+ fclose(input);+static const char subnormal[] = "SUBMIT req-8 0 1 1 1e-40 0.95\nx\n";input = bytes_input(subnormal, sizeof(subnormal) - 1);assert(coli_serve_read_command(input, &profile, &command) == COLI_SERVE_READ_OK);@@ -161,13 +176,16 @@ static void test_writer_golden_bytes(void)assert(coli_serve_format_done(done_line, sizeof(done_line), "req-7", &done) ==(int)strlen("DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n"));assert(strcmp(done_line, "DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n") == 0);+ int suffix = 5;+ assert(coli_serve_write_done_i32_suffix(output, "req-v4", &done, &suffix, 1));static const unsigned char expected[] ="\x01\x01READY\x01\x01\nSTAT 0 0.0 0.0 1.25 0 0\n""ACCEPT req-7 12\n""DATA req-7 4\nA\n\0B\n""ERROR req-7 bad frame\n"- "DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n";+ "DONE req-7 STAT 3 1.250 50.0 1.25 7 1\n"+ "DONE req-v4 STAT 3 1.250 50.0 1.25 7 1 5\n";unsigned char actual[sizeof(expected) + 16];size_t count = read_output(output, actual, sizeof(actual));assert(count == sizeof(expected) - 1);diff --git a/c/tests/test_v4_serve_framing.c b/c/tests/test_v4_serve_framing.cindex d56b406a..aca3fe95 100644--- a/c/tests/test_v4_serve_framing.c+++ b/c/tests/test_v4_serve_framing.c@@ -52,6 +52,7 @@ static void test_submit_shapes_and_controls(void)assert(v4_serve_read_request(input, output, &request, NULL) == 2);assert(request.prompt_bytes == 3 && request.extension_bytes == 4);assert(request.prefix_bytes == 2 && memcmp(request.prompt, "abc", 3) == 0);+ assert(memcmp(request.prompt + request.prompt_bytes + 1, "WXYZ", 4) == 0);assert(fgetc(input) == EOF);free(request.prompt); fclose(input); fclose(output);@@ -74,7 +75,7 @@ static void test_errors_and_writer_bytes(void)FILE *input = bytes_input(bad, sizeof(bad) - 1);FILE *output = tmpfile(); assert(output); binary_stream(output);V4ServeRequest request = {0};- assert(v4_serve_read_request(input, output, &request, NULL) == 0);+ assert(v4_serve_read_request(input, output, &request, NULL) == -2);unsigned char actual[512];size_t count = read_output(output, actual, sizeof(actual));static const char bad_want[] = "ERROR bad bad submit header\n";@@ -95,10 +96,38 @@ static void test_errors_and_writer_bytes(void)fclose(output);}+static void test_extension_rejection_precedes_accept(void)+{+ V4ServeRequest request = {+ .prompt = "abc",+ .prompt_bytes = 3,+ .max_tokens = 4,+ .extension_bytes = 4,+ };+ snprintf(request.id, sizeof(request.id), "ext");+ request.temperature = 0.0f;+ request.top_p = 1.0f;++ FILE *output = tmpfile(); assert(output); binary_stream(output);+ int saved = dup(fileno(stdout)); assert(saved >= 0);+ assert(fflush(stdout) == 0);+ assert(dup2(fileno(output), fileno(stdout)) >= 0);+ assert(v4_serve_one(NULL, NULL, &request) == 0);+ assert(fflush(stdout) == 0);+ assert(dup2(saved, fileno(stdout)) >= 0); close(saved);++ unsigned char actual[128];+ size_t count = read_output(output, actual, sizeof(actual));+ static const char want[] = "ERROR ext unsupported request extension\n";+ assert(count == sizeof(want) - 1 && memcmp(actual, want, count) == 0);+ fclose(output);+}+int main(void){test_submit_shapes_and_controls();test_errors_and_writer_bytes();+ test_extension_rejection_precedes_accept();puts("v4 serve framing baseline: ok");return 0;}