Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

test(v4): freeze serve wire framing

Mario Gutierrez5fa92da55ede
diff --git a/c/.gitignore b/c/.gitignore
index faf8f236..35552f5f 100644
--- a/c/.gitignore
+++ b/c/.gitignore
@@ -37,6 +37,7 @@ tests/test_tok_o200k
tests/test_pipe_block
tests/test_kimi_request_state
tests/test_kimi_serve_framing
+tests/test_v4_serve_framing
*.o
*.dll
cublasLt64*.dll
diff --git a/c/Makefile b/c/Makefile
index f216d547..a3c24abf 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -436,13 +436,14 @@ TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(f
# test_uring is Linux-only. V4 engine tests are appended below only on supported
# x86-64 Linux/Windows and aarch64 Linux hosts; the V4 infrastructure tests have
# unconditional rules and therefore remain portable gates.
-TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership
+TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing
TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
ifeq ($(COLI_V4_SUPPORTED),1)
-TEST_BINS += tests/test_deepseek_v4$(EXE) tests/test_v4_ownership$(EXE)
+TEST_BINS += tests/test_deepseek_v4$(EXE) tests/test_v4_ownership$(EXE) \
+ tests/test_v4_serve_framing$(EXE)
endif
# Windows CUDA/HIP DLL path: host links the loader, NOT cudart or amdhip64.
@@ -1200,6 +1201,10 @@ $(V4_OWN_DIR)/expert_store_registry.o: expert_store_registry.c expert_store_regi
tests/test_v4_ownership$(EXE): tests/test_v4_ownership.c $(V4_OWNERSHIP_OBJS)
$(CC) $(V4_OWN_CFLAGS) $< $(V4_OWNERSHIP_OBJS) -o $@ -pthread $(LDFLAGS)
+tests/test_v4_serve_framing$(EXE): tests/test_v4_serve_framing.c deepseek_v4.c \
+ deepseek_v4.h deepseek_v4_internal.h Makefile.deepseek-v4 Makefile.deepseek-v4.units
+ $(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) $@
+
endif
tests/test_route_trace$(EXE): tests/test_route_trace.c route_trace.h
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4
index c6fcee2c..6b856cda 100644
--- a/c/Makefile.deepseek-v4
+++ b/c/Makefile.deepseek-v4
@@ -167,6 +167,8 @@ TARGET_OBJS = $(addsuffix .o,$(V4_TARGET_UNITS))
TEST_UNIT_OBJS = $(addsuffix .o,$(V4_TEST_UNITS))
V4_OBJS = $(TARGET_OBJS) $(V4_WIN_EXTRA_OBJS) $(REGISTRY_OBJ)
REGISTRY_OBJ = expert_store_registry.o
+V4_SERVE_TEST := tests/test_v4_serve_framing$(if $(IS_WIN),.exe,)
+V4_SERVE_TEST_OBJS := $(filter-out COLI_V4_UNIT_GENERATE_STATS.o,$(V4_OBJS))
.PHONY: deepseek-v4 deepseek-v4-objs deepseek-v4-clean deepseek-v4-test-objs \
deepseek-v4-test-registry print-v4-objs
@@ -197,6 +199,10 @@ $(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \
native_quant_fp4_rows16.h expert_store_registry.h
$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@
+$(V4_SERVE_TEST): tests/test_v4_serve_framing.c deepseek_v4.c deepseek_v4.h \
+ deepseek_v4_internal.h $(V4_SERVE_TEST_OBJS)
+ $(CC) $(CFLAGS) $< $(V4_SERVE_TEST_OBJS) -o $@ $(LDFLAGS)
+
# The CUDA tier loader object. Compiled only on Windows, where it is appended
# to V4_OBJS and resolves the MSVC-built coli_cuda_dsv4.dll at runtime. On
# Linux it is empty so the engine links without it.
@@ -222,4 +228,4 @@ test_expert_store_registry: test_expert_store_registry.c expert_store_registry.c
$(CC) -O2 -Wall -Wextra test_expert_store_registry.c expert_store_registry.c -o $@
deepseek-v4-clean:
- rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) test_expert_store_registry
+ rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) $(V4_SERVE_TEST) test_expert_store_registry
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 988b19a7..3ec7b009 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -12598,10 +12598,11 @@ static void v4_prof_emit(double wall_s, int prompt_tokens, int completion,
fflush(stdout);
}
-static int v4_serve_read_request(V4ServeRequest *request,
+static int v4_serve_read_request(FILE *input, FILE *output,
+ V4ServeRequest *request,
const char *active_id) {
char line[512], command[16], id[64];
- if (!fgets(line, sizeof(line), stdin)) return -1;
+ if (!fgets(line, sizeof(line), input)) return -1;
if (sscanf(line, "%15s %63s", command, id) < 2) return 0;
if (!strcmp(command, "CANCEL") || !strcmp(command, "STOP"))
return active_id && !strcmp(active_id, id);
@@ -12620,22 +12621,22 @@ static int v4_serve_read_request(V4ServeRequest *request,
if (fields < 5 || slot != 0 || prompt_bytes < 0 ||
prompt_bytes > (1 << 24) || max_tokens < 1 ||
extension_bytes < 0 || extension_bytes > (1 << 24)) {
- printf("ERROR %s bad submit header\n", id);
- fflush(stdout);
+ fprintf(output, "ERROR %s bad submit header\n", id);
+ fflush(output);
return 0;
}
size_t total = (size_t)prompt_bytes + (size_t)extension_bytes;
char *payload = malloc(total + 1);
if (!payload) {
- printf("ERROR %s out of memory\n", id);
- fflush(stdout);
+ fprintf(output, "ERROR %s out of memory\n", id);
+ fflush(output);
return 0;
}
- if (fread(payload, 1, total, stdin) != total) {
+ if (fread(payload, 1, total, input) != total) {
free(payload);
return -1;
}
- (void)fgetc(stdin);
+ (void)fgetc(input);
payload[prompt_bytes] = 0;
memset(request, 0, sizeof(*request));
snprintf(request->id, sizeof(request->id), "%s", id);
@@ -12649,12 +12650,13 @@ static int v4_serve_read_request(V4ServeRequest *request,
return 2;
}
-static void v4_serve_data(const char *id, const char *data, int bytes) {
+static void v4_serve_data(FILE *output, const char *id,
+ const char *data, int bytes) {
if (bytes <= 0) return;
- printf("DATA %s %d\n", id, bytes);
- fwrite(data, 1, (size_t)bytes, stdout);
- fputc('\n', stdout);
- fflush(stdout);
+ fprintf(output, "DATA %s %d\n", id, bytes);
+ fwrite(data, 1, (size_t)bytes, output);
+ fputc('\n', output);
+ fflush(output);
}
/* Drain gateway commands queued behind an active request: CANCEL/STOP for the
@@ -12664,7 +12666,7 @@ static void v4_serve_data(const char *id, const char *data, int bytes) {
static int v4_serve_drain_commands(const char *active_id) {
while (coli_stdin_readable()) {
V4ServeRequest queued = {0};
- int result = v4_serve_read_request(&queued, active_id);
+ int result = v4_serve_read_request(stdin, stdout, &queued, active_id);
if (result < 0 || result == 1) {
free(queued.prompt);
return 1;
@@ -12688,7 +12690,7 @@ static int v4_serve_token(void *user_data, int token, float logit,
char piece[1024];
int bytes = tok_decode(&stream->session->tokenizer, &token, 1,
piece, (int)sizeof(piece) - 1);
- v4_serve_data(stream->request_id, piece, bytes);
+ v4_serve_data(stdout, stream->request_id, piece, bytes);
}
if (v4_serve_drain_commands(stream->request_id)) {
stream->cancelled = 1;
@@ -12711,19 +12713,29 @@ static int v4_serve_abort(void *user_data) {
return 0;
}
-static void v4_serve_error(const char *id, const char *message) {
+static void v4_serve_error(FILE *output, const char *id, const char *message) {
char clean[512];
snprintf(clean, sizeof(clean), "%s", message && *message ? message : "engine request failed");
for (char *p = clean; *p; p++)
if (*p == '\r' || *p == '\n') *p = ' ';
- printf("ERROR %s %s\n", id, clean);
- fflush(stdout);
+ fprintf(output, "ERROR %s %s\n", id, clean);
+ fflush(output);
+}
+
+static void v4_serve_done(FILE *output, const char *id, int completion,
+ double tokens_per_second, double hit_rate,
+ double rss, int prompt_tokens, int length_limited,
+ int prefix_reused) {
+ fprintf(output, "DONE %s STAT %d %.3f %.1f %.2f %d %d %d\n",
+ id, completion, tokens_per_second, hit_rate, rss, prompt_tokens,
+ length_limited, prefix_reused);
+ fflush(output);
}
static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
V4ServeRequest *request) {
if (request->extension_bytes) {
- v4_serve_error(request->id, "unsupported request extension");
+ v4_serve_error(stdout, request->id, "unsupported request extension");
return;
}
if (request->temperature != 0.0f)
@@ -12754,7 +12766,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
char message[256];
snprintf(message, sizeof(message), "CONTEXT_EXCEEDED %d %d",
prompt_count, prompt_capacity);
- v4_serve_error(request->id, message);
+ v4_serve_error(stdout, request->id, message);
return;
}
/* max_tokens is a CEILING, not a target (#260/#382): generation ends at
@@ -12797,7 +12809,7 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
v4_serve_token, &stream, &stats, error, sizeof(error));
double elapsed = spec_now() - started;
if (result) {
- v4_serve_error(request->id, error);
+ v4_serve_error(stdout, request->id, error);
return;
}
if (engine->experts && engine->experts->ops && engine->experts->ops->stats)
@@ -12814,12 +12826,10 @@ static void v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
* state instead of being prefilled again. Appended rather than inserted --
* openai_server.py accepts `len(fields) >= 7`, so an older reader ignores
* it and a newer one can report it. */
- printf("DONE %s STAT %d %.3f %.1f %.2f %d %d %d\n",
- request->id, completion,
- decode > 0.0 ? completion / decode : 0.0,
- hit_rate, v4_serve_rss_gb(), stats.prompt_tokens, length_limited,
- session->prefix_reused);
- fflush(stdout);
+ v4_serve_done(stdout, request->id, completion,
+ decode > 0.0 ? completion / decode : 0.0,
+ hit_rate, v4_serve_rss_gb(), stats.prompt_tokens,
+ length_limited, session->prefix_reused);
double expert_disk_s = engine->experts
? coli_v4_expert_store_disk_sec(engine->experts) - disk_before
: 0.0;
@@ -12916,7 +12926,8 @@ static int v4_serve_main(void) {
for (;;) {
V4ServeRequest request = {0};
int result;
- do result = v4_serve_read_request(&request, NULL); while (result == 0);
+ do result = v4_serve_read_request(stdin, stdout, &request, NULL);
+ while (result == 0);
if (result < 0) break;
if (result == 2) {
v4_serve_one(engine, session, &request);
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index c7127cb0..c3caa8d1 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -472,6 +472,34 @@ class FakeProcess:
class DispatcherTest(unittest.TestCase):
+ def test_v4_request_and_response_transcript_is_byte_exact(self):
+ prompt = "<|begin▁of▁sentence|>System<|User|>Hello<|Assistant|>"
+ payload = prompt.encode("utf-8")
+ prefix = len("<|begin▁of▁sentence|>System".encode("utf-8"))
+ expected = (f"SUBMIT 1 0 {len(payload)} 4 0.25 0.9 0 {prefix}\n".encode() +
+ payload + b"\n")
+
+ def respond(process, frame):
+ self.assertEqual(frame, expected)
+ process.stdout.feed(
+ b"ACCEPT 1 42\n"
+ b"DATA 1 4\nA\n\xc3\xa9\n"
+ b"DONE 1 STAT 1 2.500 50.0 1.25 42 0 17\n"
+ )
+
+ process = FakeProcess(respond)
+ with patch("openai_server.ARCH", "deepseek_v4"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("deepseek_v4", "model")
+ chunks = []
+ stats = engine.generate(prompt, 4, 0.25, 0.9, chunks.append)
+ engine.close()
+
+ self.assertEqual(process.writes, [expected])
+ self.assertEqual(chunks, ["A\né"])
+ self.assertEqual(stats["completion_tokens"], 1)
+ self.assertEqual(stats["prompt_tokens"], 42)
+
def test_kimi_request_and_response_transcript_is_byte_exact(self):
prompt = render_chat_kimi([
{"role": "system", "content": "Be precise."},
diff --git a/c/tests/test_v4_serve_framing.c b/c/tests/test_v4_serve_framing.c
new file mode 100644
index 00000000..d56b406a
--- /dev/null
+++ b/c/tests/test_v4_serve_framing.c
@@ -0,0 +1,104 @@
+/* Freeze DeepSeek V4's serving wire before adopting serve_codec.h. */
+#define COLI_V4_UNIT_GENERATE_STATS
+#define COLI_V4_SKIP_GENERATE_MAIN
+#include "../deepseek_v4.c"
+
+#include <assert.h>
+
+static void binary_stream(FILE *stream)
+{
+#ifdef _WIN32
+ assert(_setmode(_fileno(stream), _O_BINARY) != -1);
+#else
+ (void)stream;
+#endif
+}
+
+static FILE *bytes_input(const void *data, size_t size)
+{
+ FILE *stream = tmpfile();
+ assert(stream);
+ binary_stream(stream);
+ assert(fwrite(data, 1, size, stream) == size);
+ rewind(stream);
+ return stream;
+}
+
+static size_t read_output(FILE *stream, unsigned char *output, size_t capacity)
+{
+ assert(fflush(stream) == 0);
+ rewind(stream);
+ return fread(output, 1, capacity, stream);
+}
+
+static void test_submit_shapes_and_controls(void)
+{
+ static const char base[] = "SUBMIT base 0 5 4 0.7 0.95\nab\ncd\n";
+ FILE *input = bytes_input(base, sizeof(base) - 1);
+ FILE *output = tmpfile(); assert(output); binary_stream(output);
+ V4ServeRequest request = {0};
+ assert(v4_serve_read_request(input, output, &request, NULL) == 2);
+ assert(strcmp(request.id, "base") == 0);
+ assert(request.prompt_bytes == 5 && request.extension_bytes == 0);
+ assert(request.prefix_bytes == 0 && memcmp(request.prompt, "ab\ncd", 5) == 0);
+ assert(fgetc(input) == EOF);
+ free(request.prompt); fclose(input); fclose(output);
+
+ static const char extended[] =
+ "SUBMIT ext 0 3 4 0 1 4 2\nabcWXYZ\n";
+ input = bytes_input(extended, sizeof(extended) - 1);
+ output = tmpfile(); assert(output); binary_stream(output);
+ memset(&request, 0, sizeof(request));
+ assert(v4_serve_read_request(input, output, &request, NULL) == 2);
+ assert(request.prompt_bytes == 3 && request.extension_bytes == 4);
+ assert(request.prefix_bytes == 2 && memcmp(request.prompt, "abc", 3) == 0);
+ assert(fgetc(input) == EOF);
+ free(request.prompt); fclose(input); fclose(output);
+
+ static const char stop[] = "STOP live\n";
+ input = bytes_input(stop, sizeof(stop) - 1);
+ output = tmpfile(); assert(output); binary_stream(output);
+ assert(v4_serve_read_request(input, output, &request, "live") == 1);
+ fclose(input); fclose(output);
+
+ static const char cancel[] = "CANCEL live\n";
+ input = bytes_input(cancel, sizeof(cancel) - 1);
+ output = tmpfile(); assert(output); binary_stream(output);
+ assert(v4_serve_read_request(input, output, &request, "other") == 0);
+ fclose(input); fclose(output);
+}
+
+static void test_errors_and_writer_bytes(void)
+{
+ static const char bad[] = "SUBMIT bad 1 0 4 0 1\n\n";
+ FILE *input = bytes_input(bad, sizeof(bad) - 1);
+ FILE *output = tmpfile(); assert(output); binary_stream(output);
+ V4ServeRequest request = {0};
+ assert(v4_serve_read_request(input, output, &request, NULL) == 0);
+ unsigned char actual[512];
+ size_t count = read_output(output, actual, sizeof(actual));
+ static const char bad_want[] = "ERROR bad bad submit header\n";
+ assert(count == sizeof(bad_want) - 1);
+ assert(memcmp(actual, bad_want, count) == 0);
+ fclose(input); fclose(output);
+
+ output = tmpfile(); assert(output); binary_stream(output);
+ v4_serve_data(output, "req-7", "A\n\0B", 4);
+ v4_serve_error(output, "req-7", "bad\r\nframe");
+ v4_serve_done(output, "req-7", 3, 1.25, 50.0, 1.25, 7, 1, 5);
+ static const unsigned char want[] =
+ "DATA req-7 4\nA\n\0B\n"
+ "ERROR req-7 bad frame\n"
+ "DONE req-7 STAT 3 1.250 50.0 1.25 7 1 5\n";
+ count = read_output(output, actual, sizeof(actual));
+ assert(count == sizeof(want) - 1 && memcmp(actual, want, count) == 0);
+ fclose(output);
+}
+
+int main(void)
+{
+ test_submit_shapes_and_controls();
+ test_errors_and_writer_bytes();
+ puts("v4 serve framing baseline: ok");
+ return 0;
+}