Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
deepseek_v4: several conversations at once (KV_SLOTS up to 16)
JustVuggfc6ec3efee31
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 58d40fbd..4d994da8 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -6707,6 +6707,97 @@ int coli_v4_block_window_batch_ref(phase, moe_reason());return set_error(error, error_size, "hybrid batched block failed in %s", phase);}++/* Several conversations' rows through one block (KV_SLOTS, the serve's multiplexed+ * decode): row t is the next token of the conversation whose window attention state+ * is attention[t], at positions[t]. The mixes, the hyper-connections and the routed+ * experts are coli_v4_block_window_batch_ref's on the CPU (the experts through the+ * union over the rows, token-exact as the note above it says); the attention is the+ * single-token one each conversation's own decode step runs, on its own state. So+ * each row comes out with the bits its conversation's lone decode step gives. The+ * device's KV ring serves one conversation, so its marks are dropped before every+ * row's attention (a no-op without the GPU tier). */+int coli_v4_block_window_rows_ref(+ float *outputs_hc, ColiDeepSeekV4WindowAttentionState **attention,+ const ColiDeepSeekV4LayerWeights *weights,+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,+ const float *inputs_hc, const int *tokens, const int *positions, int batch,+ char *error, size_t error_size) {+ if (!outputs_hc || !attention || !weights || !config || !experts ||+ !inputs_hc || !tokens || !positions || batch < 1 || batch > 128) return -1;+ int d = config->hidden_size, hc = config->hc_mult;+ size_t hd = (size_t)hc * d;+ float *states = malloc((size_t)batch * hd * sizeof(*states));+ float *normalized = malloc((size_t)batch * d * sizeof(*normalized));+ float *branches = malloc((size_t)batch * d * sizeof(*branches));+ float *posts = malloc((size_t)batch * hc * sizeof(*posts));+ float *combs = malloc((size_t)batch * hc * hc * sizeof(*combs));+ float *reduced = malloc((size_t)d * sizeof(*reduced));+ float *ffn_normalized = malloc((size_t)batch * d * sizeof(*ffn_normalized));+ float *ffn_branch = malloc((size_t)batch * d * sizeof(*ffn_branch));+ float *ffn_post = malloc((size_t)batch * hc * sizeof(*ffn_post));+ float *ffn_comb = malloc((size_t)batch * hc * hc * sizeof(*ffn_comb));+ if (!states || !normalized || !branches || !posts || !combs || !reduced ||+ !ffn_normalized || !ffn_branch || !ffn_post || !ffn_comb) {+ free(ffn_comb); free(ffn_post); free(ffn_branch); free(ffn_normalized);+ free(reduced); free(combs); free(posts); free(branches);+ free(normalized); free(states); return -1;+ }+ int result = 0;+ const char *phase = "attention hyper-connection";+ for (int item = 0; !result && item < batch; item++)+ result = normalized_hc_pre(+ reduced, posts + (size_t)item * hc, combs + (size_t)item * hc * hc,+ normalized + (size_t)item * d, inputs_hc + (size_t)item * hd,+ weights, config, "attn", "attn_norm.weight");+ if (!result) phase = "attention";+ for (int item = 0; !result && item < batch; item++) {+#ifdef COLI_V4_GPU_TIER+ coli_v4_gpu_kv_cache_invalidate_all();+#endif+ result = coli_v4_attention_window_token_ref(+ branches + (size_t)item * d, attention[item], weights, config,+ normalized + (size_t)item * d, positions[item], error, error_size);+ }+ if (!result) phase = "attention post / FFN hyper-connection";+ for (int item = 0; !result && item < batch; item++) {+ float *state = states + (size_t)item * hd;+ result = coli_v4_hc_post(+ state, branches + (size_t)item * d, inputs_hc + (size_t)item * hd,+ posts + (size_t)item * hc, combs + (size_t)item * hc * hc, hc, d);+ if (!result) coli_bf16_round_array(state, hd);+ if (!result) result = normalized_hc_pre(+ reduced, ffn_post + (size_t)item * hc, ffn_comb + (size_t)item * hc * hc,+ ffn_normalized + (size_t)item * d, state,+ weights, config, "ffn", "ffn_norm.weight");+ }+ if (!result) phase = "MoE";+ if (!result && batch > 1 && v4_expert_union_enabled())+ result = v4_moe_batch_union(ffn_branch, weights, config, experts,+ ffn_normalized, tokens, batch);+ else+ for (int item = 0; !result && item < batch; item++)+ result = moe_token_pipeline(+ ffn_branch + (size_t)item * d, weights, config, experts,+ ffn_normalized + (size_t)item * d, tokens[item]);+ if (!result) phase = "FFN hyper-connection post";+ for (int item = 0; !result && item < batch; item++) {+ result = coli_v4_hc_post(+ outputs_hc + (size_t)item * hd, ffn_branch + (size_t)item * d,+ states + (size_t)item * hd, ffn_post + (size_t)item * hc,+ ffn_comb + (size_t)item * hc * hc, hc, d);+ if (!result) coli_bf16_round_array(outputs_hc + (size_t)item * hd, hd);+ }+ free(ffn_comb); free(ffn_post); free(ffn_branch); free(ffn_normalized);+ free(reduced); free(combs); free(posts); free(branches);+ free(normalized); free(states);+ if (!result) return 0;+ if (error && error_size && error[0]) return -1;+ if (moe_reason()[0])+ return set_error(error, error_size, "multiplexed block failed in %s: %s",+ phase, moe_reason());+ return set_error(error, error_size, "multiplexed block failed in %s", phase);+}#endif /* COLI_V4_UNIT_BLOCK_HYBRID */#ifdef COLI_V4_UNIT_COMPRESSOR_SNAPSHOT@@ -12648,6 +12739,10 @@ int coli_v4_prompt_build(char **output, size_t *output_length,#include "decode_batch.h" /* coli_logprob_tail: the numeric channel's tail, same bytes as the other engines */#include "tok.h"+/* KV_SLOTS of a serve: more than one, the conversations decode together, one row of a+ * batch each (coli_v4_sessions_step). */+static int g_v4_mux_slots = 1;+static int load_embedding(float *state, const ColiSafetensorsIndex *index,const ColiDeepSeekV4Config *config, int token) {const ColiSafetensorsTensor *embed = coli_st_find(index, "embed.weight");@@ -15140,6 +15235,29 @@ int coli_v4_session_generate(ColiV4Session *session,generated_count, options->stop_at_sentence);}double first_at = spec_now();+ if (options->prefill_only) {+ /* A multiplexed serve: coli_v4_sessions_step decodes the rest, one row of a+ * batch per step, from where this leaves it. */+ session->mux.current = current;+ session->mux.logit = current_logit;+ session->mux.last = last_processed;+ session->mux.count = generated_count;+ session->mux.max_new = max_new;+ session->mux.done = done || generated_count >= max_new;+ session->mux.logprobs = options->logprobs;+ session->mux.first_at = first_at;+ session->mux.on_token = on_token;+ session->mux.user_data = user_data;+ session->mux.on_scores = options->on_scores;+ session->mux.scores_user_data = options->scores_user_data;+ session->generated_count = generated_count;+ if (stats_out) {+ stats_out->prompt_tokens = prompt_count;+ stats_out->generated_tokens = generated_count;+ stats_out->time_to_first_token_sec = first_at - setup_done;+ }+ return 0;+ }int draft_limit = getenv("V4_DRAFT") ? atoi(getenv("V4_DRAFT")) : 0;if (draft_limit < 0) draft_limit = 0;@@ -15444,6 +15562,138 @@ int coli_v4_session_generate(ColiV4Session *session,return 0;}+/* Every head score of several hidden rows, [batch][vocab]: the batch's rows of+ * head_scores, each with the same head_bf16_dot (the CPU) as alone, the head read once+ * for the batch while it is resident. */+static int head_scores_batch(ColiV4Engine *engine, const float *hidden,+ const ColiSafetensorsIndex *index,+ const ColiDeepSeekV4Config *config, int batch,+ float *scores) {+ double t0 = spec_now();+ const ColiSafetensorsTensor *head = coli_st_find(index, "head.weight");+ int d = config->hidden_size, vocab = config->vocab_size;+ if (!head || head->dtype != COLI_ST_BF16 || batch < 1) return -1;+ const uint16_t *resident = coli_v4_head_cache_data(+ engine, coli_st_tensor_shard(index, head), (uint64_t)head->off,+ (size_t)vocab * d * sizeof(uint16_t));+ int result = 0;+ if (!resident || batch == 1) {+ for (int item = 0; !result && item < batch; item++)+ result = head_scores_impl(engine, hidden + (size_t)item * d, index,+ config, scores + (size_t)item * vocab);+ } else {+#ifdef COLI_VULKAN+ int on_device = coli_v4_vk_matmul &&+ coli_v4_vk_matmul(11, resident, NULL, 0, vocab, d, scores,+ hidden, batch) == 0;+ if (!on_device)+#endif+ #pragma omp parallel for schedule(static)+ for (int row = 0; row < vocab; row++) {+ const uint16_t *weight = resident + (size_t)row * d;+ for (int item = 0; item < batch; item++)+ scores[(size_t)item * vocab + row] = head_bf16_dot(+ weight, hidden + (size_t)item * d, d);+ }+ V4_DUMP_LOGITS(scores, (size_t)batch * vocab);+ }+ g_v4_prof_head_s += spec_now() - t0;+ return result;+}++/* One decode step of several conversations (a multiplexed serve, KV_SLOTS): each+ * session's request was started with prefill_only and is not done; its next token goes+ * through the layers as one row of a batch, at its own position, against its own+ * attention state (coli_v4_block_window_rows_ref), and the head picks each row's+ * token. The token is emitted as the session's lone decode step emits it, and the+ * session's mux cursor moves on; mux.done says the request has ended (an end of+ * sequence, its budget, or its token callback asking to stop). */+int coli_v4_sessions_step(ColiV4Session **sessions, int count,+ char *error, size_t error_size) {+ if (!sessions || count < 1 || count > 128) return -1;+ ColiV4Engine *engine = sessions[0]->engine;+ const ColiDeepSeekV4Config *config = &sessions[0]->config;+ ColiSafetensorsIndex *index = coli_v4_engine_target_index(engine);+ ColiExpertStore *experts = coli_v4_engine_expert_store(engine);+ int d = config->hidden_size, vocab = config->vocab_size;+ int layers = config->num_hidden_layers;+ size_t hd = (size_t)config->hc_mult * d;+ float *state = malloc((size_t)count * hd * sizeof(*state));+ float *next = malloc((size_t)count * hd * sizeof(*next));+ float *hidden = malloc((size_t)count * d * sizeof(*hidden));+ float *scores = malloc((size_t)count * vocab * sizeof(*scores));+ int *tokens = malloc((size_t)count * sizeof(*tokens));+ int *positions = malloc((size_t)count * sizeof(*positions));+ ColiDeepSeekV4WindowAttentionState **attention =+ malloc((size_t)count * sizeof(*attention));+ int result = -1;+ if (!state || !next || !hidden || !scores || !tokens || !positions || !attention)+ goto out;+ for (int t = 0; t < count; t++) {+ tokens[t] = sessions[t]->mux.current;+ positions[t] = sessions[t]->mux.last + 1;+ if (load_embedding(state + (size_t)t * hd, index, config, tokens[t])) {+ snprintf(error, error_size, "V4 embedding read failed");+ goto taint;+ }+ }+ double t0 = spec_now();+ for (int layer_id = 0; layer_id < layers; layer_id++) {+ ColiDeepSeekV4LayerWeights layer;+ if (coli_v4_layer_load(engine, &layer, config, index, layer_id,+ error, error_size)) goto taint;+ for (int t = 0; t < count; t++) attention[t] = sessions[t]->attention[layer_id];+ int failed = coli_v4_block_window_rows_ref(+ next, attention, &layer, config, experts, state, tokens, positions,+ count, error, error_size);+ coli_v4_layer_free(engine, &layer);+ if (failed) goto taint;+ float *swap = state; state = next; next = swap;+ }+ g_v4_prof_block_s += spec_now() - t0;+ g_v4_prof_forwards += count;+ for (int t = 0; t < count; t++) {+ /* the token is in the conversation's attention state from here on */+ kv_prefix_record(&sessions[t]->fed, &tokens[t], positions[t], 1);+ if (final_hidden(hidden + (size_t)t * d, state + (size_t)t * hd, index,+ config, error, error_size)) goto taint;+ }+ if (head_scores_batch(engine, hidden, index, config, count, scores)) {+ snprintf(error, error_size, "V4 head failed");+ goto taint;+ }+ for (int t = 0; t < count; t++) {+ ColiV4Session *session = sessions[t];+ const float *row = scores + (size_t)t * vocab;+ int current = 0;+ float logit = 0.0f;+ if (head_scores_argmax(row, vocab, ¤t, &logit)) {+ snprintf(error, error_size, "V4 head failed");+ goto taint;+ }+ session->mux.current = current;+ session->mux.logit = logit;+ session->mux.last = positions[t];+ session->generated[session->mux.count++] = current;+ session->generated_count = session->mux.count;+ if (session->mux.on_scores)+ session->mux.on_scores(session->mux.scores_user_data, positions[t],+ current, row, vocab);+ int stop = session_emit_token(session, session->mux.on_token,+ session->mux.user_data, current, logit,+ positions[t], session->mux.count, 0);+ session->mux.done = stop || session->mux.count >= session->mux.max_new;+ }+ result = 0;+ goto out;+taint:+ for (int t = 0; t < count; t++) kv_prefix_taint(&sessions[t]->fed);+out:+ free(attention); free(positions); free(tokens); free(scores);+ free(hidden); free(next); free(state);+ return result;+}+static int v4_oracle_teacher_forcing(const int *full_ids, int full_count, const int *expected, int expect_count,ColiDeepSeekV4WindowAttentionState **attention,@@ -15620,6 +15870,7 @@ typedef struct {int fatal;int logprobs;char tail[1024]; /* the next DATA frame's logprob tail, from on_scores */+ int mux; /* a multiplexed serve (KV_SLOTS): its loop reads the commands */} V4ServeStream;static const ColiServeWireProfile v4_wire = {@@ -15812,6 +16063,25 @@ static void v4_prof_emit(double wall_s, int prompt_tokens, int completion,fflush(stdout);}+/* A SUBMIT's fields, its prompt taken, the command disposed. */+static void v4_serve_take_submit(ColiServeCommand *command, V4ServeRequest *request) {+ int prefix_bytes = command->prefix_bytes;+ if (prefix_bytes < 0 || (uint64_t)prefix_bytes > command->payload_bytes)+ prefix_bytes = 0;+ memset(request, 0, sizeof(*request));+ snprintf(request->id, sizeof(request->id), "%s", command->id);+ request->prompt = (char *)coli_serve_command_take_payload(command);+ request->prompt_bytes = (int)command->payload_bytes;+ request->max_tokens = command->max_tokens;+ request->temperature = command->temperature;+ request->top_p = command->top_p;+ request->extension_bytes = (int)command->extension_bytes;+ request->prefix_bytes = prefix_bytes;+ request->logprobs = command->logprobs;+ request->pin = command->pin;+ coli_serve_command_dispose(command);+}+static int v4_serve_read_request(FILE *input, FILE *output,V4ServeRequest *request,const char *active_id) {@@ -15844,21 +16114,7 @@ static int v4_serve_read_request(FILE *input, FILE *output,coli_serve_command_dispose(&command);return -2;}- int prefix_bytes = command.prefix_bytes;- if (prefix_bytes < 0 || (uint64_t)prefix_bytes > command.payload_bytes)- prefix_bytes = 0;- memset(request, 0, sizeof(*request));- snprintf(request->id, sizeof(request->id), "%s", command.id);- request->prompt = (char *)coli_serve_command_take_payload(&command);- request->prompt_bytes = (int)command.payload_bytes;- request->max_tokens = command.max_tokens;- request->temperature = command.temperature;- request->top_p = command.top_p;- request->extension_bytes = (int)command.extension_bytes;- request->prefix_bytes = prefix_bytes;- request->logprobs = command.logprobs;- request->pin = command.pin;- coli_serve_command_dispose(&command);+ v4_serve_take_submit(&command, request);return 2;}@@ -15910,7 +16166,7 @@ static int v4_serve_token(void *user_data, int token, float logit,v4_serve_data(stdout, stream->request_id, piece, bytes);}stream->tail[0] = 0;- if (v4_serve_drain_commands(stream)) {+ if (!stream->mux && v4_serve_drain_commands(stream)) {stream->cancelled = 1;return 1;}@@ -15973,8 +16229,9 @@ static void v4_serve_scores(void *user_data, int position, int token,stream->logprobs);}-static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,- V4ServeRequest *request) {+/* A request's checks before its prefill: 1 with its ACCEPT written, 0 with its ERROR. */+static int v4_serve_admit(ColiV4Engine *engine, ColiV4Session *session,+ V4ServeRequest *request) {if (request->extension_bytes) {v4_serve_error(stdout, request->id, "unsupported request extension");return 0;@@ -16024,22 +16281,46 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,request->max_tokens = context - prompt_count;}coli_serve_write_accept(stdout, request->id, prompt_count);+ return 1;+}- ColiExpertStoreStats before = {0}, after = {0};+/* Where the counters stood when a request started: its DONE and PROF lines report what+ * moved since. */+typedef struct {+ ColiExpertStoreStats experts;+ double disk, matmul, wait, block, head, started;+ long long forwards;+} V4ServeMarks;++static void v4_serve_mark(ColiV4Engine *engine, V4ServeMarks *marks) {+ memset(marks, 0, sizeof(*marks));if (engine->experts && engine->experts->ops && engine->experts->ops->stats)- engine->experts->ops->stats(engine->experts, &before);- double disk_before =+ engine->experts->ops->stats(engine->experts, &marks->experts);+ marks->disk =engine->experts ? coli_v4_expert_store_disk_sec(engine->experts) : 0.0;- double matmul_before =+ marks->matmul =engine->experts ? coli_v4_expert_store_matmul_sec(engine->experts) : 0.0;- double wait_before =+ marks->wait =engine->experts ? coli_v4_expert_store_wait_sec(engine->experts) : 0.0;- double block_before = g_v4_prof_block_s, head_before = g_v4_prof_head_s;- long long forwards_before = g_v4_prof_forwards;- V4ServeStream stream = {session, request->id, 0, 0, request->logprobs, {0}};+ marks->block = g_v4_prof_block_s;+ marks->head = g_v4_prof_head_s;+ marks->forwards = g_v4_prof_forwards;+ marks->started = spec_now();+}++static void v4_serve_finish(ColiV4Engine *engine, ColiV4Session *session,+ const char *id, const V4ServeMarks *marks,+ int generated, int eos_stopped, int cancelled,+ int max_tokens, int prompt_tokens, double decode_sec);++static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,+ V4ServeRequest *request) {+ if (!v4_serve_admit(engine, session, request)) return 0;+ V4ServeMarks marks;+ v4_serve_mark(engine, &marks);+ V4ServeStream stream = {session, request->id, 0, 0, request->logprobs, {0}, 0};ColiV4SessionGenerateStats stats = {0};char error[512] = {0};- double started = spec_now();int result = coli_v4_session_generate(session, request->prompt, (size_t)request->prompt_bytes,&(ColiV4SessionGenerateOptions){@@ -16056,48 +16337,61 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,.scores_user_data = &stream,},v4_serve_token, &stream, &stats, error, sizeof(error));- double elapsed = spec_now() - started;if (stream.fatal) return -1;if (result) {v4_serve_error(stdout, request->id, error);return 0;}+ v4_serve_finish(engine, session, request->id, &marks, stats.generated_tokens,+ stats.eos_stopped, stream.cancelled, request->max_tokens,+ stats.prompt_tokens, stats.decode_sec);+ return 0;+}++/* A request's DONE and PROF lines, then the turn's reports: `generated` tokens, the+ * last one an end of sequence when eos_stopped; decode_sec from the first token. */+static void v4_serve_finish(ColiV4Engine *engine, ColiV4Session *session,+ const char *id, const V4ServeMarks *marks,+ int generated, int eos_stopped, int cancelled,+ int max_tokens, int prompt_tokens, double decode_sec) {+ double elapsed = spec_now() - marks->started;+ ColiExpertStoreStats after = {0};if (engine->experts && engine->experts->ops && engine->experts->ops->stats)engine->experts->ops->stats(engine->experts, &after);- uint64_t hits = after.hits - before.hits;- uint64_t misses = after.misses - before.misses;+ uint64_t hits = after.hits - marks->experts.hits;+ uint64_t misses = after.misses - marks->experts.misses;double hit_rate = hits + misses ? 100.0 * hits / (hits + misses) : 0.0;- int completion = stats.generated_tokens - (stats.eos_stopped ? 1 : 0);+ int completion = generated - (eos_stopped ? 1 : 0);if (completion < 0) completion = 0;- int length_limited = !stream.cancelled && !stats.eos_stopped &&- request->max_tokens > 0 && /* a read-only request is not cut short */- stats.generated_tokens >= request->max_tokens;- double decode = stats.decode_sec > 0.0 ? stats.decode_sec : elapsed;+ int length_limited = !cancelled && !eos_stopped &&+ max_tokens > 0 && /* a read-only request is not cut short */+ generated >= max_tokens;+ double decode = decode_sec > 0.0 ? decode_sec : elapsed;/* Trailing field: prompt tokens served from the previous turn's attention* state instead of being prefilled again. Appended rather than inserted --* openai_server.py accepts `len(fields) >= 7`, so an older reader ignores* it and a newer one can report it. */- v4_serve_done(stdout, request->id, completion,+ v4_serve_done(stdout, id, completion,decode > 0.0 ? completion / decode : 0.0,- hit_rate, v4_serve_rss_gb(), stats.prompt_tokens,+ hit_rate, v4_serve_rss_gb(), prompt_tokens,length_limited, session->prefix_reused);double expert_disk_s = engine->experts- ? coli_v4_expert_store_disk_sec(engine->experts) - disk_before+ ? coli_v4_expert_store_disk_sec(engine->experts) - marks->disk: 0.0;double expert_matmul_s = engine->experts- ? coli_v4_expert_store_matmul_sec(engine->experts) - matmul_before+ ? coli_v4_expert_store_matmul_sec(engine->experts) - marks->matmul: 0.0;double expert_wait_s = engine->experts- ? coli_v4_expert_store_wait_sec(engine->experts) - wait_before+ ? coli_v4_expert_store_wait_sec(engine->experts) - marks->wait: 0.0;/* Block time minus the expert compute and the expert wait measured inside it.* The store's disk seconds are NOT subtracted: summed across loader lanes,* they exceeded the wall on a cold tiny run (0.073 s of disk in a 0.034 s turn). */- double attention_s = (g_v4_prof_block_s - block_before) - expert_matmul_s - expert_wait_s;+ double attention_s = (g_v4_prof_block_s - marks->block) - expert_matmul_s - expert_wait_s;if (attention_s < 0.0) attention_s = 0.0;- v4_prof_emit(elapsed, stats.prompt_tokens, completion,+ v4_prof_emit(elapsed, prompt_tokens, completion,expert_disk_s, expert_wait_s, expert_matmul_s, attention_s,- g_v4_prof_head_s - head_before, g_v4_prof_forwards - forwards_before);+ g_v4_prof_head_s - marks->head, g_v4_prof_forwards - marks->forwards);#ifdef COLI_V4_GPU_TIERif (coli_v4_hybrid_enabled() && (g_v4_hyb_gpu_n + g_v4_hyb_cpu_n +g_v4_hyb_upload_n + g_v4_hyb_skip_n))@@ -16132,7 +16426,153 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,&g_v4_mir_nread[r], __ATOMIC_RELAXED));fprintf(stderr, "%s\n", line);}- return 0;+}++/* ---- several conversations at once (KV_SLOTS>1) ---------------------------------+ * The gateway's cache slots, each a conversation with a session of its own. A SUBMIT+ * on a free slot is prefilled at once on that slot's session (its prompt cache works+ * as a lone serve's) up to its first token. Then every step decodes the next token of+ * each active request as one row of a batch (coli_v4_sessions_step). A request's frames+ * are a lone request's; they interleave by id. As alone, STOP and CANCEL end a request+ * with DONE. Nothing drafts: DSpark stays unloaded. */+typedef struct {+ V4ServeRequest request;+ V4ServeStream stream;+ V4ServeMarks marks;+ ColiV4Session *session;+ int active, stop;+} V4MuxRequest;++static void v4_mux_finish(ColiV4Engine *engine, V4MuxRequest *r, int cancelled) {+ ColiV4Session *session = r->session;+ int count = session->mux.count;+ int eos = count > 0 && session->generated[count - 1] == 1;+ r->active = 0;+ v4_serve_finish(engine, session, r->request.id, &r->marks, count, eos, cancelled,+ r->request.max_tokens, session->prompt_count,+ count ? spec_now() - session->mux.first_at : 0.0);+}++static void v4_mux_start(ColiV4Engine *engine, V4MuxRequest *r) {+ if (!v4_serve_admit(engine, r->session, &r->request)) return;+ v4_serve_mark(engine, &r->marks);+ r->stream = (V4ServeStream){r->session, r->request.id, 0, 0, r->request.logprobs, {0}, 1};+#ifdef COLI_V4_GPU_TIER+ coli_v4_gpu_kv_cache_invalidate_all(); /* the device's KV ring held another conversation */+#endif+ ColiV4SessionGenerateStats stats = {0};+ char error[512] = {0};+ int result = coli_v4_session_generate(+ r->session, r->request.prompt, (size_t)r->request.prompt_bytes,+ &(ColiV4SessionGenerateOptions){+ .max_new_tokens = r->request.max_tokens,+ .no_dspark = 1,+ .prefix_bytes = (size_t)r->request.prefix_bytes,+ .logprobs = r->request.logprobs,+ .pin = r->request.pin,+ .on_echo = r->request.logprobs > 0 ? v4_serve_echo : NULL,+ .on_scores = r->request.logprobs > 0 ? v4_serve_scores : NULL,+ .scores_user_data = &r->stream,+ .prefill_only = 1,+ },+ v4_serve_token, &r->stream, &stats, error, sizeof(error));+ if (result) {+ v4_serve_error(stdout, r->request.id, error);+ return;+ }+ if (r->session->mux.done) v4_mux_finish(engine, r, 0);+ else r->active = 1;+}++static void v4_serve_mux(ColiV4Engine *engine, ColiV4Session **sessions, int n) {+ V4MuxRequest *requests = calloc((size_t)n, sizeof(*requests));+ ColiV4Session **rows = calloc((size_t)n, sizeof(*rows));+ int *row_slot = calloc((size_t)n, sizeof(*row_slot));+ if (!requests || !rows || !row_slot) {+ fprintf(stderr, "[V4] out of memory for %d conversations\n", n);+ free(row_slot); free(rows); free(requests);+ return;+ }+ unsigned long long steps = 0, row_count = 0;+ int input_eof = 0;+ fprintf(stderr, "[V4] serving %d conversations at once (KV_SLOTS)\n", n);+ for (;;) {+ int active = 0;+ for (int i = 0; i < n; i++) active += requests[i].active;+ /* idle: wait for a command; decoding: take one only when one is there */+ if (!input_eof && (!active || coli_stdin_readable())) {+ ColiServeCommand command;+ ColiServeReadResult result = coli_serve_read_command(stdin, &v4_wire, &command);+ if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) {+ input_eof = 1;+ } else if (result == COLI_SERVE_READ_NOMEM) {+ coli_serve_write_error(stdout, command.id, "out of memory");+ input_eof = 1;+ } else if (result == COLI_SERVE_READ_BAD_REQUEST) {+ if (command.kind == COLI_SERVE_COMMAND_SUBMIT)+ coli_serve_write_error(stdout, command.id, "bad submit header");+ coli_serve_command_dispose(&command);+ } else if (result != COLI_SERVE_READ_OK) {+ /* nothing to do */+ } else if (command.kind == COLI_SERVE_COMMAND_STOP ||+ command.kind == COLI_SERVE_COMMAND_CANCEL) {+ for (int i = 0; i < n; i++)+ if (requests[i].active && !strcmp(requests[i].request.id, command.id))+ requests[i].stop = 1;+ coli_serve_command_dispose(&command);+ } else if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {+ coli_serve_command_dispose(&command);+ } else if (command.slot < 0 || command.slot >= n || requests[command.slot].active) {+ coli_serve_write_error(stdout, command.id,+ command.slot < 0 || command.slot >= n+ ? "invalid cache slot" : "SLOT_BUSY");+ coli_serve_command_dispose(&command);+ } else {+ V4MuxRequest *r = &requests[command.slot];+ memset(r, 0, sizeof(*r));+ r->session = sessions[command.slot];+ v4_serve_take_submit(&command, &r->request);+ v4_mux_start(engine, r);+ free(r->request.prompt);+ r->request.prompt = NULL;+ }+ }+ active = 0;+ for (int i = 0; i < n; i++) active += requests[i].active;+ if (!active) {+ if (input_eof) break;+ continue;+ }+ int count = 0;+ for (int i = 0; i < n; i++) {+ if (!requests[i].active) continue;+ if (requests[i].stop) {+ v4_mux_finish(engine, &requests[i], 1);+ continue;+ }+ rows[count] = sessions[i];+ row_slot[count++] = i;+ }+ if (!count) continue;+ char error[512] = {0};+ if (coli_v4_sessions_step(rows, count, error, sizeof(error))) {+ /* the step left every row's attention state part way: each request ends,+ * and its conversation starts over (its prompt cache is tainted) */+ for (int k = 0; k < count; k++) {+ v4_serve_error(stdout, requests[row_slot[k]].request.id, error);+ requests[row_slot[k]].active = 0;+ }+ continue;+ }+ steps++;+ row_count += (unsigned long long)count;+ for (int k = 0; k < count; k++)+ if (sessions[row_slot[k]]->mux.done)+ v4_mux_finish(engine, &requests[row_slot[k]], 0);+ }+ fprintf(stderr, "[V4] KV_SLOTS=%d: %llu decode steps, %llu rows (%.2f a step)\n", n,+ steps, row_count, steps ? (double)row_count / (double)steps : 0.0);+ free(row_slot); free(rows); free(requests);}static int v4_serve_main(void) {@@ -16145,14 +16585,25 @@ static int v4_serve_main(void) {int max_tokens = getenv("NGEN") ? atoi(getenv("NGEN")) : 1024;if (context < 2) context = 4096;if (max_tokens < 1) max_tokens = 1024;+ const char *slots_env = getenv("KV_SLOTS"); /* the conversations decoded at once */+ if (slots_env && *slots_env) {+ char *end = NULL;+ long slots = strtol(slots_env, &end, 10);+ if (end == slots_env || *end || slots < 1 || slots > 16) {+ fprintf(stderr, "KV_SLOTS must be between 1 and 16\n");+ return 2;+ }+ g_v4_mux_slots = (int)slots;+ }+ int n = g_v4_mux_slots;char error[512] = {0};ColiV4Engine *engine = NULL;- ColiV4Session *session = NULL;+ ColiV4Session *sessions[16] = {0};ColiV4EngineOpenOptions open_options = {.target_model_dir = model_dir,.context_tokens = context,.pin_slots_per_layer = -1,- .no_dspark = 0,+ .no_dspark = n > 1, /* several conversations: nothing drafts */};const char *ram = getenv("RAM_GB");if (ram && atof(ram) > 0.0)@@ -16165,17 +16616,20 @@ static int v4_serve_main(void) {}v4_vk_open(engine);context = engine->runtime.context_tokens;- if (coli_v4_session_create(- &session, engine,- &(ColiV4SessionCreateOptions){- .max_prompt_tokens = context,- .max_new_tokens_cap = max_tokens,- },- error, sizeof(error))) {- fprintf(stderr, "%s\n", error);- coli_v4_engine_destroy(engine);- return 1;- }+ for (int i = 0; i < n; i++)+ if (coli_v4_session_create(+ &sessions[i], engine,+ &(ColiV4SessionCreateOptions){+ .max_prompt_tokens = context,+ .max_new_tokens_cap = max_tokens,+ },+ error, sizeof(error))) {+ fprintf(stderr, "%s\n", error);+ for (int k = 0; k < i; k++) coli_v4_session_destroy(sessions[k]);+ coli_v4_engine_destroy(engine);+ return 1;+ }+ ColiV4Session *session = sessions[0];/* Eagerly load all dense layers so GPU upload happens at startup.* This avoids the 25s wall-clock gap between layers during the first@@ -16196,7 +16650,8 @@ static int v4_serve_main(void) {v4_hwinfo_emit();coli_v4_expert_store_emit_tiers(engine->experts);coli_v4_expert_store_emit_emap(engine->experts);- for (;;) {+ if (n > 1) v4_serve_mux(engine, sessions, n);+ for (; n == 1;) {V4ServeRequest request = {0};int result;do result = v4_serve_read_request(stdin, stdout, &request, NULL);@@ -16209,7 +16664,7 @@ static int v4_serve_main(void) {}}v4_vk_close();- coli_v4_session_destroy(session);+ for (int i = 0; i < n; i++) coli_v4_session_destroy(sessions[i]);coli_v4_engine_destroy(engine);return 0;}diff --git a/c/deepseek_v4.h b/c/deepseek_v4.hindex c0f6ef79..a5750f13 100644--- a/c/deepseek_v4.h+++ b/c/deepseek_v4.h@@ -154,6 +154,9 @@ typedef struct {* that token boundary during this prefill so later conversations that* share it start there. 0 = unknown. */size_t prefix_bytes;+ /* Return after the prompt and its first token, the decoding's state kept in the+ * session for coli_v4_sessions_step (a multiplexed serve, KV_SLOTS): no draft. */+ int prefill_only;} ColiV4SessionGenerateOptions;typedef struct {@@ -182,6 +185,11 @@ int coli_v4_session_generate(ColiV4Session *session,ColiV4SessionGenerateStats *stats,char *error, size_t error_size);+/* One decode step of several sessions whose requests were started with prefill_only:+ * each one's next token, as a row of one batch (a multiplexed serve, KV_SLOTS). */+int coli_v4_sessions_step(ColiV4Session **sessions, int count,+ char *error, size_t error_size);+int coli_v4_session_generated_text(const ColiV4Session *session,char *buffer, size_t buffer_size,size_t *out_length);diff --git a/c/deepseek_v4_chain.h b/c/deepseek_v4_chain.hindex b375ad1d..09ec337c 100644--- a/c/deepseek_v4_chain.h+++ b/c/deepseek_v4_chain.h@@ -1361,7 +1361,13 @@ static int v4c_decide(const ColiV4Engine *engine) {if (g_v4c_decision >= 0) return g_v4c_decision;if (!g_v4_vk_ready || !engine) return 0;int tier_on = vkt_wanted() && !(engine->experts && engine->experts->gpu) && engine->config.n_routed_experts > 0;- return g_v4c_decision = coli_vk_chain_decide("deepseek_v4", tier_on, COLI_VK_CHAIN_UNMEASURED);+ g_v4c_decision = coli_vk_chain_decide("deepseek_v4", tier_on, COLI_VK_CHAIN_UNMEASURED);+ if (g_v4c_decision && g_v4_mux_slots > 1) {+ g_v4c_decision = 0;+ fprintf(stderr, "[VK] deepseek_v4: KV_SLOTS=%d: the dense chain is off (it keeps one conversation's state on the device); "+ "the expert tier runs every conversation's experts\n", g_v4_mux_slots);+ }+ return g_v4c_decision;}static void v4c_start(const ColiV4Engine *cengine) {ColiV4Engine *engine = (ColiV4Engine *)cengine;diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex cf826374..296b658a 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -814,6 +814,13 @@ int coli_v4_block_window_batch_ref(const ColiDeepSeekV4Config *config, ColiExpertStore *experts,const float *inputs_hc, const int *tokens, int start_position, int batch,char *error, size_t error_size);+/* rows of several conversations, each with its attention state and position */+int coli_v4_block_window_rows_ref(+ float *outputs_hc, ColiDeepSeekV4WindowAttentionState **attention,+ const ColiDeepSeekV4LayerWeights *weights,+ const ColiDeepSeekV4Config *config, ColiExpertStore *experts,+ const float *inputs_hc, const int *tokens, const int *positions, int batch,+ char *error, size_t error_size);/* ==== end deepseek_v4_block_batch.h ==== *//* ==== begin deepseek_v4_resource_plan.h ==== */@@ -1126,6 +1133,17 @@ struct ColiV4Session {* early returns of generate() leave nothing behind. */float *echo_hidden;float *echo_scores;+ /* A request started with prefill_only (a multiplexed serve, KV_SLOTS): where+ * its decoding stands, for coli_v4_sessions_step to go on from. */+ struct {+ int current, last, count, max_new, done, logprobs;+ float logit;+ double first_at;+ ColiV4SessionTokenFn on_token;+ void *user_data;+ ColiV4SessionScoresFn on_scores;+ void *scores_user_data;+ } mux;};/* RAM-tiered expert open used by coli_v4_engine_open (replaces ld --wrap).