Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
perf(dsv4): hybrid GPU/CPU expert split on decode misses (opt-in)
JustVuggbc44ff0e941e
diff --git a/c/Makefile b/c/Makefileindex 9ef02fd3..a142d590 100644--- a/c/Makefile+++ b/c/Makefile@@ -457,6 +457,7 @@ ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endififeq ($(COLI_V4_SUPPORTED),1)+TEST_BINS += tests/test_v4_hybrid_policy$(EXE)TEST_BINS += tests/test_deepseek_v4$(EXE) tests/test_v4_ownership$(EXE) \tests/test_v4_serve_framing$(EXE)endif@@ -1335,6 +1336,9 @@ tests/test_corpus_draft$(EXE): tests/test_corpus_draft.c colibri.c st.h uring.htests/test_cap_precedence$(EXE): tests/test_cap_precedence.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_v4_hybrid_policy$(EXE): tests/test_v4_hybrid_policy.c deepseek_v4_hybrid.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_ram_clamp$(EXE): tests/test_ram_clamp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 229eab38..527d2991 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -4202,6 +4202,7 @@ int coli_v4_sparse_attention_ref(float *output, const float *queries,#define coli_v4_block_window_token_ref coli_v4_block_window_token_serial_ref/* ---- begin include deepseek_v4_block.c ---- */#include "deepseek_v4_internal.h"+#include "deepseek_v4_hybrid.h"#include <stdarg.h>#include <stdint.h>@@ -4831,6 +4832,29 @@ static int profiled_expert_load_finish(ExpertLoadHandle *handle) {return result;}+#ifdef COLI_V4_GPU_TIER+/* DSV4_HYBRID=1 (opt-in): on a decode miss, split the missing experts between+ * a PCIe-fill set (upload, compute on GPU with the residents) and a host set+ * (compute on CPU) instead of today's all-or-nothing, sized by the q* policy+ * in deepseek_v4_hybrid.h from live bandwidth EMAs. Off by default until the+ * split is validated on real GPUs; when off, the engine byte-for-byte keeps+ * its historical behaviour. */+int coli_v4_hybrid_enabled(void) {+ static int on = -1;+ if (on < 0) {+ const char *setting = getenv("DSV4_HYBRID");+ on = setting && atoi(setting) != 0;+ }+ return on;+}+/* Non-static on purpose: the serve unit prints these counters, and under the+ * per-unit amalgam build that is a different object file. */+double g_v4_hyb_fill_bw; /* experts/s uploaded, EMA */+double g_v4_hyb_host_bw; /* experts/s host-computed, EMA */+unsigned long long g_v4_hyb_gpu_n, g_v4_hyb_cpu_n;+unsigned long long g_v4_hyb_upload_n, g_v4_hyb_skip_n;+#endif+static int moe_token_pipeline(float *output,const ColiDeepSeekV4LayerWeights *weights,const ColiDeepSeekV4Config *config,@@ -4984,8 +5008,14 @@ static int moe_token_pipeline(float *output,if (jobs[slot].result) { result = -1; break; }views[current] = jobs[slot].view;#ifdef COLI_V4_GPU_TIER- if (store->gpu)- coli_v4_gpu_expert_attach(store, &views[current]);+ if (store->gpu) {+ if (coli_v4_hybrid_enabled())+ /* peek only: uploads are decided once the token's full+ * miss count is known, by the q* pass below */+ coli_v4_gpu_expert_peek(store, &views[current]);+ else+ coli_v4_gpu_expert_attach(store, &views[current]);+ }#endifif (!views[current].gate.gpu || !views[current].up.gpu ||!views[current].down.gpu)@@ -5013,6 +5043,51 @@ static int moe_token_pipeline(float *output,}}#ifdef COLI_V4_GPU_TIER+ int hybrid_gpu_count = 0;+ if (!result && coli_v4_hybrid_enabled() && store->gpu && !gpu_compute) {+ /* q* pass: the peeks above established which of the token's experts+ * are already resident. Upload only fill = q*(m) of the m misses;+ * the rest stay host-side on purpose. Uploads are timed to feed the+ * fill-bandwidth EMA. While the host bandwidth is still unmeasured,+ * leave one expert on the CPU so it CAN be measured, otherwise+ * fill == m forever and the policy never engages. */+ int *missing = malloc((size_t)selected * sizeof(*missing));+ if (missing) {+ int miss_n = 0;+ for (int i = 0; i < selected; i++)+ if (!views[i].gate.gpu || !views[i].up.gpu ||+ !views[i].down.gpu)+ missing[miss_n++] = i;+ int fill = coli_v4_hybrid_fill_count(+ miss_n, g_v4_hyb_fill_bw, g_v4_hyb_host_bw);+ if (g_v4_hyb_host_bw <= 0.0 && fill >= miss_n && miss_n > 1)+ fill = miss_n - 1;+ for (int i = 0; i < fill; i++) {+ int v = missing[i];+ struct timespec f0, f1;+ clock_gettime(CLOCK_MONOTONIC, &f0);+ if (coli_v4_gpu_expert_attach(store, &views[v]) == 0 &&+ views[v].gate.gpu && views[v].up.gpu &&+ views[v].down.gpu) {+ clock_gettime(CLOCK_MONOTONIC, &f1);+ double dt = (f1.tv_sec - f0.tv_sec) ++ (f1.tv_nsec - f0.tv_nsec) * 1e-9;+ if (dt > 0.0)+ g_v4_hyb_fill_bw = coli_v4_hybrid_ema(+ g_v4_hyb_fill_bw, 1.0 / dt);+ g_v4_hyb_upload_n++;+ }+ }+ if (miss_n > fill)+ g_v4_hyb_skip_n += (unsigned long long)(miss_n - fill);+ free(missing);+ }+ for (int i = 0; i < selected; i++)+ if (views[i].gate.gpu && views[i].up.gpu && views[i].down.gpu)+ hybrid_gpu_count++;+ if (hybrid_gpu_count == selected)+ gpu_compute = 1; /* every expert made it: use the fused path */+ }if (!result && gpu_compute && store->gpu) {void *sg = coli_v4_layer_gpu(weights, "ffn.shared_experts.w1");void *su = coli_v4_layer_gpu(weights, "ffn.shared_experts.w2");@@ -5055,6 +5130,73 @@ static int moe_token_pipeline(float *output,output[i] = coli_bf16_round(expert_output[i]);}free(gates); free(ups); free(downs);+ } else if (!result && hybrid_gpu_count > 0 && store->gpu) {+ /* Hybrid split: the GPU computes the resident subset as one weighted+ * group while the CPU computes the rest; the two partial sums merge+ * by addition, the same order-of-addition class that already+ * separates the fused GPU path from the CPU reference. The CPU half+ * is timed to feed the host-bandwidth EMA. A backend failure on the+ * GPU half degrades that subset to the CPU loop instead of failing+ * the token. */+ int gpu_n = hybrid_gpu_count, cpu_n = selected - hybrid_gpu_count;+ void **gates = malloc((size_t)gpu_n * sizeof(*gates));+ void **ups = malloc((size_t)gpu_n * sizeof(*ups));+ void **downs = malloc((size_t)gpu_n * sizeof(*downs));+ float *gpu_weights = malloc((size_t)gpu_n * sizeof(*gpu_weights));+ float *gpu_sum = malloc((size_t)d * sizeof(*gpu_sum));+ int gpu_ok = gates && ups && downs && gpu_weights && gpu_sum;+ if (gpu_ok) {+ int k = 0;+ for (int i = 0; i < selected; i++)+ if (views[i].gate.gpu && views[i].up.gpu &&+ views[i].down.gpu) {+ gates[k] = views[i].gate.gpu;+ ups[k] = views[i].up.gpu;+ downs[k] = views[i].down.gpu;+ gpu_weights[k] = expert_weights[i];+ k++;+ }+ extern int dsv4_cuda_expert_group(+ void *const *gate, void *const *up, void *const *down,+ const float *weights, int count, float limit,+ float *y, const float *x);+ if (!dsv4_cuda_expert_group(gates, ups, downs, gpu_weights,+ gpu_n, config->swiglu_limit, gpu_sum, input)) gpu_ok = 0;+ }+ struct timespec h0, h1;+ clock_gettime(CLOCK_MONOTONIC, &h0);+ for (int current = 0; !result && current < selected; current++) {+ int on_gpu = views[current].gate.gpu && views[current].up.gpu &&+ views[current].down.gpu;+ if (on_gpu && gpu_ok) continue; /* covered by the group above */+ result = coli_v4_expert_forward_ref(+ expert_output, &views[current], input,+ expert_weights[current], config->swiglu_limit);+ if (!result)+ for (int i = 0; i < d; i++) output[i] += expert_output[i];+ }+ if (!result && gpu_ok && cpu_n > 0) {+ clock_gettime(CLOCK_MONOTONIC, &h1);+ double dt = (h1.tv_sec - h0.tv_sec) ++ (h1.tv_nsec - h0.tv_nsec) * 1e-9;+ if (dt > 0.0)+ g_v4_hyb_host_bw = coli_v4_hybrid_ema(+ g_v4_hyb_host_bw, (double)cpu_n / dt);+ }+ if (!result) {+ if (gpu_ok) {+ for (int i = 0; i < d; i++)+ output[i] = coli_bf16_round(+ output[i] + gpu_sum[i] + shared_output[i]);+ g_v4_hyb_gpu_n += (unsigned long long)gpu_n;+ g_v4_hyb_cpu_n += (unsigned long long)cpu_n;+ } else {+ for (int i = 0; i < d; i++)+ output[i] = coli_bf16_round(output[i] + shared_output[i]);+ }+ }+ free(gates); free(ups); free(downs);+ free(gpu_weights); free(gpu_sum);} else#endif{@@ -10023,6 +10165,32 @@ int coli_v4_gpu_expert_attach(ColiExpertStore *store, ColiExpertView *view) {(V4GpuExpertMirrorCache *)store->gpu, view);}+/* Lookup-only twin of attach: report whether {layer, expert} is already+ * mirrored, touching its LRU stamp, but NEVER uploading on a miss. The+ * hybrid decode split peeks the whole token first and decides its uploads+ * afterwards with the q* policy, instead of paying a blocking PCIe copy per+ * miss the moment it is discovered. */+int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view) {+ if (!store || !view || !store->gpu) return -1;+ V4GpuExpertMirrorCache *cache = (V4GpuExpertMirrorCache *)store->gpu;+ pthread_mutex_lock(&cache->mutex);+ for (int i = 0; i < cache->count; i++) {+ if (cache->entries[i].layer == view->key.layer &&+ cache->entries[i].expert == view->key.expert &&+ cache->entries[i].gate && cache->entries[i].up &&+ cache->entries[i].down) {+ cache->entries[i].clock = ++cache->clock;+ view->gate.gpu = cache->entries[i].gate;+ view->up.gpu = cache->entries[i].up;+ view->down.gpu = cache->entries[i].down;+ pthread_mutex_unlock(&cache->mutex);+ return 0;+ }+ }+ pthread_mutex_unlock(&cache->mutex);+ return -1;+}+int coli_v4_gpu_dspark_expert_attach(void *cache, ColiExpertView *view) {if (!view) return -1;return v4_gpu_expert_attach_cached((V4GpuExpertMirrorCache *)cache, view);@@ -13394,6 +13562,15 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,: 0.0;v4_prof_emit(elapsed, stats.prompt_tokens, completion,expert_disk_s, expert_matmul_s);+#ifdef COLI_V4_GPU_TIER+ if (coli_v4_hybrid_enabled() && (g_v4_hyb_gpu_n + g_v4_hyb_cpu_n ++ g_v4_hyb_upload_n + g_v4_hyb_skip_n))+ fprintf(stderr, "v4_hybrid gpu=%llu cpu=%llu uploads=%llu "+ "skipped=%llu fill_bw=%.1f/s host_bw=%.1f/s "+ "(cumulative)\n",+ g_v4_hyb_gpu_n, g_v4_hyb_cpu_n, g_v4_hyb_upload_n,+ g_v4_hyb_skip_n, g_v4_hyb_fill_bw, g_v4_hyb_host_bw);+#endifcoli_v4_expert_store_emit_hits(engine->experts);coli_v4_expert_store_emit_emap(engine->experts);coli_v4_expert_store_emit_tiers(engine->experts);diff --git a/c/deepseek_v4_hybrid.h b/c/deepseek_v4_hybrid.hnew file mode 100644index 00000000..3f3046bc--- /dev/null+++ b/c/deepseek_v4_hybrid.h@@ -0,0 +1,54 @@+/* deepseek_v4_hybrid.h β the q* fill/execute split for GPU expert misses.+ *+ * On a decode step some routed experts are resident in VRAM and some are not.+ * Today the engine is all-or-nothing: one missing expert sends the WHOLE+ * token's routed MoE to the CPU, and every miss is uploaded over PCIe+ * unconditionally. The hybrid split partitions the m missing experts into a+ * fill set (upload to VRAM, compute on GPU) and a host set (compute on CPU),+ * sized so the two branches take the same time:+ *+ * T_fill(q) ~= q * S / B_P (PCIe transfer of q experts)+ * T_host(m-q)~= (m-q) * S / (B_H-B_P) (host compute of the rest)+ *+ * Balancing the two gives the closed form+ *+ * q* = m * B_P / B_H+ *+ * with B_P the measured fill bandwidth and B_H the measured host execution+ * bandwidth. As B_H approaches B_P, q* approaches m and the split degenerates+ * into plain upload-everything β which is also the answer while the+ * bandwidths are still unmeasured. Both bandwidths are live EMAs of the+ * engine's own uploads and expert forwards, never assumptions.+ *+ * Everything in this header is pure arithmetic so the CI can test the policy+ * on machines with no GPU at all; the CUDA wiring lives in deepseek_v4.c+ * behind DSV4_HYBRID=1. */+#ifndef DEEPSEEK_V4_HYBRID_H+#define DEEPSEEK_V4_HYBRID_H++/* How many of `missing` experts to upload-and-run-on-GPU; the caller computes+ * the rest on the CPU. Bandwidths are in experts/second (only their RATIO+ * matters, so any consistent unit works). Unmeasured or nonsensical+ * bandwidths fall back to `missing` β the engine's historical+ * upload-everything behaviour, never something new. */+static inline int coli_v4_hybrid_fill_count(int missing, double fill_bw,+ double host_bw) {+ if (missing <= 0) return 0;+ if (fill_bw <= 0.0 || host_bw <= 0.0) return missing;+ double ideal = (double)missing * fill_bw / host_bw; /* q* = m*B_P/B_H */+ int fill = (int)(ideal + 0.5);+ if (fill < 0) fill = 0;+ if (fill > missing) fill = missing;+ return fill;+}++/* One-pole EMA for the live bandwidth estimates. A non-positive sample is+ * ignored (a failed or zero-length measurement must not poison the state);+ * the first valid sample seeds the average directly. */+static inline double coli_v4_hybrid_ema(double previous, double sample) {+ if (sample <= 0.0) return previous;+ if (previous <= 0.0) return sample;+ return previous * 0.8 + sample * 0.2;+}++#endif /* DEEPSEEK_V4_HYBRID_H */diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex 6873fe8c..111c5d5c 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -848,6 +848,14 @@ int coli_v4_gpu_route(float *route_weights, int *indices, const float *input,* fp4 path. Only block_rows==1 views are mirrored; rows16-packed slots are* skipped. */int coli_v4_gpu_expert_attach(ColiExpertStore *store, ColiExpertView *view);+/* lookup-only twin: reports residency, never uploads (hybrid q* split) */+int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view);+/* DSV4_HYBRID=1 gate plus its cross-unit counters/EMAs: defined in the block+ * unit, read by the serve unit's per-turn stderr line. */+int coli_v4_hybrid_enabled(void);+extern double g_v4_hyb_fill_bw, g_v4_hyb_host_bw;+extern unsigned long long g_v4_hyb_gpu_n, g_v4_hyb_cpu_n;+extern unsigned long long g_v4_hyb_upload_n, g_v4_hyb_skip_n;/* Dspark (MTP) resident-expert mirrors: a separate bounded LRU so drafting can* never evict the target model's learned expert mirrors. ensure() lazily* allocates the cache on the first V4_MTP_GPU=1 draft; attach mirrors onediff --git a/c/tests/test_v4_hybrid_policy.c b/c/tests/test_v4_hybrid_policy.cnew file mode 100644index 00000000..5d6d4844--- /dev/null+++ b/c/tests/test_v4_hybrid_policy.c@@ -0,0 +1,56 @@+/* test_v4_hybrid_policy.c β the q* fill/execute split contract.+ *+ * coli_v4_hybrid_fill_count() decides, out of m missing experts, how many to+ * upload-and-run-on-GPU versus compute on the host: q* = m * B_P / B_H,+ * clamped to [0, m], with every unmeasured or degenerate input falling back+ * to m β the engine's historical upload-everything behaviour, never a new+ * one. Pure arithmetic, so this runs on machines with no GPU: the CUDA+ * wiring in deepseek_v4.c is validated on real hardware separately. */+#include <stdio.h>+#include "../deepseek_v4_hybrid.h"++static int failures;+#define CHECK(cond, ...) do { if (!(cond)) { \+ fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \+ fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); failures++; } } while (0)++int main(void) {+ /* no misses: nothing to decide */+ CHECK(coli_v4_hybrid_fill_count(0, 5.0, 5.0) == 0, "m=0 must fill 0");+ CHECK(coli_v4_hybrid_fill_count(-3, 5.0, 5.0) == 0, "m<0 must fill 0");++ /* unmeasured bandwidths: legacy upload-everything, never something new */+ CHECK(coli_v4_hybrid_fill_count(6, 0.0, 7.0) == 6, "no fill bw -> all");+ CHECK(coli_v4_hybrid_fill_count(6, 7.0, 0.0) == 6, "no host bw -> all");+ CHECK(coli_v4_hybrid_fill_count(6, -1.0, -1.0) == 6, "bad bw -> all");++ /* the closed form: q* = m * B_P / B_H */+ CHECK(coli_v4_hybrid_fill_count(6, 5.0, 10.0) == 3, "half ratio -> m/2");+ CHECK(coli_v4_hybrid_fill_count(6, 1.0, 4.0) == 2, "6*0.25 -> 2 (round)");+ CHECK(coli_v4_hybrid_fill_count(6, 1.0, 1000.0) == 0,+ "host vastly faster -> compute everything there");++ /* degenerate limit: as B_H approaches (or drops below) B_P the split+ * collapses into plain upload-everything */+ CHECK(coli_v4_hybrid_fill_count(6, 5.0, 5.0) == 6, "equal bw -> all fill");+ CHECK(coli_v4_hybrid_fill_count(6, 10.0, 5.0) == 6, "bus faster -> clamp m");++ /* monotone in the fill bandwidth: a faster bus never fills fewer */+ int previous = 0;+ for (int step = 1; step <= 20; step++) {+ int fill = coli_v4_hybrid_fill_count(8, 0.5 * step, 10.0);+ CHECK(fill >= previous, "fill must be monotone in B_P (step %d)", step);+ previous = fill;+ }++ /* EMA: first valid sample seeds, invalid samples never poison */+ CHECK(coli_v4_hybrid_ema(0.0, 4.0) == 4.0, "first sample seeds");+ CHECK(coli_v4_hybrid_ema(4.0, 0.0) == 4.0, "zero sample ignored");+ CHECK(coli_v4_hybrid_ema(4.0, -1.0) == 4.0, "negative sample ignored");+ double smoothed = coli_v4_hybrid_ema(4.0, 8.0);+ CHECK(smoothed > 4.0 && smoothed < 8.0, "EMA moves between old and new");++ if (failures) { fprintf(stderr, "%d failure(s)\n", failures); return 1; }+ printf("test_v4_hybrid_policy: ok\n");+ return 0;+}