Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(dsv4): hybrid GPU/CPU expert split on decode misses (opt-in)

JustVuggbc44ff0e941e
diff --git a/c/Makefile b/c/Makefile
index 9ef02fd3..a142d590 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -457,6 +457,7 @@ ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
ifeq ($(COLI_V4_SUPPORTED),1)
+TEST_BINS += tests/test_v4_hybrid_policy$(EXE)
TEST_BINS += tests/test_deepseek_v4$(EXE) tests/test_v4_ownership$(EXE) \
tests/test_v4_serve_framing$(EXE)
endif
@@ -1335,6 +1336,9 @@ tests/test_corpus_draft$(EXE): tests/test_corpus_draft.c colibri.c st.h uring.h
tests/test_cap_precedence$(EXE): tests/test_cap_precedence.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_v4_hybrid_policy$(EXE): tests/test_v4_hybrid_policy.c deepseek_v4_hybrid.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_ram_clamp$(EXE): tests/test_ram_clamp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 229eab38..527d2991 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -4202,6 +4202,7 @@ int coli_v4_sparse_attention_ref(float *output, const float *queries,
#define coli_v4_block_window_token_ref coli_v4_block_window_token_serial_ref
/* ---- begin include deepseek_v4_block.c ---- */
#include "deepseek_v4_internal.h"
+#include "deepseek_v4_hybrid.h"
#include <stdarg.h>
#include <stdint.h>
@@ -4831,6 +4832,29 @@ static int profiled_expert_load_finish(ExpertLoadHandle *handle) {
return result;
}
+#ifdef COLI_V4_GPU_TIER
+/* DSV4_HYBRID=1 (opt-in): on a decode miss, split the missing experts between
+ * a PCIe-fill set (upload, compute on GPU with the residents) and a host set
+ * (compute on CPU) instead of today's all-or-nothing, sized by the q* policy
+ * in deepseek_v4_hybrid.h from live bandwidth EMAs. Off by default until the
+ * split is validated on real GPUs; when off, the engine byte-for-byte keeps
+ * its historical behaviour. */
+int coli_v4_hybrid_enabled(void) {
+ static int on = -1;
+ if (on < 0) {
+ const char *setting = getenv("DSV4_HYBRID");
+ on = setting && atoi(setting) != 0;
+ }
+ return on;
+}
+/* Non-static on purpose: the serve unit prints these counters, and under the
+ * per-unit amalgam build that is a different object file. */
+double g_v4_hyb_fill_bw; /* experts/s uploaded, EMA */
+double g_v4_hyb_host_bw; /* experts/s host-computed, EMA */
+unsigned long long g_v4_hyb_gpu_n, g_v4_hyb_cpu_n;
+unsigned long long g_v4_hyb_upload_n, g_v4_hyb_skip_n;
+#endif
+
static int moe_token_pipeline(float *output,
const ColiDeepSeekV4LayerWeights *weights,
const ColiDeepSeekV4Config *config,
@@ -4984,8 +5008,14 @@ static int moe_token_pipeline(float *output,
if (jobs[slot].result) { result = -1; break; }
views[current] = jobs[slot].view;
#ifdef COLI_V4_GPU_TIER
- if (store->gpu)
- coli_v4_gpu_expert_attach(store, &views[current]);
+ if (store->gpu) {
+ if (coli_v4_hybrid_enabled())
+ /* peek only: uploads are decided once the token's full
+ * miss count is known, by the q* pass below */
+ coli_v4_gpu_expert_peek(store, &views[current]);
+ else
+ coli_v4_gpu_expert_attach(store, &views[current]);
+ }
#endif
if (!views[current].gate.gpu || !views[current].up.gpu ||
!views[current].down.gpu)
@@ -5013,6 +5043,51 @@ static int moe_token_pipeline(float *output,
}
}
#ifdef COLI_V4_GPU_TIER
+ int hybrid_gpu_count = 0;
+ if (!result && coli_v4_hybrid_enabled() && store->gpu && !gpu_compute) {
+ /* q* pass: the peeks above established which of the token's experts
+ * are already resident. Upload only fill = q*(m) of the m misses;
+ * the rest stay host-side on purpose. Uploads are timed to feed the
+ * fill-bandwidth EMA. While the host bandwidth is still unmeasured,
+ * leave one expert on the CPU so it CAN be measured, otherwise
+ * fill == m forever and the policy never engages. */
+ int *missing = malloc((size_t)selected * sizeof(*missing));
+ if (missing) {
+ int miss_n = 0;
+ for (int i = 0; i < selected; i++)
+ if (!views[i].gate.gpu || !views[i].up.gpu ||
+ !views[i].down.gpu)
+ missing[miss_n++] = i;
+ int fill = coli_v4_hybrid_fill_count(
+ miss_n, g_v4_hyb_fill_bw, g_v4_hyb_host_bw);
+ if (g_v4_hyb_host_bw <= 0.0 && fill >= miss_n && miss_n > 1)
+ fill = miss_n - 1;
+ for (int i = 0; i < fill; i++) {
+ int v = missing[i];
+ struct timespec f0, f1;
+ clock_gettime(CLOCK_MONOTONIC, &f0);
+ if (coli_v4_gpu_expert_attach(store, &views[v]) == 0 &&
+ views[v].gate.gpu && views[v].up.gpu &&
+ views[v].down.gpu) {
+ clock_gettime(CLOCK_MONOTONIC, &f1);
+ double dt = (f1.tv_sec - f0.tv_sec) +
+ (f1.tv_nsec - f0.tv_nsec) * 1e-9;
+ if (dt > 0.0)
+ g_v4_hyb_fill_bw = coli_v4_hybrid_ema(
+ g_v4_hyb_fill_bw, 1.0 / dt);
+ g_v4_hyb_upload_n++;
+ }
+ }
+ if (miss_n > fill)
+ g_v4_hyb_skip_n += (unsigned long long)(miss_n - fill);
+ free(missing);
+ }
+ for (int i = 0; i < selected; i++)
+ if (views[i].gate.gpu && views[i].up.gpu && views[i].down.gpu)
+ hybrid_gpu_count++;
+ if (hybrid_gpu_count == selected)
+ gpu_compute = 1; /* every expert made it: use the fused path */
+ }
if (!result && gpu_compute && store->gpu) {
void *sg = coli_v4_layer_gpu(weights, "ffn.shared_experts.w1");
void *su = coli_v4_layer_gpu(weights, "ffn.shared_experts.w2");
@@ -5055,6 +5130,73 @@ static int moe_token_pipeline(float *output,
output[i] = coli_bf16_round(expert_output[i]);
}
free(gates); free(ups); free(downs);
+ } else if (!result && hybrid_gpu_count > 0 && store->gpu) {
+ /* Hybrid split: the GPU computes the resident subset as one weighted
+ * group while the CPU computes the rest; the two partial sums merge
+ * by addition, the same order-of-addition class that already
+ * separates the fused GPU path from the CPU reference. The CPU half
+ * is timed to feed the host-bandwidth EMA. A backend failure on the
+ * GPU half degrades that subset to the CPU loop instead of failing
+ * the token. */
+ int gpu_n = hybrid_gpu_count, cpu_n = selected - hybrid_gpu_count;
+ void **gates = malloc((size_t)gpu_n * sizeof(*gates));
+ void **ups = malloc((size_t)gpu_n * sizeof(*ups));
+ void **downs = malloc((size_t)gpu_n * sizeof(*downs));
+ float *gpu_weights = malloc((size_t)gpu_n * sizeof(*gpu_weights));
+ float *gpu_sum = malloc((size_t)d * sizeof(*gpu_sum));
+ int gpu_ok = gates && ups && downs && gpu_weights && gpu_sum;
+ if (gpu_ok) {
+ int k = 0;
+ for (int i = 0; i < selected; i++)
+ if (views[i].gate.gpu && views[i].up.gpu &&
+ views[i].down.gpu) {
+ gates[k] = views[i].gate.gpu;
+ ups[k] = views[i].up.gpu;
+ downs[k] = views[i].down.gpu;
+ gpu_weights[k] = expert_weights[i];
+ k++;
+ }
+ extern int dsv4_cuda_expert_group(
+ void *const *gate, void *const *up, void *const *down,
+ const float *weights, int count, float limit,
+ float *y, const float *x);
+ if (!dsv4_cuda_expert_group(gates, ups, downs, gpu_weights,
+ gpu_n, config->swiglu_limit, gpu_sum, input)) gpu_ok = 0;
+ }
+ struct timespec h0, h1;
+ clock_gettime(CLOCK_MONOTONIC, &h0);
+ for (int current = 0; !result && current < selected; current++) {
+ int on_gpu = views[current].gate.gpu && views[current].up.gpu &&
+ views[current].down.gpu;
+ if (on_gpu && gpu_ok) continue; /* covered by the group above */
+ result = coli_v4_expert_forward_ref(
+ expert_output, &views[current], input,
+ expert_weights[current], config->swiglu_limit);
+ if (!result)
+ for (int i = 0; i < d; i++) output[i] += expert_output[i];
+ }
+ if (!result && gpu_ok && cpu_n > 0) {
+ clock_gettime(CLOCK_MONOTONIC, &h1);
+ double dt = (h1.tv_sec - h0.tv_sec) +
+ (h1.tv_nsec - h0.tv_nsec) * 1e-9;
+ if (dt > 0.0)
+ g_v4_hyb_host_bw = coli_v4_hybrid_ema(
+ g_v4_hyb_host_bw, (double)cpu_n / dt);
+ }
+ if (!result) {
+ if (gpu_ok) {
+ for (int i = 0; i < d; i++)
+ output[i] = coli_bf16_round(
+ output[i] + gpu_sum[i] + shared_output[i]);
+ g_v4_hyb_gpu_n += (unsigned long long)gpu_n;
+ g_v4_hyb_cpu_n += (unsigned long long)cpu_n;
+ } else {
+ for (int i = 0; i < d; i++)
+ output[i] = coli_bf16_round(output[i] + shared_output[i]);
+ }
+ }
+ free(gates); free(ups); free(downs);
+ free(gpu_weights); free(gpu_sum);
} else
#endif
{
@@ -10023,6 +10165,32 @@ int coli_v4_gpu_expert_attach(ColiExpertStore *store, ColiExpertView *view) {
(V4GpuExpertMirrorCache *)store->gpu, view);
}
+/* Lookup-only twin of attach: report whether {layer, expert} is already
+ * mirrored, touching its LRU stamp, but NEVER uploading on a miss. The
+ * hybrid decode split peeks the whole token first and decides its uploads
+ * afterwards with the q* policy, instead of paying a blocking PCIe copy per
+ * miss the moment it is discovered. */
+int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view) {
+ if (!store || !view || !store->gpu) return -1;
+ V4GpuExpertMirrorCache *cache = (V4GpuExpertMirrorCache *)store->gpu;
+ pthread_mutex_lock(&cache->mutex);
+ for (int i = 0; i < cache->count; i++) {
+ if (cache->entries[i].layer == view->key.layer &&
+ cache->entries[i].expert == view->key.expert &&
+ cache->entries[i].gate && cache->entries[i].up &&
+ cache->entries[i].down) {
+ cache->entries[i].clock = ++cache->clock;
+ view->gate.gpu = cache->entries[i].gate;
+ view->up.gpu = cache->entries[i].up;
+ view->down.gpu = cache->entries[i].down;
+ pthread_mutex_unlock(&cache->mutex);
+ return 0;
+ }
+ }
+ pthread_mutex_unlock(&cache->mutex);
+ return -1;
+}
+
int coli_v4_gpu_dspark_expert_attach(void *cache, ColiExpertView *view) {
if (!view) return -1;
return v4_gpu_expert_attach_cached((V4GpuExpertMirrorCache *)cache, view);
@@ -13394,6 +13562,15 @@ static int v4_serve_one(ColiV4Engine *engine, ColiV4Session *session,
: 0.0;
v4_prof_emit(elapsed, stats.prompt_tokens, completion,
expert_disk_s, expert_matmul_s);
+#ifdef COLI_V4_GPU_TIER
+ if (coli_v4_hybrid_enabled() && (g_v4_hyb_gpu_n + g_v4_hyb_cpu_n +
+ g_v4_hyb_upload_n + g_v4_hyb_skip_n))
+ fprintf(stderr, "v4_hybrid gpu=%llu cpu=%llu uploads=%llu "
+ "skipped=%llu fill_bw=%.1f/s host_bw=%.1f/s "
+ "(cumulative)\n",
+ g_v4_hyb_gpu_n, g_v4_hyb_cpu_n, g_v4_hyb_upload_n,
+ g_v4_hyb_skip_n, g_v4_hyb_fill_bw, g_v4_hyb_host_bw);
+#endif
coli_v4_expert_store_emit_hits(engine->experts);
coli_v4_expert_store_emit_emap(engine->experts);
coli_v4_expert_store_emit_tiers(engine->experts);
diff --git a/c/deepseek_v4_hybrid.h b/c/deepseek_v4_hybrid.h
new file mode 100644
index 00000000..3f3046bc
--- /dev/null
+++ b/c/deepseek_v4_hybrid.h
@@ -0,0 +1,54 @@
+/* deepseek_v4_hybrid.h β€” the q* fill/execute split for GPU expert misses.
+ *
+ * On a decode step some routed experts are resident in VRAM and some are not.
+ * Today the engine is all-or-nothing: one missing expert sends the WHOLE
+ * token's routed MoE to the CPU, and every miss is uploaded over PCIe
+ * unconditionally. The hybrid split partitions the m missing experts into a
+ * fill set (upload to VRAM, compute on GPU) and a host set (compute on CPU),
+ * sized so the two branches take the same time:
+ *
+ * T_fill(q) ~= q * S / B_P (PCIe transfer of q experts)
+ * T_host(m-q)~= (m-q) * S / (B_H-B_P) (host compute of the rest)
+ *
+ * Balancing the two gives the closed form
+ *
+ * q* = m * B_P / B_H
+ *
+ * with B_P the measured fill bandwidth and B_H the measured host execution
+ * bandwidth. As B_H approaches B_P, q* approaches m and the split degenerates
+ * into plain upload-everything β€” which is also the answer while the
+ * bandwidths are still unmeasured. Both bandwidths are live EMAs of the
+ * engine's own uploads and expert forwards, never assumptions.
+ *
+ * Everything in this header is pure arithmetic so the CI can test the policy
+ * on machines with no GPU at all; the CUDA wiring lives in deepseek_v4.c
+ * behind DSV4_HYBRID=1. */
+#ifndef DEEPSEEK_V4_HYBRID_H
+#define DEEPSEEK_V4_HYBRID_H
+
+/* How many of `missing` experts to upload-and-run-on-GPU; the caller computes
+ * the rest on the CPU. Bandwidths are in experts/second (only their RATIO
+ * matters, so any consistent unit works). Unmeasured or nonsensical
+ * bandwidths fall back to `missing` β€” the engine's historical
+ * upload-everything behaviour, never something new. */
+static inline int coli_v4_hybrid_fill_count(int missing, double fill_bw,
+ double host_bw) {
+ if (missing <= 0) return 0;
+ if (fill_bw <= 0.0 || host_bw <= 0.0) return missing;
+ double ideal = (double)missing * fill_bw / host_bw; /* q* = m*B_P/B_H */
+ int fill = (int)(ideal + 0.5);
+ if (fill < 0) fill = 0;
+ if (fill > missing) fill = missing;
+ return fill;
+}
+
+/* One-pole EMA for the live bandwidth estimates. A non-positive sample is
+ * ignored (a failed or zero-length measurement must not poison the state);
+ * the first valid sample seeds the average directly. */
+static inline double coli_v4_hybrid_ema(double previous, double sample) {
+ if (sample <= 0.0) return previous;
+ if (previous <= 0.0) return sample;
+ return previous * 0.8 + sample * 0.2;
+}
+
+#endif /* DEEPSEEK_V4_HYBRID_H */
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index 6873fe8c..111c5d5c 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -848,6 +848,14 @@ int coli_v4_gpu_route(float *route_weights, int *indices, const float *input,
* fp4 path. Only block_rows==1 views are mirrored; rows16-packed slots are
* skipped. */
int coli_v4_gpu_expert_attach(ColiExpertStore *store, ColiExpertView *view);
+/* lookup-only twin: reports residency, never uploads (hybrid q* split) */
+int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view);
+/* DSV4_HYBRID=1 gate plus its cross-unit counters/EMAs: defined in the block
+ * unit, read by the serve unit's per-turn stderr line. */
+int coli_v4_hybrid_enabled(void);
+extern double g_v4_hyb_fill_bw, g_v4_hyb_host_bw;
+extern unsigned long long g_v4_hyb_gpu_n, g_v4_hyb_cpu_n;
+extern unsigned long long g_v4_hyb_upload_n, g_v4_hyb_skip_n;
/* Dspark (MTP) resident-expert mirrors: a separate bounded LRU so drafting can
* never evict the target model's learned expert mirrors. ensure() lazily
* allocates the cache on the first V4_MTP_GPU=1 draft; attach mirrors one
diff --git a/c/tests/test_v4_hybrid_policy.c b/c/tests/test_v4_hybrid_policy.c
new file mode 100644
index 00000000..5d6d4844
--- /dev/null
+++ b/c/tests/test_v4_hybrid_policy.c
@@ -0,0 +1,56 @@
+/* test_v4_hybrid_policy.c β€” the q* fill/execute split contract.
+ *
+ * coli_v4_hybrid_fill_count() decides, out of m missing experts, how many to
+ * upload-and-run-on-GPU versus compute on the host: q* = m * B_P / B_H,
+ * clamped to [0, m], with every unmeasured or degenerate input falling back
+ * to m β€” the engine's historical upload-everything behaviour, never a new
+ * one. Pure arithmetic, so this runs on machines with no GPU: the CUDA
+ * wiring in deepseek_v4.c is validated on real hardware separately. */
+#include <stdio.h>
+#include "../deepseek_v4_hybrid.h"
+
+static int failures;
+#define CHECK(cond, ...) do { if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); failures++; } } while (0)
+
+int main(void) {
+ /* no misses: nothing to decide */
+ CHECK(coli_v4_hybrid_fill_count(0, 5.0, 5.0) == 0, "m=0 must fill 0");
+ CHECK(coli_v4_hybrid_fill_count(-3, 5.0, 5.0) == 0, "m<0 must fill 0");
+
+ /* unmeasured bandwidths: legacy upload-everything, never something new */
+ CHECK(coli_v4_hybrid_fill_count(6, 0.0, 7.0) == 6, "no fill bw -> all");
+ CHECK(coli_v4_hybrid_fill_count(6, 7.0, 0.0) == 6, "no host bw -> all");
+ CHECK(coli_v4_hybrid_fill_count(6, -1.0, -1.0) == 6, "bad bw -> all");
+
+ /* the closed form: q* = m * B_P / B_H */
+ CHECK(coli_v4_hybrid_fill_count(6, 5.0, 10.0) == 3, "half ratio -> m/2");
+ CHECK(coli_v4_hybrid_fill_count(6, 1.0, 4.0) == 2, "6*0.25 -> 2 (round)");
+ CHECK(coli_v4_hybrid_fill_count(6, 1.0, 1000.0) == 0,
+ "host vastly faster -> compute everything there");
+
+ /* degenerate limit: as B_H approaches (or drops below) B_P the split
+ * collapses into plain upload-everything */
+ CHECK(coli_v4_hybrid_fill_count(6, 5.0, 5.0) == 6, "equal bw -> all fill");
+ CHECK(coli_v4_hybrid_fill_count(6, 10.0, 5.0) == 6, "bus faster -> clamp m");
+
+ /* monotone in the fill bandwidth: a faster bus never fills fewer */
+ int previous = 0;
+ for (int step = 1; step <= 20; step++) {
+ int fill = coli_v4_hybrid_fill_count(8, 0.5 * step, 10.0);
+ CHECK(fill >= previous, "fill must be monotone in B_P (step %d)", step);
+ previous = fill;
+ }
+
+ /* EMA: first valid sample seeds, invalid samples never poison */
+ CHECK(coli_v4_hybrid_ema(0.0, 4.0) == 4.0, "first sample seeds");
+ CHECK(coli_v4_hybrid_ema(4.0, 0.0) == 4.0, "zero sample ignored");
+ CHECK(coli_v4_hybrid_ema(4.0, -1.0) == 4.0, "negative sample ignored");
+ double smoothed = coli_v4_hybrid_ema(4.0, 8.0);
+ CHECK(smoothed > 4.0 && smoothed < 8.0, "EMA moves between old and new");
+
+ if (failures) { fprintf(stderr, "%d failure(s)\n", failures); return 1; }
+ printf("test_v4_hybrid_policy: ok\n");
+ return 0;
+}