Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(dsv4): overlap hybrid fill DMA with host compute; pin slabs on Linux

JustVugg605d98bed061
diff --git a/c/backend_cuda_dsv4.cu b/c/backend_cuda_dsv4.cu
index 6ae6d96d..b31df925 100644
--- a/c/backend_cuda_dsv4.cu
+++ b/c/backend_cuda_dsv4.cu
@@ -10,6 +10,9 @@
#include <cstring>
#include <climits>
#include <vector>
+#ifndef _WIN32
+#include <unistd.h>
+#endif
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
@@ -1918,10 +1921,37 @@ static bool host_pinned(const void*p,size_t len){
g_pin_bytes+=size;g_pin_regions.push_back({base,size});
return true;
#else
- (void)a;return false;
+ /* Linux/WSL: no VirtualQuery, but none is needed β€” cudaHostRegister pins
+ * whole pages, so register the requested range rounded out to page
+ * boundaries. The slot slabs are stable for the process lifetime (one
+ * posix_memalign per slot, freed only at store destroy), exactly like the
+ * Windows branch assumes. Overlapping ranges collapse into the
+ * already-registered success case; any failure (WSL builds without
+ * pinning support included) lands in g_pin_failed and every later upload
+ * from that slab takes the pageable path, same as before this branch
+ * existed. */
+ long page=sysconf(_SC_PAGESIZE);if(page<=0)page=4096;
+ uintptr_t base=a&~((uintptr_t)page-1);
+ size_t size=((a+len+page-1)&~((uintptr_t)page-1))-base;
+ static const size_t cap=(size_t)20<<30;
+ if(g_pin_bytes+size>cap){g_pin_failed.push_back({base,size});return false;}
+ cudaError_t err=cudaHostRegister((void*)base,size,cudaHostRegisterPortable);
+ if(err==cudaErrorHostMemoryAlreadyRegistered){cudaGetLastError();g_pin_regions.push_back({base,size});return true;}
+ if(err!=cudaSuccess){cudaGetLastError();g_pin_failed.push_back({base,size});return false;}
+ g_pin_bytes+=size;g_pin_regions.push_back({base,size});
+ return true;
#endif
}
+/* Drain the device's expert stream: the hybrid decode split enqueues its
+ * fill-set uploads asynchronously, computes the CPU subset while the DMA is
+ * in flight, and closes the pipeline here before the GPU group runs. */
+extern "C" int dsv4_cuda_stream_drain(int device){
+ Dev*c=ctx(device);if(!c)return 0;
+ if(!ok(cudaSetDevice(device),"select drain device"))return 0;
+ return ok(cudaStreamSynchronize(c->stream),"hybrid drain")?1:0;
+}
+
extern "C" int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet*set,int e,
const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,const uint8_t*dw,const uint8_t*ds,
Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down){
diff --git a/c/backend_cuda_dsv4.h b/c/backend_cuda_dsv4.h
index 5566bf66..416c989e 100644
--- a/c/backend_cuda_dsv4.h
+++ b/c/backend_cuda_dsv4.h
@@ -64,6 +64,10 @@ int dsv4_cuda_fp8_ref_matmul(int device,const uint8_t *w,const float *bscale,
int dsv4_cuda_backend_arch_ok(int device);
const char *dsv4_cuda_backend_name(void);
long long dsv4_cuda_mem_free_mb(int device);
+/* Drain the device's expert stream (hybrid split: async fill DMA is enqueued,
+ * the CPU subset computes meanwhile, this closes the pipeline). Returns 1 on
+ * success, 0 when the backend is unavailable. */
+int dsv4_cuda_stream_drain(int device);
int dsv4_cuda_kv_ring_append(int device,int layer,const float *rows,int start_pos,
int count,int window,int dim);
int dsv4_cuda_kv_comp_append(int device,int layer,const float *rows,int start_idx,
diff --git a/c/backend_loader_dsv4.c b/c/backend_loader_dsv4.c
index 76d12f37..215989e2 100644
--- a/c/backend_loader_dsv4.c
+++ b/c/backend_loader_dsv4.c
@@ -172,6 +172,7 @@ typedef int (*fn_tensor_refill_fp4)(Dsv4CudaTensor *t, const uint8_t
typedef int (*fn_backend_arch_ok)(int device);
typedef const char *(*fn_backend_name)(void);
typedef long long (*fn_mem_free_mb)(int device);
+typedef int (*fn_stream_drain)(int device);
typedef int (*fn_kv_ring_append)(int device, int layer, const float *rows,
int start_pos, int count, int window, int dim);
typedef int (*fn_kv_comp_append)(int device, int layer, const float *rows,
@@ -307,6 +308,7 @@ static struct {
fn_backend_name backend_name; /* optional */
char loaded_name[64];
fn_mem_free_mb mem_free_mb;
+ fn_stream_drain stream_drain; /* optional (older DLLs) */
fn_kv_ring_append kv_ring_append;
fn_kv_comp_append kv_comp_append;
fn_head_argmax head_argmax;
@@ -485,6 +487,9 @@ static int dsv4_cuda_resolve(const char *dllname){
_Pragma("GCC diagnostic push")
_Pragma("GCC diagnostic ignored \"-Wcast-function-type\"")
g_dsv4.backend_arch_ok = (fn_backend_arch_ok)GetProcAddress(g_dsv4.dll, "dsv4_cuda_backend_arch_ok");
+ /* optional: an older DLL without the export keeps the whole tier alive;
+ * the engine probes this and simply stays on synchronous attaches. */
+ g_dsv4.stream_drain = (fn_stream_drain)GetProcAddress(g_dsv4.dll, "dsv4_cuda_stream_drain");
g_dsv4.backend_name = (fn_backend_name)GetProcAddress(g_dsv4.dll, "dsv4_cuda_backend_name");
_Pragma("GCC diagnostic pop")
return 1;
@@ -832,6 +837,16 @@ long long dsv4_cuda_mem_free_mb(int device){
return g_dsv4.mem_free_mb(device);
}
+int dsv4_cuda_stream_drain(int device){
+ /* Without the DLL there is never in-flight DMA to wait for, so a drain
+ * trivially succeeds. With an older DLL that lacks the export, report
+ * failure instead: the engine probes this once and keeps every attach
+ * synchronous, so nothing is ever enqueued that could not be drained. */
+ if(!g_dsv4.available) return 1;
+ if(!g_dsv4.stream_drain) return 0;
+ return g_dsv4.stream_drain(device);
+}
+
int dsv4_cuda_kv_ring_append(int device, int layer, const float *rows, int start_pos,
int count, int window, int dim){
if(!g_dsv4.available) return 0;
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 527d2991..3fc01ff5 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -5044,6 +5044,9 @@ static int moe_token_pipeline(float *output,
}
#ifdef COLI_V4_GPU_TIER
int hybrid_gpu_count = 0;
+ int hybrid_pending_drain = 0; /* async DMA enqueued, not yet drained */
+ int hybrid_uploads = 0;
+ struct timespec hybrid_fill_t0 = {0, 0};
if (!result && coli_v4_hybrid_enabled() && store->gpu && !gpu_compute) {
/* q* pass: the peeks above established which of the token's experts
* are already resident. Upload only fill = q*(m) of the m misses;
@@ -5062,19 +5065,29 @@ static int moe_token_pipeline(float *output,
miss_n, g_v4_hyb_fill_bw, g_v4_hyb_host_bw);
if (g_v4_hyb_host_bw <= 0.0 && fill >= miss_n && miss_n > 1)
fill = miss_n - 1;
+ /* Fill uploads are ENQUEUED, not drained: the CPU subset below
+ * computes while the DMA is in flight β€” the very overlap the q*
+ * balance assumes. The fill branch is timed as a whole at the
+ * drain point; per-upload wall clocks here would only measure
+ * enqueue latency. */
+ /* Async needs a drainable stream. An older Windows DLL exports
+ * the refill but not the drain: probe once (a drain on an empty
+ * stream is a no-op) and stay fully synchronous there, so
+ * nothing is ever enqueued that could not be waited on. */
+ static int hybrid_async_ok = -1;
+ if (hybrid_async_ok < 0)
+ hybrid_async_ok = coli_v4_gpu_expert_drain(store) == 0;
+ clock_gettime(CLOCK_MONOTONIC, &hybrid_fill_t0);
for (int i = 0; i < fill; i++) {
int v = missing[i];
- struct timespec f0, f1;
- clock_gettime(CLOCK_MONOTONIC, &f0);
- if (coli_v4_gpu_expert_attach(store, &views[v]) == 0 &&
+ int attached = hybrid_async_ok
+ ? coli_v4_gpu_expert_attach_async(store, &views[v])
+ : coli_v4_gpu_expert_attach(store, &views[v]);
+ if (attached == 0 &&
views[v].gate.gpu && views[v].up.gpu &&
views[v].down.gpu) {
- clock_gettime(CLOCK_MONOTONIC, &f1);
- double dt = (f1.tv_sec - f0.tv_sec) +
- (f1.tv_nsec - f0.tv_nsec) * 1e-9;
- if (dt > 0.0)
- g_v4_hyb_fill_bw = coli_v4_hybrid_ema(
- g_v4_hyb_fill_bw, 1.0 / dt);
+ hybrid_uploads++;
+ hybrid_pending_drain = hybrid_async_ok;
g_v4_hyb_upload_n++;
}
}
@@ -5145,37 +5158,23 @@ static int moe_token_pipeline(float *output,
float *gpu_weights = malloc((size_t)gpu_n * sizeof(*gpu_weights));
float *gpu_sum = malloc((size_t)d * sizeof(*gpu_sum));
int gpu_ok = gates && ups && downs && gpu_weights && gpu_sum;
- if (gpu_ok) {
- int k = 0;
- for (int i = 0; i < selected; i++)
- if (views[i].gate.gpu && views[i].up.gpu &&
- views[i].down.gpu) {
- gates[k] = views[i].gate.gpu;
- ups[k] = views[i].up.gpu;
- downs[k] = views[i].down.gpu;
- gpu_weights[k] = expert_weights[i];
- k++;
- }
- extern int dsv4_cuda_expert_group(
- void *const *gate, void *const *up, void *const *down,
- const float *weights, int count, float limit,
- float *y, const float *x);
- if (!dsv4_cuda_expert_group(gates, ups, downs, gpu_weights,
- gpu_n, config->swiglu_limit, gpu_sum, input)) gpu_ok = 0;
- }
+ /* CPU subset FIRST: it computes while the fill DMA enqueued by the
+ * q* pass is still in flight. The host bandwidth EMA is measured
+ * right here, under bus contention β€” which is exactly the number the
+ * balance needs. */
struct timespec h0, h1;
clock_gettime(CLOCK_MONOTONIC, &h0);
for (int current = 0; !result && current < selected; current++) {
int on_gpu = views[current].gate.gpu && views[current].up.gpu &&
views[current].down.gpu;
- if (on_gpu && gpu_ok) continue; /* covered by the group above */
+ if (on_gpu && gpu_ok) continue; /* covered by the group below */
result = coli_v4_expert_forward_ref(
expert_output, &views[current], input,
expert_weights[current], config->swiglu_limit);
if (!result)
for (int i = 0; i < d; i++) output[i] += expert_output[i];
}
- if (!result && gpu_ok && cpu_n > 0) {
+ if (!result && cpu_n > 0) {
clock_gettime(CLOCK_MONOTONIC, &h1);
double dt = (h1.tv_sec - h0.tv_sec) +
(h1.tv_nsec - h0.tv_nsec) * 1e-9;
@@ -5183,6 +5182,58 @@ static int moe_token_pipeline(float *output,
g_v4_hyb_host_bw = coli_v4_hybrid_ema(
g_v4_hyb_host_bw, (double)cpu_n / dt);
}
+ /* Close the fill pipeline. The whole window (enqueue -> drained) is
+ * the fill branch's wall time; a sample is taken only when the drain
+ * actually waited, otherwise the transfers finished under the CPU
+ * work and the window would say nothing about the bus. */
+ if (hybrid_pending_drain) {
+ struct timespec d0, d1;
+ clock_gettime(CLOCK_MONOTONIC, &d0);
+ coli_v4_gpu_expert_drain(store);
+ clock_gettime(CLOCK_MONOTONIC, &d1);
+ hybrid_pending_drain = 0;
+ double waited = (d1.tv_sec - d0.tv_sec) +
+ (d1.tv_nsec - d0.tv_nsec) * 1e-9;
+ double window = (d1.tv_sec - hybrid_fill_t0.tv_sec) +
+ (d1.tv_nsec - hybrid_fill_t0.tv_nsec) * 1e-9;
+ if (hybrid_uploads > 0 && window > 0.0 &&
+ waited > window * 0.05)
+ g_v4_hyb_fill_bw = coli_v4_hybrid_ema(
+ g_v4_hyb_fill_bw, (double)hybrid_uploads / window);
+ }
+ if (gpu_ok) {
+ int k = 0;
+ for (int i = 0; i < selected; i++)
+ if (views[i].gate.gpu && views[i].up.gpu &&
+ views[i].down.gpu) {
+ gates[k] = views[i].gate.gpu;
+ ups[k] = views[i].up.gpu;
+ downs[k] = views[i].down.gpu;
+ gpu_weights[k] = expert_weights[i];
+ k++;
+ }
+ extern int dsv4_cuda_expert_group(
+ void *const *gate, void *const *up, void *const *down,
+ const float *weights, int count, float limit,
+ float *y, const float *x);
+ if (!dsv4_cuda_expert_group(gates, ups, downs, gpu_weights,
+ gpu_n, config->swiglu_limit, gpu_sum, input)) gpu_ok = 0;
+ }
+ /* Backend failure on the group: the CPU loop above deliberately
+ * skipped these experts, so compute them here β€” degrade, don't drop
+ * (nor fail the token). */
+ if (!gpu_ok)
+ for (int current = 0; !result && current < selected; current++) {
+ if (!(views[current].gate.gpu && views[current].up.gpu &&
+ views[current].down.gpu))
+ continue; /* already computed by the CPU loop above */
+ result = coli_v4_expert_forward_ref(
+ expert_output, &views[current], input,
+ expert_weights[current], config->swiglu_limit);
+ if (!result)
+ for (int i = 0; i < d; i++)
+ output[i] += expert_output[i];
+ }
if (!result) {
if (gpu_ok) {
for (int i = 0; i < d; i++)
@@ -5211,6 +5262,13 @@ static int moe_token_pipeline(float *output,
for (int i = 0; i < d; i++)
output[i] = coli_bf16_round(output[i] + shared_output[i]);
}
+#ifdef COLI_V4_GPU_TIER
+ /* If an error or the fused path skipped the hybrid branch while async
+ * fill DMA was still enqueued, drain before releasing the host slabs the
+ * copies read from. (The fused kernels sync internally, so this is only
+ * ever a wait on already-finished work β€” never a correctness gamble.) */
+ if (hybrid_pending_drain) coli_v4_gpu_expert_drain(store);
+#endif
for (int current = 0; current < selected; current++)
coli_expert_release(store, &views[current]);
free(views);
@@ -9482,8 +9540,8 @@ static V4GpuExpertMirrorCache *v4_gpu_expert_mirrors_create_capacity(
static V4GpuExpertMirrorCache *v4_gpu_expert_mirrors_create(int device,
int suggested);
static void v4_gpu_expert_mirrors_free(V4GpuExpertMirrorCache *cache);
-static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,
- ColiExpertView *view);
+static int v4_gpu_expert_attach_cached_ex(V4GpuExpertMirrorCache *cache,
+ ColiExpertView *view, int sync);
int coli_v4_gpu_engine_open(ColiV4Engine *engine) {
if (!engine) return -1;
@@ -10015,8 +10073,8 @@ static void v4_gpu_expert_mirrors_free(V4GpuExpertMirrorCache *cache) {
free(cache);
}
-static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,
- ColiExpertView *view) {
+static int v4_gpu_expert_attach_cached_ex(V4GpuExpertMirrorCache *cache,
+ ColiExpertView *view, int sync) {
if (!cache || !view) return -1;
if (view->gate.block_rows != 1 || view->up.block_rows != 1 ||
view->down.block_rows != 1)
@@ -10093,7 +10151,7 @@ static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,
dsv4_cuda_tensor_refill_fp4(
cache->entries[found].down, (const uint8_t *)view->down.data,
(const uint8_t *)view->down.scales, (int)view->down.rows,
- (int)view->down.columns, 1)) {
+ (int)view->down.columns, sync)) {
cache->entries[found].layer = view->key.layer;
cache->entries[found].expert = view->key.expert;
cache->entries[found].clock = ++cache->clock;
@@ -10161,8 +10219,27 @@ static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,
int coli_v4_gpu_expert_attach(ColiExpertStore *store, ColiExpertView *view) {
if (!store || !view) return -1;
- return v4_gpu_expert_attach_cached(
- (V4GpuExpertMirrorCache *)store->gpu, view);
+ return v4_gpu_expert_attach_cached_ex(
+ (V4GpuExpertMirrorCache *)store->gpu, view, 1);
+}
+
+/* Async twin: the upload is ENQUEUED on the device stream and may still be
+ * in flight when this returns. Safe because later work on the same stream
+ * (the expert group, the fused MoE) is ordered after it, and because the
+ * caller holds the expert leases until after compute; anyone releasing the
+ * host slabs earlier must drain first (coli_v4_gpu_expert_drain). */
+int coli_v4_gpu_expert_attach_async(ColiExpertStore *store,
+ ColiExpertView *view) {
+ if (!store || !view) return -1;
+ return v4_gpu_expert_attach_cached_ex(
+ (V4GpuExpertMirrorCache *)store->gpu, view, 0);
+}
+
+extern int dsv4_cuda_stream_drain(int device);
+int coli_v4_gpu_expert_drain(ColiExpertStore *store) {
+ if (!store || !store->gpu) return 0;
+ V4GpuExpertMirrorCache *cache = (V4GpuExpertMirrorCache *)store->gpu;
+ return dsv4_cuda_stream_drain(cache->device) ? 0 : -1;
}
/* Lookup-only twin of attach: report whether {layer, expert} is already
@@ -10193,7 +10270,7 @@ int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view) {
int coli_v4_gpu_dspark_expert_attach(void *cache, ColiExpertView *view) {
if (!view) return -1;
- return v4_gpu_expert_attach_cached((V4GpuExpertMirrorCache *)cache, view);
+ return v4_gpu_expert_attach_cached_ex((V4GpuExpertMirrorCache *)cache, view, 1);
}
/* Lazy dspark mirror cache. Kept separate from the target model's expert
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index 111c5d5c..b61b26e3 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -853,6 +853,10 @@ int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view);
/* DSV4_HYBRID=1 gate plus its cross-unit counters/EMAs: defined in the block
* unit, read by the serve unit's per-turn stderr line. */
int coli_v4_hybrid_enabled(void);
+/* async attach (enqueue only; drain closes the pipeline before compute) */
+int coli_v4_gpu_expert_attach_async(ColiExpertStore *store,
+ ColiExpertView *view);
+int coli_v4_gpu_expert_drain(ColiExpertStore *store);
extern double g_v4_hyb_fill_bw, g_v4_hyb_host_bw;
extern unsigned long long g_v4_hyb_gpu_n, g_v4_hyb_cpu_n;
extern unsigned long long g_v4_hyb_upload_n, g_v4_hyb_skip_n;
diff --git a/c/dsv4.def b/c/dsv4.def
index 9407b10e..cff4beb8 100644
--- a/c/dsv4.def
+++ b/c/dsv4.def
@@ -53,6 +53,7 @@ EXPORTS
dsv4_cuda_backend_name
dsv4_cuda_kv_ring_append
dsv4_cuda_mem_free_mb
+ dsv4_cuda_stream_drain
dsv4_cuda_kv_comp_append
dsv4_cuda_head_argmax
dsv4_cuda_final_argmax