Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
perf(dsv4): overlap hybrid fill DMA with host compute; pin slabs on Linux
JustVugg605d98bed061
diff --git a/c/backend_cuda_dsv4.cu b/c/backend_cuda_dsv4.cuindex 6ae6d96d..b31df925 100644--- a/c/backend_cuda_dsv4.cu+++ b/c/backend_cuda_dsv4.cu@@ -10,6 +10,9 @@#include <cstring>#include <climits>#include <vector>+#ifndef _WIN32+#include <unistd.h>+#endif#ifdef _WIN32#define WIN32_LEAN_AND_MEAN#define NOMINMAX@@ -1918,10 +1921,37 @@ static bool host_pinned(const void*p,size_t len){g_pin_bytes+=size;g_pin_regions.push_back({base,size});return true;#else- (void)a;return false;+ /* Linux/WSL: no VirtualQuery, but none is needed β cudaHostRegister pins+ * whole pages, so register the requested range rounded out to page+ * boundaries. The slot slabs are stable for the process lifetime (one+ * posix_memalign per slot, freed only at store destroy), exactly like the+ * Windows branch assumes. Overlapping ranges collapse into the+ * already-registered success case; any failure (WSL builds without+ * pinning support included) lands in g_pin_failed and every later upload+ * from that slab takes the pageable path, same as before this branch+ * existed. */+ long page=sysconf(_SC_PAGESIZE);if(page<=0)page=4096;+ uintptr_t base=a&~((uintptr_t)page-1);+ size_t size=((a+len+page-1)&~((uintptr_t)page-1))-base;+ static const size_t cap=(size_t)20<<30;+ if(g_pin_bytes+size>cap){g_pin_failed.push_back({base,size});return false;}+ cudaError_t err=cudaHostRegister((void*)base,size,cudaHostRegisterPortable);+ if(err==cudaErrorHostMemoryAlreadyRegistered){cudaGetLastError();g_pin_regions.push_back({base,size});return true;}+ if(err!=cudaSuccess){cudaGetLastError();g_pin_failed.push_back({base,size});return false;}+ g_pin_bytes+=size;g_pin_regions.push_back({base,size});+ return true;#endif}+/* Drain the device's expert stream: the hybrid decode split enqueues its+ * fill-set uploads asynchronously, computes the CPU subset while the DMA is+ * in flight, and closes the pipeline here before the GPU group runs. */+extern "C" int dsv4_cuda_stream_drain(int device){+ Dev*c=ctx(device);if(!c)return 0;+ if(!ok(cudaSetDevice(device),"select drain device"))return 0;+ return ok(cudaStreamSynchronize(c->stream),"hybrid drain")?1:0;+}+extern "C" int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet*set,int e,const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,const uint8_t*dw,const uint8_t*ds,Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down){diff --git a/c/backend_cuda_dsv4.h b/c/backend_cuda_dsv4.hindex 5566bf66..416c989e 100644--- a/c/backend_cuda_dsv4.h+++ b/c/backend_cuda_dsv4.h@@ -64,6 +64,10 @@ int dsv4_cuda_fp8_ref_matmul(int device,const uint8_t *w,const float *bscale,int dsv4_cuda_backend_arch_ok(int device);const char *dsv4_cuda_backend_name(void);long long dsv4_cuda_mem_free_mb(int device);+/* Drain the device's expert stream (hybrid split: async fill DMA is enqueued,+ * the CPU subset computes meanwhile, this closes the pipeline). Returns 1 on+ * success, 0 when the backend is unavailable. */+int dsv4_cuda_stream_drain(int device);int dsv4_cuda_kv_ring_append(int device,int layer,const float *rows,int start_pos,int count,int window,int dim);int dsv4_cuda_kv_comp_append(int device,int layer,const float *rows,int start_idx,diff --git a/c/backend_loader_dsv4.c b/c/backend_loader_dsv4.cindex 76d12f37..215989e2 100644--- a/c/backend_loader_dsv4.c+++ b/c/backend_loader_dsv4.c@@ -172,6 +172,7 @@ typedef int (*fn_tensor_refill_fp4)(Dsv4CudaTensor *t, const uint8_ttypedef int (*fn_backend_arch_ok)(int device);typedef const char *(*fn_backend_name)(void);typedef long long (*fn_mem_free_mb)(int device);+typedef int (*fn_stream_drain)(int device);typedef int (*fn_kv_ring_append)(int device, int layer, const float *rows,int start_pos, int count, int window, int dim);typedef int (*fn_kv_comp_append)(int device, int layer, const float *rows,@@ -307,6 +308,7 @@ static struct {fn_backend_name backend_name; /* optional */char loaded_name[64];fn_mem_free_mb mem_free_mb;+ fn_stream_drain stream_drain; /* optional (older DLLs) */fn_kv_ring_append kv_ring_append;fn_kv_comp_append kv_comp_append;fn_head_argmax head_argmax;@@ -485,6 +487,9 @@ static int dsv4_cuda_resolve(const char *dllname){_Pragma("GCC diagnostic push")_Pragma("GCC diagnostic ignored \"-Wcast-function-type\"")g_dsv4.backend_arch_ok = (fn_backend_arch_ok)GetProcAddress(g_dsv4.dll, "dsv4_cuda_backend_arch_ok");+ /* optional: an older DLL without the export keeps the whole tier alive;+ * the engine probes this and simply stays on synchronous attaches. */+ g_dsv4.stream_drain = (fn_stream_drain)GetProcAddress(g_dsv4.dll, "dsv4_cuda_stream_drain");g_dsv4.backend_name = (fn_backend_name)GetProcAddress(g_dsv4.dll, "dsv4_cuda_backend_name");_Pragma("GCC diagnostic pop")return 1;@@ -832,6 +837,16 @@ long long dsv4_cuda_mem_free_mb(int device){return g_dsv4.mem_free_mb(device);}+int dsv4_cuda_stream_drain(int device){+ /* Without the DLL there is never in-flight DMA to wait for, so a drain+ * trivially succeeds. With an older DLL that lacks the export, report+ * failure instead: the engine probes this once and keeps every attach+ * synchronous, so nothing is ever enqueued that could not be drained. */+ if(!g_dsv4.available) return 1;+ if(!g_dsv4.stream_drain) return 0;+ return g_dsv4.stream_drain(device);+}+int dsv4_cuda_kv_ring_append(int device, int layer, const float *rows, int start_pos,int count, int window, int dim){if(!g_dsv4.available) return 0;diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 527d2991..3fc01ff5 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -5044,6 +5044,9 @@ static int moe_token_pipeline(float *output,}#ifdef COLI_V4_GPU_TIERint hybrid_gpu_count = 0;+ int hybrid_pending_drain = 0; /* async DMA enqueued, not yet drained */+ int hybrid_uploads = 0;+ struct timespec hybrid_fill_t0 = {0, 0};if (!result && coli_v4_hybrid_enabled() && store->gpu && !gpu_compute) {/* q* pass: the peeks above established which of the token's experts* are already resident. Upload only fill = q*(m) of the m misses;@@ -5062,19 +5065,29 @@ static int moe_token_pipeline(float *output,miss_n, g_v4_hyb_fill_bw, g_v4_hyb_host_bw);if (g_v4_hyb_host_bw <= 0.0 && fill >= miss_n && miss_n > 1)fill = miss_n - 1;+ /* Fill uploads are ENQUEUED, not drained: the CPU subset below+ * computes while the DMA is in flight β the very overlap the q*+ * balance assumes. The fill branch is timed as a whole at the+ * drain point; per-upload wall clocks here would only measure+ * enqueue latency. */+ /* Async needs a drainable stream. An older Windows DLL exports+ * the refill but not the drain: probe once (a drain on an empty+ * stream is a no-op) and stay fully synchronous there, so+ * nothing is ever enqueued that could not be waited on. */+ static int hybrid_async_ok = -1;+ if (hybrid_async_ok < 0)+ hybrid_async_ok = coli_v4_gpu_expert_drain(store) == 0;+ clock_gettime(CLOCK_MONOTONIC, &hybrid_fill_t0);for (int i = 0; i < fill; i++) {int v = missing[i];- struct timespec f0, f1;- clock_gettime(CLOCK_MONOTONIC, &f0);- if (coli_v4_gpu_expert_attach(store, &views[v]) == 0 &&+ int attached = hybrid_async_ok+ ? coli_v4_gpu_expert_attach_async(store, &views[v])+ : coli_v4_gpu_expert_attach(store, &views[v]);+ if (attached == 0 &&views[v].gate.gpu && views[v].up.gpu &&views[v].down.gpu) {- clock_gettime(CLOCK_MONOTONIC, &f1);- double dt = (f1.tv_sec - f0.tv_sec) +- (f1.tv_nsec - f0.tv_nsec) * 1e-9;- if (dt > 0.0)- g_v4_hyb_fill_bw = coli_v4_hybrid_ema(- g_v4_hyb_fill_bw, 1.0 / dt);+ hybrid_uploads++;+ hybrid_pending_drain = hybrid_async_ok;g_v4_hyb_upload_n++;}}@@ -5145,37 +5158,23 @@ static int moe_token_pipeline(float *output,float *gpu_weights = malloc((size_t)gpu_n * sizeof(*gpu_weights));float *gpu_sum = malloc((size_t)d * sizeof(*gpu_sum));int gpu_ok = gates && ups && downs && gpu_weights && gpu_sum;- if (gpu_ok) {- int k = 0;- for (int i = 0; i < selected; i++)- if (views[i].gate.gpu && views[i].up.gpu &&- views[i].down.gpu) {- gates[k] = views[i].gate.gpu;- ups[k] = views[i].up.gpu;- downs[k] = views[i].down.gpu;- gpu_weights[k] = expert_weights[i];- k++;- }- extern int dsv4_cuda_expert_group(- void *const *gate, void *const *up, void *const *down,- const float *weights, int count, float limit,- float *y, const float *x);- if (!dsv4_cuda_expert_group(gates, ups, downs, gpu_weights,- gpu_n, config->swiglu_limit, gpu_sum, input)) gpu_ok = 0;- }+ /* CPU subset FIRST: it computes while the fill DMA enqueued by the+ * q* pass is still in flight. The host bandwidth EMA is measured+ * right here, under bus contention β which is exactly the number the+ * balance needs. */struct timespec h0, h1;clock_gettime(CLOCK_MONOTONIC, &h0);for (int current = 0; !result && current < selected; current++) {int on_gpu = views[current].gate.gpu && views[current].up.gpu &&views[current].down.gpu;- if (on_gpu && gpu_ok) continue; /* covered by the group above */+ if (on_gpu && gpu_ok) continue; /* covered by the group below */result = coli_v4_expert_forward_ref(expert_output, &views[current], input,expert_weights[current], config->swiglu_limit);if (!result)for (int i = 0; i < d; i++) output[i] += expert_output[i];}- if (!result && gpu_ok && cpu_n > 0) {+ if (!result && cpu_n > 0) {clock_gettime(CLOCK_MONOTONIC, &h1);double dt = (h1.tv_sec - h0.tv_sec) +(h1.tv_nsec - h0.tv_nsec) * 1e-9;@@ -5183,6 +5182,58 @@ static int moe_token_pipeline(float *output,g_v4_hyb_host_bw = coli_v4_hybrid_ema(g_v4_hyb_host_bw, (double)cpu_n / dt);}+ /* Close the fill pipeline. The whole window (enqueue -> drained) is+ * the fill branch's wall time; a sample is taken only when the drain+ * actually waited, otherwise the transfers finished under the CPU+ * work and the window would say nothing about the bus. */+ if (hybrid_pending_drain) {+ struct timespec d0, d1;+ clock_gettime(CLOCK_MONOTONIC, &d0);+ coli_v4_gpu_expert_drain(store);+ clock_gettime(CLOCK_MONOTONIC, &d1);+ hybrid_pending_drain = 0;+ double waited = (d1.tv_sec - d0.tv_sec) ++ (d1.tv_nsec - d0.tv_nsec) * 1e-9;+ double window = (d1.tv_sec - hybrid_fill_t0.tv_sec) ++ (d1.tv_nsec - hybrid_fill_t0.tv_nsec) * 1e-9;+ if (hybrid_uploads > 0 && window > 0.0 &&+ waited > window * 0.05)+ g_v4_hyb_fill_bw = coli_v4_hybrid_ema(+ g_v4_hyb_fill_bw, (double)hybrid_uploads / window);+ }+ if (gpu_ok) {+ int k = 0;+ for (int i = 0; i < selected; i++)+ if (views[i].gate.gpu && views[i].up.gpu &&+ views[i].down.gpu) {+ gates[k] = views[i].gate.gpu;+ ups[k] = views[i].up.gpu;+ downs[k] = views[i].down.gpu;+ gpu_weights[k] = expert_weights[i];+ k++;+ }+ extern int dsv4_cuda_expert_group(+ void *const *gate, void *const *up, void *const *down,+ const float *weights, int count, float limit,+ float *y, const float *x);+ if (!dsv4_cuda_expert_group(gates, ups, downs, gpu_weights,+ gpu_n, config->swiglu_limit, gpu_sum, input)) gpu_ok = 0;+ }+ /* Backend failure on the group: the CPU loop above deliberately+ * skipped these experts, so compute them here β degrade, don't drop+ * (nor fail the token). */+ if (!gpu_ok)+ for (int current = 0; !result && current < selected; current++) {+ if (!(views[current].gate.gpu && views[current].up.gpu &&+ views[current].down.gpu))+ continue; /* already computed by the CPU loop above */+ result = coli_v4_expert_forward_ref(+ expert_output, &views[current], input,+ expert_weights[current], config->swiglu_limit);+ if (!result)+ for (int i = 0; i < d; i++)+ output[i] += expert_output[i];+ }if (!result) {if (gpu_ok) {for (int i = 0; i < d; i++)@@ -5211,6 +5262,13 @@ static int moe_token_pipeline(float *output,for (int i = 0; i < d; i++)output[i] = coli_bf16_round(output[i] + shared_output[i]);}+#ifdef COLI_V4_GPU_TIER+ /* If an error or the fused path skipped the hybrid branch while async+ * fill DMA was still enqueued, drain before releasing the host slabs the+ * copies read from. (The fused kernels sync internally, so this is only+ * ever a wait on already-finished work β never a correctness gamble.) */+ if (hybrid_pending_drain) coli_v4_gpu_expert_drain(store);+#endiffor (int current = 0; current < selected; current++)coli_expert_release(store, &views[current]);free(views);@@ -9482,8 +9540,8 @@ static V4GpuExpertMirrorCache *v4_gpu_expert_mirrors_create_capacity(static V4GpuExpertMirrorCache *v4_gpu_expert_mirrors_create(int device,int suggested);static void v4_gpu_expert_mirrors_free(V4GpuExpertMirrorCache *cache);-static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,- ColiExpertView *view);+static int v4_gpu_expert_attach_cached_ex(V4GpuExpertMirrorCache *cache,+ ColiExpertView *view, int sync);int coli_v4_gpu_engine_open(ColiV4Engine *engine) {if (!engine) return -1;@@ -10015,8 +10073,8 @@ static void v4_gpu_expert_mirrors_free(V4GpuExpertMirrorCache *cache) {free(cache);}-static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,- ColiExpertView *view) {+static int v4_gpu_expert_attach_cached_ex(V4GpuExpertMirrorCache *cache,+ ColiExpertView *view, int sync) {if (!cache || !view) return -1;if (view->gate.block_rows != 1 || view->up.block_rows != 1 ||view->down.block_rows != 1)@@ -10093,7 +10151,7 @@ static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,dsv4_cuda_tensor_refill_fp4(cache->entries[found].down, (const uint8_t *)view->down.data,(const uint8_t *)view->down.scales, (int)view->down.rows,- (int)view->down.columns, 1)) {+ (int)view->down.columns, sync)) {cache->entries[found].layer = view->key.layer;cache->entries[found].expert = view->key.expert;cache->entries[found].clock = ++cache->clock;@@ -10161,8 +10219,27 @@ static int v4_gpu_expert_attach_cached(V4GpuExpertMirrorCache *cache,int coli_v4_gpu_expert_attach(ColiExpertStore *store, ColiExpertView *view) {if (!store || !view) return -1;- return v4_gpu_expert_attach_cached(- (V4GpuExpertMirrorCache *)store->gpu, view);+ return v4_gpu_expert_attach_cached_ex(+ (V4GpuExpertMirrorCache *)store->gpu, view, 1);+}++/* Async twin: the upload is ENQUEUED on the device stream and may still be+ * in flight when this returns. Safe because later work on the same stream+ * (the expert group, the fused MoE) is ordered after it, and because the+ * caller holds the expert leases until after compute; anyone releasing the+ * host slabs earlier must drain first (coli_v4_gpu_expert_drain). */+int coli_v4_gpu_expert_attach_async(ColiExpertStore *store,+ ColiExpertView *view) {+ if (!store || !view) return -1;+ return v4_gpu_expert_attach_cached_ex(+ (V4GpuExpertMirrorCache *)store->gpu, view, 0);+}++extern int dsv4_cuda_stream_drain(int device);+int coli_v4_gpu_expert_drain(ColiExpertStore *store) {+ if (!store || !store->gpu) return 0;+ V4GpuExpertMirrorCache *cache = (V4GpuExpertMirrorCache *)store->gpu;+ return dsv4_cuda_stream_drain(cache->device) ? 0 : -1;}/* Lookup-only twin of attach: report whether {layer, expert} is already@@ -10193,7 +10270,7 @@ int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view) {int coli_v4_gpu_dspark_expert_attach(void *cache, ColiExpertView *view) {if (!view) return -1;- return v4_gpu_expert_attach_cached((V4GpuExpertMirrorCache *)cache, view);+ return v4_gpu_expert_attach_cached_ex((V4GpuExpertMirrorCache *)cache, view, 1);}/* Lazy dspark mirror cache. Kept separate from the target model's expertdiff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex 111c5d5c..b61b26e3 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -853,6 +853,10 @@ int coli_v4_gpu_expert_peek(ColiExpertStore *store, ColiExpertView *view);/* DSV4_HYBRID=1 gate plus its cross-unit counters/EMAs: defined in the block* unit, read by the serve unit's per-turn stderr line. */int coli_v4_hybrid_enabled(void);+/* async attach (enqueue only; drain closes the pipeline before compute) */+int coli_v4_gpu_expert_attach_async(ColiExpertStore *store,+ ColiExpertView *view);+int coli_v4_gpu_expert_drain(ColiExpertStore *store);extern double g_v4_hyb_fill_bw, g_v4_hyb_host_bw;extern unsigned long long g_v4_hyb_gpu_n, g_v4_hyb_cpu_n;extern unsigned long long g_v4_hyb_upload_n, g_v4_hyb_skip_n;diff --git a/c/dsv4.def b/c/dsv4.defindex 9407b10e..cff4beb8 100644--- a/c/dsv4.def+++ b/c/dsv4.def@@ -53,6 +53,7 @@ EXPORTSdsv4_cuda_backend_namedsv4_cuda_kv_ring_appenddsv4_cuda_mem_free_mb+ dsv4_cuda_stream_draindsv4_cuda_kv_comp_appenddsv4_cuda_head_argmaxdsv4_cuda_final_argmax