Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(dsv4): double-buffered expert bank prefetch for GPU prefill (COLI_CUDA_MOE_DOUBLE=1)

JustVugg9876c4db277c
diff --git a/c/Makefile b/c/Makefile
index 24eee2b1..f5152865 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -457,7 +457,7 @@ ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
ifeq ($(COLI_V4_SUPPORTED),1)
-TEST_BINS += tests/test_v4_hybrid_policy$(EXE) tests/test_k3_fill_budget$(EXE)
+TEST_BINS += tests/test_v4_hybrid_policy$(EXE) tests/test_k3_fill_budget$(EXE) tests/test_v4_bank_pair$(EXE)
TEST_BINS += tests/test_deepseek_v4$(EXE) tests/test_v4_ownership$(EXE) \
tests/test_v4_serve_framing$(EXE)
endif
@@ -1356,6 +1356,9 @@ tests/test_v4_hybrid_policy$(EXE): tests/test_v4_hybrid_policy.c deepseek_v4_hyb
tests/test_k3_fill_budget$(EXE): tests/test_k3_fill_budget.c hybrid_split.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_v4_bank_pair$(EXE): tests/test_v4_bank_pair.c deepseek_v4_bank_pair.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_ram_clamp$(EXE): tests/test_ram_clamp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/backend_cuda_dsv4.cu b/c/backend_cuda_dsv4.cu
index b31df925..055e9e97 100644
--- a/c/backend_cuda_dsv4.cu
+++ b/c/backend_cuda_dsv4.cu
@@ -1952,49 +1952,59 @@ extern "C" int dsv4_cuda_stream_drain(int device){
return ok(cudaStreamSynchronize(c->stream),"hybrid drain")?1:0;
}
-extern "C" int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet*set,int e,
+static int bank_upload_stream(Dsv4CudaExpertSet*set,int e,
const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,const uint8_t*dw,const uint8_t*ds,
- Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down){
+ Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down,int use_aux){
if(!set||e<0||e>=set->count||!gw||!gs||!uw||!us||!dw||!ds||!gate||!up||!down||*gate||*up||*down||
!ok(cudaSetDevice(set->device),"select expert bank upload device"))return 0;
int H=set->H,I=set->I;size_t w1=(size_t)I*(H/2),s1=(size_t)I*(H/32),w2=(size_t)H*(I/2),s2=(size_t)H*(I/32);
uint8_t*fc1=set->bank_fc1+(size_t)e*2*w1,*fc1s=set->bank_fc1_scale+(size_t)e*2*s1;
uint8_t*fc2=set->bank_fc2+(size_t)e*w2,*fc2s=set->bank_fc2_scale+(size_t)e*s2;
- Dev*c=ctx(set->device);
- bool pinned=c&&host_pinned(gw,w1)&&host_pinned(uw,w1)&&host_pinned(gs,s1)&&
- host_pinned(us,s1)&&host_pinned(dw,w2)&&host_pinned(ds,s2);
- if(pinned){
- /* Pure DMA on the device stream; the caller releases the host slab
- * right after we return, so drain before returning. */
- if(!ok(cudaMemcpyAsync(fc1,gw,w1,cudaMemcpyHostToDevice,c->stream),"expert gate upload")||
- !ok(cudaMemcpyAsync(fc1+w1,uw,w1,cudaMemcpyHostToDevice,c->stream),"expert up upload")||
- !ok(cudaMemcpyAsync(fc1s,gs,s1,cudaMemcpyHostToDevice,c->stream),"expert gate scale upload")||
- !ok(cudaMemcpyAsync(fc1s+s1,us,s1,cudaMemcpyHostToDevice,c->stream),"expert up scale upload")||
- !ok(cudaMemcpyAsync(fc2,dw,w2,cudaMemcpyHostToDevice,c->stream),"expert down upload")||
- !ok(cudaMemcpyAsync(fc2s,ds,s2,cudaMemcpyHostToDevice,c->stream),"expert down scale upload")||
- !ok(cudaStreamSynchronize(c->stream),"expert upload drain"))return 0;
- }else
- if(!ok(cudaMemcpy(fc1,gw,w1,cudaMemcpyHostToDevice),"expert gate upload")||
- !ok(cudaMemcpy(fc1+w1,uw,w1,cudaMemcpyHostToDevice),"expert up upload")||
- !ok(cudaMemcpy(fc1s,gs,s1,cudaMemcpyHostToDevice),"expert gate scale upload")||
- !ok(cudaMemcpy(fc1s+s1,us,s1,cudaMemcpyHostToDevice),"expert up scale upload")||
- !ok(cudaMemcpy(fc2,dw,w2,cudaMemcpyHostToDevice),"expert down upload")||
- !ok(cudaMemcpy(fc2s,ds,s2,cudaMemcpyHostToDevice),"expert down scale upload"))return 0;
+ Dev*c=ctx(set->device);if(!c)return 0;
+ /* use_aux: the double-buffer prefetch worker uploads layer L+1 on the
+ * aux stream WHILE the compute stream chews layer L. Everything β€”
+ * copies, DeepGEMM scale packing, the pointer table β€” rides the chosen
+ * stream and drains it before returning, so the caller may release the
+ * host slab immediately and the compute stream never stalls. Pageable
+ * sources are safe under cudaMemcpyAsync (the copy is staged before it
+ * returns with respect to the host buffer). */
+ cudaStream_t s=use_aux?c->aux:c->stream;
+ if(!ok(cudaMemcpyAsync(fc1,gw,w1,cudaMemcpyHostToDevice,s),"expert gate upload")||
+ !ok(cudaMemcpyAsync(fc1+w1,uw,w1,cudaMemcpyHostToDevice,s),"expert up upload")||
+ !ok(cudaMemcpyAsync(fc1s,gs,s1,cudaMemcpyHostToDevice,s),"expert gate scale upload")||
+ !ok(cudaMemcpyAsync(fc1s+s1,us,s1,cudaMemcpyHostToDevice,s),"expert up scale upload")||
+ !ok(cudaMemcpyAsync(fc2,dw,w2,cudaMemcpyHostToDevice,s),"expert down upload")||
+ !ok(cudaMemcpyAsync(fc2s,ds,s2,cudaMemcpyHostToDevice,s),"expert down scale upload"))return 0;
#ifdef COLI_DSV4_DEEPGEMM
- dg_pack_weight_scale<<<(2*I*(H/32/4)+255)/256,256>>>(set->bank_fc1_dg_scale,fc1s,e,2*I,H/32);
- dg_pack_weight_scale<<<(H*(I/32/4)+255)/256,256>>>(set->bank_fc2_dg_scale,fc2s,e,H,I/32);
+ dg_pack_weight_scale<<<(2*I*(H/32/4)+255)/256,256,0,s>>>(set->bank_fc1_dg_scale,fc1s,e,2*I,H/32);
+ dg_pack_weight_scale<<<(H*(I/32/4)+255)/256,256,0,s>>>(set->bank_fc2_dg_scale,fc2s,e,H,I/32);
if(!ok(cudaGetLastError(),"expert DeepGEMM scale packing"))return 0;
#endif
Dsv4CudaTensor*g=expert_view(fc1,fc1s,I,H,set->device),*u=expert_view(fc1+w1,fc1s+s1,I,H,set->device),*d=expert_view(fc2,fc2s,H,I,set->device);
if(!g||!u||!d){free(g);free(u);free(d);return 0;}
ExpertPtr p[3]={{(uint8_t*)g->w,g->scale},{(uint8_t*)u->w,u->scale},{(uint8_t*)d->w,d->scale}};
- if(!ok(cudaMemcpy(set->table+e,p,sizeof(*p),cudaMemcpyHostToDevice),"expert gate table upload")||
- !ok(cudaMemcpy(set->table+set->count+e,p+1,sizeof(*p),cudaMemcpyHostToDevice),"expert up table upload")||
- !ok(cudaMemcpy(set->table+2*set->count+e,p+2,sizeof(*p),cudaMemcpyHostToDevice),"expert down table upload")){
+ if(!ok(cudaMemcpyAsync(set->table+e,p,sizeof(*p),cudaMemcpyHostToDevice,s),"expert gate table upload")||
+ !ok(cudaMemcpyAsync(set->table+set->count+e,p+1,sizeof(*p),cudaMemcpyHostToDevice,s),"expert up table upload")||
+ !ok(cudaMemcpyAsync(set->table+2*set->count+e,p+2,sizeof(*p),cudaMemcpyHostToDevice,s),"expert down table upload")||
+ !ok(cudaStreamSynchronize(s),"expert upload drain")){
free(g);free(u);free(d);return 0;
}
*gate=g;*up=u;*down=d;return 1;
}
+
+extern "C" int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet*set,int e,
+ const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,const uint8_t*dw,const uint8_t*ds,
+ Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down){
+ return bank_upload_stream(set,e,gw,gs,uw,us,dw,ds,gate,up,down,0);
+}
+
+/* Double-buffer prefetch: same upload, aux stream (overlaps the compute
+ * stream), drained before return. */
+extern "C" int dsv4_cuda_expert_bank_upload_aux(Dsv4CudaExpertSet*set,int e,
+ const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,const uint8_t*dw,const uint8_t*ds,
+ Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down){
+ return bank_upload_stream(set,e,gw,gs,uw,us,dw,ds,gate,up,down,1);
+}
extern "C" int dsv4_cuda_expert_bank_upload_tp2(Dsv4CudaExpertSet*set,int e,int rank,
const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,const uint8_t*dw,const uint8_t*ds){
if(!set||e<0||e>=set->count||(rank!=0&&rank!=1)||!gw||!gs||!uw||!us||!dw||!ds||
diff --git a/c/backend_cuda_dsv4.h b/c/backend_cuda_dsv4.h
index 416c989e..beae36f9 100644
--- a/c/backend_cuda_dsv4.h
+++ b/c/backend_cuda_dsv4.h
@@ -197,6 +197,12 @@ int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet *set,int expert,
Dsv4CudaTensor **gate,Dsv4CudaTensor **up,Dsv4CudaTensor **down);
int dsv4_cuda_expert_bank_set_shared(Dsv4CudaExpertSet *set,Dsv4CudaTensor *sg,
Dsv4CudaTensor *su,Dsv4CudaTensor *sd);
+/* Same upload on the device's aux stream: overlaps the compute stream (the
+ * double-buffer prefetch worker uses it), drained before returning. */
+int dsv4_cuda_expert_bank_upload_aux(Dsv4CudaExpertSet *set,int expert,
+ const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,
+ const uint8_t*dw,const uint8_t*ds,
+ Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down);
int dsv4_cuda_expert_bank_upload_tp2(Dsv4CudaExpertSet *set,int expert,int rank,
const uint8_t *gate_weight,const uint8_t *gate_scale,
const uint8_t *up_weight,const uint8_t *up_scale,
diff --git a/c/backend_loader_dsv4.c b/c/backend_loader_dsv4.c
index 215989e2..e3391385 100644
--- a/c/backend_loader_dsv4.c
+++ b/c/backend_loader_dsv4.c
@@ -215,6 +215,10 @@ typedef int (*fn_expert_bank_upload)(Dsv4CudaExpertSet *set, int exp
Dsv4CudaTensor **gate, Dsv4CudaTensor **up, Dsv4CudaTensor **down);
typedef int (*fn_expert_bank_set_shared)(Dsv4CudaExpertSet *set, Dsv4CudaTensor *sg,
Dsv4CudaTensor *su, Dsv4CudaTensor *sd);
+typedef int (*fn_expert_bank_upload_aux)(Dsv4CudaExpertSet *set, int expert,
+ const uint8_t *gw, const uint8_t *gs, const uint8_t *uw, const uint8_t *us,
+ const uint8_t *dw, const uint8_t *ds,
+ Dsv4CudaTensor **gate, Dsv4CudaTensor **up, Dsv4CudaTensor **down);
typedef int (*fn_expert_bank_upload_tp2)(Dsv4CudaExpertSet *set, int expert, int rank,
const uint8_t *gate_weight, const uint8_t *gate_scale,
const uint8_t *up_weight, const uint8_t *up_scale,
@@ -323,6 +327,7 @@ static struct {
fn_expert_bank_upload expert_bank_upload;
fn_expert_bank_set_shared expert_bank_set_shared;
fn_expert_bank_upload_tp2 expert_bank_upload_tp2;
+ fn_expert_bank_upload_aux expert_bank_upload_aux; /* optional (older DLLs) */
fn_expert_set_free expert_set_free;
fn_expert_set_upload_hash expert_set_upload_hash;
fn_route_moe route_moe;
@@ -490,6 +495,7 @@ static int dsv4_cuda_resolve(const char *dllname){
/* optional: an older DLL without the export keeps the whole tier alive;
* the engine probes this and simply stays on synchronous attaches. */
g_dsv4.stream_drain = (fn_stream_drain)GetProcAddress(g_dsv4.dll, "dsv4_cuda_stream_drain");
+ g_dsv4.expert_bank_upload_aux = (fn_expert_bank_upload_aux)GetProcAddress(g_dsv4.dll, "dsv4_cuda_expert_bank_upload_aux");
g_dsv4.backend_name = (fn_backend_name)GetProcAddress(g_dsv4.dll, "dsv4_cuda_backend_name");
_Pragma("GCC diagnostic pop")
return 1;
@@ -932,6 +938,18 @@ int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet *set, int expert,
down_weight, down_scale, gate, up, down);
}
+int dsv4_cuda_expert_bank_upload_aux(Dsv4CudaExpertSet *set, int expert,
+ const uint8_t *gate_weight, const uint8_t *gate_scale,
+ const uint8_t *up_weight, const uint8_t *up_scale,
+ const uint8_t *down_weight, const uint8_t *down_scale,
+ Dsv4CudaTensor **gate, Dsv4CudaTensor **up, Dsv4CudaTensor **down){
+ /* Optional export: an older DLL simply fails the first prefetch upload,
+ * and the engine falls back to the single-bank path for good. */
+ if(!g_dsv4.available || !g_dsv4.expert_bank_upload_aux) return 0;
+ return g_dsv4.expert_bank_upload_aux(set, expert, gate_weight, gate_scale, up_weight, up_scale,
+ down_weight, down_scale, gate, up, down);
+}
+
int dsv4_cuda_expert_bank_set_shared(Dsv4CudaExpertSet *set, Dsv4CudaTensor *sg,
Dsv4CudaTensor *su, Dsv4CudaTensor *sd){
if(!g_dsv4.available) return 0;
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 82566901..aab39182 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -10382,16 +10382,120 @@ int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,
* it when the prefill loop finishes; the next large prefill re-creates and
* re-fills it (per-layer refill cost only, on prompts that already run tens
* of seconds). */
+#include "deepseek_v4_bank_pair.h"
+
static Dsv4CudaExpertSet *v4_moe_bank;
static int v4_moe_bank_layer = -1;
static int v4_moe_bank_hash_layer = -1;
static unsigned char v4_moe_bank_valid[256];
static int v4_moe_bank_failed;
+/* ---- double-buffered bank (COLI_CUDA_MOE_DOUBLE=1, opt-in) ----
+ * While the compute stream chews layer L from the active bank, a worker
+ * thread loads layer L+1's complete expert set into the second bank over the
+ * aux stream β€” the transfer starts before L+1's routing is known. On the
+ * layer switch the banks swap; the per-expert valid map travels with the
+ * swap, so a PARTIAL prefetch is still profit (the route-aware refill tops
+ * up only the holes). Every failure path β€” second-bank allocation, an aux
+ * upload API that an older Windows DLL does not export, a fully failed
+ * layer β€” degrades to today's single-bank behaviour and says so once.
+ * The worker holds at most ONE store lease at a time (the expert cache's
+ * pin slots are bounded), and layer L+1's lookups live in L+1's own store
+ * partition, so the prefetch does not evict the computing layer. */
+static Dsv4CudaExpertSet *v4_moe_bank2;
+static int v4_bank2_layer = -1; /* prefetched layer; read only post-join */
+static unsigned char v4_bank2_valid[256];
+static pthread_t v4_bank2_thread;
+static int v4_bank2_running;
+static int v4_bank2_disabled;
+static unsigned long long v4_bank2_swaps, v4_bank2_prefetched;
+static struct { ColiExpertStore *store; int layer; } v4_bank2_job;
+
+static int v4_bank_double_on(void) {
+ static int on = -1;
+ if (on < 0) {
+ const char *setting = getenv("COLI_CUDA_MOE_DOUBLE");
+ on = setting && atoi(setting) != 0;
+ }
+ return on;
+}
+
+static void *v4_bank2_worker(void *argument) {
+ (void)argument;
+ ColiExpertStore *store = v4_bank2_job.store;
+ int layer = v4_bank2_job.layer;
+ memset(v4_bank2_valid, 0, sizeof(v4_bank2_valid));
+ int loaded = 0;
+ for (int expert = 0; expert < 256; expert++) {
+ ColiExpertView view;
+ memset(&view, 0, sizeof(view));
+ if (coli_expert_lookup(store, (ColiExpertKey){layer, expert},
+ &view) != 0)
+ continue;
+ Dsv4CudaTensor *bg = NULL, *bu = NULL, *bd = NULL;
+ int uploaded =
+ view.gate.data && view.gate.scales && view.gate.block_rows == 1 &&
+ view.up.data && view.up.scales && view.up.block_rows == 1 &&
+ view.down.data && view.down.scales && view.down.block_rows == 1 &&
+ view.gate.rows == 2048 && view.gate.columns == 4096 &&
+ view.up.rows == 2048 && view.up.columns == 4096 &&
+ view.down.rows == 4096 && view.down.columns == 2048 &&
+ dsv4_cuda_expert_bank_upload_aux(
+ v4_moe_bank2, expert,
+ (const uint8_t *)view.gate.data,
+ (const uint8_t *)view.gate.scales,
+ (const uint8_t *)view.up.data,
+ (const uint8_t *)view.up.scales,
+ (const uint8_t *)view.down.data,
+ (const uint8_t *)view.down.scales,
+ &bg, &bu, &bd);
+ coli_expert_release(store, &view);
+ if (bg) dsv4_cuda_tensor_free(bg);
+ if (bu) dsv4_cuda_tensor_free(bu);
+ if (bd) dsv4_cuda_tensor_free(bd);
+ if (uploaded) { v4_bank2_valid[expert] = 1; loaded++; }
+ else if (!loaded)
+ /* First attempted upload failed with nothing loaded yet: the
+ * aux path is structurally unavailable (older Windows DLL, or
+ * geometry off) β€” stop before reading the whole layer for
+ * nothing. Failures AFTER a success keep going: partial banks
+ * are profit. */
+ break;
+ }
+ v4_bank2_prefetched += (unsigned long long)loaded;
+ v4_bank2_layer = loaded ? layer : -1;
+ return NULL;
+}
+
+static void v4_bank2_join(void) {
+ if (!v4_bank2_running) return;
+ pthread_join(v4_bank2_thread, NULL);
+ v4_bank2_running = 0;
+ if (v4_bank2_layer < 0 && !v4_bank2_disabled) {
+ /* A fully failed layer (aux upload unavailable, e.g. an older Windows DLL,
+ * or the store refusing every lookup) will fail every layer: stop
+ * paying for workers and stay single-bank. */
+ v4_bank2_disabled = 1;
+ fprintf(stderr, "v4_gpu moe-double=off (prefetch produced nothing; "
+ "single-bank stays)\n");
+ }
+}
+
void coli_v4_gpu_moe_batch_release(void) {
/* A create failure is retried after every release: the freed VRAM is
* exactly what the next attempt needs. */
v4_moe_bank_failed = 0;
+ v4_bank2_join();
+ if (v4_bank2_swaps || v4_bank2_prefetched)
+ fprintf(stderr, "v4_gpu moe-double swaps=%llu prefetched=%llu\n",
+ v4_bank2_swaps, v4_bank2_prefetched);
+ v4_bank2_swaps = 0;
+ v4_bank2_prefetched = 0;
+ v4_bank2_layer = -1;
+ if (v4_moe_bank2) {
+ dsv4_cuda_expert_set_free(v4_moe_bank2);
+ v4_moe_bank2 = NULL;
+ }
if (!v4_moe_bank) return;
dsv4_cuda_expert_set_free(v4_moe_bank);
v4_moe_bank = NULL;
@@ -10465,9 +10569,51 @@ int coli_v4_gpu_moe_batch_union(float *outputs,
* re-reads the same experts several times per layer under prefill's
* expert-major sweeps (measured ~4-6x the layer's expert bytes). */
if (bank_layer != weights->plan.layer) {
+ v4_bank2_join();
+ int double_on = v4_bank_double_on() && !v4_bank2_disabled;
+ if (coli_v4_bank_pair_decide(double_on, v4_bank2_layer,
+ weights->plan.layer) == V4_BANK_SWAP) {
+ /* The prefetched bank becomes the active one; its valid map
+ * travels with the swap, so a partial prefetch is topped up by
+ * the route-aware refill below instead of thrown away. */
+ Dsv4CudaExpertSet *spare = bank;
+ bank = v4_moe_bank2;
+ v4_moe_bank2 = spare;
+ memcpy(bank_valid, v4_bank2_valid, sizeof(v4_moe_bank_valid));
+ v4_bank2_swaps++;
+ } else {
+ memset(bank_valid, 0, sizeof(bank_valid));
+ }
+ v4_bank2_layer = -1;
if (!dsv4_cuda_expert_bank_set_shared(bank, sg, su, sd)) return -1;
- memset(bank_valid, 0, sizeof(bank_valid));
bank_layer = weights->plan.layer;
+ if (double_on) {
+ int next = coli_v4_bank_pair_prefetch_target(
+ 1, bank_layer, config->num_hidden_layers);
+ if (next >= 0) {
+ if (!v4_moe_bank2) {
+ v4_moe_bank2 = dsv4_cuda_expert_bank_create(
+ 256, 4096, 2048, device, sg, su, sd);
+ if (!v4_moe_bank2) {
+ /* Not enough VRAM for two full banks: stay on the
+ * proven single-bank path, permanently and loudly. */
+ v4_bank2_disabled = 1;
+ fprintf(stderr,
+ "v4_gpu moe-double=off (second bank "
+ "allocation failed; single-bank stays)\n");
+ } else {
+ fprintf(stderr, "v4_gpu moe-double=on\n");
+ }
+ }
+ if (v4_moe_bank2) {
+ v4_bank2_job.store = store;
+ v4_bank2_job.layer = next;
+ if (pthread_create(&v4_bank2_thread, NULL,
+ v4_bank2_worker, NULL) == 0)
+ v4_bank2_running = 1;
+ }
+ }
+ }
}
long long elements = (long long)batch * config->hidden_size;
if (!in_mirror) {
diff --git a/c/deepseek_v4_bank_pair.h b/c/deepseek_v4_bank_pair.h
new file mode 100644
index 00000000..da02c82f
--- /dev/null
+++ b/c/deepseek_v4_bank_pair.h
@@ -0,0 +1,52 @@
+/* deepseek_v4_bank_pair.h β€” decision logic for the double-buffered prefill
+ * expert bank (COLI_CUDA_MOE_DOUBLE=1).
+ *
+ * While the GPU computes layer L's chunks from the active bank, a worker
+ * thread loads layer L+1's COMPLETE expert set into the other bank over the
+ * device's aux stream β€” the transfer starts before L+1's routing is known,
+ * which is exactly what loading the full layer buys. On the layer switch the
+ * banks swap roles. If the prefetched layer does not match (segment restart,
+ * bootstrap, worker failure) the engine falls back to the route-aware
+ * on-demand refill it uses today; a PARTIAL prefetch still helps, because
+ * the swap carries the per-expert valid map and the existing refill tops up
+ * only the holes.
+ *
+ * This header holds the pure decisions so the sequencing β€” where
+ * double-buffer bugs actually live β€” is unit-tested on machines with no GPU;
+ * the CUDA glue in deepseek_v4.c wires threads and streams around it. */
+#ifndef DEEPSEEK_V4_BANK_PAIR_H
+#define DEEPSEEK_V4_BANK_PAIR_H
+
+typedef enum {
+ V4_BANK_SWAP = 0, /* other bank holds this layer: swap roles */
+ V4_BANK_LEGACY = 1, /* refill the active bank on demand */
+} V4BankPairAction;
+
+/* What to do when a chunk arrives for `incoming_layer`. `other_layer` is the
+ * layer the inactive bank was prefetched for (-1 = none/failed), valid only
+ * once the caller has JOINED the prefetch worker β€” deciding on a bank whose
+ * upload is still in flight is the classic double-buffer bug, so the caller
+ * must join first and only then consult this. */
+static inline V4BankPairAction coli_v4_bank_pair_decide(int double_on,
+ int other_layer,
+ int incoming_layer) {
+ if (!double_on) return V4_BANK_LEGACY;
+ if (other_layer < 0 || other_layer != incoming_layer)
+ return V4_BANK_LEGACY;
+ return V4_BANK_SWAP;
+}
+
+/* Which layer to prefetch while `current_layer` computes: the next one, or
+ * -1 when there is none (last layer) or the feature is off. Prefetching
+ * anything OTHER than current+1 is never right: target_batch walks layers in
+ * ascending order and a segment restart goes back to 0, which the mismatch
+ * path above absorbs. */
+static inline int coli_v4_bank_pair_prefetch_target(int double_on,
+ int current_layer,
+ int num_layers) {
+ if (!double_on || current_layer < 0) return -1;
+ if (current_layer + 1 >= num_layers) return -1;
+ return current_layer + 1;
+}
+
+#endif /* DEEPSEEK_V4_BANK_PAIR_H */
diff --git a/c/dsv4.def b/c/dsv4.def
index cff4beb8..2e18ccc3 100644
--- a/c/dsv4.def
+++ b/c/dsv4.def
@@ -67,6 +67,7 @@ EXPORTS
dsv4_cuda_expert_bank_upload
dsv4_cuda_expert_bank_set_shared
dsv4_cuda_expert_bank_upload_tp2
+ dsv4_cuda_expert_bank_upload_aux
dsv4_cuda_expert_set_free
dsv4_cuda_expert_set_upload_hash
dsv4_cuda_route_moe
diff --git a/c/tests/test_v4_bank_pair.c b/c/tests/test_v4_bank_pair.c
new file mode 100644
index 00000000..d0f8c8bd
--- /dev/null
+++ b/c/tests/test_v4_bank_pair.c
@@ -0,0 +1,50 @@
+/* test_v4_bank_pair.c β€” sequencing contract of the double-buffered bank.
+ * Pure decisions, GPU-less: the classic double-buffer failure modes (swap on
+ * a mismatched or failed prefetch, prefetch past the last layer, feature
+ * leaking when off) must be impossible by construction. */
+#include <stdio.h>
+#include "../deepseek_v4_bank_pair.h"
+
+static int failures;
+#define CHECK(cond, ...) do { if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); failures++; } } while (0)
+
+int main(void) {
+ /* off: never anything but legacy, never a prefetch */
+ CHECK(coli_v4_bank_pair_decide(0, 5, 5) == V4_BANK_LEGACY, "off must be legacy");
+ CHECK(coli_v4_bank_pair_prefetch_target(0, 3, 61) == -1, "off must not prefetch");
+
+ /* bootstrap: nothing prefetched yet */
+ CHECK(coli_v4_bank_pair_decide(1, -1, 0) == V4_BANK_LEGACY, "bootstrap legacy");
+ CHECK(coli_v4_bank_pair_prefetch_target(1, 0, 61) == 1, "then prefetch L+1");
+
+ /* steady state: match swaps, and the next prefetch follows */
+ CHECK(coli_v4_bank_pair_decide(1, 1, 1) == V4_BANK_SWAP, "match must swap");
+ CHECK(coli_v4_bank_pair_prefetch_target(1, 1, 61) == 2, "prefetch L+2");
+
+ /* segment restart: bank holds layer 8, chunk arrives for layer 0 */
+ CHECK(coli_v4_bank_pair_decide(1, 8, 0) == V4_BANK_LEGACY, "mismatch legacy");
+
+ /* worker failed mid-prefetch: other_layer reported -1 */
+ CHECK(coli_v4_bank_pair_decide(1, -1, 7) == V4_BANK_LEGACY, "failure legacy");
+
+ /* last layer: nothing to prefetch */
+ CHECK(coli_v4_bank_pair_prefetch_target(1, 60, 61) == -1, "no L past last");
+ CHECK(coli_v4_bank_pair_prefetch_target(1, -1, 61) == -1, "no current layer");
+
+ /* full sweep: every ascending walk is swap after the bootstrap layer */
+ int other = -1;
+ int swaps = 0, legacies = 0;
+ for (int layer = 0; layer < 61; layer++) {
+ if (coli_v4_bank_pair_decide(1, other, layer) == V4_BANK_SWAP) swaps++;
+ else legacies++;
+ other = coli_v4_bank_pair_prefetch_target(1, layer, 61);
+ }
+ CHECK(legacies == 1 && swaps == 60,
+ "ascending walk: 1 bootstrap + 60 swaps, got %d + %d", legacies, swaps);
+
+ if (failures) { fprintf(stderr, "%d failure(s)\n", failures); return 1; }
+ printf("test_v4_bank_pair: ok\n");
+ return 0;
+}