Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
cuda: COLI_CUDA_RESID=1 β expert-group results stay on device (#431 PR-C0)
ZacharyZcR0f8cbe290417
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex a0083f40..23123782 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -38,6 +38,7 @@ typedef struct {float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap;float *pipe_buf[27]; size_t pipe_cap[27]; /* scratch persistenti del resident pipeline */cudaStream_t stream;+ cudaEvent_t ev_done; int ev_done_ok; /* resident-group issue completion (#431 PR-C0) */void *group_desc; size_t group_desc_cap;size_t tensor_count, tensor_bytes;int group_pending; size_t group_pending_bytes; /* async expert-group in flight (Inc.4) */@@ -1294,6 +1295,106 @@ extern "C" int coli_cuda_pipe_router(int device,const float *x_dev,memcpy(keff_host,buf+Ksel*(sizeof(int)+sizeof(float)),sizeof(int));return 1;}+/* ---- resident expert-group accumulation (#431 PR-C0) ----------------------+ * Decode-time (S=1) expert groups without the host round-trip: the input row+ * is P2P'd from the layer's home device, the group runs through the grouped-W4+ * kernels on its own stream, the down-projection outputs are weighted and+ * reduced ON DEVICE (fixed expert order), and the device's partial sum is+ * peer-pushed into a per-issue slot on the home device. take() makes the home+ * legacy stream wait on every issue event and reduces the slots in issue order+ * β deterministic, no atomics, no host bytes. The CPU tier overlaps with all+ * of it exactly as before. */+__global__ static void bcast_row(float *dst,const float *src,int count,int D){+ for(int i=blockIdx.x*blockDim.x+threadIdx.x;i<D;i+=gridDim.x*blockDim.x){+ float v=src[i];+ for(int c=0;c<count;c++) dst[(size_t)c*D+i]=v;+ }+}+__global__ static void weighted_sum_rows(float *out,const float *y,const float *w,+ int count,int D){+ for(int i=blockIdx.x*blockDim.x+threadIdx.x;i<D;i+=gridDim.x*blockDim.x){+ float acc=0.f;+ for(int c=0;c<count;c++) acc+=w[c]*y[(size_t)c*D+i]; /* fixed order */+ out[i]=acc;+ }+}+__global__ static void sum_slots(float *dst,const float *slots,int n,int D){+ for(int i=blockIdx.x*blockDim.x+threadIdx.x;i<D;i+=gridDim.x*blockDim.x){+ float acc=0.f;+ for(int s=0;s<n;s++) acc+=slots[(size_t)s*D+i]; /* issue order */+ dst[i]=acc;+ }+}+extern "C" int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates,+ ColiCudaTensor *const *ups, ColiCudaTensor *const *downs,+ const float *weights, int count,+ int home_device, const float *x_src_dev, float *partial_slot_dev){+ if(!gates||!ups||!downs||!weights||count<1||count>64||!x_src_dev||!partial_slot_dev) return 0;+ ColiCudaTensor *first=gates[0]; if(!first) return 0;+ int device=first->device,D=first->I,I=first->O;+ GroupDesc host[64];+ int total=0,all_s4=1;+ for(int c=0;c<count;c++){+ ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];+ if(!g||!u||!d||g->device!=device||u->device!=device||d->device!=device||+ g->I!=D||u->I!=D||g->O!=I||u->O!=I||d->I!=I||d->O!=D) return 0;+ host[c]={g->weights,u->weights,d->weights,g->scales,u->scales,d->scales,+ g->fmt,u->fmt,d->fmt,1,total};+ all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2;+ total++;+ }+ if(!all_s4) return 0; /* resident path: per-row int4 only */+ DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0;+ if(!ctx->ev_done_ok){+ if(!cuda_ok(cudaEventCreateWithFlags(&ctx->ev_done,cudaEventDisableTiming),+ "resident group event")) return 0;+ ctx->ev_done_ok=1;+ }+ /* size for the 64-expert cap, not for `count`: reserve() reallocs on growth,+ * and a realloc here could free a buffer the PREVIOUS layer's still-queued+ * async work on this stream reads. Fixed caps make re-issue realloc-free. */+ size_t xb=(size_t)64*D*sizeof(float), ib=(size_t)64*I*sizeof(float);+ if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)||+ !reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)||+ !reserve(&ctx->ac,&ctx->ac_cap,(size_t)(D+64)*sizeof(float))||+ !reserve_bytes(&ctx->group_desc,&ctx->group_desc_cap,(size_t)64*sizeof(GroupDesc)))+ return 0;+ float *w_dev=ctx->ac+D, *partial_local=ctx->ac;+ if(!cuda_ok(cudaMemcpyAsync(ctx->group_desc,host,(size_t)count*sizeof(GroupDesc),+ cudaMemcpyHostToDevice,ctx->stream),"resident group desc")||+ !cuda_ok(cudaMemcpyAsync(w_dev,weights,(size_t)count*sizeof(float),+ cudaMemcpyHostToDevice,ctx->stream),"resident group weights"))+ return 0;+ /* input row: P2P from the home device. The caller guarantees x_src_dev is+ * materialized (the pre-moe nrm download already synced the home stream). */+ if(!cuda_ok(cudaMemcpyPeerAsync(ctx->x,device,x_src_dev,home_device,+ (size_t)D*sizeof(float),ctx->stream),"resident group x p2p"))+ return 0;+ bcast_row<<<64,256,0,ctx->stream>>>(ctx->x,ctx->x,count,D); /* row 0 -> rows 1..count-1 (in-place safe: row 0 rewritten with itself) */+ GroupDesc *dev=(GroupDesc*)ctx->group_desc;+ dim3 hg((unsigned)I,1,(unsigned)count),og((unsigned)D,1,(unsigned)count);+ grouped_hidden_w4_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->up,ctx->x,dev,I,D);+ silu_mul<<<(unsigned)(((size_t)count*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)count*I);+ grouped_down_w4<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);+ weighted_sum_rows<<<48,256,0,ctx->stream>>>(partial_local,ctx->y,w_dev,count,D);+ if(!cuda_ok(cudaMemcpyPeerAsync(partial_slot_dev,home_device,partial_local,device,+ (size_t)D*sizeof(float),ctx->stream),"resident partial p2p"))+ return 0;+ if(!cuda_ok(cudaEventRecord(ctx->ev_done,ctx->stream),"resident event record")) return 0;+ return cuda_ok(cudaGetLastError(),"resident group launch");+}+extern "C" int coli_cuda_expert_group_resident_take(int home_device,const int *devices,int n_issued,+ float *slots_dev,float *acc_dev,int D){+ if(n_issued<1||!slots_dev||!acc_dev||D<1) return 0;+ DeviceContext *home=find_ctx(home_device); if(!select_ctx(home)) return 0;+ for(int i=0;i<n_issued;i++){+ DeviceContext *src=find_ctx(devices[i]);+ if(!src||!src->ev_done_ok) return 0;+ if(!cuda_ok(cudaStreamWaitEvent(0,src->ev_done,0),"resident take wait")) return 0;+ }+ sum_slots<<<48,256>>>(acc_dev,slots_dev,n_issued,D); /* legacy stream: ordered with pipe_* */+ return cuda_ok(cudaGetLastError(),"resident take reduce");+}extern "C" int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,int spitch,int width,int height){DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0;diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex dfe38091..aad43f78 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -139,6 +139,12 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const fint xstride,int ystride);COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,int stride,int offset,int R,int heads,float theta);+COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates,+ ColiCudaTensor *const *ups, ColiCudaTensor *const *downs,+ const float *weights, int count,+ int home_device, const float *x_src_dev, float *partial_slot_dev);+COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_take(int home_device,const int *devices,+ int n_issued,float *slots_dev,float *acc_dev,int D);COLI_CUDA_DLLEXPORT int coli_cuda_pipe_router(int device,const float *x_dev,const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,float topp,int norm_topk,float routed_scale,diff --git a/c/backend_loader.c b/c/backend_loader.cindex feb908e6..e8f03541 100644--- a/c/backend_loader.c+++ b/c/backend_loader.c@@ -82,6 +82,8 @@ typedef int (*fn_pipe_gemm)(ColiCudaTensor *t,float *y_dev,const float *x_dev,intypedef int (*fn_pipe_peer_copy)(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes);typedef int (*fn_pipe_rmsnorm)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps);typedef int (*fn_pipe_rmsnorm_s)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride);+typedef int (*fn_group_resident_issue)(ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,ColiCudaTensor *const *downs,const float *weights,int count,int home_device,const float *x_src_dev,float *partial_slot_dev);+typedef int (*fn_group_resident_take)(int home_device,const int *devices,int n_issued,float *slots_dev,float *acc_dev,int D);typedef int (*fn_pipe_router)(int device,const float *x_dev,const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,float topp,int norm_topk,float routed_scale,int *idx_host,float *w_host,int *keff_host);typedef int (*fn_pipe_rope)(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta);typedef int (*fn_pipe_rope_base)(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta);@@ -133,6 +135,8 @@ static struct {fn_pipe_peer_copy pipe_peer_copy;fn_pipe_rmsnorm pipe_rmsnorm;fn_pipe_rmsnorm_s pipe_rmsnorm_s;+ fn_group_resident_issue expert_group_resident_issue;+ fn_group_resident_take expert_group_resident_take;fn_pipe_router pipe_router;fn_pipe_rope pipe_rope;fn_pipe_rope_base pipe_rope_base;@@ -231,6 +235,8 @@ static int coli_cuda_load(void){RESOLVE(pipe_peer_copy, fn_pipe_peer_copy)RESOLVE(pipe_rmsnorm, fn_pipe_rmsnorm)RESOLVE(pipe_rmsnorm_s, fn_pipe_rmsnorm_s)+ RESOLVE(expert_group_resident_issue, fn_group_resident_issue)+ RESOLVE(expert_group_resident_take, fn_group_resident_take)RESOLVE(pipe_router, fn_pipe_router)RESOLVE(pipe_rope, fn_pipe_rope)RESOLVE(pipe_rope_base, fn_pipe_rope_base)@@ -440,6 +446,16 @@ int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const floreturn g_cuda.pipe_rmsnorm(device, y_dev, x_dev, w_dev, S, D, eps);}+int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,ColiCudaTensor *const *downs,const float *weights,int count,int home_device,const float *x_src_dev,float *partial_slot_dev){+ if(!g_cuda.available || !g_cuda.expert_group_issue){ return 0; }+ return g_cuda.expert_group_resident_issue(gates, ups, downs, weights, count, home_device, x_src_dev, partial_slot_dev);+}++int coli_cuda_expert_group_resident_take(int home_device,const int *devices,int n_issued,float *slots_dev,float *acc_dev,int D){+ if(!g_cuda.available || !g_cuda.expert_group_take){ return 0; }+ return g_cuda.expert_group_resident_take(home_device, devices, n_issued, slots_dev, acc_dev, D);+}+int coli_cuda_pipe_router(int device,const float *x_dev,const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,float topp,int norm_topk,float routed_scale,int *idx_host,float *w_host,int *keff_host){if(!g_cuda.available || !g_cuda.pipe_router){ return 0; }return g_cuda.pipe_router(device, x_dev, rw_dev, rb_dev, D, E, Ksel, topp, norm_topk, routed_scale, idx_host, w_host, keff_host);diff --git a/c/colibri.c b/c/colibri.cindex ef32b016..d182e59e 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -1913,6 +1913,13 @@ static int g_absorb=-1;#ifdef COLI_CUDAstatic int g_cuda_pipe=0; /* COLI_CUDA_PIPE=1: prefill attention chain resident on the layer home device */static int g_cuda_router=0; /* COLI_CUDA_ROUTER=1 (#431 PR-A): router on the layer home device at decode */+static int g_cuda_resid=0; /* COLI_CUDA_RESID=1 (#431 PR-C0): expert-group results stay on device */+/* pipe -> moe handoff for the resident group path (set per layer by+ * pipe_layer_sparse, consumed by moe()'s group dispatch, cleared after take) */+static int g_pres_home=-1; /* home device, -1 = path off */+static const float *g_pres_xsrc; /* nrm_d on the home device */+static float *g_pres_slots; /* [ndev][D] partial slots on home */+static int g_pres_used[COLI_CUDA_MAX_DEVICES], g_pres_nused;#endif /* ABSORB: -1 auto (decode S<=4), 0 mai, 1 sempre (test) */static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min(k,T)=denso) */static int cmp_fdesc(const void *a,const void *b){@@ -3085,14 +3092,31 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(issued[di]) coli_cuda_expert_group_take(g_cuda_devices[di]);}if(!async_done)+ /* resident path (#431 PR-C0): issue the group on its device with the input+ * P2P'd from the home device and the weighted partial pushed back there β+ * no host bytes, no per-device sync; take() runs after moe() returns.+ * Issue is serial (it only queues async work); failure falls back to the+ * synchronous host-round-trip call below, per device. */+ if(g_pres_home>=0 && S==1){+ for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]){+ float wbuf[64];+ for(int q=0;q<dev_nc[di];q++) wbuf[q]=group_weight[(int64_t)dev_which[di][q]*S];+ if(coli_cuda_expert_group_resident_issue(dev_g[di],dev_u[di],dev_d[di],wbuf,dev_nc[di],+ g_pres_home,g_pres_xsrc,g_pres_slots+(int64_t)g_pres_nused*D)){+ g_pres_used[g_pres_nused++]=g_cuda_devices[di];+ dev_ok[di]=2; /* handled on device: skip host collection */+ }+ }+ }#pragma omp parallel for if(g_cuda_ndev>1) schedule(static)- for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]){+ for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]&&dev_ok[di]!=2){double td=g_prof?now_s():0;dev_ok[di]=coli_cuda_expert_group(dev_g[di],dev_u[di],dev_d[di],dev_rows[di],dev_nc[di],group_y+(int64_t)dev_off[di]*D,group_x+(int64_t)dev_off[di]*D);if(g_prof)dev_time[di]=now_s()-td;}for(int di=0;di<g_cuda_ndev;di++){+ if(dev_ok[di]==2) continue; /* results live on the home device (#431 PR-C0) */int off=dev_off[di];for(int q=0;q<dev_nc[di];q++){int gi=dev_which[di][q],nr=group_n[gi]; ESlot *e=group_e[gi];@@ -3631,8 +3655,29 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, inm->t_emm += now_s()-te; /* shared-expert GPU dispatch only *//* expert routed su CPU/gruppi GPU come oggi (shared saltata: la fa il device) */if(dev_routed){ g_pre_idx=lr_idx; g_pre_w=lr_w; g_pre_keff=lr_keff; }+ float *res_acc=NULL;+ if(g_cuda_resid && S==1){+ g_pres_slots=coli_cuda_pipe_scratch(dev,25,(size_t)g_cuda_ndev*D*sizeof(float));+ res_acc =coli_cuda_pipe_scratch(dev,26,(size_t)D*sizeof(float));+ if(g_pres_slots && res_acc){ g_pres_home=dev; g_pres_xsrc=nrm_d; g_pres_nused=0; }+ else { g_pres_slots=NULL; res_acc=NULL; }+ }moe(m,l,li,nrm_host,S,out_host,0); /* self-times its own t_emm */if(dev_routed){ g_pre_idx=NULL; g_pre_w=NULL; g_pre_keff=NULL; }+ if(g_pres_home>=0){+ int nused=g_pres_nused; g_pres_home=-1; g_pres_xsrc=NULL;+ if(nused>0){+ /* partials are in flight toward our slots; take() orders the legacy+ * stream behind every issue event and reduces in issue order. A+ * failure here would silently drop routed experts β that is a wrong+ * answer, not a slow one, so it is fatal by design. */+ if(!coli_cuda_expert_group_resident_take(dev,g_pres_used,nused,g_pres_slots,res_acc,D)||+ !coli_cuda_pipe_add(dev,x_dev,res_acc,(size_t)D)){+ fprintf(stderr,"[CUDA] resident expert take failed β refusing to drop routed experts\n");+ exit(1);+ }+ }+ }te=now_s();if(!coli_cuda_pipe_upload(dev,y_d,out_host,xb)) return 0; /* sync: waits for moe */if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0; /* routed residual (async) */@@ -5883,6 +5928,7 @@ int main(int argc, char **argv){g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0;g_cuda_pipe=getenv("COLI_CUDA_PIPE")?atoi(getenv("COLI_CUDA_PIPE")):0;g_cuda_router=getenv("COLI_CUDA_ROUTER")?atoi(getenv("COLI_CUDA_ROUTER")):0;+ g_cuda_resid=getenv("COLI_CUDA_RESID")?atoi(getenv("COLI_CUDA_RESID")):0;const char *cuda_expert=getenv("CUDA_EXPERT_GB");g_cuda_expert_auto=cuda_expert&&!strcmp(cuda_expert,"auto");g_cuda_expert_gb=cuda_expert&&!g_cuda_expert_auto?atof(cuda_expert):0;