Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: COLI_CUDA_RESID=1 β€” expert-group results stay on device (#431 PR-C0)

ZacharyZcR0f8cbe290417
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index a0083f40..23123782 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -38,6 +38,7 @@ typedef struct {
float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap;
float *pipe_buf[27]; size_t pipe_cap[27]; /* scratch persistenti del resident pipeline */
cudaStream_t stream;
+ cudaEvent_t ev_done; int ev_done_ok; /* resident-group issue completion (#431 PR-C0) */
void *group_desc; size_t group_desc_cap;
size_t tensor_count, tensor_bytes;
int group_pending; size_t group_pending_bytes; /* async expert-group in flight (Inc.4) */
@@ -1294,6 +1295,106 @@ extern "C" int coli_cuda_pipe_router(int device,const float *x_dev,
memcpy(keff_host,buf+Ksel*(sizeof(int)+sizeof(float)),sizeof(int));
return 1;
}
+/* ---- resident expert-group accumulation (#431 PR-C0) ----------------------
+ * Decode-time (S=1) expert groups without the host round-trip: the input row
+ * is P2P'd from the layer's home device, the group runs through the grouped-W4
+ * kernels on its own stream, the down-projection outputs are weighted and
+ * reduced ON DEVICE (fixed expert order), and the device's partial sum is
+ * peer-pushed into a per-issue slot on the home device. take() makes the home
+ * legacy stream wait on every issue event and reduces the slots in issue order
+ * β€” deterministic, no atomics, no host bytes. The CPU tier overlaps with all
+ * of it exactly as before. */
+__global__ static void bcast_row(float *dst,const float *src,int count,int D){
+ for(int i=blockIdx.x*blockDim.x+threadIdx.x;i<D;i+=gridDim.x*blockDim.x){
+ float v=src[i];
+ for(int c=0;c<count;c++) dst[(size_t)c*D+i]=v;
+ }
+}
+__global__ static void weighted_sum_rows(float *out,const float *y,const float *w,
+ int count,int D){
+ for(int i=blockIdx.x*blockDim.x+threadIdx.x;i<D;i+=gridDim.x*blockDim.x){
+ float acc=0.f;
+ for(int c=0;c<count;c++) acc+=w[c]*y[(size_t)c*D+i]; /* fixed order */
+ out[i]=acc;
+ }
+}
+__global__ static void sum_slots(float *dst,const float *slots,int n,int D){
+ for(int i=blockIdx.x*blockDim.x+threadIdx.x;i<D;i+=gridDim.x*blockDim.x){
+ float acc=0.f;
+ for(int s=0;s<n;s++) acc+=slots[(size_t)s*D+i]; /* issue order */
+ dst[i]=acc;
+ }
+}
+extern "C" int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates,
+ ColiCudaTensor *const *ups, ColiCudaTensor *const *downs,
+ const float *weights, int count,
+ int home_device, const float *x_src_dev, float *partial_slot_dev){
+ if(!gates||!ups||!downs||!weights||count<1||count>64||!x_src_dev||!partial_slot_dev) return 0;
+ ColiCudaTensor *first=gates[0]; if(!first) return 0;
+ int device=first->device,D=first->I,I=first->O;
+ GroupDesc host[64];
+ int total=0,all_s4=1;
+ for(int c=0;c<count;c++){
+ ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];
+ if(!g||!u||!d||g->device!=device||u->device!=device||d->device!=device||
+ g->I!=D||u->I!=D||g->O!=I||u->O!=I||d->I!=I||d->O!=D) return 0;
+ host[c]={g->weights,u->weights,d->weights,g->scales,u->scales,d->scales,
+ g->fmt,u->fmt,d->fmt,1,total};
+ all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2;
+ total++;
+ }
+ if(!all_s4) return 0; /* resident path: per-row int4 only */
+ DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0;
+ if(!ctx->ev_done_ok){
+ if(!cuda_ok(cudaEventCreateWithFlags(&ctx->ev_done,cudaEventDisableTiming),
+ "resident group event")) return 0;
+ ctx->ev_done_ok=1;
+ }
+ /* size for the 64-expert cap, not for `count`: reserve() reallocs on growth,
+ * and a realloc here could free a buffer the PREVIOUS layer's still-queued
+ * async work on this stream reads. Fixed caps make re-issue realloc-free. */
+ size_t xb=(size_t)64*D*sizeof(float), ib=(size_t)64*I*sizeof(float);
+ if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)||
+ !reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)||
+ !reserve(&ctx->ac,&ctx->ac_cap,(size_t)(D+64)*sizeof(float))||
+ !reserve_bytes(&ctx->group_desc,&ctx->group_desc_cap,(size_t)64*sizeof(GroupDesc)))
+ return 0;
+ float *w_dev=ctx->ac+D, *partial_local=ctx->ac;
+ if(!cuda_ok(cudaMemcpyAsync(ctx->group_desc,host,(size_t)count*sizeof(GroupDesc),
+ cudaMemcpyHostToDevice,ctx->stream),"resident group desc")||
+ !cuda_ok(cudaMemcpyAsync(w_dev,weights,(size_t)count*sizeof(float),
+ cudaMemcpyHostToDevice,ctx->stream),"resident group weights"))
+ return 0;
+ /* input row: P2P from the home device. The caller guarantees x_src_dev is
+ * materialized (the pre-moe nrm download already synced the home stream). */
+ if(!cuda_ok(cudaMemcpyPeerAsync(ctx->x,device,x_src_dev,home_device,
+ (size_t)D*sizeof(float),ctx->stream),"resident group x p2p"))
+ return 0;
+ bcast_row<<<64,256,0,ctx->stream>>>(ctx->x,ctx->x,count,D); /* row 0 -> rows 1..count-1 (in-place safe: row 0 rewritten with itself) */
+ GroupDesc *dev=(GroupDesc*)ctx->group_desc;
+ dim3 hg((unsigned)I,1,(unsigned)count),og((unsigned)D,1,(unsigned)count);
+ grouped_hidden_w4_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->up,ctx->x,dev,I,D);
+ silu_mul<<<(unsigned)(((size_t)count*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)count*I);
+ grouped_down_w4<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);
+ weighted_sum_rows<<<48,256,0,ctx->stream>>>(partial_local,ctx->y,w_dev,count,D);
+ if(!cuda_ok(cudaMemcpyPeerAsync(partial_slot_dev,home_device,partial_local,device,
+ (size_t)D*sizeof(float),ctx->stream),"resident partial p2p"))
+ return 0;
+ if(!cuda_ok(cudaEventRecord(ctx->ev_done,ctx->stream),"resident event record")) return 0;
+ return cuda_ok(cudaGetLastError(),"resident group launch");
+}
+extern "C" int coli_cuda_expert_group_resident_take(int home_device,const int *devices,int n_issued,
+ float *slots_dev,float *acc_dev,int D){
+ if(n_issued<1||!slots_dev||!acc_dev||D<1) return 0;
+ DeviceContext *home=find_ctx(home_device); if(!select_ctx(home)) return 0;
+ for(int i=0;i<n_issued;i++){
+ DeviceContext *src=find_ctx(devices[i]);
+ if(!src||!src->ev_done_ok) return 0;
+ if(!cuda_ok(cudaStreamWaitEvent(0,src->ev_done,0),"resident take wait")) return 0;
+ }
+ sum_slots<<<48,256>>>(acc_dev,slots_dev,n_issued,D); /* legacy stream: ordered with pipe_* */
+ return cuda_ok(cudaGetLastError(),"resident take reduce");
+}
extern "C" int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,
int spitch,int width,int height){
DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0;
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index dfe38091..aad43f78 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -139,6 +139,12 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const f
int xstride,int ystride);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,
int stride,int offset,int R,int heads,float theta);
+COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gates,
+ ColiCudaTensor *const *ups, ColiCudaTensor *const *downs,
+ const float *weights, int count,
+ int home_device, const float *x_src_dev, float *partial_slot_dev);
+COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_resident_take(int home_device,const int *devices,
+ int n_issued,float *slots_dev,float *acc_dev,int D);
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_router(int device,const float *x_dev,
const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,
float topp,int norm_topk,float routed_scale,
diff --git a/c/backend_loader.c b/c/backend_loader.c
index feb908e6..e8f03541 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -82,6 +82,8 @@ typedef int (*fn_pipe_gemm)(ColiCudaTensor *t,float *y_dev,const float *x_dev,in
typedef int (*fn_pipe_peer_copy)(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes);
typedef int (*fn_pipe_rmsnorm)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps);
typedef int (*fn_pipe_rmsnorm_s)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride);
+typedef int (*fn_group_resident_issue)(ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,ColiCudaTensor *const *downs,const float *weights,int count,int home_device,const float *x_src_dev,float *partial_slot_dev);
+typedef int (*fn_group_resident_take)(int home_device,const int *devices,int n_issued,float *slots_dev,float *acc_dev,int D);
typedef int (*fn_pipe_router)(int device,const float *x_dev,const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,float topp,int norm_topk,float routed_scale,int *idx_host,float *w_host,int *keff_host);
typedef int (*fn_pipe_rope)(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta);
typedef int (*fn_pipe_rope_base)(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta);
@@ -133,6 +135,8 @@ static struct {
fn_pipe_peer_copy pipe_peer_copy;
fn_pipe_rmsnorm pipe_rmsnorm;
fn_pipe_rmsnorm_s pipe_rmsnorm_s;
+ fn_group_resident_issue expert_group_resident_issue;
+ fn_group_resident_take expert_group_resident_take;
fn_pipe_router pipe_router;
fn_pipe_rope pipe_rope;
fn_pipe_rope_base pipe_rope_base;
@@ -231,6 +235,8 @@ static int coli_cuda_load(void){
RESOLVE(pipe_peer_copy, fn_pipe_peer_copy)
RESOLVE(pipe_rmsnorm, fn_pipe_rmsnorm)
RESOLVE(pipe_rmsnorm_s, fn_pipe_rmsnorm_s)
+ RESOLVE(expert_group_resident_issue, fn_group_resident_issue)
+ RESOLVE(expert_group_resident_take, fn_group_resident_take)
RESOLVE(pipe_router, fn_pipe_router)
RESOLVE(pipe_rope, fn_pipe_rope)
RESOLVE(pipe_rope_base, fn_pipe_rope_base)
@@ -440,6 +446,16 @@ int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const flo
return g_cuda.pipe_rmsnorm(device, y_dev, x_dev, w_dev, S, D, eps);
}
+int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,ColiCudaTensor *const *downs,const float *weights,int count,int home_device,const float *x_src_dev,float *partial_slot_dev){
+ if(!g_cuda.available || !g_cuda.expert_group_issue){ return 0; }
+ return g_cuda.expert_group_resident_issue(gates, ups, downs, weights, count, home_device, x_src_dev, partial_slot_dev);
+}
+
+int coli_cuda_expert_group_resident_take(int home_device,const int *devices,int n_issued,float *slots_dev,float *acc_dev,int D){
+ if(!g_cuda.available || !g_cuda.expert_group_take){ return 0; }
+ return g_cuda.expert_group_resident_take(home_device, devices, n_issued, slots_dev, acc_dev, D);
+}
+
int coli_cuda_pipe_router(int device,const float *x_dev,const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,float topp,int norm_topk,float routed_scale,int *idx_host,float *w_host,int *keff_host){
if(!g_cuda.available || !g_cuda.pipe_router){ return 0; }
return g_cuda.pipe_router(device, x_dev, rw_dev, rb_dev, D, E, Ksel, topp, norm_topk, routed_scale, idx_host, w_host, keff_host);
diff --git a/c/colibri.c b/c/colibri.c
index ef32b016..d182e59e 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -1913,6 +1913,13 @@ static int g_absorb=-1;
#ifdef COLI_CUDA
static int g_cuda_pipe=0; /* COLI_CUDA_PIPE=1: prefill attention chain resident on the layer home device */
static int g_cuda_router=0; /* COLI_CUDA_ROUTER=1 (#431 PR-A): router on the layer home device at decode */
+static int g_cuda_resid=0; /* COLI_CUDA_RESID=1 (#431 PR-C0): expert-group results stay on device */
+/* pipe -> moe handoff for the resident group path (set per layer by
+ * pipe_layer_sparse, consumed by moe()'s group dispatch, cleared after take) */
+static int g_pres_home=-1; /* home device, -1 = path off */
+static const float *g_pres_xsrc; /* nrm_d on the home device */
+static float *g_pres_slots; /* [ndev][D] partial slots on home */
+static int g_pres_used[COLI_CUDA_MAX_DEVICES], g_pres_nused;
#endif /* ABSORB: -1 auto (decode S<=4), 0 mai, 1 sempre (test) */
static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min(k,T)=denso) */
static int cmp_fdesc(const void *a,const void *b){
@@ -3085,14 +3092,31 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(issued[di]) coli_cuda_expert_group_take(g_cuda_devices[di]);
}
if(!async_done)
+ /* resident path (#431 PR-C0): issue the group on its device with the input
+ * P2P'd from the home device and the weighted partial pushed back there β€”
+ * no host bytes, no per-device sync; take() runs after moe() returns.
+ * Issue is serial (it only queues async work); failure falls back to the
+ * synchronous host-round-trip call below, per device. */
+ if(g_pres_home>=0 && S==1){
+ for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]){
+ float wbuf[64];
+ for(int q=0;q<dev_nc[di];q++) wbuf[q]=group_weight[(int64_t)dev_which[di][q]*S];
+ if(coli_cuda_expert_group_resident_issue(dev_g[di],dev_u[di],dev_d[di],wbuf,dev_nc[di],
+ g_pres_home,g_pres_xsrc,g_pres_slots+(int64_t)g_pres_nused*D)){
+ g_pres_used[g_pres_nused++]=g_cuda_devices[di];
+ dev_ok[di]=2; /* handled on device: skip host collection */
+ }
+ }
+ }
#pragma omp parallel for if(g_cuda_ndev>1) schedule(static)
- for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]){
+ for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di]&&dev_ok[di]!=2){
double td=g_prof?now_s():0;
dev_ok[di]=coli_cuda_expert_group(dev_g[di],dev_u[di],dev_d[di],dev_rows[di],dev_nc[di],
group_y+(int64_t)dev_off[di]*D,group_x+(int64_t)dev_off[di]*D);
if(g_prof)dev_time[di]=now_s()-td;
}
for(int di=0;di<g_cuda_ndev;di++){
+ if(dev_ok[di]==2) continue; /* results live on the home device (#431 PR-C0) */
int off=dev_off[di];
for(int q=0;q<dev_nc[di];q++){
int gi=dev_which[di][q],nr=group_n[gi]; ESlot *e=group_e[gi];
@@ -3631,8 +3655,29 @@ static int pipe_layer_sparse(Model *m, Layer *l, int li, float *x_dev, int S, in
m->t_emm += now_s()-te; /* shared-expert GPU dispatch only */
/* expert routed su CPU/gruppi GPU come oggi (shared saltata: la fa il device) */
if(dev_routed){ g_pre_idx=lr_idx; g_pre_w=lr_w; g_pre_keff=lr_keff; }
+ float *res_acc=NULL;
+ if(g_cuda_resid && S==1){
+ g_pres_slots=coli_cuda_pipe_scratch(dev,25,(size_t)g_cuda_ndev*D*sizeof(float));
+ res_acc =coli_cuda_pipe_scratch(dev,26,(size_t)D*sizeof(float));
+ if(g_pres_slots && res_acc){ g_pres_home=dev; g_pres_xsrc=nrm_d; g_pres_nused=0; }
+ else { g_pres_slots=NULL; res_acc=NULL; }
+ }
moe(m,l,li,nrm_host,S,out_host,0); /* self-times its own t_emm */
if(dev_routed){ g_pre_idx=NULL; g_pre_w=NULL; g_pre_keff=NULL; }
+ if(g_pres_home>=0){
+ int nused=g_pres_nused; g_pres_home=-1; g_pres_xsrc=NULL;
+ if(nused>0){
+ /* partials are in flight toward our slots; take() orders the legacy
+ * stream behind every issue event and reduces in issue order. A
+ * failure here would silently drop routed experts β€” that is a wrong
+ * answer, not a slow one, so it is fatal by design. */
+ if(!coli_cuda_expert_group_resident_take(dev,g_pres_used,nused,g_pres_slots,res_acc,D)||
+ !coli_cuda_pipe_add(dev,x_dev,res_acc,(size_t)D)){
+ fprintf(stderr,"[CUDA] resident expert take failed β€” refusing to drop routed experts\n");
+ exit(1);
+ }
+ }
+ }
te=now_s();
if(!coli_cuda_pipe_upload(dev,y_d,out_host,xb)) return 0; /* sync: waits for moe */
if(!coli_cuda_pipe_add(dev,x_dev,y_d,(size_t)S*D)) return 0; /* routed residual (async) */
@@ -5883,6 +5928,7 @@ int main(int argc, char **argv){
g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0;
g_cuda_pipe=getenv("COLI_CUDA_PIPE")?atoi(getenv("COLI_CUDA_PIPE")):0;
g_cuda_router=getenv("COLI_CUDA_ROUTER")?atoi(getenv("COLI_CUDA_ROUTER")):0;
+ g_cuda_resid=getenv("COLI_CUDA_RESID")?atoi(getenv("COLI_CUDA_RESID")):0;
const char *cuda_expert=getenv("CUDA_EXPERT_GB");
g_cuda_expert_auto=cuda_expert&&!strcmp(cuda_expert,"auto");
g_cuda_expert_gb=cuda_expert&&!g_cuda_expert_auto?atof(cuda_expert):0;