Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda: add paged ragged KV research runtime
ZacharyZcR2c5be24fb818
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 0ccd1496..9ce0a650 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -27,10 +27,25 @@struct RaggedKVEntry {const void *key;const float *host_l,*host_r;- float *latent,*rope;- int length,capacity,K,R;+ float **latent_pages,**rope_pages;+ int length,page_count,K,R;};+#ifndef COLI_KV_PAGE_TOKENS+#define COLI_KV_PAGE_TOKENS 64+#endif++static void ragged_kv_clear(RaggedKVEntry *e) {+ for (int i=0;i<e->page_count;i++) {+ if (e->latent_pages[i]) cudaFree(e->latent_pages[i]);+ if (e->rope_pages[i]) cudaFree(e->rope_pages[i]);+ }+ std::free(e->latent_pages);+ std::free(e->rope_pages);+ e->latent_pages=e->rope_pages=nullptr;+ e->length=e->page_count=0;+}+struct ColiCudaTensor {void *weights;float *scales;@@ -86,6 +101,12 @@ static DeviceContext g_ctx[COLI_CUDA_MAX_DEVICES];static int g_nctx;static uint64_t g_group_calls,g_group_experts,g_group_rows;static double g_group_h2d_ms,g_group_kernel_ms,g_group_d2h_ms;+static uint64_t g_device_group_calls[COLI_CUDA_MAX_DEVICES];+static uint64_t g_device_group_experts[COLI_CUDA_MAX_DEVICES];+static uint64_t g_device_group_rows[COLI_CUDA_MAX_DEVICES];+static double g_device_group_h2d_ms[COLI_CUDA_MAX_DEVICES];+static double g_device_group_kernel_ms[COLI_CUDA_MAX_DEVICES];+static double g_device_group_d2h_ms[COLI_CUDA_MAX_DEVICES];static std::mutex g_group_stats_mu;#ifdef COLI_ANSstatic FILE *g_ans_sidecar;@@ -634,18 +655,18 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q,__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,const float *const *latent,const float *const *rope,const int *lengths,const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R,- int V,int K,int T,float scale,int gs,int ng){+ int V,int K,int T,int page_stride,float scale,int gs,int ng){int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V);if(s>=S||nt<1||nt>T)return;extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T;const float *qs=q+((size_t)s*H+h)*(Q+R);- const float *ls=latent[s],*rs=rope[s];for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int d=0;d<Q;d++)a+=qs[d]*weight_at(weights,fmt,(size_t)(rbase+d)*row_bytes(fmt,K),k)*absorb_scale(wscale,fmt,gs,ng,rbase+d,k);qa[k]=a;}__syncthreads();- for(int t=tid;t<nt;t+=blockDim.x){float a=0;const float *lt=ls+(size_t)t*K;- const float *rt=rs+(size_t)t*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];+ for(int t=tid;t<nt;t+=blockDim.x){float a=0;int pg=t/COLI_KV_PAGE_TOKENS,pt=t%COLI_KV_PAGE_TOKENS;+ const float *lt=latent[(size_t)s*page_stride+pg]+(size_t)pt*K;+ const float *rt=rope[(size_t)s*page_stride+pg]+(size_t)pt*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];for(int d=0;d<R;d++)a+=qs[Q+d]*rt[d];scores[t]=a*scale;}__syncthreads();float local=-3.402823466e+38F;for(int t=tid;t<nt;t+=blockDim.x)local=fmaxf(local,scores[t]);@@ -656,7 +677,10 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,for(int n=blockDim.x>>1;n;n>>=1){if(tid<n)red[tid]+=red[tid+n];__syncthreads();}float inv=1.f/red[0];for(int t=tid;t<nt;t+=blockDim.x)scores[t]*=inv;__syncthreads();- for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++)a+=scores[t]*ls[(size_t)t*K+k];cl[k]=a;}+ for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++){+ const float *lt=latent[(size_t)s*page_stride+t/COLI_KV_PAGE_TOKENS]++ (size_t)(t%COLI_KV_PAGE_TOKENS)*K;+ a+=scores[t]*lt[k];}cl[k]=a;}__syncthreads();for(int v=tid;v<V;v+=blockDim.x){int row=rbase+Q+v;float a=0;size_t rb=row_bytes(fmt,K);for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k)*@@ -665,11 +689,15 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,}__global__ static void ragged_kv_append(float *const *latent,float *const *rope,- const float *packed,const int *old_len,const int *add,const int *offset,int K,int R){+ const float *packed,const int *old_len,const int *add,const int *offset,+ int K,int R,int page_stride){int s=blockIdx.x,n=add[s],base=offset[s];- for(int i=threadIdx.x;i<n*(K+R);i+=blockDim.x){- if(i<n*K)latent[s][(size_t)old_len[s]*K+i]=packed[base+i];- else rope[s][(size_t)old_len[s]*R+i-n*K]=packed[base+i];+ for(int t=0;t<n;t++){+ int pos=old_len[s]+t,pg=pos/COLI_KV_PAGE_TOKENS,pt=pos%COLI_KV_PAGE_TOKENS;+ float *lp=latent[(size_t)s*page_stride+pg]+(size_t)pt*K;+ float *rp=rope[(size_t)s*page_stride+pg]+(size_t)pt*R;+ for(int k=threadIdx.x;k<K;k+=blockDim.x)lp[k]=packed[base+(size_t)t*K+k];+ for(int r=threadIdx.x;r<R;r+=blockDim.x)rp[r]=packed[base+(size_t)n*K+(size_t)t*R+r];}}@@ -907,6 +935,20 @@ extern "C" void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64if(d2h_ms) *d2h_ms=g_group_d2h_ms;}+extern "C" void coli_cuda_group_stats_device(+ int device, uint64_t *calls, uint64_t *experts, uint64_t *rows,+ double *h2d_ms, double *kernel_ms, double *d2h_ms) {+ std::lock_guard<std::mutex> lock(g_group_stats_mu);+ int index=-1;+ for(int i=0;i<g_nctx;i++) if(g_ctx[i].device==device){ index=i; break; }+ if(calls) *calls=index<0?0:g_device_group_calls[index];+ if(experts) *experts=index<0?0:g_device_group_experts[index];+ if(rows) *rows=index<0?0:g_device_group_rows[index];+ if(h2d_ms) *h2d_ms=index<0?0:g_device_group_h2d_ms[index];+ if(kernel_ms) *kernel_ms=index<0?0:g_device_group_kernel_ms[index];+ if(d2h_ms) *d2h_ms=index<0?0:g_device_group_d2h_ms[index];+}+/* group size for the NEXT upload on this thread (fmt=4): routed through a* thread_local so the widely-wired upload signature (and the Windows DLL ABI)* stays untouched. pin_load uploads in parallel, hence thread_local. */@@ -1423,11 +1465,18 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,cudaEventElapsedTime(&a,ev[0],ev[1]); cudaEventElapsedTime(&b,ev[1],ev[2]);cudaEventElapsedTime(&c,ev[2],ev[3]);{ std::lock_guard<std::mutex> lock(g_group_stats_mu);- g_group_h2d_ms+=a; g_group_kernel_ms+=b; g_group_d2h_ms+=c; }+ int index=(int)(ctx-g_ctx);+ g_group_h2d_ms+=a; g_group_kernel_ms+=b; g_group_d2h_ms+=c;+ g_device_group_h2d_ms[index]+=a;+ g_device_group_kernel_ms[index]+=b;+ g_device_group_d2h_ms[index]+=c; }for(int i=0;i<4;i++) cudaEventDestroy(ev[i]);}{ std::lock_guard<std::mutex> lock(g_group_stats_mu);- g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total; }+ int index=(int)(ctx-g_ctx);+ g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total;+ g_device_group_calls[index]++; g_device_group_experts[index]+=(uint64_t)count;+ g_device_group_rows[index]+=(uint64_t)total; }return 1;}@@ -1516,7 +1565,10 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,"expert group issue download")) return 0;ctx->group_pending=1; ctx->group_pending_bytes=xb;{ std::lock_guard<std::mutex> lock(g_group_stats_mu);- g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total; }+ int index=(int)(ctx-g_ctx);+ g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total;+ g_device_group_calls[index]++; g_device_group_experts[index]+=(uint64_t)count;+ g_device_group_rows[index]+=(uint64_t)total; }return 1;}@@ -1607,50 +1659,66 @@ extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensproj->device!=w->device||proj->I!=H*V)return 0;DeviceContext *dc=find_ctx(w->device);if(!select_ctx(dc))return 0;- float **dl=(float**)std::malloc((size_t)S*sizeof(*dl));- float **dr=(float**)std::malloc((size_t)S*sizeof(*dr));int *old=(int*)std::malloc((size_t)S*sizeof(*old));int *add=(int*)std::malloc((size_t)S*sizeof(*add));int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0;- if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}+ if(!old||!add||!off){std::free(old);std::free(add);std::free(off);return 0;}+ int page_stride=0;for(int s=0;s<S;s++){- if(!keys[s]||lengths[s]<1||lengths[s]>T){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}+ if(!keys[s]||lengths[s]<1||lengths[s]>T){std::free(old);std::free(add);std::free(off);return 0;}RaggedKVEntry *e=nullptr;for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;}if(!e){- if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}+ if(w->ragged_count>=512){std::free(old);std::free(add);std::free(off);return 0;}e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s];}if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]<e->length){- if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);- e->latent=e->rope=nullptr;e->length=e->capacity=0;+ ragged_kv_clear(e);e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s];}- if(lengths[s]>e->capacity){- int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr;- if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")||- !cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){- if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;+ int need=(lengths[s]+COLI_KV_PAGE_TOKENS-1)/COLI_KV_PAGE_TOKENS;+ if(need>e->page_count){+ float **nl=(float**)std::calloc((size_t)need,sizeof(*nl));+ float **nr=(float**)std::calloc((size_t)need,sizeof(*nr));+ if(!nl||!nr){std::free(nl);std::free(nr);std::free(old);std::free(add);std::free(off);return 0;}+ for(int i=0;i<e->page_count;i++){nl[i]=e->latent_pages[i];nr[i]=e->rope_pages[i];}+ int made=e->page_count;+ for(;made<need;made++){+ if(!cuda_ok(cudaMalloc(&nl[made],(size_t)COLI_KV_PAGE_TOKENS*K*sizeof(float)),"ragged KV latent page")||+ !cuda_ok(cudaMalloc(&nr[made],(size_t)COLI_KV_PAGE_TOKENS*R*sizeof(float)),"ragged KV rope page"))break;}- if(e->length){- cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);- cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);+ if(made<need){+ if(nl[made])cudaFree(nl[made]);if(nr[made])cudaFree(nr[made]);+ for(int i=e->page_count;i<made;i++){cudaFree(nl[i]);cudaFree(nr[i]);}+ std::free(nl);std::free(nr);std::free(old);std::free(add);std::free(off);return 0;}- if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);- e->latent=nl;e->rope=nr;e->capacity=cap;+ std::free(e->latent_pages);std::free(e->rope_pages);+ e->latent_pages=nl;e->rope_pages=nr;e->page_count=need;}- dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length;+ if(e->page_count>page_stride)page_stride=e->page_count;+ old[s]=e->length;add[s]=lengths[s]-e->length;off[s]=packed_n;packed_n+=add[s]*(K+R);}+ size_t table_n=(size_t)S*page_stride;+ float **dl=(float**)std::calloc(table_n,sizeof(*dl));+ float **dr=(float**)std::calloc(table_n,sizeof(*dr));+ if(!dl||!dr){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}+ for(int s=0;s<S;s++)for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){+ for(int p=0;p<w->ragged[i].page_count;p++){+ dl[(size_t)s*page_stride+p]=w->ragged[i].latent_pages[p];+ dr[(size_t)s*page_stride+p]=w->ragged[i].rope_pages[p];+ }+ break;+ }size_t qb=(size_t)S*H*(Q+R)*sizeof(float);size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float);size_t pb=(size_t)packed_n*sizeof(float);- size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int));+ size_t desc=2*table_n*sizeof(float*)+(size_t)S*4*sizeof(int);int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&&reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&&(!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb)));- char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S;- int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S;+ char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+table_n;+ int *dn=(int*)(ddr+table_n),*dold=dn+S,*dadd=dold+S,*doff=dadd+S;if(ok&&pb){for(int s=0;s<S;s++)if(add[s]){float *p=dc->host_kv+off[s];@@ -1660,20 +1728,20 @@ extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload");}if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&&- cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&&- cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&&+ cuda_ok(cudaMemcpyAsync(ddl,dl,table_n*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent page table")&&+ cuda_ok(cudaMemcpyAsync(ddr,dr,table_n*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope page table")&&cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&&cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&&cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&&cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets");- if(ok&&pb)ragged_kv_append<<<S,256,0,dc->stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R);+ if(ok&&pb)ragged_kv_append<<<S,256,0,dc->stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R,page_stride);if(ok)for(int s=0;s<S;s++){for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;}}std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0;size_t shared=(size_t)(2*K+T+256)*sizeof(float);attention_absorb_ragged_kernel<<<dim3(H,S),256,shared,dc->stream>>>(dc->ac,dc->aq,ddl,ddr,- dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng);+ dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,page_stride,scale,w->gs,w->ng);quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng);return cuda_ok(cudaGetLastError(),"ragged attention launch")&&@@ -1702,10 +1770,7 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {}if (tensor->weights&&tensor->weights_owned) cudaFree(tensor->weights);if (tensor->scales) cudaFree(tensor->scales);- for(int i=0;i<tensor->ragged_count;i++){- if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent);- if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope);- }+ for(int i=0;i<tensor->ragged_count;i++)ragged_kv_clear(&tensor->ragged[i]);std::free(tensor);}diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex 9e2fc86a..03b13696 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -35,6 +35,10 @@ COLI_CUDA_DLLEXPORT int coli_cuda_device_integrated(int device);COLI_CUDA_DLLEXPORT void coli_cuda_stats(int device, size_t *tensor_count, size_t *tensor_bytes);COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,double *h2d_ms, double *kernel_ms, double *d2h_ms);+/* Per-device form of coli_cuda_group_stats; unknown devices return zeros. */+COLI_CUDA_DLLEXPORT void coli_cuda_group_stats_device(+ int device, uint64_t *calls, uint64_t *experts, uint64_t *rows,+ double *h2d_ms, double *kernel_ms, double *d2h_ms);/* Publish the E8 codebook (quant.h's e8_grid, 256x4 bytes) to every configured* device. Must be called after coli_cuda_init and before any fmt=6 upload; thediff --git a/c/colibri.c b/c/colibri.cindex da180db6..5bd83416 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -604,6 +604,13 @@ static void cuda_stats_print(void){"(%.2f expert/call)%s\n",(unsigned long long)calls,(unsigned long long)experts,(unsigned long long)rows,(double)experts/calls,getenv("COLI_CUDA_PROFILE")?"; timing sotto":"");+ if(calls&&g_cuda_ndev>1) for(int i=0;i<g_cuda_ndev;i++){+ uint64_t dc=0,de=0,dr=0;+ coli_cuda_group_stats_device(g_cuda_devices[i],&dc,&de,&dr,NULL,NULL,NULL);+ fprintf(stderr,"[CUDA] device %d groups: %llu call, %llu expert, %llu rows "+ "(%.2f expert/call)\n",g_cuda_devices[i],(unsigned long long)dc,+ (unsigned long long)de,(unsigned long long)dr,dc?(double)de/dc:0.0);+ }if(calls&&getenv("COLI_CUDA_PROFILE")) fprintf(stderr,"[CUDA] expert groups timing: H2D %.1f ms | kernel %.1f ms | D2H %.1f ms\n",h2d,kernel,d2h);if(g_ovl_issue+g_ovl_cpu+g_ovl_take>0) fprintf(stderr,@@ -4311,6 +4318,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* no pipe_wait is needed here; the CPU loop keeps its own waits. Any issue* failure drops the layer back to the collect-in-loop + sync-group path. */int early_issued=0, done_j[64]={0};+ double early_issue_start=0;{static int g_group_async2=-1;if(g_group_async2<0) g_group_async2=getenv("COLI_GROUP_ASYNC")?atoi(getenv("COLI_GROUP_ASYNC")):0;@@ -4355,7 +4363,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(all){static int announced2;if(!announced2){ announced2=1; fprintf(stderr,"[CUDA] expert group overlap active\n"); }- early_issued=1; g_ovl_mark=now_s();+ early_issued=1; early_issue_start=tg0; g_ovl_mark=now_s();for(int q=0;q<npg;q++) done_j[pg_j[q]]=1;/* stash packing for the take phase */for(int di=0;di<g_cuda_ndev;di++){ dev_nc0[di]=pd_nc[di]; dev_off0[di]=pd_off[di]; dev_total0[di]=pd_total[di];@@ -4659,6 +4667,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}}m->t_emm+=now_s()-tg1; g_ovl_take+=now_s()-tg1;+ if(g_prof&&early_issue_start>0) m->t_egpu+=now_s()-early_issue_start;}ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64];ColiCudaTensor *dev_d[COLI_CUDA_MAX_DEVICES][64];@@ -4705,6 +4714,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intmemcpy(group_y+(int64_t)dev_off[di]*D,hy,(size_t)dev_total[di]*D*sizeof(float)); }else dev_ok[di]=0; /* per-device sync failure: CPU fallback below */}+ if(g_prof) m->t_egpu+=now_s()-tg;} else for(int di=0;di<g_cuda_ndev;di++)if(issued[di]) coli_cuda_expert_group_take(g_cuda_devices[di]);}@@ -6064,6 +6074,10 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo* EN: soft MTP guard (#163): 24-proposal window, pause and re-arm instead of the* permanent latch — a transient collapse no longer kills MTP for the whole session. */enum { GUARD_PAUSE_TOKENS = 256 };+ int gd_min_pct=getenv("COLI_MTP_GUARD_PCT")?atoi(getenv("COLI_MTP_GUARD_PCT")):70;+ int gd_window=getenv("COLI_MTP_GUARD_WINDOW")?atoi(getenv("COLI_MTP_GUARD_WINDOW")):24;+ if(gd_min_pct<0)gd_min_pct=0;if(gd_min_pct>100)gd_min_pct=100;+ if(gd_window<4)gd_window=4;if(gd_window>256)gd_window=256;uint64_t gd_prop0=m->mtp_prop, gd_acc0=m->mtp_acc; int gd_pause=0;uint64_t cp_prop0=g_corp_prop, cp_acc0=g_corp_acc; int cp_pause=0;while(emitted<n_new && !done && !g_intr && !g_mux_stop && !g_mux_cancel){@@ -6109,7 +6123,8 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo* ma il vecchio g_draft=0 era permanente. EN: adaptive pause; the old* g_draft=0 latch was permanent. */if(gd_pause>0){ gd_pause--; if(!gd_pause){ gd_prop0=m->mtp_prop; gd_acc0=m->mtp_acc; } }- else if(m->mtp_prop-gd_prop0>=24 && (m->mtp_acc-gd_acc0)*10 < m->mtp_prop-gd_prop0){+ else if(m->mtp_prop-gd_prop0>=(uint64_t)gd_window &&+ (m->mtp_acc-gd_acc0)*100 < (m->mtp_prop-gd_prop0)*(uint64_t)gd_min_pct){fprintf(stderr,"[MTP] %.0f%% acceptance over the last %llu proposals: drafts paused for %d tokens\n",100.0*(m->mtp_acc-gd_acc0)/(m->mtp_prop-gd_prop0),(unsigned long long)(m->mtp_prop-gd_prop0), (int)GUARD_PAUSE_TOKENS);@@ -6575,6 +6590,7 @@ static void run_replay(Model *m, const int *full, int nfull, int np){double dt=now_s()-t0, tot=m->hits+m->miss;printf("REPLAY decode: %d tokens in %.3fs | %.2f tok/s | expert hit %.1f%%\n",steps,dt,steps/dt,tot?100.0*m->hits/tot:0.0);+ if(g_cp_enq) printf("couple: %ld cross-layer prefetch hints enqueued\n",g_cp_enq);profile_print(m,dt);if(g_prof) prof_report(m,&pb,dt,steps,stdout);#ifdef COLI_CUDA@@ -8039,6 +8055,9 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){* count and pinned only the leftovers (measured: 9,280 VRAM + 1,721 RAM* on a box whose RAM fits ~11k — the cold tail then paid disk forever). */double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0};+ double placed_w[COLI_CUDA_MAX_DEVICES]={0};+ int load_balance=getenv("CUDA_EXPERT_LOAD_BALANCE")?+ atoi(getenv("CUDA_EXPERT_LOAD_BALANCE")):0;int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0, prefix_est=0;double budget=g_cuda_expert_gb*1e9, safe_total=0;if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;i<g_cuda_ndev;i++){@@ -8140,7 +8159,10 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){for(int attempt=0;attempt<g_cuda_ndev && !placed;attempt++){int best=-1;for(int i=0;i<g_cuda_ndev;i++) if(!tried[i] && remaining[i]>=projected &&- (best<0||placed_b[i]<placed_b[best])) best=i;+ (best<0||+ (load_balance && (placed_w[i]<placed_w[best] ||+ (placed_w[i]==placed_w[best]&&placed_b[i]<placed_b[best])))||+ (!load_balance&&placed_b[i]<placed_b[best]))) best=i;if(best<0) break;tried[best]=1;s->g.cuda_device=s->u.cuda_device=s->d.cuda_device=g_cuda_devices[best];@@ -8163,6 +8185,7 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);m->gpu_expert_count++; m->gpu_expert_bytes+=actual;remaining[best]-=actual; placed_b[best]+=actual; placed_n[best]++;+ placed_w[best]+=(double)r[a].c;if(g_cuda_release_host) expert_host_release(m,s);placed=1;} else {@@ -8186,6 +8209,8 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){g_cuda_expert_auto?safe_total/1e9:budget/1e9,g_cuda_expert_auto?", auto":"",g_cuda_reserve_gb);+ if(load_balance) fprintf(stderr,+ "[CUDA] expert device assignment: frequency-load balanced (experimental)\n");for(int i=0;i<g_cuda_ndev;i++) fprintf(stderr,"[CUDA] device %d: %d experts, %.2f GB\n",g_cuda_devices[i],placed_n[i],placed_b[i]/1e9);/* #653: on integrated / unified-memory GPUs (Grace-Blackwell GB10, Jetson) thediff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cuindex 34e89b3b..34a819c0 100644--- a/c/tests/test_ragged_attention.cu+++ b/c/tests/test_ragged_attention.cu@@ -1,12 +1,18 @@#include "../backend_cuda.h"+#include <chrono>#include <cmath>#include <cstdio>#include <vector>+#ifndef COLI_KV_PAGE_TOKENS+#define COLI_KV_PAGE_TOKENS 64+#endif+int main(){int dev=0;if(!coli_cuda_init(&dev,1))return 77;- constexpr int S=3,H=2,Q=2,R=1,V=2,K=128,D=H*V,O=3,T=3;+ constexpr int S=3,H=2,Q=2,R=1,V=2,K=128,D=H*V,O=3;+ constexpr int P=COLI_KV_PAGE_TOKENS,T=P+6,B=1024;std::vector<float> w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R));for(size_t i=0;i<w.size();i++)w[i]=((int)(i%11)-5)*.07f;for(size_t i=0;i<p.size();i++)p[i]=((int)(i%7)-3)*.09f;@@ -14,20 +20,23 @@ int main(){ColiCudaTensor *tw=nullptr,*tp=nullptr,*tg=nullptr;if(!coli_cuda_tensor_upload(&tw,w.data(),nullptr,0,K,H*(Q+V),dev)||!coli_cuda_tensor_upload(&tp,p.data(),nullptr,0,D,O,dev))return 1;- int n[S]={1,2,3};std::vector<std::vector<float>> l(S),r(S);+ int n[S]={1,P+1,P+6};std::vector<std::vector<float>> l(S),r(S);const float *lp[S],*rp[S];const void *keys[S];for(int s=0;s<S;s++){- l[s].resize(n[s]*K);r[s].resize(n[s]*R);+ l[s].resize(B*K);r[s].resize(B*R);for(size_t i=0;i<l[s].size();i++)l[s][i]=((int)((i+s*3)%9)-4)*.08f;for(size_t i=0;i<r[s].size();i++)r[s][i]=((int)((i+s)%5)-2)*.06f;lp[s]=l[s].data();rp[s]=r[s].data();keys[s]=&l[s];}float got[S*O],ref[S*O],warm[S*O];- int first[S]={1,1,1};+ int first[S]={1,P-1,P};+ auto t0=std::chrono::steady_clock::now();if(!coli_cuda_attention_project_ragged(tw,tp,warm,q.data(),keys,lp,rp,first,- S,H,Q,R,V,K,1,.2f))return 2;+ S,H,Q,R,V,K,T,.2f))return 2;+ auto t1=std::chrono::steady_clock::now();if(!coli_cuda_attention_project_ragged(tw,tp,got,q.data(),keys,lp,rp,n,S,H,Q,R,V,K,T,.2f))return 2;+ auto t2=std::chrono::steady_clock::now();for(int s=0;s<S;s++)if(!coli_cuda_attention_project_batch(tw,tp,ref+s*O,q.data()+s*H*(Q+R),lp[s],rp[s],1,H,Q,R,V,K,n[s],.2f))return 3;double e=0,z=0;for(int i=0;i<S*O;i++){@@ -55,7 +64,21 @@ int main(){double d=got[i]-ref[i];ge+=d*d;gz+=(double)ref[i]*ref[i];}double grms=std::sqrt(ge/(gz+1e-30));- std::printf("ragged_relative_rms=%.9g ragged_g64_relative_rms=%.9g\n",rms,grms);+ int bench_n[S]={B,B,B};+ auto b0=std::chrono::steady_clock::now();+ if(!coli_cuda_attention_project_ragged(tw,tp,warm,q.data(),keys,lp,rp,bench_n,+ S,H,Q,R,V,K,B,.2f))return 2;+ auto b1=std::chrono::steady_clock::now();+ auto t3=std::chrono::steady_clock::now();+ for(int i=0;i<100;i++)if(!coli_cuda_attention_project_ragged(+ tw,tp,warm,q.data(),keys,lp,rp,bench_n,S,H,Q,R,V,K,B,.2f))return 2;+ auto t4=std::chrono::steady_clock::now();+ std::printf("ragged_relative_rms=%.9g ragged_g64_relative_rms=%.9g warm_ms=%.3f cross_page_growth_ms=%.3f grow_1024_ms=%.3f steady_1024_ms=%.3f\n",+ rms,grms,+ std::chrono::duration<double,std::milli>(t1-t0).count(),+ std::chrono::duration<double,std::milli>(t2-t1).count(),+ std::chrono::duration<double,std::milli>(b1-b0).count(),+ std::chrono::duration<double,std::milli>(t4-t3).count()/100.0);coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tg);coli_cuda_tensor_free(tp);coli_cuda_shutdown();return rms<1e-6&&grms<1e-6?0:4;}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 1d1484f2..399732b6 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -157,11 +157,14 @@ See [docs/vulkan.md](vulkan.md). On multi-core boxes also set `COLI_NO_OMP_TUNE=| `COLI_GPU` / `COLI_GPUS` | unset | Device selection (`auto`, `none`, or a list like `0,1`). Requires `COLI_CUDA=1`. || `CUDA_DENSE` | `0` | Place dense (non-expert) matmuls on the GPU. Off by default the engine reports `routed experts only (resident dense on CPU)`: on a host where the CPU is the limiter this leaves the dense path of every layer on the CPU while the VRAM tier serves experts only. Measured x2.8 on a 4x A6000 / 24-core host (1.53 -> 4.26 tok/s). || `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. |+| `CUDA_EXPERT_LOAD_BALANCE` | `0` (off) | Experimental multi-GPU expert assignment: keep the same frequency-ranked GPU prefix, but greedily distribute it by accumulated profile weight instead of resident bytes alone. On one 6×RTX 5090 fixed replay its three-run median was +2.9%, with large variance; leave off unless validated on the target workload. || `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. || `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. || `COLI_CUDA_ATTN_PREFIX` | off | Reuse one uploaded decode activation across `q_a` and `kv_a` while preserving the stock CPU RMSNorm path. || `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). || `COLI_CUDA_PROFILE` | off | Emit CUDA timing. |+| `COLI_MTP_GUARD_PCT` | `70` | Pause MTP after the guard window when recent acceptance falls below this percentage. |+| `COLI_MTP_GUARD_WINDOW` | `24` | Number of MTP proposals used by the soft acceptance guard. || `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. || `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). || `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. |