Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: add paged ragged KV research runtime

ZacharyZcR2c5be24fb818
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 0ccd1496..9ce0a650 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -27,10 +27,25 @@
struct RaggedKVEntry {
const void *key;
const float *host_l,*host_r;
- float *latent,*rope;
- int length,capacity,K,R;
+ float **latent_pages,**rope_pages;
+ int length,page_count,K,R;
};
+#ifndef COLI_KV_PAGE_TOKENS
+#define COLI_KV_PAGE_TOKENS 64
+#endif
+
+static void ragged_kv_clear(RaggedKVEntry *e) {
+ for (int i=0;i<e->page_count;i++) {
+ if (e->latent_pages[i]) cudaFree(e->latent_pages[i]);
+ if (e->rope_pages[i]) cudaFree(e->rope_pages[i]);
+ }
+ std::free(e->latent_pages);
+ std::free(e->rope_pages);
+ e->latent_pages=e->rope_pages=nullptr;
+ e->length=e->page_count=0;
+}
+
struct ColiCudaTensor {
void *weights;
float *scales;
@@ -86,6 +101,12 @@ static DeviceContext g_ctx[COLI_CUDA_MAX_DEVICES];
static int g_nctx;
static uint64_t g_group_calls,g_group_experts,g_group_rows;
static double g_group_h2d_ms,g_group_kernel_ms,g_group_d2h_ms;
+static uint64_t g_device_group_calls[COLI_CUDA_MAX_DEVICES];
+static uint64_t g_device_group_experts[COLI_CUDA_MAX_DEVICES];
+static uint64_t g_device_group_rows[COLI_CUDA_MAX_DEVICES];
+static double g_device_group_h2d_ms[COLI_CUDA_MAX_DEVICES];
+static double g_device_group_kernel_ms[COLI_CUDA_MAX_DEVICES];
+static double g_device_group_d2h_ms[COLI_CUDA_MAX_DEVICES];
static std::mutex g_group_stats_mu;
#ifdef COLI_ANS
static FILE *g_ans_sidecar;
@@ -634,18 +655,18 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q,
__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
const float *const *latent,const float *const *rope,const int *lengths,
const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R,
- int V,int K,int T,float scale,int gs,int ng){
+ int V,int K,int T,int page_stride,float scale,int gs,int ng){
int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V);
if(s>=S||nt<1||nt>T)return;
extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T;
const float *qs=q+((size_t)s*H+h)*(Q+R);
- const float *ls=latent[s],*rs=rope[s];
for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int d=0;d<Q;d++)
a+=qs[d]*weight_at(weights,fmt,(size_t)(rbase+d)*row_bytes(fmt,K),k)*
absorb_scale(wscale,fmt,gs,ng,rbase+d,k);qa[k]=a;}
__syncthreads();
- for(int t=tid;t<nt;t+=blockDim.x){float a=0;const float *lt=ls+(size_t)t*K;
- const float *rt=rs+(size_t)t*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];
+ for(int t=tid;t<nt;t+=blockDim.x){float a=0;int pg=t/COLI_KV_PAGE_TOKENS,pt=t%COLI_KV_PAGE_TOKENS;
+ const float *lt=latent[(size_t)s*page_stride+pg]+(size_t)pt*K;
+ const float *rt=rope[(size_t)s*page_stride+pg]+(size_t)pt*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];
for(int d=0;d<R;d++)a+=qs[Q+d]*rt[d];scores[t]=a*scale;}
__syncthreads();
float local=-3.402823466e+38F;for(int t=tid;t<nt;t+=blockDim.x)local=fmaxf(local,scores[t]);
@@ -656,7 +677,10 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
for(int n=blockDim.x>>1;n;n>>=1){if(tid<n)red[tid]+=red[tid+n];__syncthreads();}
float inv=1.f/red[0];for(int t=tid;t<nt;t+=blockDim.x)scores[t]*=inv;
__syncthreads();
- for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++)a+=scores[t]*ls[(size_t)t*K+k];cl[k]=a;}
+ for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++){
+ const float *lt=latent[(size_t)s*page_stride+t/COLI_KV_PAGE_TOKENS]+
+ (size_t)(t%COLI_KV_PAGE_TOKENS)*K;
+ a+=scores[t]*lt[k];}cl[k]=a;}
__syncthreads();
for(int v=tid;v<V;v+=blockDim.x){int row=rbase+Q+v;float a=0;size_t rb=row_bytes(fmt,K);
for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k)*
@@ -665,11 +689,15 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
}
__global__ static void ragged_kv_append(float *const *latent,float *const *rope,
- const float *packed,const int *old_len,const int *add,const int *offset,int K,int R){
+ const float *packed,const int *old_len,const int *add,const int *offset,
+ int K,int R,int page_stride){
int s=blockIdx.x,n=add[s],base=offset[s];
- for(int i=threadIdx.x;i<n*(K+R);i+=blockDim.x){
- if(i<n*K)latent[s][(size_t)old_len[s]*K+i]=packed[base+i];
- else rope[s][(size_t)old_len[s]*R+i-n*K]=packed[base+i];
+ for(int t=0;t<n;t++){
+ int pos=old_len[s]+t,pg=pos/COLI_KV_PAGE_TOKENS,pt=pos%COLI_KV_PAGE_TOKENS;
+ float *lp=latent[(size_t)s*page_stride+pg]+(size_t)pt*K;
+ float *rp=rope[(size_t)s*page_stride+pg]+(size_t)pt*R;
+ for(int k=threadIdx.x;k<K;k+=blockDim.x)lp[k]=packed[base+(size_t)t*K+k];
+ for(int r=threadIdx.x;r<R;r+=blockDim.x)rp[r]=packed[base+(size_t)n*K+(size_t)t*R+r];
}
}
@@ -907,6 +935,20 @@ extern "C" void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64
if(d2h_ms) *d2h_ms=g_group_d2h_ms;
}
+extern "C" void coli_cuda_group_stats_device(
+ int device, uint64_t *calls, uint64_t *experts, uint64_t *rows,
+ double *h2d_ms, double *kernel_ms, double *d2h_ms) {
+ std::lock_guard<std::mutex> lock(g_group_stats_mu);
+ int index=-1;
+ for(int i=0;i<g_nctx;i++) if(g_ctx[i].device==device){ index=i; break; }
+ if(calls) *calls=index<0?0:g_device_group_calls[index];
+ if(experts) *experts=index<0?0:g_device_group_experts[index];
+ if(rows) *rows=index<0?0:g_device_group_rows[index];
+ if(h2d_ms) *h2d_ms=index<0?0:g_device_group_h2d_ms[index];
+ if(kernel_ms) *kernel_ms=index<0?0:g_device_group_kernel_ms[index];
+ if(d2h_ms) *d2h_ms=index<0?0:g_device_group_d2h_ms[index];
+}
+
/* group size for the NEXT upload on this thread (fmt=4): routed through a
* thread_local so the widely-wired upload signature (and the Windows DLL ABI)
* stays untouched. pin_load uploads in parallel, hence thread_local. */
@@ -1423,11 +1465,18 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
cudaEventElapsedTime(&a,ev[0],ev[1]); cudaEventElapsedTime(&b,ev[1],ev[2]);
cudaEventElapsedTime(&c,ev[2],ev[3]);
{ std::lock_guard<std::mutex> lock(g_group_stats_mu);
- g_group_h2d_ms+=a; g_group_kernel_ms+=b; g_group_d2h_ms+=c; }
+ int index=(int)(ctx-g_ctx);
+ g_group_h2d_ms+=a; g_group_kernel_ms+=b; g_group_d2h_ms+=c;
+ g_device_group_h2d_ms[index]+=a;
+ g_device_group_kernel_ms[index]+=b;
+ g_device_group_d2h_ms[index]+=c; }
for(int i=0;i<4;i++) cudaEventDestroy(ev[i]);
}
{ std::lock_guard<std::mutex> lock(g_group_stats_mu);
- g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total; }
+ int index=(int)(ctx-g_ctx);
+ g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total;
+ g_device_group_calls[index]++; g_device_group_experts[index]+=(uint64_t)count;
+ g_device_group_rows[index]+=(uint64_t)total; }
return 1;
}
@@ -1516,7 +1565,10 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
"expert group issue download")) return 0;
ctx->group_pending=1; ctx->group_pending_bytes=xb;
{ std::lock_guard<std::mutex> lock(g_group_stats_mu);
- g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total; }
+ int index=(int)(ctx-g_ctx);
+ g_group_calls++; g_group_experts+=(uint64_t)count; g_group_rows+=(uint64_t)total;
+ g_device_group_calls[index]++; g_device_group_experts[index]+=(uint64_t)count;
+ g_device_group_rows[index]+=(uint64_t)total; }
return 1;
}
@@ -1607,50 +1659,66 @@ extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTens
proj->device!=w->device||proj->I!=H*V)return 0;
DeviceContext *dc=find_ctx(w->device);
if(!select_ctx(dc))return 0;
- float **dl=(float**)std::malloc((size_t)S*sizeof(*dl));
- float **dr=(float**)std::malloc((size_t)S*sizeof(*dr));
int *old=(int*)std::malloc((size_t)S*sizeof(*old));
int *add=(int*)std::malloc((size_t)S*sizeof(*add));
int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0;
- if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
+ if(!old||!add||!off){std::free(old);std::free(add);std::free(off);return 0;}
+ int page_stride=0;
for(int s=0;s<S;s++){
- if(!keys[s]||lengths[s]<1||lengths[s]>T){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
+ if(!keys[s]||lengths[s]<1||lengths[s]>T){std::free(old);std::free(add);std::free(off);return 0;}
RaggedKVEntry *e=nullptr;
for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;}
if(!e){
- if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
+ if(w->ragged_count>=512){std::free(old);std::free(add);std::free(off);return 0;}
e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s];
}
if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]<e->length){
- if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);
- e->latent=e->rope=nullptr;e->length=e->capacity=0;
+ ragged_kv_clear(e);
e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s];
}
- if(lengths[s]>e->capacity){
- int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr;
- if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")||
- !cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){
- if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;
+ int need=(lengths[s]+COLI_KV_PAGE_TOKENS-1)/COLI_KV_PAGE_TOKENS;
+ if(need>e->page_count){
+ float **nl=(float**)std::calloc((size_t)need,sizeof(*nl));
+ float **nr=(float**)std::calloc((size_t)need,sizeof(*nr));
+ if(!nl||!nr){std::free(nl);std::free(nr);std::free(old);std::free(add);std::free(off);return 0;}
+ for(int i=0;i<e->page_count;i++){nl[i]=e->latent_pages[i];nr[i]=e->rope_pages[i];}
+ int made=e->page_count;
+ for(;made<need;made++){
+ if(!cuda_ok(cudaMalloc(&nl[made],(size_t)COLI_KV_PAGE_TOKENS*K*sizeof(float)),"ragged KV latent page")||
+ !cuda_ok(cudaMalloc(&nr[made],(size_t)COLI_KV_PAGE_TOKENS*R*sizeof(float)),"ragged KV rope page"))break;
}
- if(e->length){
- cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);
- cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);
+ if(made<need){
+ if(nl[made])cudaFree(nl[made]);if(nr[made])cudaFree(nr[made]);
+ for(int i=e->page_count;i<made;i++){cudaFree(nl[i]);cudaFree(nr[i]);}
+ std::free(nl);std::free(nr);std::free(old);std::free(add);std::free(off);return 0;
}
- if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);
- e->latent=nl;e->rope=nr;e->capacity=cap;
+ std::free(e->latent_pages);std::free(e->rope_pages);
+ e->latent_pages=nl;e->rope_pages=nr;e->page_count=need;
}
- dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length;
+ if(e->page_count>page_stride)page_stride=e->page_count;
+ old[s]=e->length;add[s]=lengths[s]-e->length;
off[s]=packed_n;packed_n+=add[s]*(K+R);
}
+ size_t table_n=(size_t)S*page_stride;
+ float **dl=(float**)std::calloc(table_n,sizeof(*dl));
+ float **dr=(float**)std::calloc(table_n,sizeof(*dr));
+ if(!dl||!dr){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
+ for(int s=0;s<S;s++)for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){
+ for(int p=0;p<w->ragged[i].page_count;p++){
+ dl[(size_t)s*page_stride+p]=w->ragged[i].latent_pages[p];
+ dr[(size_t)s*page_stride+p]=w->ragged[i].rope_pages[p];
+ }
+ break;
+ }
size_t qb=(size_t)S*H*(Q+R)*sizeof(float);
size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float);
size_t pb=(size_t)packed_n*sizeof(float);
- size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int));
+ size_t desc=2*table_n*sizeof(float*)+(size_t)S*4*sizeof(int);
int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&&
reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&&
(!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb)));
- char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S;
- int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S;
+ char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+table_n;
+ int *dn=(int*)(ddr+table_n),*dold=dn+S,*dadd=dold+S,*doff=dadd+S;
if(ok&&pb){
for(int s=0;s<S;s++)if(add[s]){
float *p=dc->host_kv+off[s];
@@ -1660,20 +1728,20 @@ extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTens
ok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload");
}
if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&&
- cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&&
- cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&&
+ cuda_ok(cudaMemcpyAsync(ddl,dl,table_n*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent page table")&&
+ cuda_ok(cudaMemcpyAsync(ddr,dr,table_n*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope page table")&&
cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&&
cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&&
cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&&
cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets");
- if(ok&&pb)ragged_kv_append<<<S,256,0,dc->stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R);
+ if(ok&&pb)ragged_kv_append<<<S,256,0,dc->stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R,page_stride);
if(ok)for(int s=0;s<S;s++){
for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;}
}
std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0;
size_t shared=(size_t)(2*K+T+256)*sizeof(float);
attention_absorb_ragged_kernel<<<dim3(H,S),256,shared,dc->stream>>>(dc->ac,dc->aq,ddl,ddr,
- dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng);
+ dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,page_stride,scale,w->gs,w->ng);
quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,
proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng);
return cuda_ok(cudaGetLastError(),"ragged attention launch")&&
@@ -1702,10 +1770,7 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
}
if (tensor->weights&&tensor->weights_owned) cudaFree(tensor->weights);
if (tensor->scales) cudaFree(tensor->scales);
- for(int i=0;i<tensor->ragged_count;i++){
- if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent);
- if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope);
- }
+ for(int i=0;i<tensor->ragged_count;i++)ragged_kv_clear(&tensor->ragged[i]);
std::free(tensor);
}
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index 9e2fc86a..03b13696 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -35,6 +35,10 @@ COLI_CUDA_DLLEXPORT int coli_cuda_device_integrated(int device);
COLI_CUDA_DLLEXPORT void coli_cuda_stats(int device, size_t *tensor_count, size_t *tensor_bytes);
COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,
double *h2d_ms, double *kernel_ms, double *d2h_ms);
+/* Per-device form of coli_cuda_group_stats; unknown devices return zeros. */
+COLI_CUDA_DLLEXPORT void coli_cuda_group_stats_device(
+ int device, uint64_t *calls, uint64_t *experts, uint64_t *rows,
+ double *h2d_ms, double *kernel_ms, double *d2h_ms);
/* Publish the E8 codebook (quant.h's e8_grid, 256x4 bytes) to every configured
* device. Must be called after coli_cuda_init and before any fmt=6 upload; the
diff --git a/c/colibri.c b/c/colibri.c
index da180db6..5bd83416 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -604,6 +604,13 @@ static void cuda_stats_print(void){
"(%.2f expert/call)%s\n",(unsigned long long)calls,(unsigned long long)experts,
(unsigned long long)rows,(double)experts/calls,
getenv("COLI_CUDA_PROFILE")?"; timing sotto":"");
+ if(calls&&g_cuda_ndev>1) for(int i=0;i<g_cuda_ndev;i++){
+ uint64_t dc=0,de=0,dr=0;
+ coli_cuda_group_stats_device(g_cuda_devices[i],&dc,&de,&dr,NULL,NULL,NULL);
+ fprintf(stderr,"[CUDA] device %d groups: %llu call, %llu expert, %llu rows "
+ "(%.2f expert/call)\n",g_cuda_devices[i],(unsigned long long)dc,
+ (unsigned long long)de,(unsigned long long)dr,dc?(double)de/dc:0.0);
+ }
if(calls&&getenv("COLI_CUDA_PROFILE")) fprintf(stderr,
"[CUDA] expert groups timing: H2D %.1f ms | kernel %.1f ms | D2H %.1f ms\n",h2d,kernel,d2h);
if(g_ovl_issue+g_ovl_cpu+g_ovl_take>0) fprintf(stderr,
@@ -4311,6 +4318,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
* no pipe_wait is needed here; the CPU loop keeps its own waits. Any issue
* failure drops the layer back to the collect-in-loop + sync-group path. */
int early_issued=0, done_j[64]={0};
+ double early_issue_start=0;
{
static int g_group_async2=-1;
if(g_group_async2<0) g_group_async2=getenv("COLI_GROUP_ASYNC")?atoi(getenv("COLI_GROUP_ASYNC")):0;
@@ -4355,7 +4363,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(all){
static int announced2;
if(!announced2){ announced2=1; fprintf(stderr,"[CUDA] expert group overlap active\n"); }
- early_issued=1; g_ovl_mark=now_s();
+ early_issued=1; early_issue_start=tg0; g_ovl_mark=now_s();
for(int q=0;q<npg;q++) done_j[pg_j[q]]=1;
/* stash packing for the take phase */
for(int di=0;di<g_cuda_ndev;di++){ dev_nc0[di]=pd_nc[di]; dev_off0[di]=pd_off[di]; dev_total0[di]=pd_total[di];
@@ -4659,6 +4667,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
}
}
m->t_emm+=now_s()-tg1; g_ovl_take+=now_s()-tg1;
+ if(g_prof&&early_issue_start>0) m->t_egpu+=now_s()-early_issue_start;
}
ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64];
ColiCudaTensor *dev_d[COLI_CUDA_MAX_DEVICES][64];
@@ -4705,6 +4714,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
memcpy(group_y+(int64_t)dev_off[di]*D,hy,(size_t)dev_total[di]*D*sizeof(float)); }
else dev_ok[di]=0; /* per-device sync failure: CPU fallback below */
}
+ if(g_prof) m->t_egpu+=now_s()-tg;
} else for(int di=0;di<g_cuda_ndev;di++)
if(issued[di]) coli_cuda_expert_group_take(g_cuda_devices[di]);
}
@@ -6064,6 +6074,10 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
* EN: soft MTP guard (#163): 24-proposal window, pause and re-arm instead of the
* permanent latch — a transient collapse no longer kills MTP for the whole session. */
enum { GUARD_PAUSE_TOKENS = 256 };
+ int gd_min_pct=getenv("COLI_MTP_GUARD_PCT")?atoi(getenv("COLI_MTP_GUARD_PCT")):70;
+ int gd_window=getenv("COLI_MTP_GUARD_WINDOW")?atoi(getenv("COLI_MTP_GUARD_WINDOW")):24;
+ if(gd_min_pct<0)gd_min_pct=0;if(gd_min_pct>100)gd_min_pct=100;
+ if(gd_window<4)gd_window=4;if(gd_window>256)gd_window=256;
uint64_t gd_prop0=m->mtp_prop, gd_acc0=m->mtp_acc; int gd_pause=0;
uint64_t cp_prop0=g_corp_prop, cp_acc0=g_corp_acc; int cp_pause=0;
while(emitted<n_new && !done && !g_intr && !g_mux_stop && !g_mux_cancel){
@@ -6109,7 +6123,8 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
* ma il vecchio g_draft=0 era permanente. EN: adaptive pause; the old
* g_draft=0 latch was permanent. */
if(gd_pause>0){ gd_pause--; if(!gd_pause){ gd_prop0=m->mtp_prop; gd_acc0=m->mtp_acc; } }
- else if(m->mtp_prop-gd_prop0>=24 && (m->mtp_acc-gd_acc0)*10 < m->mtp_prop-gd_prop0){
+ else if(m->mtp_prop-gd_prop0>=(uint64_t)gd_window &&
+ (m->mtp_acc-gd_acc0)*100 < (m->mtp_prop-gd_prop0)*(uint64_t)gd_min_pct){
fprintf(stderr,"[MTP] %.0f%% acceptance over the last %llu proposals: drafts paused for %d tokens\n",
100.0*(m->mtp_acc-gd_acc0)/(m->mtp_prop-gd_prop0),
(unsigned long long)(m->mtp_prop-gd_prop0), (int)GUARD_PAUSE_TOKENS);
@@ -6575,6 +6590,7 @@ static void run_replay(Model *m, const int *full, int nfull, int np){
double dt=now_s()-t0, tot=m->hits+m->miss;
printf("REPLAY decode: %d tokens in %.3fs | %.2f tok/s | expert hit %.1f%%\n",
steps,dt,steps/dt,tot?100.0*m->hits/tot:0.0);
+ if(g_cp_enq) printf("couple: %ld cross-layer prefetch hints enqueued\n",g_cp_enq);
profile_print(m,dt);
if(g_prof) prof_report(m,&pb,dt,steps,stdout);
#ifdef COLI_CUDA
@@ -8039,6 +8055,9 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
* count and pinned only the leftovers (measured: 9,280 VRAM + 1,721 RAM
* on a box whose RAM fits ~11k — the cold tail then paid disk forever). */
double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0};
+ double placed_w[COLI_CUDA_MAX_DEVICES]={0};
+ int load_balance=getenv("CUDA_EXPERT_LOAD_BALANCE")?
+ atoi(getenv("CUDA_EXPERT_LOAD_BALANCE")):0;
int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0, prefix_est=0;
double budget=g_cuda_expert_gb*1e9, safe_total=0;
if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;i<g_cuda_ndev;i++){
@@ -8140,7 +8159,10 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
for(int attempt=0;attempt<g_cuda_ndev && !placed;attempt++){
int best=-1;
for(int i=0;i<g_cuda_ndev;i++) if(!tried[i] && remaining[i]>=projected &&
- (best<0||placed_b[i]<placed_b[best])) best=i;
+ (best<0||
+ (load_balance && (placed_w[i]<placed_w[best] ||
+ (placed_w[i]==placed_w[best]&&placed_b[i]<placed_b[best])))||
+ (!load_balance&&placed_b[i]<placed_b[best]))) best=i;
if(best<0) break;
tried[best]=1;
s->g.cuda_device=s->u.cuda_device=s->d.cuda_device=g_cuda_devices[best];
@@ -8163,6 +8185,7 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);
m->gpu_expert_count++; m->gpu_expert_bytes+=actual;
remaining[best]-=actual; placed_b[best]+=actual; placed_n[best]++;
+ placed_w[best]+=(double)r[a].c;
if(g_cuda_release_host) expert_host_release(m,s);
placed=1;
} else {
@@ -8186,6 +8209,8 @@ static void pin_load(Model *m, const char *statspath, double gb, int trusted){
g_cuda_expert_auto?safe_total/1e9:budget/1e9,
g_cuda_expert_auto?", auto":"",
g_cuda_reserve_gb);
+ if(load_balance) fprintf(stderr,
+ "[CUDA] expert device assignment: frequency-load balanced (experimental)\n");
for(int i=0;i<g_cuda_ndev;i++) fprintf(stderr,"[CUDA] device %d: %d experts, %.2f GB\n",
g_cuda_devices[i],placed_n[i],placed_b[i]/1e9);
/* #653: on integrated / unified-memory GPUs (Grace-Blackwell GB10, Jetson) the
diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cu
index 34e89b3b..34a819c0 100644
--- a/c/tests/test_ragged_attention.cu
+++ b/c/tests/test_ragged_attention.cu
@@ -1,12 +1,18 @@
#include "../backend_cuda.h"
+#include <chrono>
#include <cmath>
#include <cstdio>
#include <vector>
+#ifndef COLI_KV_PAGE_TOKENS
+#define COLI_KV_PAGE_TOKENS 64
+#endif
+
int main(){
int dev=0;if(!coli_cuda_init(&dev,1))return 77;
- constexpr int S=3,H=2,Q=2,R=1,V=2,K=128,D=H*V,O=3,T=3;
+ constexpr int S=3,H=2,Q=2,R=1,V=2,K=128,D=H*V,O=3;
+ constexpr int P=COLI_KV_PAGE_TOKENS,T=P+6,B=1024;
std::vector<float> w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R));
for(size_t i=0;i<w.size();i++)w[i]=((int)(i%11)-5)*.07f;
for(size_t i=0;i<p.size();i++)p[i]=((int)(i%7)-3)*.09f;
@@ -14,20 +20,23 @@ int main(){
ColiCudaTensor *tw=nullptr,*tp=nullptr,*tg=nullptr;
if(!coli_cuda_tensor_upload(&tw,w.data(),nullptr,0,K,H*(Q+V),dev)||
!coli_cuda_tensor_upload(&tp,p.data(),nullptr,0,D,O,dev))return 1;
- int n[S]={1,2,3};std::vector<std::vector<float>> l(S),r(S);
+ int n[S]={1,P+1,P+6};std::vector<std::vector<float>> l(S),r(S);
const float *lp[S],*rp[S];
const void *keys[S];
for(int s=0;s<S;s++){
- l[s].resize(n[s]*K);r[s].resize(n[s]*R);
+ l[s].resize(B*K);r[s].resize(B*R);
for(size_t i=0;i<l[s].size();i++)l[s][i]=((int)((i+s*3)%9)-4)*.08f;
for(size_t i=0;i<r[s].size();i++)r[s][i]=((int)((i+s)%5)-2)*.06f;
lp[s]=l[s].data();rp[s]=r[s].data();keys[s]=&l[s];
}
float got[S*O],ref[S*O],warm[S*O];
- int first[S]={1,1,1};
+ int first[S]={1,P-1,P};
+ auto t0=std::chrono::steady_clock::now();
if(!coli_cuda_attention_project_ragged(tw,tp,warm,q.data(),keys,lp,rp,first,
- S,H,Q,R,V,K,1,.2f))return 2;
+ S,H,Q,R,V,K,T,.2f))return 2;
+ auto t1=std::chrono::steady_clock::now();
if(!coli_cuda_attention_project_ragged(tw,tp,got,q.data(),keys,lp,rp,n,S,H,Q,R,V,K,T,.2f))return 2;
+ auto t2=std::chrono::steady_clock::now();
for(int s=0;s<S;s++)if(!coli_cuda_attention_project_batch(tw,tp,ref+s*O,
q.data()+s*H*(Q+R),lp[s],rp[s],1,H,Q,R,V,K,n[s],.2f))return 3;
double e=0,z=0;for(int i=0;i<S*O;i++){
@@ -55,7 +64,21 @@ int main(){
double d=got[i]-ref[i];ge+=d*d;gz+=(double)ref[i]*ref[i];
}
double grms=std::sqrt(ge/(gz+1e-30));
- std::printf("ragged_relative_rms=%.9g ragged_g64_relative_rms=%.9g\n",rms,grms);
+ int bench_n[S]={B,B,B};
+ auto b0=std::chrono::steady_clock::now();
+ if(!coli_cuda_attention_project_ragged(tw,tp,warm,q.data(),keys,lp,rp,bench_n,
+ S,H,Q,R,V,K,B,.2f))return 2;
+ auto b1=std::chrono::steady_clock::now();
+ auto t3=std::chrono::steady_clock::now();
+ for(int i=0;i<100;i++)if(!coli_cuda_attention_project_ragged(
+ tw,tp,warm,q.data(),keys,lp,rp,bench_n,S,H,Q,R,V,K,B,.2f))return 2;
+ auto t4=std::chrono::steady_clock::now();
+ std::printf("ragged_relative_rms=%.9g ragged_g64_relative_rms=%.9g warm_ms=%.3f cross_page_growth_ms=%.3f grow_1024_ms=%.3f steady_1024_ms=%.3f\n",
+ rms,grms,
+ std::chrono::duration<double,std::milli>(t1-t0).count(),
+ std::chrono::duration<double,std::milli>(t2-t1).count(),
+ std::chrono::duration<double,std::milli>(b1-b0).count(),
+ std::chrono::duration<double,std::milli>(t4-t3).count()/100.0);
coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tg);coli_cuda_tensor_free(tp);coli_cuda_shutdown();
return rms<1e-6&&grms<1e-6?0:4;
}
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 1d1484f2..399732b6 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -157,11 +157,14 @@ See [docs/vulkan.md](vulkan.md). On multi-core boxes also set `COLI_NO_OMP_TUNE=
| `COLI_GPU` / `COLI_GPUS` | unset | Device selection (`auto`, `none`, or a list like `0,1`). Requires `COLI_CUDA=1`. |
| `CUDA_DENSE` | `0` | Place dense (non-expert) matmuls on the GPU. Off by default the engine reports `routed experts only (resident dense on CPU)`: on a host where the CPU is the limiter this leaves the dense path of every layer on the CPU while the VRAM tier serves experts only. Measured x2.8 on a 4x A6000 / 24-core host (1.53 -> 4.26 tok/s). |
| `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. |
+| `CUDA_EXPERT_LOAD_BALANCE` | `0` (off) | Experimental multi-GPU expert assignment: keep the same frequency-ranked GPU prefix, but greedily distribute it by accumulated profile weight instead of resident bytes alone. On one 6×RTX 5090 fixed replay its three-run median was +2.9%, with large variance; leave off unless validated on the target workload. |
| `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. |
| `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. |
| `COLI_CUDA_ATTN_PREFIX` | off | Reuse one uploaded decode activation across `q_a` and `kv_a` while preserving the stock CPU RMSNorm path. |
| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). |
| `COLI_CUDA_PROFILE` | off | Emit CUDA timing. |
+| `COLI_MTP_GUARD_PCT` | `70` | Pause MTP after the guard window when recent acceptance falls below this percentage. |
+| `COLI_MTP_GUARD_WINDOW` | `24` | Number of MTP proposals used by the soft acceptance guard. |
| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. |
| `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). |
| `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. |