Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: keep async packed-int4 groups token exact

ZacharyZcR455b071ebcae
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 5a739cd9..a85c7adb 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -1085,7 +1085,7 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
if (!gates || !ups || !downs || !rows || !x || count < 1 || count > 64) return 0;
ColiCudaTensor *first=gates[0];
if (!first) return 0;
- int device=first->device,D=first->I,I=first->O,total=0;
+ int device=first->device,D=first->I,I=first->O,total=0,max_rows=0,all_s4=1;
GroupDesc host[64];
for(int c=0;c<count;c++){
ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];
@@ -1094,19 +1094,37 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
host[c]={g->weights,u->weights,d->weights,g->scales,u->scales,d->scales,
g->fmt,u->fmt,d->fmt,rows[c],total,
g->gs,u->gs,d->gs};
- total+=rows[c];
+ all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2;
+ total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];
}
if(total>8) return 0; /* decode-scale only */
DeviceContext *ctx=find_ctx(device); if(!ctx||ctx->group_pending||!select_ctx(ctx)) return 0;
size_t xb=(size_t)total*D*sizeof(float), ib=(size_t)total*I*sizeof(float);
if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)||
!reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)||
+ !reserve_bytes(&ctx->group_desc,&ctx->group_desc_cap,(size_t)count*sizeof(GroupDesc))||
!reserve_pinned(&ctx->host_x,&ctx->host_x_cap,xb)||
!reserve_pinned(&ctx->host_y,&ctx->host_y_cap,xb)) return 0;
std::memcpy(ctx->host_x,x,xb);
- if(!cuda_ok(cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream),
+ if(!cuda_ok(cudaMemcpyAsync(ctx->group_desc,host,(size_t)count*sizeof(GroupDesc),
+ cudaMemcpyHostToDevice,ctx->stream),
+ "expert group issue descriptors")||
+ !cuda_ok(cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream),
"expert group issue upload")) return 0;
- for(int c=0;c<count;c++){
+ if(all_s4&&(!getenv("COLI_CUDA_W4_PACKED")||atoi(getenv("COLI_CUDA_W4_PACKED")))){
+ GroupDesc *dev=(GroupDesc*)ctx->group_desc;
+ dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count);
+ dim3 og((unsigned)D,(unsigned)max_rows,(unsigned)count);
+ int dual=!getenv("COLI_CUDA_DUAL_PROJ")||atoi(getenv("COLI_CUDA_DUAL_PROJ"));
+ if(dual) grouped_hidden_w4_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->up,ctx->x,dev,I,D);
+ else {
+ grouped_hidden_w4<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->x,dev,I,D,0);
+ grouped_hidden_w4<<<hg,256,0,ctx->stream>>>(ctx->up,ctx->x,dev,I,D,1);
+ silu_mul<<<(unsigned)(((size_t)total*I+255)/256),256,0,ctx->stream>>>(
+ ctx->gate,ctx->up,(size_t)total*I);
+ }
+ grouped_down_w4<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);
+ } else for(int c=0;c<count;c++){
int r=rows[c];
float *g16=ctx->gate+(size_t)host[c].offset*I,*u16=ctx->up+(size_t)host[c].offset*I;
float *x16=ctx->x+(size_t)host[c].offset*D,*y16=ctx->y+(size_t)host[c].offset*D;
diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cu
index 0253cf60..f1537750 100644
--- a/c/tests/test_backend_cuda.cu
+++ b/c/tests/test_backend_cuda.cu
@@ -341,7 +341,7 @@ int main(int argc, char **argv) {
/* Native s4 WMMA path: compare the quantized-activation result against the
existing FP32-activation/s4-weight grouped implementation. */
- uint8_t w4[32*32/2]; float ws4[32], gx4[64], scalar4[64], tensor4[64];
+ uint8_t w4[32*32/2]; float ws4[32], gx4[64], scalar4[64], async4[64], tensor4[64];
for(int i=0;i<(int)sizeof(w4);i++){
int lo=((i%15)-7)&15,hi=(((i*3)%15)-7)&15;
w4[i]=(uint8_t)(lo|(hi<<4));
@@ -354,6 +354,14 @@ int main(int argc, char **argv) {
!coli_cuda_tensor_upload_g(&d4,w4,ws4,2,32,32,d0,0))return 1;
ColiCudaTensor *gg4[2]={g4,g4},*ug4[2]={u4,u4},*dg4[2]={d4,d4};
if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,scalar4,gx4))return 1;
+ if(!coli_cuda_expert_group_issue(gg4,ug4,dg4,group_rows,2,gx4))return 1;
+ const float *async_result=coli_cuda_expert_group_take(d0);
+ if(!async_result)return 1;
+ std::memcpy(async4,async_result,sizeof(async4));
+ if(std::memcmp(async4,scalar4,sizeof(async4))){
+ std::fprintf(stderr,"async packed-s4 group differs from sync path\n");
+ return 1;
+ }
setenv("COLI_CUDA_TC_INT4","1",1);
setenv("COLI_CUDA_TC_MIN_ROWS","1",1);
if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,tensor4,gx4)||
@@ -363,7 +371,7 @@ int main(int argc, char **argv) {
coli_cuda_tensor_free(g4);coli_cuda_tensor_free(u4);coli_cuda_tensor_free(d4);
uint64_t group_calls=0,group_experts=0,group_total_rows=0;
coli_cuda_group_stats(&group_calls,&group_experts,&group_total_rows,nullptr,nullptr,nullptr);
- if(group_calls!=3||group_experts!=6||group_total_rows!=6) return 1;
+ if(group_calls!=4||group_experts!=8||group_total_rows!=8) return 1;
coli_cuda_stats(-1, &count, &bytes);
if (count != 7 || bytes != 166) {