Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda: keep async packed-int4 groups token exact
ZacharyZcR455b071ebcae
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 5a739cd9..a85c7adb 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -1085,7 +1085,7 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,if (!gates || !ups || !downs || !rows || !x || count < 1 || count > 64) return 0;ColiCudaTensor *first=gates[0];if (!first) return 0;- int device=first->device,D=first->I,I=first->O,total=0;+ int device=first->device,D=first->I,I=first->O,total=0,max_rows=0,all_s4=1;GroupDesc host[64];for(int c=0;c<count;c++){ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];@@ -1094,19 +1094,37 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,host[c]={g->weights,u->weights,d->weights,g->scales,u->scales,d->scales,g->fmt,u->fmt,d->fmt,rows[c],total,g->gs,u->gs,d->gs};- total+=rows[c];+ all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2;+ total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];}if(total>8) return 0; /* decode-scale only */DeviceContext *ctx=find_ctx(device); if(!ctx||ctx->group_pending||!select_ctx(ctx)) return 0;size_t xb=(size_t)total*D*sizeof(float), ib=(size_t)total*I*sizeof(float);if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)||!reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)||+ !reserve_bytes(&ctx->group_desc,&ctx->group_desc_cap,(size_t)count*sizeof(GroupDesc))||!reserve_pinned(&ctx->host_x,&ctx->host_x_cap,xb)||!reserve_pinned(&ctx->host_y,&ctx->host_y_cap,xb)) return 0;std::memcpy(ctx->host_x,x,xb);- if(!cuda_ok(cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream),+ if(!cuda_ok(cudaMemcpyAsync(ctx->group_desc,host,(size_t)count*sizeof(GroupDesc),+ cudaMemcpyHostToDevice,ctx->stream),+ "expert group issue descriptors")||+ !cuda_ok(cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream),"expert group issue upload")) return 0;- for(int c=0;c<count;c++){+ if(all_s4&&(!getenv("COLI_CUDA_W4_PACKED")||atoi(getenv("COLI_CUDA_W4_PACKED")))){+ GroupDesc *dev=(GroupDesc*)ctx->group_desc;+ dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count);+ dim3 og((unsigned)D,(unsigned)max_rows,(unsigned)count);+ int dual=!getenv("COLI_CUDA_DUAL_PROJ")||atoi(getenv("COLI_CUDA_DUAL_PROJ"));+ if(dual) grouped_hidden_w4_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->up,ctx->x,dev,I,D);+ else {+ grouped_hidden_w4<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->x,dev,I,D,0);+ grouped_hidden_w4<<<hg,256,0,ctx->stream>>>(ctx->up,ctx->x,dev,I,D,1);+ silu_mul<<<(unsigned)(((size_t)total*I+255)/256),256,0,ctx->stream>>>(+ ctx->gate,ctx->up,(size_t)total*I);+ }+ grouped_down_w4<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);+ } else for(int c=0;c<count;c++){int r=rows[c];float *g16=ctx->gate+(size_t)host[c].offset*I,*u16=ctx->up+(size_t)host[c].offset*I;float *x16=ctx->x+(size_t)host[c].offset*D,*y16=ctx->y+(size_t)host[c].offset*D;diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cuindex 0253cf60..f1537750 100644--- a/c/tests/test_backend_cuda.cu+++ b/c/tests/test_backend_cuda.cu@@ -341,7 +341,7 @@ int main(int argc, char **argv) {/* Native s4 WMMA path: compare the quantized-activation result against theexisting FP32-activation/s4-weight grouped implementation. */- uint8_t w4[32*32/2]; float ws4[32], gx4[64], scalar4[64], tensor4[64];+ uint8_t w4[32*32/2]; float ws4[32], gx4[64], scalar4[64], async4[64], tensor4[64];for(int i=0;i<(int)sizeof(w4);i++){int lo=((i%15)-7)&15,hi=(((i*3)%15)-7)&15;w4[i]=(uint8_t)(lo|(hi<<4));@@ -354,6 +354,14 @@ int main(int argc, char **argv) {!coli_cuda_tensor_upload_g(&d4,w4,ws4,2,32,32,d0,0))return 1;ColiCudaTensor *gg4[2]={g4,g4},*ug4[2]={u4,u4},*dg4[2]={d4,d4};if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,scalar4,gx4))return 1;+ if(!coli_cuda_expert_group_issue(gg4,ug4,dg4,group_rows,2,gx4))return 1;+ const float *async_result=coli_cuda_expert_group_take(d0);+ if(!async_result)return 1;+ std::memcpy(async4,async_result,sizeof(async4));+ if(std::memcmp(async4,scalar4,sizeof(async4))){+ std::fprintf(stderr,"async packed-s4 group differs from sync path\n");+ return 1;+ }setenv("COLI_CUDA_TC_INT4","1",1);setenv("COLI_CUDA_TC_MIN_ROWS","1",1);if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,tensor4,gx4)||@@ -363,7 +371,7 @@ int main(int argc, char **argv) {coli_cuda_tensor_free(g4);coli_cuda_tensor_free(u4);coli_cuda_tensor_free(d4);uint64_t group_calls=0,group_experts=0,group_total_rows=0;coli_cuda_group_stats(&group_calls,&group_experts,&group_total_rows,nullptr,nullptr,nullptr);- if(group_calls!=3||group_experts!=6||group_total_rows!=6) return 1;+ if(group_calls!=4||group_experts!=8||group_total_rows!=8) return 1;coli_cuda_stats(-1, &count, &bytes);if (count != 7 || bytes != 166) {