Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: add native grouped E8 expert kernels

ZacharyZcRa842d5088f46
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 21fdf1fa..1ecec638 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -403,6 +403,48 @@ __global__ static void grouped_down(float *y,const float *x,const GroupDesc *des
if(!threadIdx.x) y[(size_t)(d.offset+s)*D+o]=p[0]*(d.df?d.ds[o]:1.f);
}
+/* Native fmt=6 expert groups. One block owns one (expert,row,output) and
+ * expands each 32-weight E8 sub-block once for both gate/up projections. */
+__global__ static void grouped_hidden_e8_dual(float *gate,const float *x,
+ const GroupDesc *desc,int I,int D){
+ int o=blockIdx.x,s=blockIdx.y,c=blockIdx.z;GroupDesc d=desc[c];if(s>=d.rows)return;
+ size_t rb=row_bytes(6,D);
+ const uint8_t *gr=(const uint8_t*)d.g+(size_t)o*rb;
+ const uint8_t *ur=(const uint8_t*)d.u+(size_t)o*rb;
+ const float *xs=x+(size_t)(d.offset+s)*D;float ga=0,ua=0;
+ int nsub=(D+COLI_E8_SUB-1)/COLI_E8_SUB;
+ for(int sb=threadIdx.x;sb<nsub;sb+=blockDim.x){
+ int ib=sb%(COLI_E8_QK/COLI_E8_SUB);
+ const uint8_t *gb=gr+(size_t)(sb/(COLI_E8_QK/COLI_E8_SUB))*COLI_E8_BBYTES;
+ const uint8_t *ub=ur+(size_t)(sb/(COLI_E8_QK/COLI_E8_SUB))*COLI_E8_BBYTES;
+ float gw[COLI_E8_SUB],uw[COLI_E8_SUB];
+ e8_expand_sub_dev(gb,ib,e8_fp16(gb+96),gw);
+ e8_expand_sub_dev(ub,ib,e8_fp16(ub+96),uw);
+ int off=sb*COLI_E8_SUB,n=D-off<COLI_E8_SUB?D-off:COLI_E8_SUB;
+ for(int k=0;k<n;k++){float v=xs[off+k];ga+=v*gw[k];ua+=v*uw[k];}
+ }
+ __shared__ float gp[256],up[256];gp[threadIdx.x]=ga;up[threadIdx.x]=ua;__syncthreads();
+ for(int n=128;n;n>>=1){if(threadIdx.x<n){gp[threadIdx.x]+=gp[threadIdx.x+n];up[threadIdx.x]+=up[threadIdx.x+n];}__syncthreads();}
+ if(!threadIdx.x){float g=gp[0];gate[(size_t)(d.offset+s)*I+o]=(g/(1.f+expf(-g)))*up[0];}
+}
+
+__global__ static void grouped_down_e8(float *y,const float *x,const GroupDesc *desc,int D,int I){
+ int o=blockIdx.x,s=blockIdx.y,c=blockIdx.z;GroupDesc d=desc[c];if(s>=d.rows)return;
+ size_t rb=row_bytes(6,I);const uint8_t *wr=(const uint8_t*)d.d+(size_t)o*rb;
+ const float *xs=x+(size_t)(d.offset+s)*I;float sum=0;
+ int nsub=(I+COLI_E8_SUB-1)/COLI_E8_SUB;
+ for(int sb=threadIdx.x;sb<nsub;sb+=blockDim.x){
+ int ib=sb%(COLI_E8_QK/COLI_E8_SUB);
+ const uint8_t *blk=wr+(size_t)(sb/(COLI_E8_QK/COLI_E8_SUB))*COLI_E8_BBYTES;
+ float w[COLI_E8_SUB];e8_expand_sub_dev(blk,ib,e8_fp16(blk+96),w);
+ int off=sb*COLI_E8_SUB,n=I-off<COLI_E8_SUB?I-off:COLI_E8_SUB;
+ for(int k=0;k<n;k++)sum+=xs[off+k]*w[k];
+ }
+ __shared__ float p[256];p[threadIdx.x]=sum;__syncthreads();
+ for(int n=128;n;n>>=1){if(threadIdx.x<n)p[threadIdx.x]+=p[threadIdx.x+n];__syncthreads();}
+ if(!threadIdx.x)y[(size_t)(d.offset+s)*D+o]=p[0];
+}
+
__device__ static void unpack_s4(uint8_t v,float *lo,float *hi){
int a=v&15,b=v>>4; *lo=(float)(a&8?a-16:a); *hi=(float)(b&8?b-16:b);
}
@@ -939,7 +981,7 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
if (!first) return 0;
int device=first->device,D=first->I,I=first->O,total=0,max_rows=0;
GroupDesc host[64]; if(count>64) return 0;
- int all_s4=1,all_q4=1,any_g4=0,any_e8=0;
+ int all_s4=1,all_q4=1,any_g4=0,any_e8=0,all_e8=1;
for(int c=0;c<count;c++){
ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];
if(!g||!u||!d||rows[c]<1||g->device!=device||u->device!=device||d->device!=device||
@@ -952,13 +994,11 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
!(g->gs&1)&&!(u->gs&1)&&!(d->gs&1); /* even gs: a packed byte never straddles groups */
any_g4|=g->fmt==4||u->fmt==4||d->fmt==4;
any_e8|=g->fmt==6||u->fmt==6||d->fmt==6;
+ all_e8&=g->fmt==6&&u->fmt==6&&d->fmt==6;
total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];
}
- /* The grouped kernels decode formats 0..4 only. Sending fmt=6 through
- * grouped_hidden's element decoder reads the 98-byte E8 blocks as int2 and
- * runs past the real row stride on GLM-sized tensors. Keep the proven E8
- * expert kernel until a native grouped E8 kernel exists. */
- if(any_e8){
+ /* Mixed E8 groups cannot use either homogeneous grouped kernel. */
+ if(any_e8&&!all_e8){
int off=0;
for(int c=0;c<count;c++){
if(!coli_cuda_expert_mlp(gates[c],ups[c],downs[c],
@@ -1000,7 +1040,12 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
tc=tc&&COLI_GPU_HAS_WMMA&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;
int tc_min=getenv("COLI_CUDA_TC_MIN_ROWS")?atoi(getenv("COLI_CUDA_TC_MIN_ROWS")):8;
for(int c=0;c<count&&tc;c++)tc=rows[c]>=tc_min;
- if(tc){
+ if(all_e8){
+ dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count),og((unsigned)D,(unsigned)max_rows,(unsigned)count);
+ grouped_hidden_e8_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->x,dev,I,D);
+ if(!e8_rot_rows_dev(ctx->gate,total,I,ctx->stream))return 0;
+ grouped_down_e8<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);
+ }else if(tc){
size_t qb=(size_t)(total+7)*(size_t)(D>I?D:I)/2;
if(!reserve_bytes((void**)&ctx->qx,&ctx->qx_cap,qb)||
!reserve(&ctx->qscale,&ctx->qscale_cap,(size_t)(total+7)*sizeof(float)))return 0;
@@ -1116,7 +1161,7 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
if (!gates || !ups || !downs || !rows || !x || count < 1 || count > 64) return 0;
ColiCudaTensor *first=gates[0];
if (!first) return 0;
- int device=first->device,D=first->I,I=first->O,total=0,max_rows=0,all_s4=1,any_e8=0;
+ int device=first->device,D=first->I,I=first->O,total=0,max_rows=0,all_s4=1,any_e8=0,all_e8=1;
GroupDesc host[64];
for(int c=0;c<count;c++){
ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];
@@ -1127,9 +1172,10 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
g->gs,u->gs,d->gs};
all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2;
any_e8|=g->fmt==6||u->fmt==6||d->fmt==6;
+ all_e8&=g->fmt==6&&u->fmt==6&&d->fmt==6;
total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];
}
- if(any_e8) return 0; /* sync path uses expert_mlp safely */
+ if(any_e8&&!all_e8) return 0;
if(total>8) return 0; /* decode-scale only */
DeviceContext *ctx=find_ctx(device); if(!ctx||ctx->group_pending||!select_ctx(ctx)) return 0;
size_t xb=(size_t)total*D*sizeof(float), ib=(size_t)total*I*sizeof(float);
@@ -1144,7 +1190,14 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
"expert group issue descriptors")||
!cuda_ok(cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream),
"expert group issue upload")) return 0;
- if(all_s4&&(!getenv("COLI_CUDA_W4_PACKED")||atoi(getenv("COLI_CUDA_W4_PACKED")))){
+ if(all_e8){
+ GroupDesc *dev=(GroupDesc*)ctx->group_desc;
+ dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count);
+ dim3 og((unsigned)D,(unsigned)max_rows,(unsigned)count);
+ grouped_hidden_e8_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->x,dev,I,D);
+ if(!e8_rot_rows_dev(ctx->gate,total,I,ctx->stream))return 0;
+ grouped_down_e8<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);
+ }else if(all_s4&&(!getenv("COLI_CUDA_W4_PACKED")||atoi(getenv("COLI_CUDA_W4_PACKED")))){
GroupDesc *dev=(GroupDesc*)ctx->group_desc;
dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count);
dim3 og((unsigned)D,(unsigned)max_rows,(unsigned)count);