Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda: add native grouped E8 expert kernels
ZacharyZcRa842d5088f46
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 21fdf1fa..1ecec638 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -403,6 +403,48 @@ __global__ static void grouped_down(float *y,const float *x,const GroupDesc *desif(!threadIdx.x) y[(size_t)(d.offset+s)*D+o]=p[0]*(d.df?d.ds[o]:1.f);}+/* Native fmt=6 expert groups. One block owns one (expert,row,output) and+ * expands each 32-weight E8 sub-block once for both gate/up projections. */+__global__ static void grouped_hidden_e8_dual(float *gate,const float *x,+ const GroupDesc *desc,int I,int D){+ int o=blockIdx.x,s=blockIdx.y,c=blockIdx.z;GroupDesc d=desc[c];if(s>=d.rows)return;+ size_t rb=row_bytes(6,D);+ const uint8_t *gr=(const uint8_t*)d.g+(size_t)o*rb;+ const uint8_t *ur=(const uint8_t*)d.u+(size_t)o*rb;+ const float *xs=x+(size_t)(d.offset+s)*D;float ga=0,ua=0;+ int nsub=(D+COLI_E8_SUB-1)/COLI_E8_SUB;+ for(int sb=threadIdx.x;sb<nsub;sb+=blockDim.x){+ int ib=sb%(COLI_E8_QK/COLI_E8_SUB);+ const uint8_t *gb=gr+(size_t)(sb/(COLI_E8_QK/COLI_E8_SUB))*COLI_E8_BBYTES;+ const uint8_t *ub=ur+(size_t)(sb/(COLI_E8_QK/COLI_E8_SUB))*COLI_E8_BBYTES;+ float gw[COLI_E8_SUB],uw[COLI_E8_SUB];+ e8_expand_sub_dev(gb,ib,e8_fp16(gb+96),gw);+ e8_expand_sub_dev(ub,ib,e8_fp16(ub+96),uw);+ int off=sb*COLI_E8_SUB,n=D-off<COLI_E8_SUB?D-off:COLI_E8_SUB;+ for(int k=0;k<n;k++){float v=xs[off+k];ga+=v*gw[k];ua+=v*uw[k];}+ }+ __shared__ float gp[256],up[256];gp[threadIdx.x]=ga;up[threadIdx.x]=ua;__syncthreads();+ for(int n=128;n;n>>=1){if(threadIdx.x<n){gp[threadIdx.x]+=gp[threadIdx.x+n];up[threadIdx.x]+=up[threadIdx.x+n];}__syncthreads();}+ if(!threadIdx.x){float g=gp[0];gate[(size_t)(d.offset+s)*I+o]=(g/(1.f+expf(-g)))*up[0];}+}++__global__ static void grouped_down_e8(float *y,const float *x,const GroupDesc *desc,int D,int I){+ int o=blockIdx.x,s=blockIdx.y,c=blockIdx.z;GroupDesc d=desc[c];if(s>=d.rows)return;+ size_t rb=row_bytes(6,I);const uint8_t *wr=(const uint8_t*)d.d+(size_t)o*rb;+ const float *xs=x+(size_t)(d.offset+s)*I;float sum=0;+ int nsub=(I+COLI_E8_SUB-1)/COLI_E8_SUB;+ for(int sb=threadIdx.x;sb<nsub;sb+=blockDim.x){+ int ib=sb%(COLI_E8_QK/COLI_E8_SUB);+ const uint8_t *blk=wr+(size_t)(sb/(COLI_E8_QK/COLI_E8_SUB))*COLI_E8_BBYTES;+ float w[COLI_E8_SUB];e8_expand_sub_dev(blk,ib,e8_fp16(blk+96),w);+ int off=sb*COLI_E8_SUB,n=I-off<COLI_E8_SUB?I-off:COLI_E8_SUB;+ for(int k=0;k<n;k++)sum+=xs[off+k]*w[k];+ }+ __shared__ float p[256];p[threadIdx.x]=sum;__syncthreads();+ for(int n=128;n;n>>=1){if(threadIdx.x<n)p[threadIdx.x]+=p[threadIdx.x+n];__syncthreads();}+ if(!threadIdx.x)y[(size_t)(d.offset+s)*D+o]=p[0];+}+__device__ static void unpack_s4(uint8_t v,float *lo,float *hi){int a=v&15,b=v>>4; *lo=(float)(a&8?a-16:a); *hi=(float)(b&8?b-16:b);}@@ -939,7 +981,7 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,if (!first) return 0;int device=first->device,D=first->I,I=first->O,total=0,max_rows=0;GroupDesc host[64]; if(count>64) return 0;- int all_s4=1,all_q4=1,any_g4=0,any_e8=0;+ int all_s4=1,all_q4=1,any_g4=0,any_e8=0,all_e8=1;for(int c=0;c<count;c++){ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];if(!g||!u||!d||rows[c]<1||g->device!=device||u->device!=device||d->device!=device||@@ -952,13 +994,11 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,!(g->gs&1)&&!(u->gs&1)&&!(d->gs&1); /* even gs: a packed byte never straddles groups */any_g4|=g->fmt==4||u->fmt==4||d->fmt==4;any_e8|=g->fmt==6||u->fmt==6||d->fmt==6;+ all_e8&=g->fmt==6&&u->fmt==6&&d->fmt==6;total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];}- /* The grouped kernels decode formats 0..4 only. Sending fmt=6 through- * grouped_hidden's element decoder reads the 98-byte E8 blocks as int2 and- * runs past the real row stride on GLM-sized tensors. Keep the proven E8- * expert kernel until a native grouped E8 kernel exists. */- if(any_e8){+ /* Mixed E8 groups cannot use either homogeneous grouped kernel. */+ if(any_e8&&!all_e8){int off=0;for(int c=0;c<count;c++){if(!coli_cuda_expert_mlp(gates[c],ups[c],downs[c],@@ -1000,7 +1040,12 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,tc=tc&&COLI_GPU_HAS_WMMA&&all_s4&&D%32==0&&I%32==0&&D%8==0&&I%8==0;int tc_min=getenv("COLI_CUDA_TC_MIN_ROWS")?atoi(getenv("COLI_CUDA_TC_MIN_ROWS")):8;for(int c=0;c<count&&tc;c++)tc=rows[c]>=tc_min;- if(tc){+ if(all_e8){+ dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count),og((unsigned)D,(unsigned)max_rows,(unsigned)count);+ grouped_hidden_e8_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->x,dev,I,D);+ if(!e8_rot_rows_dev(ctx->gate,total,I,ctx->stream))return 0;+ grouped_down_e8<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);+ }else if(tc){size_t qb=(size_t)(total+7)*(size_t)(D>I?D:I)/2;if(!reserve_bytes((void**)&ctx->qx,&ctx->qx_cap,qb)||!reserve(&ctx->qscale,&ctx->qscale_cap,(size_t)(total+7)*sizeof(float)))return 0;@@ -1116,7 +1161,7 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,if (!gates || !ups || !downs || !rows || !x || count < 1 || count > 64) return 0;ColiCudaTensor *first=gates[0];if (!first) return 0;- int device=first->device,D=first->I,I=first->O,total=0,max_rows=0,all_s4=1,any_e8=0;+ int device=first->device,D=first->I,I=first->O,total=0,max_rows=0,all_s4=1,any_e8=0,all_e8=1;GroupDesc host[64];for(int c=0;c<count;c++){ColiCudaTensor *g=gates[c],*u=ups[c],*d=downs[c];@@ -1127,9 +1172,10 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,g->gs,u->gs,d->gs};all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2;any_e8|=g->fmt==6||u->fmt==6||d->fmt==6;+ all_e8&=g->fmt==6&&u->fmt==6&&d->fmt==6;total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];}- if(any_e8) return 0; /* sync path uses expert_mlp safely */+ if(any_e8&&!all_e8) return 0;if(total>8) return 0; /* decode-scale only */DeviceContext *ctx=find_ctx(device); if(!ctx||ctx->group_pending||!select_ctx(ctx)) return 0;size_t xb=(size_t)total*D*sizeof(float), ib=(size_t)total*I*sizeof(float);@@ -1144,7 +1190,14 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,"expert group issue descriptors")||!cuda_ok(cudaMemcpyAsync(ctx->x,ctx->host_x,xb,cudaMemcpyHostToDevice,ctx->stream),"expert group issue upload")) return 0;- if(all_s4&&(!getenv("COLI_CUDA_W4_PACKED")||atoi(getenv("COLI_CUDA_W4_PACKED")))){+ if(all_e8){+ GroupDesc *dev=(GroupDesc*)ctx->group_desc;+ dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count);+ dim3 og((unsigned)D,(unsigned)max_rows,(unsigned)count);+ grouped_hidden_e8_dual<<<hg,256,0,ctx->stream>>>(ctx->gate,ctx->x,dev,I,D);+ if(!e8_rot_rows_dev(ctx->gate,total,I,ctx->stream))return 0;+ grouped_down_e8<<<og,256,0,ctx->stream>>>(ctx->y,ctx->gate,dev,D,I);+ }else if(all_s4&&(!getenv("COLI_CUDA_W4_PACKED")||atoi(getenv("COLI_CUDA_W4_PACKED")))){GroupDesc *dev=(GroupDesc*)ctx->group_desc;dim3 hg((unsigned)I,(unsigned)max_rows,(unsigned)count);dim3 og((unsigned)D,(unsigned)max_rows,(unsigned)count);