Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(cuda): apply grouped scales in ragged attention
ZacharyZcR67a07fdbeb75
diff --git a/c/Makefile b/c/Makefileindex 71b4fd30..63b3f568 100644--- a/c/Makefile+++ b/c/Makefile@@ -385,10 +385,12 @@ rans: $(RANSLIB)$(RANSLIB): tools/rans_ctypes.c rans.h$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu+cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)./backend_cuda_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)+ ./ragged_attention_test$(EXE)# convenience alias: kernel correctness on AMD (same test, hipcc toolchain)hip-test:@@ -398,6 +400,7 @@ hip-test:# runners with the toolchain but no GPU). Pass CUDA_ARCH/HIP=1+HIP_ARCH.gpu-compile: backend_cuda.o"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 6642eb96..beb8821b 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -549,7 +549,7 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q,__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,const float *const *latent,const float *const *rope,const int *lengths,const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R,- int V,int K,int T,float scale){+ int V,int K,int T,float scale,int gs,int ng){int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V);if(s>=S||nt<1||nt>T)return;extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T;@@ -557,7 +557,7 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,const float *ls=latent[s],*rs=rope[s];for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int d=0;d<Q;d++)a+=qs[d]*weight_at(weights,fmt,(size_t)(rbase+d)*row_bytes(fmt,K),k)*- (fmt?wscale[rbase+d]:1.f);qa[k]=a;}+ absorb_scale(wscale,fmt,gs,ng,rbase+d,k);qa[k]=a;}__syncthreads();for(int t=tid;t<nt;t+=blockDim.x){float a=0;const float *lt=ls+(size_t)t*K;const float *rt=rs+(size_t)t*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];@@ -574,8 +574,9 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++)a+=scores[t]*ls[(size_t)t*K+k];cl[k]=a;}__syncthreads();for(int v=tid;v<V;v+=blockDim.x){int row=rbase+Q+v;float a=0;size_t rb=row_bytes(fmt,K);- for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k);- ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);}+ for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k)*+ absorb_scale(wscale,fmt,gs,ng,row,k);+ ctx[((size_t)s*H+h)*V+v]=a;}}__global__ static void ragged_kv_append(float *const *latent,float *const *rope,@@ -1312,7 +1313,7 @@ extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensstd::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0;size_t shared=(size_t)(2*K+T+256)*sizeof(float);attention_absorb_ragged_kernel<<<dim3(H,S),256,shared,dc->stream>>>(dc->ac,dc->aq,ddl,ddr,- dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale);+ dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng);quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng);return cuda_ok(cudaGetLastError(),"ragged attention launch")&&diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cuindex dd1e2f89..34e89b3b 100644--- a/c/tests/test_ragged_attention.cu+++ b/c/tests/test_ragged_attention.cu@@ -6,12 +6,12 @@int main(){int dev=0;if(!coli_cuda_init(&dev,1))return 77;- constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3;+ constexpr int S=3,H=2,Q=2,R=1,V=2,K=128,D=H*V,O=3,T=3;std::vector<float> w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R));for(size_t i=0;i<w.size();i++)w[i]=((int)(i%11)-5)*.07f;for(size_t i=0;i<p.size();i++)p[i]=((int)(i%7)-3)*.09f;for(size_t i=0;i<q.size();i++)q[i]=((int)(i%13)-6)*.05f;- ColiCudaTensor *tw=nullptr,*tp=nullptr;+ ColiCudaTensor *tw=nullptr,*tp=nullptr,*tg=nullptr;if(!coli_cuda_tensor_upload(&tw,w.data(),nullptr,0,K,H*(Q+V),dev)||!coli_cuda_tensor_upload(&tp,p.data(),nullptr,0,D,O,dev))return 1;int n[S]={1,2,3};std::vector<std::vector<float>> l(S),r(S);@@ -33,7 +33,29 @@ int main(){double e=0,z=0;for(int i=0;i<S*O;i++){double d=got[i]-ref[i];e+=d*d;z+=(double)ref[i]*ref[i];}- double rms=std::sqrt(e/(z+1e-30));std::printf("ragged_relative_rms=%.9g\n",rms);- coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tp);coli_cuda_shutdown();- return rms<1e-6?0:4;+ double rms=std::sqrt(e/(z+1e-30));++ constexpr int ROWS=H*(Q+V),GS=64,NG=(K+GS-1)/GS;+ std::vector<unsigned char> q4((size_t)ROWS*((K+1)/2));+ std::vector<float> scales((size_t)ROWS*NG);+ for(int row=0;row<ROWS;row++){+ for(int g=0;g<NG;g++)scales[(size_t)row*NG+g]=.01f*(1+row+3*g);+ for(int k=0;k<K;k++){+ int value=(row*3+k*5)%15-7,packed=value+8;+ unsigned char &byte=q4[(size_t)row*((K+1)/2)+(k>>1)];+ if(k&1)byte|=(unsigned char)(packed<<4);else byte=(unsigned char)packed;+ }+ }+ if(!coli_cuda_tensor_upload_g(&tg,q4.data(),scales.data(),4,K,ROWS,dev,GS))return 5;+ if(!coli_cuda_attention_project_ragged(tg,tp,got,q.data(),keys,lp,rp,n,+ S,H,Q,R,V,K,T,.2f))return 6;+ for(int s=0;s<S;s++)if(!coli_cuda_attention_project_batch(tg,tp,ref+s*O,+ q.data()+s*H*(Q+R),lp[s],rp[s],1,H,Q,R,V,K,n[s],.2f))return 7;+ double ge=0,gz=0;for(int i=0;i<S*O;i++){+ double d=got[i]-ref[i];ge+=d*d;gz+=(double)ref[i]*ref[i];+ }+ double grms=std::sqrt(ge/(gz+1e-30));+ std::printf("ragged_relative_rms=%.9g ragged_g64_relative_rms=%.9g\n",rms,grms);+ coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tg);coli_cuda_tensor_free(tp);coli_cuda_shutdown();+ return rms<1e-6&&grms<1e-6?0:4;}