Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(cuda): apply grouped scales in ragged attention

ZacharyZcR67a07fdbeb75
diff --git a/c/Makefile b/c/Makefile
index 71b4fd30..63b3f568 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -385,10 +385,12 @@ rans: $(RANSLIB)
$(RANSLIB): tools/rans_ctypes.c rans.h
$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)
-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu
+cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)
./backend_cuda_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)
+ ./ragged_attention_test$(EXE)
# convenience alias: kernel correctness on AMD (same test, hipcc toolchain)
hip-test:
@@ -398,6 +400,7 @@ hip-test:
# runners with the toolchain but no GPU). Pass CUDA_ARCH/HIP=1+HIP_ARCH.
gpu-compile: backend_cuda.o
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)
cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 6642eb96..beb8821b 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -549,7 +549,7 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q,
__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
const float *const *latent,const float *const *rope,const int *lengths,
const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R,
- int V,int K,int T,float scale){
+ int V,int K,int T,float scale,int gs,int ng){
int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V);
if(s>=S||nt<1||nt>T)return;
extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T;
@@ -557,7 +557,7 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
const float *ls=latent[s],*rs=rope[s];
for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int d=0;d<Q;d++)
a+=qs[d]*weight_at(weights,fmt,(size_t)(rbase+d)*row_bytes(fmt,K),k)*
- (fmt?wscale[rbase+d]:1.f);qa[k]=a;}
+ absorb_scale(wscale,fmt,gs,ng,rbase+d,k);qa[k]=a;}
__syncthreads();
for(int t=tid;t<nt;t+=blockDim.x){float a=0;const float *lt=ls+(size_t)t*K;
const float *rt=rs+(size_t)t*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];
@@ -574,8 +574,9 @@ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++)a+=scores[t]*ls[(size_t)t*K+k];cl[k]=a;}
__syncthreads();
for(int v=tid;v<V;v+=blockDim.x){int row=rbase+Q+v;float a=0;size_t rb=row_bytes(fmt,K);
- for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k);
- ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);}
+ for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k)*
+ absorb_scale(wscale,fmt,gs,ng,row,k);
+ ctx[((size_t)s*H+h)*V+v]=a;}
}
__global__ static void ragged_kv_append(float *const *latent,float *const *rope,
@@ -1312,7 +1313,7 @@ extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTens
std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0;
size_t shared=(size_t)(2*K+T+256)*sizeof(float);
attention_absorb_ragged_kernel<<<dim3(H,S),256,shared,dc->stream>>>(dc->ac,dc->aq,ddl,ddr,
- dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale);
+ dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale,w->gs,w->ng);
quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,
proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng);
return cuda_ok(cudaGetLastError(),"ragged attention launch")&&
diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cu
index dd1e2f89..34e89b3b 100644
--- a/c/tests/test_ragged_attention.cu
+++ b/c/tests/test_ragged_attention.cu
@@ -6,12 +6,12 @@
int main(){
int dev=0;if(!coli_cuda_init(&dev,1))return 77;
- constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3;
+ constexpr int S=3,H=2,Q=2,R=1,V=2,K=128,D=H*V,O=3,T=3;
std::vector<float> w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R));
for(size_t i=0;i<w.size();i++)w[i]=((int)(i%11)-5)*.07f;
for(size_t i=0;i<p.size();i++)p[i]=((int)(i%7)-3)*.09f;
for(size_t i=0;i<q.size();i++)q[i]=((int)(i%13)-6)*.05f;
- ColiCudaTensor *tw=nullptr,*tp=nullptr;
+ ColiCudaTensor *tw=nullptr,*tp=nullptr,*tg=nullptr;
if(!coli_cuda_tensor_upload(&tw,w.data(),nullptr,0,K,H*(Q+V),dev)||
!coli_cuda_tensor_upload(&tp,p.data(),nullptr,0,D,O,dev))return 1;
int n[S]={1,2,3};std::vector<std::vector<float>> l(S),r(S);
@@ -33,7 +33,29 @@ int main(){
double e=0,z=0;for(int i=0;i<S*O;i++){
double d=got[i]-ref[i];e+=d*d;z+=(double)ref[i]*ref[i];
}
- double rms=std::sqrt(e/(z+1e-30));std::printf("ragged_relative_rms=%.9g\n",rms);
- coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tp);coli_cuda_shutdown();
- return rms<1e-6?0:4;
+ double rms=std::sqrt(e/(z+1e-30));
+
+ constexpr int ROWS=H*(Q+V),GS=64,NG=(K+GS-1)/GS;
+ std::vector<unsigned char> q4((size_t)ROWS*((K+1)/2));
+ std::vector<float> scales((size_t)ROWS*NG);
+ for(int row=0;row<ROWS;row++){
+ for(int g=0;g<NG;g++)scales[(size_t)row*NG+g]=.01f*(1+row+3*g);
+ for(int k=0;k<K;k++){
+ int value=(row*3+k*5)%15-7,packed=value+8;
+ unsigned char &byte=q4[(size_t)row*((K+1)/2)+(k>>1)];
+ if(k&1)byte|=(unsigned char)(packed<<4);else byte=(unsigned char)packed;
+ }
+ }
+ if(!coli_cuda_tensor_upload_g(&tg,q4.data(),scales.data(),4,K,ROWS,dev,GS))return 5;
+ if(!coli_cuda_attention_project_ragged(tg,tp,got,q.data(),keys,lp,rp,n,
+ S,H,Q,R,V,K,T,.2f))return 6;
+ for(int s=0;s<S;s++)if(!coli_cuda_attention_project_batch(tg,tp,ref+s*O,
+ q.data()+s*H*(Q+R),lp[s],rp[s],1,H,Q,R,V,K,n[s],.2f))return 7;
+ double ge=0,gz=0;for(int i=0;i<S*O;i++){
+ double d=got[i]-ref[i];ge+=d*d;gz+=(double)ref[i]*ref[i];
+ }
+ double grms=std::sqrt(ge/(gz+1e-30));
+ std::printf("ragged_relative_rms=%.9g ragged_g64_relative_rms=%.9g\n",rms,grms);
+ coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tg);coli_cuda_tensor_free(tp);coli_cuda_shutdown();
+ return rms<1e-6&&grms<1e-6?0:4;
}