Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
merge: dev (#298 fmt=4 per-group scales) β HIP verified, test call sites repaired
noobdev-phfe6c164119ef
diff --cc c/backend_cuda.cuindex 6fe66caf,327915e4..4937b528--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@@ -687,8 -807,16 +806,16 @@@ extern "C" int coli_cuda_expert_group(Csilu_mul<<<(unsigned)(((size_t)total*I+255)/256),256,0,ctx->stream>>>(ctx->gate,ctx->up,(size_t)total*I);quantize_s4_rows<<<total,256,0,ctx->stream>>>(ctx->qx,ctx->qscale,ctx->gate,total,I);grouped_s4_wmma<<<dim3((unsigned)((D+63)/64),(unsigned)count),256,0,ctx->stream>>>(ctx->y,ctx->qx,ctx->qscale,dev,I,D,2);- }else if(all_s4&&ctx->compute_major>=7&&getenv("COLI_CUDA_TC_W4A16")&&+ }else if(all_s4&&COLI_GPU_HAS_WMMA&&ctx->compute_major>=7&&getenv("COLI_CUDA_TC_W4A16")&&- atoi(getenv("COLI_CUDA_TC_W4A16"))){+ atoi(getenv("COLI_CUDA_TC_W4A16"))&&+ [&]{ int tc16_min=getenv("COLI_CUDA_TC_W4A16_MIN")?atoi(getenv("COLI_CUDA_TC_W4A16_MIN")):16;+ for(int c=0;c<count;c++) if(rows[c]>=tc16_min) return 1;+ return 0; }()){+ /* At least one expert has enough rows for a Tensor Core tile. Groups+ * where EVERY expert is below the threshold (decode: r=1) fall through+ * to the grouped-W4 path below β 3 launches for the whole group instead+ * of 4 per expert (#431: the launch flood measured at ~981 micro-kernels+ * per token came from decode riding this branch's per-expert fallback). *//* W4A16 Tensor Core per gruppo: attivazioni fp16 per tile (lossless al* contrario del path W4A4), un lancio per expert dentro lo stream β* l'overhead di lancio e' trascurabile rispetto ai GEMM. */diff --cc c/tests/test_backend_cuda.cuindex 5550bdef,b0b5a3e8..1a826b8f--- a/c/tests/test_backend_cuda.cu+++ b/c/tests/test_backend_cuda.cu@@@ -49,7 -49,53 +49,53 @@@ int main(int argc, char **argv)if (!coli_cuda_tensor_upload(&t8, q8, s8, 1, 4, 2, d0)) return 1;if (coli_cuda_tensor_upload(&t8, q8, s8, 1, 5, 2, d0)) return 1;if (ndev > 1 && coli_cuda_tensor_upload(&t8, q8, s8, 1, 4, 2, d1)) return 1;-- if (!coli_cuda_matmul(&t8, got, x, q8, s8, 1, 2, 4, 2, d0) || !close_enough(got, want8, 4)) return 1;++ if (!coli_cuda_matmul(&t8, got, x, q8, s8, 1, 2, 4, 2, d0, 0) || !close_enough(got, want8, 4)) return 1;+ /* Cached tensor must stay callable without live host pointers+ * (CUDA_RELEASE_HOST slots null theirs after upload) β including+ * SUSTAINED reuse, not just the first call. */+ for (int rep = 0; rep < 64; rep++)- if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) ||++ if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) ||+ !close_enough(got, want8, 4)) return 1;+ /* A tensor uploaded from a TEMPORARY host buffer must survive the buffer+ * being scribbled and freed (the release-host lifecycle). */+ {+ int8_t *tmpw = static_cast<int8_t *>(std::malloc(8));+ float *tmps = static_cast<float *>(std::malloc(2 * sizeof(float)));+ if (!tmpw || !tmps) return 2;+ for (int i = 0; i < 8; i++) tmpw[i] = q8[i];+ tmps[0] = s8[0]; tmps[1] = s8[1];+ ColiCudaTensor *tt = nullptr;+ if (!coli_cuda_tensor_upload(&tt, tmpw, tmps, 1, 4, 2, d0)) return 1;+ for (int i = 0; i < 8; i++) tmpw[i] = 99;+ std::free(tmpw); std::free(tmps);- if (!coli_cuda_matmul(&tt, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) ||++ if (!coli_cuda_matmul(&tt, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) ||+ !close_enough(got, want8, 4)) return 1;+ coli_cuda_tensor_free(tt);+ }+ /* Upload failures must be graceful and must not corrupt accounting β+ * and must not poison LATER healthy launches (sticky-error regression). */+ {+ size_t c0 = 0, b0 = 0, c1 = 0, b1 = 0;+ coli_cuda_stats(-1, &c0, &b0);+ ColiCudaTensor *bad = nullptr;+ if (coli_cuda_tensor_upload(&bad, q8, s8, 1, 4, 2, 9999)) return 1;+ if (coli_cuda_tensor_upload(&bad, q8, s8, 7, 4, 2, d0)) return 1;+ if (coli_cuda_tensor_upload(&bad, q8, nullptr, 1, 4, 2, d0)) return 1;+ if (coli_cuda_tensor_upload(&bad, nullptr, s8, 1, 4, 2, d0)) return 1;+ if (coli_cuda_tensor_upload(&bad, q8, s8, 1, 1 << 20, 1 << 24, d0)) return 1; /* ~16 TB */+ if (bad) return 1;+ coli_cuda_stats(-1, &c1, &b1);+ if (c0 != c1 || b0 != b1) return 1;+ /* healthy launch immediately after the failed allocation */- if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) ||++ if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) ||+ !close_enough(got, want8, 4)) return 1;+ }+ /* Fault injection hook: on/off, restores cleanly. */+ if (setenv("COLI_GPU_FAIL_AFTER", "0", 1)) return 2;- if (coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0)) return 1;++ if (coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0)) return 1;+ if (unsetenv("COLI_GPU_FAIL_AFTER")) return 2;- if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0) ||++ if (!coli_cuda_matmul(&t8, got, x, nullptr, nullptr, 1, 2, 4, 2, d0, 0) ||+ !close_enough(got, want8, 4)) return 1;const int8_t q8b[8]={-1,-2,-3,-4, 1,-2,3,-4};const float s8b[2]={1.f,.5f},want8b[4]={10.f,15.f,-3.f,-2.5f};if(!coli_cuda_tensor_update(t8,q8b,s8b)||@@@ -78,9 -124,9 +124,9 @@@const float eu[8] = {1,0,0,0, 0,1,0,0};const float ed[8] = {1,0, 0,1, 1,1, 1,-1};ColiCudaTensor *tg=nullptr,*tu=nullptr,*td=nullptr;- if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0) ||- !coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0) ||- !coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0)) return 1;- if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0,0) ||- !coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0,0) ||- !coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0,0)) return 1;++ if (!coli_cuda_tensor_upload_g(&tg,eg,nullptr,0,4,2,d0,0) ||++ !coli_cuda_tensor_upload_g(&tu,eu,nullptr,0,4,2,d0,0) ||++ !coli_cuda_tensor_upload_g(&td,ed,nullptr,0,2,4,d0,0)) return 1;float expert[8], want_expert[8];for(int s=0;s<2;s++){float a=x[s*4], b=x[s*4+1];@@@ -98,7 -144,7 +144,7 @@@const float aw[16]={1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1};const float aq[4]={1,2,.5f,-.5f},al[12]={1,0,0,0, 0,1,0,0, 0,0,1,0};const float ar[6]={1,0, 0,1, 1,1};float actx[2],aref[2];- ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0))return 1;- ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0,0))return 1;++ ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload_g(&at,aw,nullptr,0,4,4,d0,0))return 1;float score[3];for(int t=0;t<3;t++)score[t]=aq[0]*al[t*4]+aq[1]*al[t*4+1]+aq[2]*ar[t*2]+aq[3]*ar[t*2+1];float mx=score[0],z=0;for(int t=1;t<3;t++)mx=score[t]>mx?score[t]:mx;for(int t=0;t<3;t++){score[t]=std::exp(score[t]-mx);z+=score[t];}for(int t=0;t<3;t++)score[t]/=z;@@@ -117,9 -163,9 +163,9 @@@for(int i=0;i<32;i++)ws4[i]=0.01f+(i%5)*0.002f;for(int i=0;i<64;i++)gx4[i]=std::sin((float)(i+1)*0.17f)*2.f;ColiCudaTensor *g4=nullptr,*u4=nullptr,*d4=nullptr;- if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0)||- !coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0)||- !coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0))return 1;- if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0,0)||- !coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0,0)||- !coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0,0))return 1;++ if(!coli_cuda_tensor_upload_g(&g4,w4,ws4,2,32,32,d0,0)||++ !coli_cuda_tensor_upload_g(&u4,w4,ws4,2,32,32,d0,0)||++ !coli_cuda_tensor_upload_g(&d4,w4,ws4,2,32,32,d0,0))return 1;ColiCudaTensor *gg4[2]={g4,g4},*ug4[2]={u4,u4},*dg4[2]={d4,d4};if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,scalar4,gx4))return 1;setenv("COLI_CUDA_TC_INT4","1",1);