Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tests: fmt=8 warp-kernel oracle (sweep, census/tail shapes, S-invariance)

monotophic9aa32f2bfb9a
diff --git a/c/Makefile b/c/Makefile
index e05be77c..c6a2e732 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -689,7 +689,7 @@ rans: $(RANSLIB)
$(RANSLIB): tools/rans_ctypes.c rans.h
$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)
-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c
+cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)
./backend_cuda_test$(EXE)
@@ -701,6 +701,11 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen
$(CC) $(CFLAGS) -c tests/mxfp4_ref.c -o tests/mxfp4_ref.o
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE) $(GPU_OMP_LINK)
./mxfp4_cuda_test$(EXE)
+ # fmt=8 warp kernels (COLI_CUDA_F8_WARP): decode sweep, census + tail
+ # shapes, S-invariance, NaN/denormal policy. Includes backend_cuda.cu
+ # itself (kernel-level oracle), so only the test file is compiled.
+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)
+ ./fp8_warp_test$(EXE)
# convenience alias: kernel correctness on AMD (same test, hipcc toolchain)
hip-test:
@@ -711,6 +716,7 @@ hip-test:
gpu-compile: backend_cuda.o
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) tests/test_fp8_warp_cuda.cu -o fp8_warp_test$(EXE) $(ANS_NVCC_LIBS)
cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
diff --git a/c/tests/test_fp8_warp_cuda.cu b/c/tests/test_fp8_warp_cuda.cu
new file mode 100644
index 00000000..fa41d6e8
--- /dev/null
+++ b/c/tests/test_fp8_warp_cuda.cu
@@ -0,0 +1,379 @@
+/* fmt=8 (fp8-e4m3) warp-kernel oracle (COLI_CUDA_F8_WARP rework).
+ *
+ * Complements tests/test_fp8_cuda.cu (which pins the ORIGINAL kernels and the
+ * upload API on small tail shapes) with the contracts the warp rework adds:
+ *
+ * 1. 256-value decode sweep: every byte decoded on-device through each
+ * compiled path (shared-LUT, and the cuda_fp8.h cvt path where built)
+ * against an arithmetic host reference β€” bit-compare, NaN via isnan,
+ * plus a proof-of-bite pass with one mutated LUT entry.
+ * 2. LUT gate: fmt=8 upload refused before coli_cuda_fp8_set_lut.
+ * 3. Grouped-vs-CPU on the census shapes (gate/up 6144x2048, down
+ * 2048x6144) AND a synthetic tail shape (O,I not %128), relative RMS +
+ * max-ulp logged; dense quant_matmul branch on the census shape too.
+ * 4. Old-vs-new: COLI_CUDA_F8_WARP=0 reruns the same group through the
+ * original kernels; both must sit inside the CPU-reference bound (the
+ * two paths are NOT bit-identical β€” the accumulation convention is the
+ * point of the rework).
+ * 5. Mixed-group fallback: fmt=8 + fmt=2 group == per-expert MLP, bitwise.
+ * 6. issue/take async parity vs the sync path, bitwise.
+ * 7. S-invariance: rows=64 in one call vs 64 rows=1 calls, bitwise (the
+ * s-tile changes concurrency, never the order within a dot).
+ * 8. NaN-byte injection: a planted 0x7F reaches the output as NaN exactly
+ * where the CPU reference produces one (no kernel-side scrubbing).
+ * 9. Denormal-scale contribution: scale x subnormal-decode landing in the
+ * f32 denormal range must match the CPU bit for bit (catches -ftz/
+ * fast-math contamination of the build, not just kernel bugs).
+ *
+ * The e4m3 reference decoder is computed arithmetically here, so it
+ * cross-checks the LUT rather than assuming it.
+ *
+ * Build: nvcc -O2 -std=c++17 -arch=native tests/test_fp8_warp_cuda.cu -o tests/test_fp8_warp
+ */
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cmath>
+#include <cstdint>
+#include <cuda_runtime.h>
+
+#include "../backend_cuda.cu"
+
+#ifdef _WIN32
+/* MSVC has no POSIX setenv/unsetenv */
+static int setenv(const char *name, const char *value, int overwrite) {
+ (void)overwrite; return _putenv_s(name, value);
+}
+static int unsetenv(const char *name) { return _putenv_s(name, ""); }
+#endif
+
+static float e4m3_ref(uint8_t b){
+ int s=b>>7, e=(b>>3)&15, m=b&7;
+ if(e==15&&m==7) return NAN; /* E4M3-FN: only NaN, no inf */
+ float v = e ? ldexpf(1.f+m/8.f, e-7) : ldexpf(m/8.f, -6);
+ return s ? -v : v;
+}
+
+static uint8_t rnd_e4m3(void){ /* any byte except the two NaNs */
+ uint8_t b=(uint8_t)(rand()&255);
+ if((b&0x7F)==0x7F) b&=(uint8_t)~1;
+ return b;
+}
+
+/* matmul_fp8's accumulation, one output element: float within a 128-block,
+ * double across blocks, block scale applied on the block subtotal. */
+static void cpu_gemv_f8(const uint8_t *q,const float *sc,int K,int O,
+ const float *x,float *y){
+ int nblk=(K+127)/128;
+ for(int o=0;o<O;o++){
+ const uint8_t *w=q+(size_t)o*K;
+ const float *scl=sc+(size_t)(o/128)*nblk;
+ double a=0;
+ for(int b=0;b*128<K;b++){
+ int base=b*128, len=K-base<128?K-base:128; float acc=0;
+ for(int i=base;i<base+len;i++) acc+=e4m3_ref(w[i])*x[i];
+ a+=(double)acc*scl[b];
+ }
+ y[o]=(float)a;
+ }
+}
+
+/* relative RMS + max ulp distance, both logged (the ulp figure is
+ * informational: across a sign flip it is not a meaningful distance). */
+static int rms_named(const char *name,const float *got,const float *want,
+ size_t n,float limit){
+ double err=0,ref=0; long mu=0;
+ for(size_t i=0;i<n;i++){
+ double d=(double)got[i]-want[i]; err+=d*d; ref+=(double)want[i]*want[i];
+ int32_t a,b; memcpy(&a,&got[i],4); memcpy(&b,&want[i],4);
+ long u=labs((long)a-(long)b); if(u>mu)mu=u;
+ }
+ float r=(float)sqrt(err/(ref+1e-20));
+ printf("%s: relative RMS %.3g, max ulp %ld\n",name,r,mu);
+ if(r>limit){ printf("FAIL %s (limit %.3g)\n",name,limit); return 0; }
+ return 1;
+}
+
+template<int HW>
+__global__ static void f8_sweep_kernel(float *out){
+ __shared__ float slut[256];
+ for(int i=threadIdx.x;i<256;i+=blockDim.x) slut[i]=c_e4m3[i];
+ __syncthreads();
+ for(int i=threadIdx.x;i<256;i+=blockDim.x) out[i]=f8_dec<HW>(slut,(uint8_t)i);
+}
+
+static int sweep_check(const char *name,const float *got,const float *want){
+ int bad=0;
+ for(int i=0;i<256;i++){
+ if((i&0x7F)==0x7F){ if(!(isnan(got[i])&&isnan(want[i]))) bad++; }
+ else if(memcmp(&got[i],&want[i],4)) bad++;
+ }
+ printf("decode sweep [%s]: %d mismatches\n",name,bad);
+ return bad;
+}
+
+/* One full expert group on the GPU (sync path) + the CPU reference chain. */
+static void cpu_expert_chain(const uint8_t *g,const uint8_t *u,const uint8_t *d,
+ const float *gs,const float *us,const float *ds,int D,int I,
+ const float *x,float *y,float *h){
+ float *rg=(float*)malloc((size_t)I*4),*ru=(float*)malloc((size_t)I*4);
+ cpu_gemv_f8(g,gs,D,I,x,rg);
+ cpu_gemv_f8(u,us,D,I,x,ru);
+ for(int o=0;o<I;o++) h[o]=(rg[o]/(1.f+expf(-rg[o])))*ru[o];
+ cpu_gemv_f8(d,ds,I,D,h,y);
+ free(rg); free(ru);
+}
+
+#define NEXP 3
+int main(void){
+ srand(11);
+ int devs[1]={0};
+ if(!coli_cuda_init(devs,1)){ printf("FAIL cuda init\n"); return 1; }
+ float lut[256]; for(int i=0;i<256;i++) lut[i]=e4m3_ref((uint8_t)i);
+
+ /* ---- LUT gate: refusal before publication ------------------------------ */
+ {
+ uint8_t w[128]; float sc[1]={1.f}; ColiCudaTensor *t=nullptr;
+ for(int i=0;i<128;i++) w[i]=rnd_e4m3();
+ if(coli_cuda_tensor_upload(&t,w,sc,8,128,1,0)){ printf("FAIL gate open before LUT\n"); return 1; }
+ }
+ if(!coli_cuda_fp8_set_lut(lut)){ printf("FAIL set_lut\n"); return 1; }
+
+ /* ---- 256-value decode sweep, every compiled path ----------------------- */
+ {
+ float *dout,got[256]; cudaMalloc(&dout,256*4);
+ f8_sweep_kernel<0><<<1,256>>>(dout);
+ cudaMemcpy(got,dout,256*4,cudaMemcpyDeviceToHost);
+ if(sweep_check("shared-lut",got,lut)){ printf("FAIL\n"); return 1; }
+#if COLI_F8_HWCVT
+ f8_sweep_kernel<1><<<1,256>>>(dout);
+ cudaMemcpy(got,dout,256*4,cudaMemcpyDeviceToHost);
+ if(sweep_check("hw-cvt",got,lut)){ printf("FAIL\n"); return 1; }
+#else
+ printf("decode sweep [hw-cvt]: not compiled on this toolchain\n");
+#endif
+ /* proof of bite: a single mutated LUT entry must be caught */
+ float bad_lut[256]; memcpy(bad_lut,lut,sizeof(lut));
+ bad_lut[0x35]*=2.f;
+ cudaMemcpyToSymbol(c_e4m3,bad_lut,sizeof(bad_lut));
+ f8_sweep_kernel<0><<<1,256>>>(dout);
+ cudaMemcpy(got,dout,256*4,cudaMemcpyDeviceToHost);
+ int bit=sweep_check("shared-lut, mutated entry (must bite)",got,lut);
+ cudaMemcpyToSymbol(c_e4m3,lut,sizeof(lut));
+ cudaFree(dout);
+ if(bit!=1){ printf("FAIL sweep has no bite\n"); return 1; }
+ }
+
+ /* ---- tail shape (O,I not %128): sync group vs CPU + async parity ------- */
+ {
+ const int D=201,I=300; /* blocks: 128+73 and 2x128+44;
+ * D%4!=0 also forces the guarded
+ * byte path on full blocks */
+ const int nbD=(D+127)/128,nbI=(I+127)/128;
+ int rows[NEXP]={1,2,5},total=8; /* total 8: issue path admits it */
+ uint8_t *hg[NEXP],*hu[NEXP],*hd[NEXP]; float *hgs[NEXP],*hus[NEXP],*hds[NEXP];
+ ColiCudaTensor *tg[NEXP]={},*tu[NEXP]={},*td[NEXP]={};
+ for(int c=0;c<NEXP;c++){
+ hg[c]=(uint8_t*)malloc((size_t)I*D); hu[c]=(uint8_t*)malloc((size_t)I*D);
+ hd[c]=(uint8_t*)malloc((size_t)D*I);
+ hgs[c]=(float*)malloc((size_t)nbI*nbD*4); hus[c]=(float*)malloc((size_t)nbI*nbD*4);
+ hds[c]=(float*)malloc((size_t)nbD*nbI*4);
+ for(size_t i=0;i<(size_t)I*D;i++){ hg[c][i]=rnd_e4m3(); hu[c][i]=rnd_e4m3(); }
+ for(size_t i=0;i<(size_t)D*I;i++) hd[c][i]=rnd_e4m3();
+ for(int i=0;i<nbI*nbD;i++){ hgs[c][i]=ldexpf(1.f+rand()/(float)RAND_MAX,-12);
+ hus[c][i]=ldexpf(1.f+rand()/(float)RAND_MAX,-12); }
+ for(int i=0;i<nbD*nbI;i++) hds[c][i]=ldexpf(1.f+rand()/(float)RAND_MAX,-12);
+ if(!coli_cuda_tensor_upload(&tg[c],hg[c],hgs[c],8,D,I,0)||
+ !coli_cuda_tensor_upload(&tu[c],hu[c],hus[c],8,D,I,0)||
+ !coli_cuda_tensor_upload(&td[c],hd[c],hds[c],8,I,D,0)){
+ printf("FAIL tail upload\n"); return 1; }
+ }
+ float *x=(float*)malloc((size_t)total*D*4),*ys=(float*)malloc((size_t)total*D*4);
+ float *want=(float*)malloc((size_t)total*D*4),*h=(float*)malloc((size_t)I*4);
+ for(size_t i=0;i<(size_t)total*D;i++) x[i]=(rand()/(float)RAND_MAX-.5f)*2.f;
+ if(!coli_cuda_expert_group(tg,tu,td,rows,NEXP,ys,x)){ printf("FAIL tail sync group\n"); return 1; }
+ int off=0;
+ for(int c=0;c<NEXP;c++){ for(int s=0;s<rows[c];s++)
+ cpu_expert_chain(hg[c],hu[c],hd[c],hgs[c],hus[c],hds[c],D,I,
+ x+(size_t)(off+s)*D,want+(size_t)(off+s)*D,h); off+=rows[c]; }
+ if(!rms_named("tail 201x300 grouped vs CPU",ys,want,(size_t)total*D,1e-4f)) return 1;
+ if(!coli_cuda_expert_group_issue(tg,tu,td,rows,NEXP,x)){ printf("FAIL tail issue\n"); return 1; }
+ const float *ya=coli_cuda_expert_group_take(0);
+ if(!ya){ printf("FAIL tail take\n"); return 1; }
+ if(memcmp(ya,ys,(size_t)total*D*4)){ printf("FAIL async != sync (bitwise)\n"); return 1; }
+ printf("tail async/take parity: bitwise OK\n");
+ for(int c=0;c<NEXP;c++){ coli_cuda_tensor_free(tg[c]);coli_cuda_tensor_free(tu[c]);coli_cuda_tensor_free(td[c]);
+ free(hg[c]);free(hu[c]);free(hd[c]);free(hgs[c]);free(hus[c]);free(hds[c]); }
+ free(x);free(ys);free(want);free(h);
+ }
+
+ /* ---- census shapes: grouped (new AND old path) + dense, vs CPU --------- */
+ {
+ const int D=2048,I=6144; /* GLM-5.2 expert geometry */
+ const int nbD=D/128,nbI=I/128;
+ const int NC=2; int rows[NC]={1,3},total=4;
+ uint8_t *hg[NC],*hu[NC],*hd[NC]; float *hgs[NC],*hus[NC],*hds[NC];
+ ColiCudaTensor *tg[NC]={},*tu[NC]={},*td[NC]={};
+ for(int c=0;c<NC;c++){
+ hg[c]=(uint8_t*)malloc((size_t)I*D); hu[c]=(uint8_t*)malloc((size_t)I*D);
+ hd[c]=(uint8_t*)malloc((size_t)D*I);
+ hgs[c]=(float*)malloc((size_t)nbI*nbD*4); hus[c]=(float*)malloc((size_t)nbI*nbD*4);
+ hds[c]=(float*)malloc((size_t)nbD*nbI*4);
+ for(size_t i=0;i<(size_t)I*D;i++){ hg[c][i]=rnd_e4m3(); hu[c][i]=rnd_e4m3(); }
+ for(size_t i=0;i<(size_t)D*I;i++) hd[c][i]=rnd_e4m3();
+ for(int i=0;i<nbI*nbD;i++){ hgs[c][i]=ldexpf(1.f+rand()/(float)RAND_MAX,-12);
+ hus[c][i]=ldexpf(1.f+rand()/(float)RAND_MAX,-12); }
+ for(int i=0;i<nbD*nbI;i++) hds[c][i]=ldexpf(1.f+rand()/(float)RAND_MAX,-12);
+ if(!coli_cuda_tensor_upload(&tg[c],hg[c],hgs[c],8,D,I,0)||
+ !coli_cuda_tensor_upload(&tu[c],hu[c],hus[c],8,D,I,0)||
+ !coli_cuda_tensor_upload(&td[c],hd[c],hds[c],8,I,D,0)){
+ printf("FAIL census upload\n"); return 1; }
+ }
+ float *x=(float*)malloc((size_t)total*D*4),*ys=(float*)malloc((size_t)total*D*4);
+ float *yold=(float*)malloc((size_t)total*D*4);
+ float *want=(float*)malloc((size_t)total*D*4),*h=(float*)malloc((size_t)I*4);
+ for(size_t i=0;i<(size_t)total*D;i++) x[i]=(rand()/(float)RAND_MAX-.5f)*2.f;
+ if(!coli_cuda_expert_group(tg,tu,td,rows,NC,ys,x)){ printf("FAIL census sync group\n"); return 1; }
+ int off=0;
+ for(int c=0;c<NC;c++){ for(int s=0;s<rows[c];s++)
+ cpu_expert_chain(hg[c],hu[c],hd[c],hgs[c],hus[c],hds[c],D,I,
+ x+(size_t)(off+s)*D,want+(size_t)(off+s)*D,h); off+=rows[c]; }
+ if(!rms_named("census 6144x2048/2048x6144 grouped (warp) vs CPU",ys,want,(size_t)total*D,1e-4f)) return 1;
+ /* old kernels, same inputs: still inside the CPU bound (not bitwise β€”
+ * the accumulation convention is what changed) */
+ setenv("COLI_CUDA_F8_WARP","0",1);
+ if(!coli_cuda_expert_group(tg,tu,td,rows,NC,yold,x)){ printf("FAIL census old-path group\n"); return 1; }
+ unsetenv("COLI_CUDA_F8_WARP");
+ if(!rms_named("census grouped (old kernels) vs CPU",yold,want,(size_t)total*D,1e-4f)) return 1;
+ rms_named("census grouped old vs new (informational)",yold,ys,(size_t)total*D,1.f);
+
+ /* dense branch (quant_matmul fmt=8) on the census gate matrix */
+ {
+ const int S=2;
+ float *dx=(float*)malloc((size_t)S*D*4),*dy=(float*)malloc((size_t)S*I*4);
+ float *dwant=(float*)malloc((size_t)S*I*4);
+ ColiCudaTensor *t=nullptr;
+ for(size_t i=0;i<(size_t)S*D;i++) dx[i]=(rand()/(float)RAND_MAX-.5f)*2.f;
+ if(!coli_cuda_matmul(&t,dy,dx,hg[0],hgs[0],8,S,D,I,0,0)){ printf("FAIL census dense matmul\n"); return 1; }
+ for(int s=0;s<S;s++) cpu_gemv_f8(hg[0],hgs[0],D,I,dx+(size_t)s*D,dwant+(size_t)s*I);
+ if(!rms_named("census dense fmt=8 vs CPU",dy,dwant,(size_t)S*I,1e-4f)) return 1;
+ coli_cuda_tensor_free(t);
+ free(dx);free(dy);free(dwant);
+ }
+
+ /* ---- S-invariance: rows=64 in one call vs 64 single-row calls ------ */
+ {
+ int r64[1]={64},r1[1]={1};
+ float *sx=(float*)malloc((size_t)64*D*4),*y64=(float*)malloc((size_t)64*D*4);
+ float *y1=(float*)malloc((size_t)D*4);
+ for(size_t i=0;i<(size_t)64*D;i++) sx[i]=(rand()/(float)RAND_MAX-.5f)*2.f;
+ if(!coli_cuda_expert_group(&tg[0],&tu[0],&td[0],r64,1,y64,sx)){ printf("FAIL S=64 group\n"); return 1; }
+ int sbad=0;
+ for(int k=0;k<64;k++){
+ if(!coli_cuda_expert_group(&tg[0],&tu[0],&td[0],r1,1,y1,sx+(size_t)k*D)){ printf("FAIL S=1 group\n"); return 1; }
+ if(memcmp(y1,y64+(size_t)k*D,(size_t)D*4)) sbad++;
+ }
+ printf("S-invariance (S=64 vs 64x S=1): %d mismatching rows\n",sbad);
+ if(sbad){ printf("FAIL\n"); return 1; }
+ free(sx);free(y64);free(y1);
+ }
+ for(int c=0;c<NC;c++){ coli_cuda_tensor_free(tg[c]);coli_cuda_tensor_free(tu[c]);coli_cuda_tensor_free(td[c]);
+ free(hg[c]);free(hu[c]);free(hd[c]);free(hgs[c]);free(hus[c]);free(hds[c]); }
+ free(x);free(ys);free(yold);free(want);free(h);
+ }
+
+ /* ---- mixed group (fmt=8 + fmt=2): fallback equals per-expert MLP ------- */
+ {
+ const int D=256,I=128; const int nbD=D/128,nbI=I/128;
+ int rows[2]={2,1},total=3;
+ uint8_t *f8g=(uint8_t*)malloc((size_t)I*D),*f8u=(uint8_t*)malloc((size_t)I*D),
+ *f8d=(uint8_t*)malloc((size_t)D*I);
+ float *f8gs=(float*)malloc((size_t)nbI*nbD*4),*f8us=(float*)malloc((size_t)nbI*nbD*4),
+ *f8ds=(float*)malloc((size_t)nbD*nbI*4);
+ for(size_t i=0;i<(size_t)I*D;i++){ f8g[i]=rnd_e4m3(); f8u[i]=rnd_e4m3(); }
+ for(size_t i=0;i<(size_t)D*I;i++) f8d[i]=rnd_e4m3();
+ for(int i=0;i<nbI*nbD;i++){ f8gs[i]=ldexpf(1.f,-10); f8us[i]=ldexpf(1.f,-10); }
+ for(int i=0;i<nbD*nbI;i++) f8ds[i]=ldexpf(1.f,-10);
+ uint8_t *i4g=(uint8_t*)malloc((size_t)I*((D+1)/2)),*i4u=(uint8_t*)malloc((size_t)I*((D+1)/2)),
+ *i4d=(uint8_t*)malloc((size_t)D*((I+1)/2));
+ float *i4gs=(float*)malloc((size_t)I*4),*i4us=(float*)malloc((size_t)I*4),
+ *i4ds=(float*)malloc((size_t)D*4);
+ for(size_t i=0;i<(size_t)I*((D+1)/2);i++){ i4g[i]=(uint8_t)(rand()&255); i4u[i]=(uint8_t)(rand()&255); }
+ for(size_t i=0;i<(size_t)D*((I+1)/2);i++) i4d[i]=(uint8_t)(rand()&255);
+ for(int i=0;i<I;i++){ i4gs[i]=ldexpf(1.f,-8); i4us[i]=ldexpf(1.f,-8); }
+ for(int i=0;i<D;i++) i4ds[i]=ldexpf(1.f,-8);
+ ColiCudaTensor *tg[2]={},*tu[2]={},*td[2]={};
+ if(!coli_cuda_tensor_upload(&tg[0],f8g,f8gs,8,D,I,0)||
+ !coli_cuda_tensor_upload(&tu[0],f8u,f8us,8,D,I,0)||
+ !coli_cuda_tensor_upload(&td[0],f8d,f8ds,8,I,D,0)||
+ !coli_cuda_tensor_upload(&tg[1],i4g,i4gs,2,D,I,0)||
+ !coli_cuda_tensor_upload(&tu[1],i4u,i4us,2,D,I,0)||
+ !coli_cuda_tensor_upload(&td[1],i4d,i4ds,2,I,D,0)){
+ printf("FAIL mixed upload\n"); return 1; }
+ float *x=(float*)malloc((size_t)total*D*4),*yg=(float*)malloc((size_t)total*D*4),
+ *ym=(float*)malloc((size_t)total*D*4);
+ for(size_t i=0;i<(size_t)total*D;i++) x[i]=(rand()/(float)RAND_MAX-.5f)*2.f;
+ if(!coli_cuda_expert_group(tg,tu,td,rows,2,yg,x)){ printf("FAIL mixed group\n"); return 1; }
+ if(!coli_cuda_expert_mlp(tg[0],tu[0],td[0],ym,x,rows[0])||
+ !coli_cuda_expert_mlp(tg[1],tu[1],td[1],ym+(size_t)rows[0]*D,x+(size_t)rows[0]*D,rows[1])){
+ printf("FAIL mixed per-expert MLP\n"); return 1; }
+ if(memcmp(yg,ym,(size_t)total*D*4)){ printf("FAIL mixed group != per-expert MLP (bitwise)\n"); return 1; }
+ printf("mixed fmt=8/fmt=2 group falls back to per-expert MLP: bitwise OK\n");
+ for(int c=0;c<2;c++){ coli_cuda_tensor_free(tg[c]);coli_cuda_tensor_free(tu[c]);coli_cuda_tensor_free(td[c]); }
+ free(f8g);free(f8u);free(f8d);free(f8gs);free(f8us);free(f8ds);
+ free(i4g);free(i4u);free(i4d);free(i4gs);free(i4us);free(i4ds);
+ free(x);free(yg);free(ym);
+ }
+
+ /* ---- NaN-byte injection: planted 0x7F reaches the output as NaN -------- */
+ {
+ const int D=256,I=128; const int nbD=D/128,nbI=I/128;
+ int rows[1]={2},total=2;
+ uint8_t *g=(uint8_t*)malloc((size_t)I*D),*u=(uint8_t*)malloc((size_t)I*D),
+ *d=(uint8_t*)malloc((size_t)D*I);
+ float *gs=(float*)malloc((size_t)nbI*nbD*4),*us=(float*)malloc((size_t)nbI*nbD*4),
+ *ds=(float*)malloc((size_t)nbD*nbI*4);
+ for(size_t i=0;i<(size_t)I*D;i++){ g[i]=rnd_e4m3(); u[i]=rnd_e4m3(); }
+ for(size_t i=0;i<(size_t)D*I;i++) d[i]=rnd_e4m3();
+ for(int i=0;i<nbI*nbD;i++){ gs[i]=ldexpf(1.f,-10); us[i]=ldexpf(1.f,-10); }
+ for(int i=0;i<nbD*nbI;i++) ds[i]=ldexpf(1.f,-10);
+ d[(size_t)3*I+7]=0x7F; /* down row 3: output col 3 NaN */
+ ColiCudaTensor *tg=nullptr,*tu=nullptr,*td=nullptr;
+ if(!coli_cuda_tensor_upload(&tg,g,gs,8,D,I,0)||
+ !coli_cuda_tensor_upload(&tu,u,us,8,D,I,0)||
+ !coli_cuda_tensor_upload(&td,d,ds,8,I,D,0)){ printf("FAIL nan upload\n"); return 1; }
+ float *x=(float*)malloc((size_t)total*D*4),*y=(float*)malloc((size_t)total*D*4),
+ *want=(float*)malloc((size_t)total*D*4),*h=(float*)malloc((size_t)I*4);
+ for(size_t i=0;i<(size_t)total*D;i++) x[i]=(rand()/(float)RAND_MAX-.5f)*2.f;
+ if(!coli_cuda_expert_group(&tg,&tu,&td,rows,1,y,x)){ printf("FAIL nan group\n"); return 1; }
+ for(int s=0;s<total;s++)
+ cpu_expert_chain(g,u,d,gs,us,ds,D,I,x+(size_t)s*D,want+(size_t)s*D,h);
+ int nbad=0;
+ for(size_t i=0;i<(size_t)total*D;i++)
+ if(isnan(y[i])!=isnan(want[i])) nbad++;
+ for(int s=0;s<total;s++) if(!isnan(y[(size_t)s*D+3])) nbad++;
+ printf("NaN injection (0x7F in down row 3): %d policy mismatches\n",nbad);
+ if(nbad){ printf("FAIL\n"); return 1; }
+ coli_cuda_tensor_free(tg);coli_cuda_tensor_free(tu);coli_cuda_tensor_free(td);
+ free(g);free(u);free(d);free(gs);free(us);free(ds);free(x);free(y);free(want);free(h);
+ }
+
+ /* ---- denormal-scale contribution (dense path, bit-exact vs CPU) -------- */
+ {
+ uint8_t w[128]={}; float sc[1]={ldexpf(1.f,-120)};
+ float x[128]={},y[1]={-1.f};
+ w[0]=0x01; x[0]=1.f; /* 2^-9 * 2^-120 = 2^-129: f32 denormal */
+ ColiCudaTensor *t=nullptr;
+ if(!coli_cuda_matmul(&t,y,x,w,sc,8,1,128,1,0,0)){ printf("FAIL denormal matmul\n"); return 1; }
+ float expect=(float)((double)(e4m3_ref(0x01)*1.f)*sc[0]);
+ if(expect==0.f||memcmp(&y[0],&expect,4)){
+ printf("FAIL denormal contribution: got %a want %a (FTZ contamination?)\n",y[0],expect);
+ return 1;
+ }
+ printf("denormal-scale contribution: bit-exact (%a)\n",y[0]);
+ coli_cuda_tensor_free(t);
+ }
+
+ coli_cuda_shutdown();
+ printf("OK\n"); return 0;
+}