Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(cuda): stop fmt=8 tensor_free from overcounting freed scale bytes
monotophic3bad5a28fedd
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex ef332c6c..2af3f329 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -2287,17 +2287,19 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {DeviceContext *ctx = find_ctx(tensor->device);if (ctx) select_ctx(ctx);if (tensor->tracked && ctx) {- int ng = tensor->ng > 0 ? tensor->ng : 1;- /* Must mirror the upload's accounting exactly: fmt=6 never charged for a- * scale buffer, and over-subtracting here trips the >= guard below, which- * silently leaves the tensor's bytes on the device counter forever. */+ /* Must mirror the upload's accounting exactly -- literally the same+ * expression upload uses to charge (scale_count * sizeof(float), gated+ * on fmt=6 never having a separate scale buffer), so the two can no+ * longer drift independently. Over-subtracting here trips the >= guard+ * below, which silently leaves the tensor's bytes on the device counter+ * forever. */size_t storage_bytes =#ifdef COLI_ANStensor->compressed ? tensor->archive_bytes :#endiftensor->weight_bytes;size_t bytes = storage_bytes +- ((tensor->fmt && tensor->fmt != 6) ? (size_t)tensor->O * ng * sizeof(float) : 0);+ ((tensor->fmt && tensor->fmt != 6) ? tensor->scale_count * sizeof(float) : 0);if (ctx->tensor_count) ctx->tensor_count--;if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes;}diff --git a/c/tests/test_fp8_cuda.cu b/c/tests/test_fp8_cuda.cuindex c3965dc9..7b1bb0c7 100644--- a/c/tests/test_fp8_cuda.cu+++ b/c/tests/test_fp8_cuda.cu@@ -218,8 +218,50 @@ int main(void){for(int c=0;c<COUNT;c++){ coli_cuda_tensor_free(tg[c]);coli_cuda_tensor_free(tu[c]);coli_cuda_tensor_free(td[c]);free(hg[c]);free(hu[c]);free(hd[c]);free(hgs[c]);free(hus[c]);free(hds[c]); }free(x);free(ysync);free(hw);free(hsc);- coli_cuda_shutdown();if(api_bad){ printf("FAIL\n"); return 1; }++ /* ---- Byte-accounting regression: coli_cuda_tensor_free must undo+ * exactly what upload charged, per scale shape. fmt=8's 128x128 block+ * scale_count ((O+127)/128 blocks, not O rows) is the shape that once+ * made free() over-count and trip the tensor_bytes >= bytes guard,+ * leaving the diagnostic counter stuck non-zero forever; fmt=4 grouped+ * (O*ng row-group scales) is covered alongside since it shares the same+ * free()-side expression. */+ {+ size_t c0,b0,c1,b1;+ coli_cuda_stats(0,&c0,&b0);+ ColiCudaTensor *bt=nullptr;+ uint8_t *bw=(uint8_t*)malloc((size_t)I*D); float *bs=(float*)malloc(hs_n*4);+ for(size_t i=0;i<(size_t)I*D;i++) bw[i]=rnd_e4m3();+ for(int i=0;i<hs_n;i++) bs[i]=ldexpf(1.f+rand()/(float)RAND_MAX,-12);+ if(!coli_cuda_tensor_upload(&bt,bw,bs,8,D,I,0)){ printf("FAIL byte-check fmt8 upload\n"); return 1; }+ coli_cuda_tensor_free(bt);+ coli_cuda_stats(0,&c1,&b1);+ if(c1!=c0||b1!=b0){+ printf("FAIL byte-check fmt8: count %zu->%zu bytes %zu->%zu (want unchanged)\n",c0,c1,b0,b1);+ return 1;+ }+ free(bw);free(bs);++ const int GI=64,GO=32,GS=32;+ int gng=(GI+GS-1)/GS;+ ColiCudaTensor *gt=nullptr;+ uint8_t *gw=(uint8_t*)malloc((size_t)((GI+1)/2)*GO);+ float *gsc=(float*)malloc((size_t)GO*gng*4);+ for(size_t i=0;i<(size_t)((GI+1)/2)*GO;i++) gw[i]=(uint8_t)rand();+ for(int i=0;i<GO*gng;i++) gsc[i]=ldexpf(1.f+rand()/(float)RAND_MAX,-4);+ coli_cuda_stats(0,&c0,&b0);+ if(!coli_cuda_tensor_upload_g(>,gw,gsc,4,GI,GO,0,GS)){ printf("FAIL byte-check fmt4g upload\n"); return 1; }+ coli_cuda_tensor_free(gt);+ coli_cuda_stats(0,&c1,&b1);+ if(c1!=c0||b1!=b0){+ printf("FAIL byte-check fmt4g: count %zu->%zu bytes %zu->%zu (want unchanged)\n",c0,c1,b0,b1);+ return 1;+ }+ free(gw);free(gsc);+ printf("byte accounting: fmt=8 dense + fmt=4 grouped free() restores stats exactly\n");+ }+ coli_cuda_shutdown();}printf("OK\n"); return 0;}