Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix: extend the qdq scratch to the DUAL and BATCH units too
JustVugg45e0f991c430
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 25907576..b3066f7f 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -14641,10 +14641,10 @@ typedef struct {float *activation; size_t activation_capacity;uint8_t *scales; size_t scales_capacity;} V4QdqScratch;-static _Thread_local V4QdqScratch v4_qdq_scratch_state;-static int v4_qdq_scratch(size_t activation_count, size_t scales_count,+_Thread_local V4QdqScratch coli_v4_qdq_scratch_state;+int coli_v4_qdq_scratch(size_t activation_count, size_t scales_count,float **activation, uint8_t **scales) {- V4QdqScratch *s = &v4_qdq_scratch_state;+ V4QdqScratch *s = &coli_v4_qdq_scratch_state;if (activation_count > s->activation_capacity) {free(s->activation);s->activation = malloc(activation_count * sizeof(*s->activation));@@ -14688,7 +14688,7 @@ int coli_fp4_matvec_ref(float *output, const ColiTensorView *weight,return 0;#endiffloat *activation; uint8_t *activation_scales;- if (v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128) != 0) {@@ -14726,7 +14726,7 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,return 0;#endiffloat *activation; uint8_t *activation_scales;- if (v4_qdq_scratch(columns, scale_columns, &activation, &activation_scales))+ if (coli_v4_qdq_scratch(columns, scale_columns, &activation, &activation_scales))return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128) != 0) {@@ -14823,20 +14823,18 @@ int coli_fp4_dual_matvec_ref(float *output_a, float *output_b,coli_v4_gpu_matvec_grouped(b, output_b, input, 1) == 0)return 0;#endif- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(columns / 128);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128) != 0) {- free(activation_scales); free(activation); return -1;+ return -1;}matmul_mxfp4(output_a, activation, a->data, a->scales,1, (int)columns, (int)rows);matmul_mxfp4(output_b, activation, b->data, b->scales,1, (int)columns, (int)rows);- free(activation_scales); free(activation); return 0;+ return 0;}int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,@@ -14865,19 +14863,17 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,coli_v4_gpu_fp8_matvec(b, output_b, input) == 0)return 0;#endif- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(scale_columns);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, scale_columns, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128) != 0) {- free(activation_scales); free(activation); return -1;+ return -1;}#ifdef __AVX2__if (a->block_rows == 8) {if (rows % 8) {- free(activation_scales); free(activation); return -1;+ return -1;}float fp8[256];for (int code = 0; code < 256; code++)@@ -14912,14 +14908,14 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,_mm256_storeu_ps(output_a + (size_t)tile * 8, sum_a);_mm256_storeu_ps(output_b + (size_t)tile * 8, sum_b);}- free(activation_scales); free(activation); return 0;+ return 0;}#endifmatmul_fp8(output_a, activation, a->data, a->scales,1, (int)columns, (int)rows);matmul_fp8(output_b, activation, b->data, b->scales,1, (int)columns, (int)rows);- free(activation_scales); free(activation); return 0;+ return 0;}#endif /* COLI_V4_UNIT_NATIVE_QUANT_DUAL */@@ -14967,22 +14963,21 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,coli_v4_gpu_fp8_matmul_batch(weight, outputs, inputs, batch) == 0)return 0;#endif- float *activations = malloc((size_t)batch * columns * sizeof(*activations));- uint8_t *activation_scales = malloc((size_t)batch * scale_columns);- if (!activations || !activation_scales) {- free(activation_scales); free(activations); return -1;- }+ float *activations; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch((size_t)batch * columns, (size_t)batch * scale_columns,+ &activations, &activation_scales))+ return -1;for (int item = 0; item < batch; item++)if (coli_fp8_activation_qdq_ref(activations + (size_t)item * columns,activation_scales + (size_t)item * scale_columns,inputs + (size_t)item * columns, columns, 128) != 0) {- free(activation_scales); free(activations); return -1;+ return -1;}#ifdef __AVX2__if (weight->block_rows == 8) {if (rows % 8) {- free(activation_scales); free(activations); return -1;+ return -1;}float fp8[256];for (int code = 0; code < 256; code++)@@ -15017,12 +15012,12 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,_mm256_storeu_ps(outputs + (size_t)item * rows +(size_t)tile * 8, sums[item]);}- free(activation_scales); free(activations); return 0;+ return 0;}#endifmatmul_fp8(outputs, activations, weight->data, weight->scales,batch, (int)columns, (int)rows);- free(activation_scales); free(activations); return 0;+ return 0;}int coli_fp4_matmul_batch_ref(float *outputs, const ColiTensorView *weight,@@ -15037,21 +15032,20 @@ int coli_fp4_matmul_batch_ref(float *outputs, const ColiTensorView *weight,size_t packed_stride = columns / 2, scale_stride = columns / 32;if (weight->data_bytes != rows * packed_stride ||weight->scale_bytes != rows * scale_stride) return -1;- float *activations = malloc((size_t)batch * columns * sizeof(*activations));- uint8_t *activation_scales = malloc((size_t)batch * columns / 128);- if (!activations || !activation_scales) {- free(activation_scales); free(activations); return -1;- }+ float *activations; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch((size_t)batch * columns, (size_t)batch * columns / 128,+ &activations, &activation_scales))+ return -1;for (int item = 0; item < batch; item++)if (coli_fp8_activation_qdq_ref(activations + (size_t)item * columns,activation_scales + (size_t)item * columns / 128,inputs + (size_t)item * columns, columns, 128) != 0) {- free(activation_scales); free(activations); return -1;+ return -1;}matmul_mxfp4(outputs, activations, weight->data, weight->scales,batch, (int)columns, (int)rows);- free(activation_scales); free(activations); return 0;+ return 0;}#endif /* COLI_V4_UNIT_NATIVE_QUANT_BATCH */@@ -15248,14 +15242,12 @@ int coli_fp4_matvec_rows16_v10(float *output,if (!output || !input || !packed_valid(weight)) return -1;size_t columns = (size_t)weight->columns;size_t data_stride = columns / 2, scale_stride = columns / 32;- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(columns / 128);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128)) {- free(activation_scales); free(activation); return -1;+ return -1;}__m512 fp4_table; float e8[256]; decode_tables(&fp4_table, e8);const unsigned char *data = weight->data, *scales = weight->scales;@@ -15278,19 +15270,17 @@ int coli_fp4_matvec_rows16_v10(float *output,}_mm512_storeu_ps(output + (size_t)tile * 16, sum);}- free(activation_scales); free(activation); return 0;+ return 0;#elif defined(__AVX2__)if (!output || !input || !packed_valid(weight)) return -1;size_t columns = (size_t)weight->columns;size_t data_stride = columns / 2, scale_stride = columns / 32;- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(columns / 128);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128)) {- free(activation_scales); free(activation); return -1;+ return -1;}Avx2Rows16Tables tables; avx2_rows16_tables(&tables);const unsigned char *data = weight->data, *scales = weight->scales;@@ -15318,19 +15308,17 @@ int coli_fp4_matvec_rows16_v10(float *output,_mm256_storeu_ps(output + (size_t)tile * 16, sum[0]);_mm256_storeu_ps(output + (size_t)tile * 16 + 8, sum[1]);}- free(activation_scales); free(activation); return 0;+ return 0;#else /* __aarch64__ */if (!output || !input || !packed_valid(weight)) return -1;size_t columns = (size_t)weight->columns;size_t data_stride = columns / 2, scale_stride = columns / 32;- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(columns / 128);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128)) {- free(activation_scales); free(activation); return -1;+ return -1;}NeonRows16Tables tables;neon_rows16_tables(&tables);@@ -15360,7 +15348,7 @@ int coli_fp4_matvec_rows16_v10(float *output,for (int group = 0; group < 4; group++)vst1q_f32(output + (size_t)tile * 16 + 4 * group, sums[group]);}- free(activation_scales); free(activation); return 0;+ return 0;#endif}@@ -15376,14 +15364,12 @@ int coli_fp4_dual_matvec_rows16_v10(float *output_a, float *output_b,a->columns != b->columns) return -1;size_t columns = (size_t)a->columns;size_t data_stride = columns / 2, scale_stride = columns / 32;- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(columns / 128);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128)) {- free(activation_scales); free(activation); return -1;+ return -1;}__m512 fp4_table; float e8[256]; decode_tables(&fp4_table, e8);const unsigned char *data_a = a->data, *data_b = b->data;@@ -15416,21 +15402,19 @@ int coli_fp4_dual_matvec_rows16_v10(float *output_a, float *output_b,_mm512_storeu_ps(output_a + (size_t)tile * 16, sum_a);_mm512_storeu_ps(output_b + (size_t)tile * 16, sum_b);}- free(activation_scales); free(activation); return 0;+ return 0;#elif defined(__AVX2__)if (!output_a || !output_b || !input || !packed_valid(a) ||!packed_valid(b) || a->rows != b->rows ||a->columns != b->columns) return -1;size_t columns = (size_t)a->columns;size_t data_stride = columns / 2, scale_stride = columns / 32;- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(columns / 128);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128)) {- free(activation_scales); free(activation); return -1;+ return -1;}Avx2Rows16Tables tables; avx2_rows16_tables(&tables);const unsigned char *data_a = a->data, *data_b = b->data;@@ -15468,21 +15452,19 @@ int coli_fp4_dual_matvec_rows16_v10(float *output_a, float *output_b,_mm256_storeu_ps(output_b + (size_t)tile * 16, sum_b[0]);_mm256_storeu_ps(output_b + (size_t)tile * 16 + 8, sum_b[1]);}- free(activation_scales); free(activation); return 0;+ return 0;#else /* __aarch64__ */if (!output_a || !output_b || !input || !packed_valid(a) ||!packed_valid(b) || a->rows != b->rows ||a->columns != b->columns) return -1;size_t columns = (size_t)a->columns;size_t data_stride = columns / 2, scale_stride = columns / 32;- float *activation = malloc(columns * sizeof(*activation));- uint8_t *activation_scales = malloc(columns / 128);- if (!activation || !activation_scales) {- free(activation_scales); free(activation); return -1;- }+ float *activation; uint8_t *activation_scales;+ if (coli_v4_qdq_scratch(columns, columns / 128, &activation, &activation_scales))+ return -1;if (coli_fp8_activation_qdq_ref(activation, activation_scales,input, columns, 128)) {- free(activation_scales); free(activation); return -1;+ return -1;}NeonRows16Tables tables;neon_rows16_tables(&tables);@@ -15528,7 +15510,7 @@ int coli_fp4_dual_matvec_rows16_v10(float *output_a, float *output_b,sums_b[group]);}}- free(activation_scales); free(activation); return 0;+ return 0;#endif}#endif /* COLI_V4_UNIT_NATIVE_QUANT_ROWS16 */diff --git a/c/native_quant.h b/c/native_quant.hindex 3ee1934f..d6325def 100644--- a/c/native_quant.h+++ b/c/native_quant.h@@ -39,6 +39,13 @@ int coli_fp4_matvec_ref(float *output, const ColiTensorView *weight,int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,const float *input);+/* Growable thread-local scratch for the qdq activation buffers shared by the+ * *_ref/_pre matvec and matmul entries (defined in the NATIVE_QUANT unit).+ * Replaces the historical malloc/free pair per call; buffers are never+ * returned. Values computed from them are bit-for-bit unchanged. */+int coli_v4_qdq_scratch(size_t activation_count, size_t scales_count,+ float **activation, uint8_t **scales);+/* Optional CUDA tier (Windows engine build only, COLI_V4_GPU_TIER). The* engine-side wrappers in deepseek_v4.c resolve coli_cuda_dsv4.dll through* backend_loader_dsv4.c; the matvec_ref implementations dispatch to them when