Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(v4): eliminate dynamic heap allocations in indexer batch selection path
Thanniru Sai Tejae29ae2ad02a2
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 8d39135d..4a53e0c1 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -3514,6 +3514,8 @@ struct ColiDeepSeekV4Indexer {int capacity;int count;float *compressed;+ void *scratch_buf;+ size_t scratch_cap;};typedef struct { float score; int index; } IndexScore;@@ -3618,9 +3620,22 @@ void coli_v4_indexer_destroy(ColiDeepSeekV4Indexer *state) {if (!state) return;coli_v4_compressor_destroy(state->compressor);free(state->compressed);+ free(state->scratch_buf);free(state);}+static void *indexer_scratch_alloc(ColiDeepSeekV4Indexer *state, size_t needed) {+ if (!state) return NULL;+ if (state->scratch_cap < needed) {+ size_t new_cap = needed < 65536 ? 65536 : needed * 2;+ void *new_buf = realloc(state->scratch_buf, new_cap);+ if (!new_buf) return NULL;+ state->scratch_buf = new_buf;+ state->scratch_cap = new_cap;+ }+ return state->scratch_buf;+}+static int apply_position_rope(float *queries,const ColiDeepSeekV4Config *config,int position) {@@ -3800,19 +3815,36 @@ int coli_v4_indexer_select_batch(ColiDeepSeekV4Indexer *state, int *indices,const uint16_t *raw_weights = value(state->weights, "attn.indexer.weights_proj.weight", NULL);size_t qn = (size_t)heads * dimension;- float *queries = malloc((size_t)batch * qn * sizeof(*queries));- float *sq = malloc((size_t)need * qn * sizeof(*sq));- float *head_weights = malloc((size_t)need * heads * sizeof(*head_weights));- int *scounts = malloc((size_t)need * sizeof(*scounts));- int *stoken = malloc((size_t)need * sizeof(*stoken));- float *scores = malloc((size_t)need * max_count * sizeof(*scores));- IndexScore *ranked = malloc((size_t)max_count * sizeof(*ranked));- uint8_t *scales = malloc((size_t)dimension / 32);- float *qdq = malloc((size_t)dimension * sizeof(*qdq));+ size_t cols = (size_t)wq.columns;++ size_t sz_queries = (size_t)batch * qn * sizeof(float);+ size_t sz_sq = (size_t)need * qn * sizeof(float);+ size_t sz_head_weights = (size_t)need * heads * sizeof(float);+ size_t sz_scounts = (size_t)need * sizeof(int);+ size_t sz_stoken = (size_t)need * sizeof(int);+ size_t sz_scores = (size_t)need * max_count * sizeof(float);+ size_t sz_ranked = (size_t)max_count * sizeof(IndexScore);+ size_t sz_scales = (size_t)dimension / 32;+ size_t sz_qdq = (size_t)dimension * sizeof(float);+ size_t sz_gpu = (need <= 1024) ? ((size_t)need * cols * sizeof(float) + (size_t)need * qn * sizeof(float) + (size_t)need * (cols / 128)) : 0;++ size_t total_scratch = sz_queries + sz_sq + sz_head_weights + sz_scounts + sz_stoken ++ sz_scores + sz_ranked + sz_scales + sz_qdq + sz_gpu + 256;++ char *scratch_ptr = (char *)indexer_scratch_alloc(state, total_scratch);+ if (!scratch_ptr || !raw_weights)+ return set_error(error, error_size, "out of memory scoring indexer batch");++ float *queries = (float *)scratch_ptr; scratch_ptr += sz_queries;+ float *sq = (float *)scratch_ptr; scratch_ptr += sz_sq;+ float *head_weights = (float *)scratch_ptr; scratch_ptr += sz_head_weights;+ int *scounts = (int *)scratch_ptr; scratch_ptr += sz_scounts;+ int *stoken = (int *)scratch_ptr; scratch_ptr += sz_stoken;+ float *scores = (float *)scratch_ptr; scratch_ptr += sz_scores;+ IndexScore *ranked = (IndexScore *)scratch_ptr; scratch_ptr += sz_ranked;+ uint8_t *scales = (uint8_t *)scratch_ptr; scratch_ptr += sz_scales;+ float *qdq = (float *)scratch_ptr; scratch_ptr += sz_qdq;int result = 0;- if (!queries || !sq || !head_weights || !scounts || !stoken || !scores ||- !ranked || !scales || !qdq || !raw_weights)- result = set_error(error, error_size, "out of memory scoring indexer batch");/* Query projection. Preferred: host fp8 activation quantization (the* reference qdq, per token) + the GPU replica of the reference matmul —@@ -3823,10 +3855,9 @@ int coli_v4_indexer_select_batch(ColiDeepSeekV4Indexer *state, int *indices,int projected = 0;#ifdef COLI_V4_GPU_TIERif (!result && need <= 1024) {- size_t cols = (size_t)wq.columns;- float *xq = malloc((size_t)need * cols * sizeof(*xq));- float *yq = malloc((size_t)need * qn * sizeof(*yq));- uint8_t *xs = malloc((size_t)need * (cols / 128));+ float *xq = (float *)scratch_ptr; scratch_ptr += (size_t)need * cols * sizeof(*xq);+ float *yq = (float *)scratch_ptr; scratch_ptr += (size_t)need * qn * sizeof(*yq);+ uint8_t *xs = (uint8_t *)scratch_ptr;int qok = xq && yq && xs;if (qok) {int i = 0;@@ -3849,7 +3880,6 @@ int coli_v4_indexer_select_batch(ColiDeepSeekV4Indexer *state, int *indices,}projected = 1;}- free(xs); free(yq); free(xq);}#endif{@@ -3978,8 +4008,6 @@ int coli_v4_indexer_select_batch(ColiDeepSeekV4Indexer *state, int *indices,t_prep * 1e3, t_score * 1e3, t_sort * 1e3,gpu_scored ? "" : " (cpu-score)");#undef IDX_PROF_MARK- free(qdq); free(scales); free(ranked); free(scores); free(stoken);- free(scounts); free(head_weights); free(sq); free(queries);return result;}