Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
deepseek_v4: bound RoPE computation to requested positions
waizuichougoub5bd711eecac
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 4047e43c..e358e512 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -1077,6 +1077,7 @@ int coli_v4_expert_store_open_planned(/* ######## deepseek_v4_math.c ######## */#include "deepseek_v4_internal.h"+#include <limits.h>#include <math.h>#include <stdlib.h>@@ -1232,15 +1233,11 @@ int coli_v4_rmsnorm(float *output, const float *input, const float *weight,return 0;}-int coli_v4_rope_precompute(float *cosines, float *sines,- int dimension, int sequence_length,- int original_sequence_length, float base,- float factor, int beta_fast, int beta_slow) {- if (!cosines || !sines || dimension < 2 || (dimension & 1) ||- sequence_length < 1 || !(base > 1.0f) || !(factor > 0.0f))- return -1;- int pairs = dimension / 2;- int low = 0, high = -1;+static void rope_yarn_bounds(int dimension, int original_sequence_length,+ float base, int beta_fast, int beta_slow,+ int *low, int *high) {+ *low = 0;+ *high = -1;if (original_sequence_length > 0) {const float two_pi = 6.2831853071795864769f;float denominator = 2.0f * logf(base);@@ -1248,24 +1245,48 @@ int coli_v4_rope_precompute(float *cosines, float *sines,original_sequence_length / (beta_fast * two_pi)) / denominator;float high_value = dimension * logf(original_sequence_length / (beta_slow * two_pi)) / denominator;- low = (int)floorf(low_value);- high = (int)ceilf(high_value);- if (low < 0) low = 0;- if (high > dimension - 1) high = dimension - 1;+ *low = (int)floorf(low_value);+ *high = (int)ceilf(high_value);+ if (*low < 0) *low = 0;+ if (*high > dimension - 1) *high = dimension - 1;+ }+}++static float rope_frequency(int pair, int dimension,+ int original_sequence_length, float base,+ float factor, int low, int high) {+ float frequency = 1.0f / powf(base, (float)(2 * pair) / dimension);+ if (original_sequence_length > 0) {+ float width = high == low ? 0.001f : (float)(high - low);+ float ramp = (pair - low) / width;+ if (ramp < 0.0f) ramp = 0.0f;+ if (ramp > 1.0f) ramp = 1.0f;+ float smooth = 1.0f - ramp;+ frequency = frequency / factor * (1.0f - smooth) + frequency * smooth;}+ return frequency;+}++int coli_v4_rope_precompute_range(+ float *cosines, float *sines, int dimension, int start_position,+ int sequence_length, int original_sequence_length, float base,+ float factor, int beta_fast, int beta_slow) {+ if (!cosines || !sines || dimension < 2 || (dimension & 1) ||+ start_position < 0 || sequence_length < 1 ||+ !(base > 1.0f) || !(factor > 0.0f))+ return -1;+ if (start_position > INT_MAX - (sequence_length - 1)) return -1;+ int pairs = dimension / 2;+ int low = 0, high = -1;+ rope_yarn_bounds(dimension, original_sequence_length, base,+ beta_fast, beta_slow, &low, &high);for (int pair = 0; pair < pairs; pair++) {- float frequency = 1.0f / powf(base, (float)(2 * pair) / dimension);- if (original_sequence_length > 0) {- float width = high == low ? 0.001f : (float)(high - low);- float ramp = (pair - low) / width;- if (ramp < 0.0f) ramp = 0.0f;- if (ramp > 1.0f) ramp = 1.0f;- float smooth = 1.0f - ramp;- frequency = frequency / factor * (1.0f - smooth) + frequency * smooth;- }- for (int position = 0; position < sequence_length; position++) {- size_t index = (size_t)position * pairs + pair;- float angle = position * frequency;+ float frequency = rope_frequency(pair, dimension,+ original_sequence_length, base,+ factor, low, high);+ for (int item = 0; item < sequence_length; item++) {+ size_t index = (size_t)item * pairs + pair;+ float angle = (start_position + item) * frequency;cosines[index] = cosf(angle);sines[index] = sinf(angle);}@@ -1273,6 +1294,37 @@ int coli_v4_rope_precompute(float *cosines, float *sines,return 0;}+int coli_v4_rope_precompute(float *cosines, float *sines,+ int dimension, int sequence_length,+ int original_sequence_length, float base,+ float factor, int beta_fast, int beta_slow) {+ return coli_v4_rope_precompute_range(+ cosines, sines, dimension, 0, sequence_length,+ original_sequence_length, base, factor, beta_fast, beta_slow);+}++int coli_v4_rope_position(float *cosines, float *sines,+ int dimension, int position,+ int original_sequence_length, float base,+ float factor, int beta_fast, int beta_slow) {+ if (!cosines || !sines || dimension < 2 || (dimension & 1) ||+ position < 0 || !(base > 1.0f) || !(factor > 0.0f))+ return -1;+ int pairs = dimension / 2;+ int low = 0, high = -1;+ rope_yarn_bounds(dimension, original_sequence_length, base,+ beta_fast, beta_slow, &low, &high);+ for (int pair = 0; pair < pairs; pair++) {+ float frequency = rope_frequency(pair, dimension,+ original_sequence_length, base,+ factor, low, high);+ float angle = position * frequency;+ cosines[pair] = cosf(angle);+ sines[pair] = sinf(angle);+ }+ return 0;+}+int coli_v4_rope_apply(float *vectors, int vector_count, int dimension,const float *cosines, const float *sines, int inverse) {if (!vectors || !cosines || !sines || vector_count < 1 ||@@ -1626,22 +1678,13 @@ static int attention_token_impl(float *output,if (!result) coli_bf16_round_array(kv, (size_t)head_dim);if (!result) {- float *all_cos = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_cos));- float *all_sin = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_sin));int compressed = weights->plan.compression_ratio != 0;- if (!all_cos || !all_sin || coli_v4_rope_precompute(- all_cos, all_sin, rope_dim, position + 1,+ if (coli_v4_rope_position(+ cosines, sines, rope_dim, position,compressed ? config->original_max_position_embeddings : 0,compressed ? config->compress_rope_theta : config->rope_theta,config->rope_factor,config->rope_beta_fast, config->rope_beta_slow)) result = -1;- if (!result) {- memcpy(cosines, all_cos + (size_t)position * rope_dim / 2,- (size_t)rope_dim / 2 * sizeof(*cosines));- memcpy(sines, all_sin + (size_t)position * rope_dim / 2,- (size_t)rope_dim / 2 * sizeof(*sines));- }- free(all_sin); free(all_cos);}if (!result) {for (int head = 0; head < heads; head++) {@@ -2024,22 +2067,13 @@ static int attention_token_impl(float *output,if (!result) coli_bf16_round_array(kv, (size_t)head_dim);if (!result) {- float *all_cos = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_cos));- float *all_sin = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_sin));int compressed = weights->plan.compression_ratio != 0;- if (!all_cos || !all_sin || coli_v4_rope_precompute(- all_cos, all_sin, rope_dim, position + 1,+ if (coli_v4_rope_position(+ cosines, sines, rope_dim, position,compressed ? config->original_max_position_embeddings : 0,compressed ? config->compress_rope_theta : config->rope_theta,config->rope_factor,config->rope_beta_fast, config->rope_beta_slow)) result = -1;- if (!result) {- memcpy(cosines, all_cos + (size_t)position * rope_dim / 2,- (size_t)rope_dim / 2 * sizeof(*cosines));- memcpy(sines, all_sin + (size_t)position * rope_dim / 2,- (size_t)rope_dim / 2 * sizeof(*sines));- }- free(all_sin); free(all_cos);}if (!result) {for (int head = 0; head < heads; head++) {@@ -2211,10 +2245,11 @@ int coli_v4_attention_window_batch_ref(int *compressed_counts = calloc((size_t)batch, sizeof(*compressed_counts));int *compressed_indices = malloc((size_t)batch * config->index_topk *sizeof(*compressed_indices));- int end_position = start_position + batch;size_t rope_pairs = (size_t)rope_dim / 2;- float *cosines = malloc((size_t)end_position * rope_pairs * sizeof(*cosines));- float *sines = malloc((size_t)end_position * rope_pairs * sizeof(*sines));+ /* The batch consumes only its own positions. A table from position zero+ * made long-context prefill scale with history that is never read here. */+ float *cosines = malloc((size_t)batch * rope_pairs * sizeof(*cosines));+ float *sines = malloc((size_t)batch * rope_pairs * sizeof(*sines));if (!qa || !q || !kv || !attended || !oa || !norm || !selected_counts ||!compressed_counts || !compressed_indices || !cosines || !sines) {free(sines); free(cosines); free(compressed_indices);@@ -2287,15 +2322,14 @@ int coli_v4_attention_window_batch_ref(}int compressed = weights->plan.compression_ratio != 0;- if (!result) result = coli_v4_rope_precompute(- cosines, sines, rope_dim, end_position,+ if (!result) result = coli_v4_rope_precompute_range(+ cosines, sines, rope_dim, start_position, batch,compressed ? config->original_max_position_embeddings : 0,compressed ? config->compress_rope_theta : config->rope_theta,config->rope_factor, config->rope_beta_fast, config->rope_beta_slow);for (int item = 0; !result && item < batch; item++) {- int position = start_position + item;- const float *item_cos = cosines + (size_t)position * rope_pairs;- const float *item_sin = sines + (size_t)position * rope_pairs;+ const float *item_cos = cosines + (size_t)item * rope_pairs;+ const float *item_sin = sines + (size_t)item * rope_pairs;float *item_q = q + (size_t)item * q_width;float *item_kv = kv + (size_t)item * head_dim;for (int head = 0; head < heads; head++) {@@ -2363,8 +2397,8 @@ int coli_v4_attention_window_batch_ref(kv_count, topk, 1.0f / sqrtf((float)head_dim));}free(all_kv); free(indices);- const float *item_cos = cosines + (size_t)position * rope_pairs;- const float *item_sin = sines + (size_t)position * rope_pairs;+ const float *item_cos = cosines + (size_t)item * rope_pairs;+ const float *item_sin = sines + (size_t)item * rope_pairs;for (int head = 0; !result && head < heads; head++) {float *rope = item_attended + (size_t)head * head_dim +head_dim - rope_dim;@@ -2406,7 +2440,8 @@ int coli_v4_attention_window_batch_ref(if (!result) result = coli_fp8_matmul_batch_ref(outputs, &wo_b, oa, batch);if (!result) coli_bf16_round_array(outputs, (size_t)batch * hidden);- free(group_outputs); free(group_inputs); free(sines); free(cosines);+ free(group_outputs); free(group_inputs);+ free(sines); free(cosines);free(compressed_indices); free(compressed_counts); free(selected_counts);free(norm); free(oa); free(attended); free(kv); free(q); free(qa);return result ? set_error(error, error_size, "batched attention failed") : 0;@@ -2622,11 +2657,10 @@ int coli_v4_compressor_step(ColiDeepSeekV4CompressorState *state,int rope_position = position + 1 - state->ratio;int pairs = state->rope_dim / 2;- size_t table_count = (size_t)(rope_position + 1) * pairs;- float *cosines = malloc(table_count * sizeof(*cosines));- float *sines = malloc(table_count * sizeof(*sines));- if (!cosines || !sines || coli_v4_rope_precompute(- cosines, sines, state->rope_dim, rope_position + 1,+ float *cosines = malloc((size_t)pairs * sizeof(*cosines));+ float *sines = malloc((size_t)pairs * sizeof(*sines));+ if (!cosines || !sines || coli_v4_rope_position(+ cosines, sines, state->rope_dim, rope_position,state->config->original_max_position_embeddings,state->config->compress_rope_theta, state->config->rope_factor,state->config->rope_beta_fast, state->config->rope_beta_slow)) {@@ -2634,9 +2668,7 @@ int coli_v4_compressor_step(ColiDeepSeekV4CompressorState *state,return set_error(error, error_size, "cannot create compressor RoPE table");}float *rope = output + dimension - state->rope_dim;- coli_v4_rope_apply(rope, 1, state->rope_dim,- cosines + (size_t)rope_position * pairs,- sines + (size_t)rope_position * pairs, 0);+ coli_v4_rope_apply(rope, 1, state->rope_dim, cosines, sines, 0);coli_bf16_round_array(rope, (size_t)state->rope_dim);free(sines); free(cosines);@@ -2803,11 +2835,10 @@ static int apply_position_rope(float *queries,int position) {int heads = config->index_n_heads, dimension = config->index_head_dim;int rope_dim = config->qk_rope_head_dim, pairs = rope_dim / 2;- size_t count = (size_t)(position + 1) * pairs;- float *cosines = malloc(count * sizeof(*cosines));- float *sines = malloc(count * sizeof(*sines));- if (!cosines || !sines || coli_v4_rope_precompute(- cosines, sines, rope_dim, position + 1,+ float *cosines = malloc((size_t)pairs * sizeof(*cosines));+ float *sines = malloc((size_t)pairs * sizeof(*sines));+ if (!cosines || !sines || coli_v4_rope_position(+ cosines, sines, rope_dim, position,config->original_max_position_embeddings,config->compress_rope_theta, config->rope_factor,config->rope_beta_fast, config->rope_beta_slow)) {@@ -2816,8 +2847,7 @@ static int apply_position_rope(float *queries,for (int head = 0; head < heads; head++) {float *query = queries + (size_t)head * dimension;coli_v4_rope_apply(query + dimension - rope_dim, 1, rope_dim,- cosines + (size_t)position * pairs,- sines + (size_t)position * pairs, 0);+ cosines, sines, 0);coli_bf16_round_array(query + dimension - rope_dim, (size_t)rope_dim);}free(sines); free(cosines);@@ -4126,11 +4156,10 @@ int coli_v4_compressor_step(ColiDeepSeekV4CompressorState *state,int rope_position = position + 1 - state->ratio;int pairs = state->rope_dim / 2;- size_t table_count = (size_t)(rope_position + 1) * pairs;- float *cosines = malloc(table_count * sizeof(*cosines));- float *sines = malloc(table_count * sizeof(*sines));- if (!cosines || !sines || coli_v4_rope_precompute(- cosines, sines, state->rope_dim, rope_position + 1,+ float *cosines = malloc((size_t)pairs * sizeof(*cosines));+ float *sines = malloc((size_t)pairs * sizeof(*sines));+ if (!cosines || !sines || coli_v4_rope_position(+ cosines, sines, state->rope_dim, rope_position,state->config->original_max_position_embeddings,state->config->compress_rope_theta, state->config->rope_factor,state->config->rope_beta_fast, state->config->rope_beta_slow)) {@@ -4138,9 +4167,7 @@ int coli_v4_compressor_step(ColiDeepSeekV4CompressorState *state,return set_error(error, error_size, "cannot create compressor RoPE table");}float *rope = output + dimension - state->rope_dim;- coli_v4_rope_apply(rope, 1, state->rope_dim,- cosines + (size_t)rope_position * pairs,- sines + (size_t)rope_position * pairs, 0);+ coli_v4_rope_apply(rope, 1, state->rope_dim, cosines, sines, 0);coli_bf16_round_array(rope, (size_t)state->rope_dim);free(sines); free(cosines);@@ -4366,11 +4393,10 @@ static int apply_position_rope(float *queries,int position) {int heads = config->index_n_heads, dimension = config->index_head_dim;int rope_dim = config->qk_rope_head_dim, pairs = rope_dim / 2;- size_t count = (size_t)(position + 1) * pairs;- float *cosines = malloc(count * sizeof(*cosines));- float *sines = malloc(count * sizeof(*sines));- if (!cosines || !sines || coli_v4_rope_precompute(- cosines, sines, rope_dim, position + 1,+ float *cosines = malloc((size_t)pairs * sizeof(*cosines));+ float *sines = malloc((size_t)pairs * sizeof(*sines));+ if (!cosines || !sines || coli_v4_rope_position(+ cosines, sines, rope_dim, position,config->original_max_position_embeddings,config->compress_rope_theta, config->rope_factor,config->rope_beta_fast, config->rope_beta_slow)) {@@ -4379,8 +4405,7 @@ static int apply_position_rope(float *queries,for (int head = 0; head < heads; head++) {float *query = queries + (size_t)head * dimension;coli_v4_rope_apply(query + dimension - rope_dim, 1, rope_dim,- cosines + (size_t)position * pairs,- sines + (size_t)position * pairs, 0);+ cosines, sines, 0);coli_bf16_round_array(query + dimension - rope_dim, (size_t)rope_dim);}free(sines); free(cosines);@@ -4798,22 +4823,13 @@ static int attention_token_impl(float *output,if (!result) coli_bf16_round_array(kv, (size_t)head_dim);if (!result) {- float *all_cos = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_cos));- float *all_sin = calloc((size_t)(position + 1) * rope_dim / 2, sizeof(*all_sin));int compressed = weights->plan.compression_ratio != 0;- if (!all_cos || !all_sin || coli_v4_rope_precompute(- all_cos, all_sin, rope_dim, position + 1,+ if (coli_v4_rope_position(+ cosines, sines, rope_dim, position,compressed ? config->original_max_position_embeddings : 0,compressed ? config->compress_rope_theta : config->rope_theta,config->rope_factor,config->rope_beta_fast, config->rope_beta_slow)) result = -1;- if (!result) {- memcpy(cosines, all_cos + (size_t)position * rope_dim / 2,- (size_t)rope_dim / 2 * sizeof(*cosines));- memcpy(sines, all_sin + (size_t)position * rope_dim / 2,- (size_t)rope_dim / 2 * sizeof(*sines));- }- free(all_sin); free(all_cos);}if (!result) {for (int head = 0; head < heads; head++) {diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex 63b8a8fc..7af32c13 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -112,6 +112,17 @@ int coli_v4_rope_precompute(float *cosines, float *sines,int original_sequence_length, float base,float factor, int beta_fast, int beta_slow);+int coli_v4_rope_precompute_range(float *cosines, float *sines,+ int dimension, int start_position,+ int sequence_length,+ int original_sequence_length, float base,+ float factor, int beta_fast, int beta_slow);++int coli_v4_rope_position(float *cosines, float *sines,+ int dimension, int position,+ int original_sequence_length, float base,+ float factor, int beta_fast, int beta_slow);+int coli_v4_rope_apply(float *vectors, int vector_count, int dimension,const float *cosines, const float *sines, int inverse);diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.cindex dd87043c..8fd047a1 100644--- a/c/tests/test_deepseek_v4.c+++ b/c/tests/test_deepseek_v4.c@@ -5,6 +5,7 @@#include <assert.h>#include <fcntl.h>+#include <limits.h>#include <math.h>#include <stdint.h>#include <stdio.h>@@ -467,6 +468,14 @@ static int close_enough(float left, float right, float tolerance) {return fabsf(left - right) <= tolerance;}+static int rope_matches_fixed(const float *actual, const float *expected,+ size_t count, float tolerance) {+ for (size_t index = 0; index < count; index++)+ if (!close_enough(actual[index], expected[index], tolerance))+ return 0;+ return 1;+}+static int test_math(void) {enum { HC = 4, DIMENSION = 3, MIXES = (2 + HC) * HC };float input[HC * DIMENSION] = {@@ -536,9 +545,77 @@ static int test_math(void) {return 1;for (int index = 0; index < 8; index++)if (!close_enough(rope_values[index], rope_original[index], 1e-5f)) return 1;++ static const float reference_cos[] = {+ -0.989992499f, 0.999887526f,+ -0.653643608f, 0.999800026f,+ };+ static const float reference_sin[] = {+ 0.141120002f, 0.0149994371f,+ -0.756802499f, 0.0199986659f,+ };+ if (!rope_matches_fixed(rope_cos + 3 * 2, reference_cos,+ sizeof(reference_cos) / sizeof(*reference_cos),+ 2e-6f) ||+ !rope_matches_fixed(rope_sin + 3 * 2, reference_sin,+ sizeof(reference_sin) / sizeof(*reference_sin),+ 2e-6f))+ return 1;++ float rope_row_cos[2], rope_row_sin[2];+ if (coli_v4_rope_position(rope_row_cos, rope_row_sin, 4, 3, 4,+ 10000.0f, 2.0f, 32, 1) != 0 ||+ !rope_matches_fixed(rope_row_cos, reference_cos, 2, 2e-6f) ||+ !rope_matches_fixed(rope_row_sin, reference_sin, 2, 2e-6f))+ return 1;++ float rope_range_cos[4], rope_range_sin[4];+ if (coli_v4_rope_precompute_range(+ rope_range_cos, rope_range_sin, 4, 3, 2, 4,+ 10000.0f, 2.0f, 32, 1) != 0)+ return 1;+ if (!rope_matches_fixed(rope_range_cos, reference_cos,+ sizeof(reference_cos) / sizeof(*reference_cos),+ 2e-6f) ||+ !rope_matches_fixed(rope_range_sin, reference_sin,+ sizeof(reference_sin) / sizeof(*reference_sin),+ 2e-6f))+ return 1;++ static const float yarn_reference_cos[] = {+ 0.987353623f, 0.979488254f, 0.966957450f, -0.587573767f,+ -0.521824539f, 0.726297259f, -0.576302052f, 0.954964221f,+ -0.988605082f, -0.159120470f, -0.972486019f, -0.394434780f,+ -0.924126625f, -0.807510197f, 0.966177464f, -0.478773415f,+ -0.893718898f, -0.100252181f, 0.523291290f, 0.823492348f,+ 0.942630887f, 0.970277667f, 0.984636068f, 0.992067456f,+ 0.995906770f, 0.997888565f, 0.998911023f, 0.999438405f,+ 0.999710381f, 0.999850631f, 0.999922991f, 0.999960303f,+ };+ static const float yarn_reference_sin[] = {+ -0.158533379f, 0.201501235f, 0.254937738f, 0.809170604f,+ 0.853052855f, 0.687380731f, 0.817236781f, 0.296720892f,+ -0.150532320f, 0.987259150f, -0.232961148f, -0.918923914f,+ 0.382086396f, -0.589853585f, 0.257878065f, -0.877938509f,+ 0.448627412f, 0.994962037f, 0.852153838f, 0.567327321f,+ 0.333836794f, 0.241994366f, 0.174619019f, 0.125706837f,+ 0.0903861374f, 0.0649493411f, 0.0466561131f, 0.0335097015f,+ 0.0240655378f, 0.0172822978f, 0.0124107366f, 0.00891227461f,+ };+ float yarn_cos[32], yarn_sin[32];+ if (coli_v4_rope_position(yarn_cos, yarn_sin, 64, 1024, 4096,+ 40000.0f, 4.0f, 32, 1) != 0 ||+ !rope_matches_fixed(yarn_cos, yarn_reference_cos, 32, 3e-6f) ||+ !rope_matches_fixed(yarn_sin, yarn_reference_sin, 32, 3e-6f))+ return 1;+if (coli_v4_rope_precompute(rope_cos, rope_sin, 3, 6, 0,10000.0f, 1.0f, 32, 1) == 0)return 1;+ if (coli_v4_rope_precompute_range(+ rope_cos, rope_sin, 4, INT_MAX, 2, 0,+ 10000.0f, 1.0f, 32, 1) == 0)+ return 1;float hidden[2] = {0.5f, -1.0f};float router[8] = {1, 0, 0, 1, -1, 0, 0, -1};