Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(deepseek_v4): qdq the attention input once β€” wq_a and wkv share it

JustVugg7c6cdc8f4697
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index b3066f7f..4b970aea 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -1999,7 +1999,16 @@ static int attention_token_impl(float *output,
return set_error(error, error_size, "out of memory in attention");
}
- int result = coli_fp8_matvec_ref(qa, &wq_a, input);
+ /* wq_a e wkv consumano lo stesso input: qdq UNA volta e riuso via _pre
+ * (il dedup rinviato da #1076). Bit-identico: stessi byte qdq, stesso
+ * compute, GPU path invariato (riceve l'input raw come prima).
+ * EN: wq_a and wkv consume the same input β€” qdq once, reuse via _pre. */
+ float *input_act = malloc((size_t)wq_a.columns * sizeof(*input_act));
+ uint8_t *input_act_scales = malloc((size_t)wq_a.columns / 128 + 1);
+ int result = (!input_act || !input_act_scales ||
+ coli_fp8_activation_qdq_ref(input_act, input_act_scales, input,
+ (size_t)wq_a.columns, 128)) ? -1 : 0;
+ if (!result) result = coli_fp8_matvec_pre(qa, &wq_a, input, input_act);
coli_bf16_round_array(qa, (size_t)q_rank);
const void *q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
if (!result && (!q_norm || decode_bf16(norm_weight, q_norm, (size_t)q_rank) ||
@@ -2049,7 +2058,7 @@ static int attention_token_impl(float *output,
for (int i = 0; i < head_dim; i++) values[i] = coli_bf16_round(values[i] * scale);
}
- if (!result) result = coli_fp8_matvec_ref(kv, &wkv, input);
+ if (!result) result = coli_fp8_matvec_pre(kv, &wkv, input, input_act);
if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
const void *kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
if (!result && (!kv_norm || decode_bf16(norm_weight, kv_norm, (size_t)head_dim) ||
@@ -2220,6 +2229,7 @@ static int attention_token_impl(float *output,
free(compressed_indices);
free(sines); free(cosines); free(norm_weight); free(oa);
free(attended); free(kv); free(q); free(qa);
+ free(input_act_scales); free(input_act);
if (result) return set_error(error, error_size, "attention computation failed");
return 0;
}
@@ -2451,7 +2461,16 @@ static int attention_token_impl(float *output,
return set_error(error, error_size, "out of memory in attention");
}
- int result = coli_fp8_matvec_ref(qa, &wq_a, input);
+ /* wq_a e wkv consumano lo stesso input: qdq UNA volta e riuso via _pre
+ * (il dedup rinviato da #1076). Bit-identico: stessi byte qdq, stesso
+ * compute, GPU path invariato (riceve l'input raw come prima).
+ * EN: wq_a and wkv consume the same input β€” qdq once, reuse via _pre. */
+ float *input_act = malloc((size_t)wq_a.columns * sizeof(*input_act));
+ uint8_t *input_act_scales = malloc((size_t)wq_a.columns / 128 + 1);
+ int result = (!input_act || !input_act_scales ||
+ coli_fp8_activation_qdq_ref(input_act, input_act_scales, input,
+ (size_t)wq_a.columns, 128)) ? -1 : 0;
+ if (!result) result = coli_fp8_matvec_pre(qa, &wq_a, input, input_act);
coli_bf16_round_array(qa, (size_t)q_rank);
const void *q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
if (!result && (!q_norm || decode_bf16(norm_weight, q_norm, (size_t)q_rank) ||
@@ -2489,7 +2508,7 @@ static int attention_token_impl(float *output,
for (int i = 0; i < head_dim; i++) values[i] = coli_bf16_round(values[i] * scale);
}
- if (!result) result = coli_fp8_matvec_ref(kv, &wkv, input);
+ if (!result) result = coli_fp8_matvec_pre(kv, &wkv, input, input_act);
if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
const void *kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
if (!result && (!kv_norm || decode_bf16(norm_weight, kv_norm, (size_t)head_dim) ||
@@ -2617,6 +2636,7 @@ static int attention_token_impl(float *output,
free(compressed_indices);
free(sines); free(cosines); free(norm_weight); free(oa);
free(attended); free(kv); free(q); free(qa);
+ free(input_act_scales); free(input_act);
if (result) return set_error(error, error_size, "attention computation failed");
return 0;
}
@@ -2738,7 +2758,20 @@ int coli_v4_attention_window_batch_ref(
#define V4_ATTN_PROF_MARK(slot) do { if (prof_t) { \
double _now = v4_attn_prof_now(); (slot) += _now - prof_t; \
prof_t = _now; } } while (0)
- int result = coli_fp8_matmul_batch_ref(qa, &wq_a, inputs, batch);
+ /* wq_a e wkv consumano lo stesso batch di input: qdq UNA volta per item e
+ * riuso via _pre (dedup rinviato da #1076). Bit-identico; il path GPU
+ * riceve gli input raw come prima.
+ * EN: wq_a and wkv consume the same input batch β€” qdq once, reuse. */
+ float *inputs_act = malloc((size_t)batch * hidden * sizeof(*inputs_act));
+ uint8_t *inputs_act_scales = malloc((size_t)batch * (hidden / 128 + 1));
+ int result = (!inputs_act || !inputs_act_scales) ? -1 : 0;
+ for (int item = 0; !result && item < batch; item++)
+ if (coli_fp8_activation_qdq_ref(
+ inputs_act + (size_t)item * hidden,
+ inputs_act_scales + (size_t)item * (hidden / 128),
+ inputs + (size_t)item * hidden, (size_t)hidden, 128))
+ result = -1;
+ if (!result) result = coli_fp8_matmul_batch_pre(qa, &wq_a, inputs, inputs_act, batch);
if (!result) coli_bf16_round_array(qa, (size_t)batch * q_rank);
const void *raw_q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
if (!result && (!raw_q_norm || decode_bf16(norm, raw_q_norm, q_rank))) result = -1;
@@ -2879,7 +2912,7 @@ int coli_v4_attention_window_batch_ref(
}
V4_ATTN_PROF_MARK(prof_qb);
- if (!result) result = coli_fp8_matmul_batch_ref(kv, &wkv, inputs, batch);
+ if (!result) result = coli_fp8_matmul_batch_pre(kv, &wkv, inputs, inputs_act, batch);
if (!result) coli_bf16_round_array(kv, (size_t)batch * head_dim);
const void *raw_kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
if (!result && (!raw_kv_norm || decode_bf16(norm, raw_kv_norm, head_dim))) result = -1;
@@ -3148,6 +3181,7 @@ int coli_v4_attention_window_batch_ref(
prof_rope * 1e3, prof_attn * 1e3, prof_wo * 1e3);
#undef V4_ATTN_PROF_MARK
+ free(inputs_act_scales); free(inputs_act);
return result ? set_error(error, error_size, "batched attention failed") : 0;
}
#endif /* COLI_V4_UNIT_ATTENTION_BATCH */
@@ -6534,7 +6568,16 @@ static int attention_token_impl(float *output,
return set_error(error, error_size, "out of memory in attention");
}
- int result = coli_fp8_matvec_ref(qa, &wq_a, input);
+ /* wq_a e wkv consumano lo stesso input: qdq UNA volta e riuso via _pre
+ * (il dedup rinviato da #1076). Bit-identico: stessi byte qdq, stesso
+ * compute, GPU path invariato (riceve l'input raw come prima).
+ * EN: wq_a and wkv consume the same input β€” qdq once, reuse via _pre. */
+ float *input_act = malloc((size_t)wq_a.columns * sizeof(*input_act));
+ uint8_t *input_act_scales = malloc((size_t)wq_a.columns / 128 + 1);
+ int result = (!input_act || !input_act_scales ||
+ coli_fp8_activation_qdq_ref(input_act, input_act_scales, input,
+ (size_t)wq_a.columns, 128)) ? -1 : 0;
+ if (!result) result = coli_fp8_matvec_pre(qa, &wq_a, input, input_act);
coli_bf16_round_array(qa, (size_t)q_rank);
const void *q_norm = layer_data(weights, "attn.q_norm.weight", NULL);
if (!result && (!q_norm || decode_bf16(norm_weight, q_norm, (size_t)q_rank) ||
@@ -6572,7 +6615,7 @@ static int attention_token_impl(float *output,
for (int i = 0; i < head_dim; i++) values[i] = coli_bf16_round(values[i] * scale);
}
- if (!result) result = coli_fp8_matvec_ref(kv, &wkv, input);
+ if (!result) result = coli_fp8_matvec_pre(kv, &wkv, input, input_act);
if (!result) coli_bf16_round_array(kv, (size_t)head_dim);
const void *kv_norm = layer_data(weights, "attn.kv_norm.weight", NULL);
if (!result && (!kv_norm || decode_bf16(norm_weight, kv_norm, (size_t)head_dim) ||
@@ -6700,6 +6743,7 @@ static int attention_token_impl(float *output,
free(compressed_indices);
free(sines); free(cosines); free(norm_weight); free(oa);
free(attended); free(kv); free(q); free(qa);
+ free(input_act_scales); free(input_act);
if (result) return set_error(error, error_size, "attention computation failed");
return 0;
}
@@ -14699,10 +14743,10 @@ int coli_fp4_matvec_ref(float *output, const ColiTensorView *weight,
return 0;
}
-int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
- const float *input) {
- if (!output || !weight || !input ||
- weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
+/* Validazione condivisa fra _ref e _pre (stessi controlli di sempre).
+ * EN: shared shape/format validation between _ref and _pre. */
+static int fp8_matvec_validate(const ColiTensorView *weight) {
+ if (!weight || weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
weight->scale_format != COLI_SCALE_F32 ||
!weight->data || !weight->scales || weight->rows < 1 ||
weight->columns < 1 || weight->columns % 128 ||
@@ -14716,22 +14760,17 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
if (weight->data_bytes != rows * columns ||
weight->scale_bytes != scale_rows * scale_columns * sizeof(float))
return -1;
-#ifdef COLI_V4_GPU_TIER
- /* The resident layer mirror carries a Dsv4CudaTensor* in view->gpu. Its
- * weight bytes are the UNPACKED row-major matrix (the resident copy may be
- * rows8-packed for the AVX2 CPU path), so the handle alone drives the call;
- * only the activation vectors cross the boundary. Any backend failure falls
- * through to the CPU reference below. */
- if (weight->gpu && coli_v4_gpu_fp8_matvec(weight, output, input) == 0)
- return 0;
-#endif
- float *activation; uint8_t *activation_scales;
- if (coli_v4_qdq_scratch(columns, scale_columns, &activation, &activation_scales))
- return -1;
- if (coli_fp8_activation_qdq_ref(activation, activation_scales,
- input, columns, 128) != 0) {
- return -1;
- }
+ return 0;
+}
+
+/* Compute CPU su attivazione GIA' qdq (estratto invariato da matvec_ref).
+ * EN: CPU compute on an already-qdq'd activation, extracted verbatim. */
+static int fp8_matvec_compute(float *output, const ColiTensorView *weight,
+ const float *activation) {
+ size_t rows = (size_t)weight->rows;
+ size_t columns = (size_t)weight->columns;
+ size_t scale_columns = columns / 128;
+ (void)scale_columns;
#ifdef __AVX2__
if (weight->block_rows == 8) {
if (rows % 8) {
@@ -14769,6 +14808,50 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
1, (int)columns, (int)rows);
return 0;
}
+
+int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
+ const float *input) {
+ if (!output || !input || fp8_matvec_validate(weight))
+ return -1;
+ size_t columns = (size_t)weight->columns;
+ size_t scale_columns = columns / 128;
+#ifdef COLI_V4_GPU_TIER
+ /* The resident layer mirror carries a Dsv4CudaTensor* in view->gpu. Its
+ * weight bytes are the UNPACKED row-major matrix (the resident copy may be
+ * rows8-packed for the AVX2 CPU path), so the handle alone drives the call;
+ * only the activation vectors cross the boundary. Any backend failure falls
+ * through to the CPU reference below. */
+ if (weight->gpu && coli_v4_gpu_fp8_matvec(weight, output, input) == 0)
+ return 0;
+#endif
+ float *activation; uint8_t *activation_scales;
+ if (coli_v4_qdq_scratch(columns, scale_columns, &activation, &activation_scales))
+ return -1;
+ if (coli_fp8_activation_qdq_ref(activation, activation_scales,
+ input, columns, 128) != 0) {
+ return -1;
+ }
+ return fp8_matvec_compute(output, weight, activation);
+}
+
+/* Variante a qdq sollevata: `activation` e' l'input gia' passato da
+ * coli_fp8_activation_qdq_ref UNA volta dal chiamante (wq_a e wkv consumano lo
+ * stesso vettore: prima veniva riquantizzato due volte per token per layer).
+ * `input` resta il vettore raw per l'eventuale path GPU, che quantizza per
+ * conto suo β€” identico a _ref. Stessi controlli, stesso compute: bit-identico.
+ * EN: hoisted-qdq variant. `activation` is the input already qdq'd ONCE by the
+ * caller (wq_a and wkv consume the same vector); `input` stays raw for the GPU
+ * path, which does its own thing exactly as in _ref. Bit-identical. */
+int coli_fp8_matvec_pre(float *output, const ColiTensorView *weight,
+ const float *input, const float *activation) {
+ if (!output || !input || !activation || fp8_matvec_validate(weight))
+ return -1;
+#ifdef COLI_V4_GPU_TIER
+ if (weight->gpu && coli_v4_gpu_fp8_matvec(weight, output, input) == 0)
+ return 0;
+#endif
+ return fp8_matvec_compute(output, weight, activation);
+}
#endif /* COLI_V4_UNIT_NATIVE_QUANT */
#ifdef COLI_V4_UNIT_NATIVE_QUANT_DUAL
@@ -14939,9 +15022,10 @@ int coli_fp8_dual_matvec_ref(float *output_a, float *output_b,
#pragma GCC diagnostic pop
#endif
-int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
- const float *inputs, int batch) {
- if (!outputs || !weight || !inputs || batch < 1 || batch > 128 ||
+/* Validazione condivisa fra _ref e _pre (stessi controlli di sempre).
+ * EN: shared shape/format validation between _ref and _pre. */
+static int fp8_batch_validate(const ColiTensorView *weight, int batch) {
+ if (!weight || batch < 1 || batch > 128 ||
weight->format != COLI_TENSOR_FP8_E4M3_BLOCK ||
weight->scale_format != COLI_SCALE_F32 ||
!weight->data || !weight->scales || weight->rows < 1 ||
@@ -14953,6 +15037,17 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
size_t scale_columns = columns / 128;
if (weight->data_bytes != rows * columns ||
weight->scale_bytes != scale_rows * scale_columns * sizeof(float)) return -1;
+ return 0;
+}
+
+static int fp8_batch_compute(float *outputs, const ColiTensorView *weight,
+ const float *activations, int batch);
+
+int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
+ const float *inputs, int batch) {
+ if (!outputs || !inputs || fp8_batch_validate(weight, batch)) return -1;
+ size_t columns = (size_t)weight->columns;
+ size_t scale_columns = columns / 128;
#ifdef COLI_V4_GPU_TIER
/* Same mirror-driven dispatch as coli_fp8_matvec_ref: the resident
* layer's Dsv4CudaTensor handle rides on view->gpu and only activations
@@ -14974,6 +15069,35 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
inputs + (size_t)item * columns, columns, 128) != 0) {
return -1;
}
+ return fp8_batch_compute(outputs, weight, activations, batch);
+}
+
+/* Variante a qdq sollevata: `activations` sono gli input gia' passati da
+ * coli_fp8_activation_qdq_ref UNA volta dal chiamante (wq_a e wkv del prefill
+ * consumano lo stesso batch); `inputs` resta il batch raw per il path GPU,
+ * identico a _ref. Bit-identico.
+ * EN: hoisted-qdq variant β€” `activations` were qdq'd once by the caller,
+ * `inputs` stays raw for the GPU path exactly as in _ref. Bit-identical. */
+int coli_fp8_matmul_batch_pre(float *outputs, const ColiTensorView *weight,
+ const float *inputs, const float *activations,
+ int batch) {
+ if (!outputs || !inputs || !activations || fp8_batch_validate(weight, batch))
+ return -1;
+#ifdef COLI_V4_GPU_TIER
+ if (weight->gpu &&
+ coli_v4_gpu_fp8_matmul_batch(weight, outputs, inputs, batch) == 0)
+ return 0;
+#endif
+ return fp8_batch_compute(outputs, weight, activations, batch);
+}
+
+/* Compute CPU su attivazioni GIA' qdq (estratto invariato da batch_ref).
+ * EN: CPU compute on already-qdq'd activations, extracted verbatim. */
+static int fp8_batch_compute(float *outputs, const ColiTensorView *weight,
+ const float *activations, int batch) {
+ size_t rows = (size_t)weight->rows, columns = (size_t)weight->columns;
+ size_t scale_columns = columns / 128;
+ (void)scale_columns;
#ifdef __AVX2__
if (weight->block_rows == 8) {
if (rows % 8) {
diff --git a/c/native_quant.h b/c/native_quant.h
index d6325def..7d73262f 100644
--- a/c/native_quant.h
+++ b/c/native_quant.h
@@ -46,6 +46,13 @@ int coli_fp8_matvec_ref(float *output, const ColiTensorView *weight,
int coli_v4_qdq_scratch(size_t activation_count, size_t scales_count,
float **activation, uint8_t **scales);
+/* Hoisted-qdq FP8 matvec: `activation` is `input` already passed through
+ * coli_fp8_activation_qdq_ref once by the caller (wq_a and wkv consume the
+ * same vector); `input` stays raw for the GPU path, exactly as in _ref.
+ * Same checks, same compute: bit-identical to _ref. */
+int coli_fp8_matvec_pre(float *output, const ColiTensorView *weight,
+ const float *input, const float *activation);
+
/* Optional CUDA tier (Windows engine build only, COLI_V4_GPU_TIER). The
* engine-side wrappers in deepseek_v4.c resolve coli_cuda_dsv4.dll through
* backend_loader_dsv4.c; the matvec_ref implementations dispatch to them when
diff --git a/c/native_quant_batch.h b/c/native_quant_batch.h
index 5a97a1e8..5af078c9 100644
--- a/c/native_quant_batch.h
+++ b/c/native_quant_batch.h
@@ -10,4 +10,11 @@ int coli_fp8_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
int coli_fp4_matmul_batch_ref(float *outputs, const ColiTensorView *weight,
const float *inputs, int batch);
+/* Hoisted-qdq variant: `activations` were qdq'd once by the caller (batch-
+ * major, same layout the _ref computes internally); `inputs` stays raw for
+ * the GPU path, exactly as in _ref. Bit-identical to _ref. */
+int coli_fp8_matmul_batch_pre(float *outputs, const ColiTensorView *weight,
+ const float *inputs, const float *activations,
+ int batch);
+
#endif