Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Add MLX affine quantization reference contract

Léon Avic Simmons3233311c4c86
diff --git a/c/Makefile b/c/Makefile
index 49db8d14..5c9d7e7a 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1782,6 +1782,11 @@ tests/test_expert_store_ops$(EXE): tests/test_expert_store_ops.c expert_store.h
tests/test_native_quant$(EXE): tests/test_native_quant.c sse41_kernels.h deepseek_v4.c native_quant.h tensor.h quant.h fp8_format.h idot.h
$(CC) $(CFLAGS) -DCOLI_V4_UNIT_NATIVE_QUANT deepseek_v4.c $< -o $@ $(LDFLAGS)
+# Portable MLX affine Q4/Q8 format contract and CPU oracle. This stays
+# separate from QT fmt=4, whose signed-int4 scale-only layout is incompatible.
+tests/test_affine_quant$(EXE): tests/test_affine_quant.c affine_quant.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_edge_runtime$(EXE): tests/test_edge_runtime.c edge_runtime.c edge_runtime.h
$(CC) $(CFLAGS) tests/test_edge_runtime.c edge_runtime.c -o $@ $(LDFLAGS)
diff --git a/c/affine_quant.h b/c/affine_quant.h
new file mode 100644
index 00000000..61f4089c
--- /dev/null
+++ b/c/affine_quant.h
@@ -0,0 +1,231 @@
+/* affine_quant.h -- portable MLX affine-quantization contract and CPU oracle.
+ *
+ * This format is deliberately separate from QT fmt=4. Colibri's fmt=4 stores
+ * signed int4 values with multiplicative group scales; MLX stores unsigned
+ * uint32-packed values and reconstructs each group as scale*q + bias.
+ */
+#ifndef COLI_AFFINE_QUANT_H
+#define COLI_AFFINE_QUANT_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+ COLI_AFFINE_MLX_Q4 = 0,
+ COLI_AFFINE_MLX_Q8 = 1
+} ColiAffineFormat;
+
+typedef enum {
+ COLI_AFFINE_SCALAR_F32 = 0,
+ COLI_AFFINE_SCALAR_F16 = 1,
+ COLI_AFFINE_SCALAR_BF16 = 2
+} ColiAffineScalarFormat;
+
+typedef enum {
+ COLI_AFFINE_OK = 0,
+ COLI_AFFINE_NULL = -1,
+ COLI_AFFINE_BAD_FORMAT = -2,
+ COLI_AFFINE_BAD_SHAPE = -3,
+ COLI_AFFINE_TRUNCATED = -4,
+ COLI_AFFINE_OVERFLOW = -5
+} ColiAffineStatus;
+
+/* MLX affine weights are little-endian uint32 words, packed along input_dim.
+ * Scales and biases have identical scalar types and [output_dim, groups]
+ * layout, where groups = input_dim/group_size. Byte lengths are mandatory:
+ * qpack sections and resident tensors must be checked before GPU registration.
+ */
+typedef struct {
+ const void *weights;
+ const void *scales;
+ const void *biases;
+ size_t weight_bytes;
+ size_t scale_bytes;
+ size_t bias_bytes;
+ size_t output_dim;
+ size_t input_dim;
+ size_t group_size;
+ ColiAffineFormat format;
+ ColiAffineScalarFormat scalar_format;
+} ColiAffineQuantizedView;
+
+static inline const char *coli_affine_status_string(ColiAffineStatus status) {
+ switch (status) {
+ case COLI_AFFINE_OK: return "ok";
+ case COLI_AFFINE_NULL: return "null affine buffer";
+ case COLI_AFFINE_BAD_FORMAT: return "unsupported affine format";
+ case COLI_AFFINE_BAD_SHAPE: return "invalid affine shape";
+ case COLI_AFFINE_TRUNCATED: return "truncated affine buffer";
+ case COLI_AFFINE_OVERFLOW: return "affine size overflow";
+ }
+ return "unknown affine status";
+}
+
+static inline unsigned coli_affine_bits(ColiAffineFormat format) {
+ return format == COLI_AFFINE_MLX_Q4 ? 4u :
+ format == COLI_AFFINE_MLX_Q8 ? 8u : 0u;
+}
+
+static inline size_t coli_affine_scalar_size(ColiAffineScalarFormat format) {
+ return format == COLI_AFFINE_SCALAR_F32 ? 4u :
+ (format == COLI_AFFINE_SCALAR_F16 ||
+ format == COLI_AFFINE_SCALAR_BF16) ? 2u : 0u;
+}
+
+static inline int coli_affine_size_mul(size_t left, size_t right, size_t *out) {
+ if (left && right > SIZE_MAX / left) return 0;
+ *out = left * right;
+ return 1;
+}
+
+static inline ColiAffineStatus
+coli_affine_validate(const ColiAffineQuantizedView *view) {
+ size_t packed_words, group_count, need_weights, need_scalars;
+ unsigned bits, per_word;
+ size_t scalar_size;
+
+ if (!view || !view->weights || !view->scales || !view->biases)
+ return COLI_AFFINE_NULL;
+ bits = coli_affine_bits(view->format);
+ scalar_size = coli_affine_scalar_size(view->scalar_format);
+ if (!bits || !scalar_size) return COLI_AFFINE_BAD_FORMAT;
+ if (!view->output_dim || !view->input_dim || !view->group_size)
+ return COLI_AFFINE_BAD_SHAPE;
+
+ per_word = 32u / bits;
+ if (view->input_dim % view->group_size ||
+ view->input_dim % per_word || view->group_size % per_word)
+ return COLI_AFFINE_BAD_SHAPE;
+
+ packed_words = view->input_dim / per_word;
+ group_count = view->input_dim / view->group_size;
+ if (!coli_affine_size_mul(view->output_dim, packed_words, &need_weights) ||
+ !coli_affine_size_mul(need_weights, sizeof(uint32_t), &need_weights) ||
+ !coli_affine_size_mul(view->output_dim, group_count, &need_scalars) ||
+ !coli_affine_size_mul(need_scalars, scalar_size, &need_scalars))
+ return COLI_AFFINE_OVERFLOW;
+ if (view->weight_bytes < need_weights ||
+ view->scale_bytes < need_scalars || view->bias_bytes < need_scalars)
+ return COLI_AFFINE_TRUNCATED;
+ return COLI_AFFINE_OK;
+}
+
+static inline uint16_t coli_affine_load_le16(const uint8_t *source) {
+ return (uint16_t)source[0] | (uint16_t)((uint16_t)source[1] << 8);
+}
+
+static inline uint32_t coli_affine_load_le32(const uint8_t *source) {
+ return (uint32_t)source[0] | ((uint32_t)source[1] << 8) |
+ ((uint32_t)source[2] << 16) | ((uint32_t)source[3] << 24);
+}
+
+static inline float coli_affine_f32_from_bits(uint32_t bits) {
+ float value;
+ memcpy(&value, &bits, sizeof(value));
+ return value;
+}
+
+static inline float coli_affine_f16_to_f32(uint16_t value) {
+ uint32_t sign = ((uint32_t)value & 0x8000u) << 16;
+ int exponent = (int)(((uint32_t)value >> 10) & 0x1fu);
+ uint32_t mantissa = (uint32_t)value & 0x3ffu;
+
+ if (exponent == 0) {
+ if (!mantissa) return coli_affine_f32_from_bits(sign);
+ exponent = 1;
+ while (!(mantissa & 0x400u)) {
+ mantissa <<= 1;
+ exponent--;
+ }
+ mantissa &= 0x3ffu;
+ exponent += 127 - 15;
+ } else if (exponent == 31) {
+ exponent = 255;
+ } else {
+ exponent += 127 - 15;
+ }
+ return coli_affine_f32_from_bits(sign | ((uint32_t)exponent << 23) |
+ (mantissa << 13));
+}
+
+static inline float coli_affine_load_scalar(const void *values, size_t index,
+ ColiAffineScalarFormat format) {
+ const uint8_t *bytes = (const uint8_t *)values;
+ if (format == COLI_AFFINE_SCALAR_F32)
+ return coli_affine_f32_from_bits(coli_affine_load_le32(bytes + index * 4));
+ if (format == COLI_AFFINE_SCALAR_F16)
+ return coli_affine_f16_to_f32(coli_affine_load_le16(bytes + index * 2));
+ return coli_affine_f32_from_bits(
+ (uint32_t)coli_affine_load_le16(bytes + index * 2) << 16);
+}
+
+/* Correctness-first y[S,O] = x[S,I] @ dequant(W[O,I])^T. Accumulation is
+ * grouped exactly like Swiftlet's generic Metal kernel: two f32 reductions
+ * (q*x and x), followed by scale*qdot + bias*xsum for each group.
+ */
+static inline ColiAffineStatus
+coli_affine_matmul_ref(float *output, const float *input, size_t batch,
+ const ColiAffineQuantizedView *view) {
+ ColiAffineStatus status = coli_affine_validate(view);
+ unsigned bits, per_word;
+ uint32_t mask;
+ size_t packed_words, groups, unused;
+ const uint8_t *weights;
+
+ if (status != COLI_AFFINE_OK) return status;
+ if (!output || !input) return COLI_AFFINE_NULL;
+ if (!batch) return COLI_AFFINE_BAD_SHAPE;
+ if (!coli_affine_size_mul(batch, view->input_dim, &unused) ||
+ !coli_affine_size_mul(batch, view->output_dim, &unused))
+ return COLI_AFFINE_OVERFLOW;
+
+ bits = coli_affine_bits(view->format);
+ per_word = 32u / bits;
+ mask = (UINT32_C(1) << bits) - 1u;
+ packed_words = view->input_dim / per_word;
+ groups = view->input_dim / view->group_size;
+ weights = (const uint8_t *)view->weights;
+
+ for (size_t row = 0; row < view->output_dim; row++) {
+ const uint8_t *weight_row = weights + row * packed_words * 4;
+ for (size_t sample = 0; sample < batch; sample++) {
+ const float *input_row = input + sample * view->input_dim;
+ float sum = 0.0f;
+ for (size_t group = 0; group < groups; group++) {
+ const size_t word_base = group * (view->group_size / per_word);
+ const size_t input_base = group * view->group_size;
+ float quantized_dot = 0.0f, input_sum = 0.0f;
+
+ for (size_t local = 0; local < view->group_size; local++) {
+ const size_t column = input_base + local;
+ const size_t word_index = word_base + local / per_word;
+ const uint32_t word =
+ coli_affine_load_le32(weight_row + word_index * 4);
+ const unsigned shift = bits * (unsigned)(local % per_word);
+ const float x = input_row[column];
+ quantized_dot += (float)((word >> shift) & mask) * x;
+ input_sum += x;
+ }
+ const size_t scalar_index = row * groups + group;
+ const float scale = coli_affine_load_scalar(
+ view->scales, scalar_index, view->scalar_format);
+ const float bias = coli_affine_load_scalar(
+ view->biases, scalar_index, view->scalar_format);
+ sum += scale * quantized_dot + bias * input_sum;
+ }
+ output[sample * view->output_dim + row] = sum;
+ }
+ }
+ return COLI_AFFINE_OK;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/c/tests/test_affine_quant.c b/c/tests/test_affine_quant.c
new file mode 100644
index 00000000..939802b3
--- /dev/null
+++ b/c/tests/test_affine_quant.c
@@ -0,0 +1,214 @@
+#include "../affine_quant.h"
+
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+enum { OUTPUT_DIM = 3, INPUT_DIM = 32, GROUP_SIZE = 16, BATCH = 2 };
+
+static void store_le16(uint8_t *destination, uint16_t value) {
+ destination[0] = (uint8_t)value;
+ destination[1] = (uint8_t)(value >> 8);
+}
+
+static void store_le32(uint8_t *destination, uint32_t value) {
+ destination[0] = (uint8_t)value;
+ destination[1] = (uint8_t)(value >> 8);
+ destination[2] = (uint8_t)(value >> 16);
+ destination[3] = (uint8_t)(value >> 24);
+}
+
+static uint16_t f32_to_f16(float value) {
+ uint32_t bits;
+ uint32_t sign, mantissa;
+ int exponent;
+ memcpy(&bits, &value, sizeof(bits));
+ sign = (bits >> 16) & 0x8000u;
+ exponent = (int)((bits >> 23) & 0xffu) - 127 + 15;
+ mantissa = bits & 0x7fffffu;
+ if (exponent <= 0) return (uint16_t)sign;
+ if (exponent >= 31) return (uint16_t)(sign | 0x7c00u);
+ return (uint16_t)(sign | ((uint32_t)exponent << 10) | (mantissa >> 13));
+}
+
+static void store_scalar(uint8_t *destination, size_t index,
+ ColiAffineScalarFormat format, float value) {
+ uint32_t bits;
+ if (format == COLI_AFFINE_SCALAR_F32) {
+ memcpy(&bits, &value, sizeof(bits));
+ store_le32(destination + index * 4, bits);
+ } else if (format == COLI_AFFINE_SCALAR_F16) {
+ store_le16(destination + index * 2, f32_to_f16(value));
+ } else {
+ memcpy(&bits, &value, sizeof(bits));
+ store_le16(destination + index * 2, (uint16_t)(bits >> 16));
+ }
+}
+
+static uint32_t packed_value(size_t row, size_t column, unsigned bits) {
+ const uint32_t mask = (UINT32_C(1) << bits) - 1u;
+ return (uint32_t)(row * 17 + column * 5 + 3) & mask;
+}
+
+static float direct_reference(const ColiAffineQuantizedView *view,
+ const float *input, size_t sample, size_t row) {
+ const unsigned bits = coli_affine_bits(view->format);
+ const unsigned per_word = 32u / bits;
+ const size_t packed_words = view->input_dim / per_word;
+ const size_t groups = view->input_dim / view->group_size;
+ const uint32_t mask = (UINT32_C(1) << bits) - 1u;
+ const uint8_t *weight_row =
+ (const uint8_t *)view->weights + row * packed_words * 4;
+ double result = 0.0;
+
+ for (size_t column = 0; column < view->input_dim; column++) {
+ const uint32_t word =
+ coli_affine_load_le32(weight_row + (column / per_word) * 4);
+ const uint32_t quantized =
+ (word >> (bits * (unsigned)(column % per_word))) & mask;
+ const size_t group = row * groups + column / view->group_size;
+ const float scale = coli_affine_load_scalar(
+ view->scales, group, view->scalar_format);
+ const float bias = coli_affine_load_scalar(
+ view->biases, group, view->scalar_format);
+ const float x = input[sample * view->input_dim + column];
+ result += (double)(scale * (float)quantized + bias) * (double)x;
+ }
+ return (float)result;
+}
+
+static int run_parity(ColiAffineFormat format,
+ ColiAffineScalarFormat scalar_format) {
+ const unsigned bits = coli_affine_bits(format);
+ const unsigned per_word = 32u / bits;
+ const size_t packed_words = INPUT_DIM / per_word;
+ const size_t scalar_count = OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE);
+ const size_t scalar_size = coli_affine_scalar_size(scalar_format);
+ uint8_t weights[OUTPUT_DIM * INPUT_DIM];
+ uint8_t scales[OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE) * 4];
+ uint8_t biases[OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE) * 4];
+ float input[BATCH * INPUT_DIM], output[BATCH * OUTPUT_DIM];
+
+ memset(weights, 0, sizeof(weights));
+ memset(scales, 0, sizeof(scales));
+ memset(biases, 0, sizeof(biases));
+ for (size_t row = 0; row < OUTPUT_DIM; row++) {
+ for (size_t word_index = 0; word_index < packed_words; word_index++) {
+ uint32_t word = 0;
+ for (unsigned lane = 0; lane < per_word; lane++) {
+ const size_t column = word_index * per_word + lane;
+ word |= packed_value(row, column, bits) << (lane * bits);
+ }
+ store_le32(weights + (row * packed_words + word_index) * 4, word);
+ }
+ }
+ for (size_t index = 0; index < scalar_count; index++) {
+ const float scale = (float)((int)(index % 5) - 2) * 0.125f;
+ const float bias = (float)((int)(index % 3) - 1) * 0.0625f;
+ store_scalar(scales, index, scalar_format, scale);
+ store_scalar(biases, index, scalar_format, bias);
+ }
+ for (size_t index = 0; index < BATCH * INPUT_DIM; index++)
+ input[index] = (float)((int)(index % 13) - 6) * 0.125f;
+
+ ColiAffineQuantizedView view = {
+ weights, scales, biases,
+ OUTPUT_DIM * packed_words * 4, scalar_count * scalar_size,
+ scalar_count * scalar_size,
+ OUTPUT_DIM, INPUT_DIM, GROUP_SIZE, format, scalar_format
+ };
+ ColiAffineStatus status = coli_affine_validate(&view);
+ if (status != COLI_AFFINE_OK) {
+ fprintf(stderr, "affine validation failed bits=%u scalar=%d: %s\n",
+ bits, (int)scalar_format, coli_affine_status_string(status));
+ return 1;
+ }
+ status = coli_affine_matmul_ref(output, input, BATCH, &view);
+ if (status != COLI_AFFINE_OK) {
+ fprintf(stderr, "affine matmul failed bits=%u scalar=%d: %s\n",
+ bits, (int)scalar_format, coli_affine_status_string(status));
+ return 1;
+ }
+
+ for (size_t sample = 0; sample < BATCH; sample++) {
+ for (size_t row = 0; row < OUTPUT_DIM; row++) {
+ const float expected = direct_reference(&view, input, sample, row);
+ const float actual = output[sample * OUTPUT_DIM + row];
+ const float tolerance = 2e-6f * fmaxf(1.0f, fabsf(expected));
+ if (fabsf(actual - expected) > tolerance) {
+ fprintf(stderr,
+ "affine parity failed bits=%u scalar=%d sample=%zu row=%zu: "
+ "got %.9g want %.9g\n",
+ bits, (int)scalar_format, sample, row, actual, expected);
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+
+static int test_validation(void) {
+ uint32_t weight = 0;
+ float scales = 1.0f, bias = 0.0f;
+ float input[8] = {0}, output = 123.0f;
+ ColiAffineQuantizedView view = {
+ &weight, &scales, &bias, sizeof(weight), sizeof(scales), sizeof(bias),
+ 1, 8, 8, COLI_AFFINE_MLX_Q4, COLI_AFFINE_SCALAR_F32
+ };
+
+#define EXPECT_STATUS(expression, expected) do { \
+ ColiAffineStatus got = (expression); \
+ if (got != (expected)) { \
+ fprintf(stderr, "validation line %d: got %s, want %s\n", __LINE__, \
+ coli_affine_status_string(got), \
+ coli_affine_status_string(expected)); \
+ return 1; \
+ } \
+ } while (0)
+
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_OK);
+ view.biases = NULL;
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_NULL);
+ view.biases = &bias;
+ view.format = (ColiAffineFormat)99;
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_BAD_FORMAT);
+ view.format = COLI_AFFINE_MLX_Q4;
+ view.input_dim = 7;
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_BAD_SHAPE);
+ view.input_dim = 8;
+ view.weight_bytes--;
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);
+ view.weight_bytes++;
+ view.scale_bytes--;
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);
+ view.scale_bytes++;
+ view.bias_bytes--;
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);
+ view.bias_bytes++;
+ view.output_dim = SIZE_MAX;
+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_OVERFLOW);
+ view.output_dim = 1;
+ EXPECT_STATUS(coli_affine_matmul_ref(&output, input, 0, &view),
+ COLI_AFFINE_BAD_SHAPE);
+ EXPECT_STATUS(coli_affine_matmul_ref(&output, input, SIZE_MAX, &view),
+ COLI_AFFINE_OVERFLOW);
+#undef EXPECT_STATUS
+ if (output != 123.0f) return 1;
+ return 0;
+}
+
+int main(void) {
+ for (int format = COLI_AFFINE_MLX_Q4;
+ format <= COLI_AFFINE_MLX_Q8; format++) {
+ for (int scalar = COLI_AFFINE_SCALAR_F32;
+ scalar <= COLI_AFFINE_SCALAR_BF16; scalar++) {
+ if (run_parity((ColiAffineFormat)format,
+ (ColiAffineScalarFormat)scalar))
+ return 1;
+ }
+ }
+ if (test_validation()) return 1;
+ puts("affine quant tests: ok");
+ return 0;
+}