Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Add MLX affine quantization reference contract
Léon Avic Simmons3233311c4c86
diff --git a/c/Makefile b/c/Makefileindex 49db8d14..5c9d7e7a 100644--- a/c/Makefile+++ b/c/Makefile@@ -1782,6 +1782,11 @@ tests/test_expert_store_ops$(EXE): tests/test_expert_store_ops.c expert_store.htests/test_native_quant$(EXE): tests/test_native_quant.c sse41_kernels.h deepseek_v4.c native_quant.h tensor.h quant.h fp8_format.h idot.h$(CC) $(CFLAGS) -DCOLI_V4_UNIT_NATIVE_QUANT deepseek_v4.c $< -o $@ $(LDFLAGS)+# Portable MLX affine Q4/Q8 format contract and CPU oracle. This stays+# separate from QT fmt=4, whose signed-int4 scale-only layout is incompatible.+tests/test_affine_quant$(EXE): tests/test_affine_quant.c affine_quant.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_edge_runtime$(EXE): tests/test_edge_runtime.c edge_runtime.c edge_runtime.h$(CC) $(CFLAGS) tests/test_edge_runtime.c edge_runtime.c -o $@ $(LDFLAGS)diff --git a/c/affine_quant.h b/c/affine_quant.hnew file mode 100644index 00000000..61f4089c--- /dev/null+++ b/c/affine_quant.h@@ -0,0 +1,231 @@+/* affine_quant.h -- portable MLX affine-quantization contract and CPU oracle.+ *+ * This format is deliberately separate from QT fmt=4. Colibri's fmt=4 stores+ * signed int4 values with multiplicative group scales; MLX stores unsigned+ * uint32-packed values and reconstructs each group as scale*q + bias.+ */+#ifndef COLI_AFFINE_QUANT_H+#define COLI_AFFINE_QUANT_H++#include <stddef.h>+#include <stdint.h>+#include <string.h>++#ifdef __cplusplus+extern "C" {+#endif++typedef enum {+ COLI_AFFINE_MLX_Q4 = 0,+ COLI_AFFINE_MLX_Q8 = 1+} ColiAffineFormat;++typedef enum {+ COLI_AFFINE_SCALAR_F32 = 0,+ COLI_AFFINE_SCALAR_F16 = 1,+ COLI_AFFINE_SCALAR_BF16 = 2+} ColiAffineScalarFormat;++typedef enum {+ COLI_AFFINE_OK = 0,+ COLI_AFFINE_NULL = -1,+ COLI_AFFINE_BAD_FORMAT = -2,+ COLI_AFFINE_BAD_SHAPE = -3,+ COLI_AFFINE_TRUNCATED = -4,+ COLI_AFFINE_OVERFLOW = -5+} ColiAffineStatus;++/* MLX affine weights are little-endian uint32 words, packed along input_dim.+ * Scales and biases have identical scalar types and [output_dim, groups]+ * layout, where groups = input_dim/group_size. Byte lengths are mandatory:+ * qpack sections and resident tensors must be checked before GPU registration.+ */+typedef struct {+ const void *weights;+ const void *scales;+ const void *biases;+ size_t weight_bytes;+ size_t scale_bytes;+ size_t bias_bytes;+ size_t output_dim;+ size_t input_dim;+ size_t group_size;+ ColiAffineFormat format;+ ColiAffineScalarFormat scalar_format;+} ColiAffineQuantizedView;++static inline const char *coli_affine_status_string(ColiAffineStatus status) {+ switch (status) {+ case COLI_AFFINE_OK: return "ok";+ case COLI_AFFINE_NULL: return "null affine buffer";+ case COLI_AFFINE_BAD_FORMAT: return "unsupported affine format";+ case COLI_AFFINE_BAD_SHAPE: return "invalid affine shape";+ case COLI_AFFINE_TRUNCATED: return "truncated affine buffer";+ case COLI_AFFINE_OVERFLOW: return "affine size overflow";+ }+ return "unknown affine status";+}++static inline unsigned coli_affine_bits(ColiAffineFormat format) {+ return format == COLI_AFFINE_MLX_Q4 ? 4u :+ format == COLI_AFFINE_MLX_Q8 ? 8u : 0u;+}++static inline size_t coli_affine_scalar_size(ColiAffineScalarFormat format) {+ return format == COLI_AFFINE_SCALAR_F32 ? 4u :+ (format == COLI_AFFINE_SCALAR_F16 ||+ format == COLI_AFFINE_SCALAR_BF16) ? 2u : 0u;+}++static inline int coli_affine_size_mul(size_t left, size_t right, size_t *out) {+ if (left && right > SIZE_MAX / left) return 0;+ *out = left * right;+ return 1;+}++static inline ColiAffineStatus+coli_affine_validate(const ColiAffineQuantizedView *view) {+ size_t packed_words, group_count, need_weights, need_scalars;+ unsigned bits, per_word;+ size_t scalar_size;++ if (!view || !view->weights || !view->scales || !view->biases)+ return COLI_AFFINE_NULL;+ bits = coli_affine_bits(view->format);+ scalar_size = coli_affine_scalar_size(view->scalar_format);+ if (!bits || !scalar_size) return COLI_AFFINE_BAD_FORMAT;+ if (!view->output_dim || !view->input_dim || !view->group_size)+ return COLI_AFFINE_BAD_SHAPE;++ per_word = 32u / bits;+ if (view->input_dim % view->group_size ||+ view->input_dim % per_word || view->group_size % per_word)+ return COLI_AFFINE_BAD_SHAPE;++ packed_words = view->input_dim / per_word;+ group_count = view->input_dim / view->group_size;+ if (!coli_affine_size_mul(view->output_dim, packed_words, &need_weights) ||+ !coli_affine_size_mul(need_weights, sizeof(uint32_t), &need_weights) ||+ !coli_affine_size_mul(view->output_dim, group_count, &need_scalars) ||+ !coli_affine_size_mul(need_scalars, scalar_size, &need_scalars))+ return COLI_AFFINE_OVERFLOW;+ if (view->weight_bytes < need_weights ||+ view->scale_bytes < need_scalars || view->bias_bytes < need_scalars)+ return COLI_AFFINE_TRUNCATED;+ return COLI_AFFINE_OK;+}++static inline uint16_t coli_affine_load_le16(const uint8_t *source) {+ return (uint16_t)source[0] | (uint16_t)((uint16_t)source[1] << 8);+}++static inline uint32_t coli_affine_load_le32(const uint8_t *source) {+ return (uint32_t)source[0] | ((uint32_t)source[1] << 8) |+ ((uint32_t)source[2] << 16) | ((uint32_t)source[3] << 24);+}++static inline float coli_affine_f32_from_bits(uint32_t bits) {+ float value;+ memcpy(&value, &bits, sizeof(value));+ return value;+}++static inline float coli_affine_f16_to_f32(uint16_t value) {+ uint32_t sign = ((uint32_t)value & 0x8000u) << 16;+ int exponent = (int)(((uint32_t)value >> 10) & 0x1fu);+ uint32_t mantissa = (uint32_t)value & 0x3ffu;++ if (exponent == 0) {+ if (!mantissa) return coli_affine_f32_from_bits(sign);+ exponent = 1;+ while (!(mantissa & 0x400u)) {+ mantissa <<= 1;+ exponent--;+ }+ mantissa &= 0x3ffu;+ exponent += 127 - 15;+ } else if (exponent == 31) {+ exponent = 255;+ } else {+ exponent += 127 - 15;+ }+ return coli_affine_f32_from_bits(sign | ((uint32_t)exponent << 23) |+ (mantissa << 13));+}++static inline float coli_affine_load_scalar(const void *values, size_t index,+ ColiAffineScalarFormat format) {+ const uint8_t *bytes = (const uint8_t *)values;+ if (format == COLI_AFFINE_SCALAR_F32)+ return coli_affine_f32_from_bits(coli_affine_load_le32(bytes + index * 4));+ if (format == COLI_AFFINE_SCALAR_F16)+ return coli_affine_f16_to_f32(coli_affine_load_le16(bytes + index * 2));+ return coli_affine_f32_from_bits(+ (uint32_t)coli_affine_load_le16(bytes + index * 2) << 16);+}++/* Correctness-first y[S,O] = x[S,I] @ dequant(W[O,I])^T. Accumulation is+ * grouped exactly like Swiftlet's generic Metal kernel: two f32 reductions+ * (q*x and x), followed by scale*qdot + bias*xsum for each group.+ */+static inline ColiAffineStatus+coli_affine_matmul_ref(float *output, const float *input, size_t batch,+ const ColiAffineQuantizedView *view) {+ ColiAffineStatus status = coli_affine_validate(view);+ unsigned bits, per_word;+ uint32_t mask;+ size_t packed_words, groups, unused;+ const uint8_t *weights;++ if (status != COLI_AFFINE_OK) return status;+ if (!output || !input) return COLI_AFFINE_NULL;+ if (!batch) return COLI_AFFINE_BAD_SHAPE;+ if (!coli_affine_size_mul(batch, view->input_dim, &unused) ||+ !coli_affine_size_mul(batch, view->output_dim, &unused))+ return COLI_AFFINE_OVERFLOW;++ bits = coli_affine_bits(view->format);+ per_word = 32u / bits;+ mask = (UINT32_C(1) << bits) - 1u;+ packed_words = view->input_dim / per_word;+ groups = view->input_dim / view->group_size;+ weights = (const uint8_t *)view->weights;++ for (size_t row = 0; row < view->output_dim; row++) {+ const uint8_t *weight_row = weights + row * packed_words * 4;+ for (size_t sample = 0; sample < batch; sample++) {+ const float *input_row = input + sample * view->input_dim;+ float sum = 0.0f;+ for (size_t group = 0; group < groups; group++) {+ const size_t word_base = group * (view->group_size / per_word);+ const size_t input_base = group * view->group_size;+ float quantized_dot = 0.0f, input_sum = 0.0f;++ for (size_t local = 0; local < view->group_size; local++) {+ const size_t column = input_base + local;+ const size_t word_index = word_base + local / per_word;+ const uint32_t word =+ coli_affine_load_le32(weight_row + word_index * 4);+ const unsigned shift = bits * (unsigned)(local % per_word);+ const float x = input_row[column];+ quantized_dot += (float)((word >> shift) & mask) * x;+ input_sum += x;+ }+ const size_t scalar_index = row * groups + group;+ const float scale = coli_affine_load_scalar(+ view->scales, scalar_index, view->scalar_format);+ const float bias = coli_affine_load_scalar(+ view->biases, scalar_index, view->scalar_format);+ sum += scale * quantized_dot + bias * input_sum;+ }+ output[sample * view->output_dim + row] = sum;+ }+ }+ return COLI_AFFINE_OK;+}++#ifdef __cplusplus+}+#endif++#endifdiff --git a/c/tests/test_affine_quant.c b/c/tests/test_affine_quant.cnew file mode 100644index 00000000..939802b3--- /dev/null+++ b/c/tests/test_affine_quant.c@@ -0,0 +1,214 @@+#include "../affine_quant.h"++#include <math.h>+#include <stdint.h>+#include <stdio.h>+#include <string.h>++enum { OUTPUT_DIM = 3, INPUT_DIM = 32, GROUP_SIZE = 16, BATCH = 2 };++static void store_le16(uint8_t *destination, uint16_t value) {+ destination[0] = (uint8_t)value;+ destination[1] = (uint8_t)(value >> 8);+}++static void store_le32(uint8_t *destination, uint32_t value) {+ destination[0] = (uint8_t)value;+ destination[1] = (uint8_t)(value >> 8);+ destination[2] = (uint8_t)(value >> 16);+ destination[3] = (uint8_t)(value >> 24);+}++static uint16_t f32_to_f16(float value) {+ uint32_t bits;+ uint32_t sign, mantissa;+ int exponent;+ memcpy(&bits, &value, sizeof(bits));+ sign = (bits >> 16) & 0x8000u;+ exponent = (int)((bits >> 23) & 0xffu) - 127 + 15;+ mantissa = bits & 0x7fffffu;+ if (exponent <= 0) return (uint16_t)sign;+ if (exponent >= 31) return (uint16_t)(sign | 0x7c00u);+ return (uint16_t)(sign | ((uint32_t)exponent << 10) | (mantissa >> 13));+}++static void store_scalar(uint8_t *destination, size_t index,+ ColiAffineScalarFormat format, float value) {+ uint32_t bits;+ if (format == COLI_AFFINE_SCALAR_F32) {+ memcpy(&bits, &value, sizeof(bits));+ store_le32(destination + index * 4, bits);+ } else if (format == COLI_AFFINE_SCALAR_F16) {+ store_le16(destination + index * 2, f32_to_f16(value));+ } else {+ memcpy(&bits, &value, sizeof(bits));+ store_le16(destination + index * 2, (uint16_t)(bits >> 16));+ }+}++static uint32_t packed_value(size_t row, size_t column, unsigned bits) {+ const uint32_t mask = (UINT32_C(1) << bits) - 1u;+ return (uint32_t)(row * 17 + column * 5 + 3) & mask;+}++static float direct_reference(const ColiAffineQuantizedView *view,+ const float *input, size_t sample, size_t row) {+ const unsigned bits = coli_affine_bits(view->format);+ const unsigned per_word = 32u / bits;+ const size_t packed_words = view->input_dim / per_word;+ const size_t groups = view->input_dim / view->group_size;+ const uint32_t mask = (UINT32_C(1) << bits) - 1u;+ const uint8_t *weight_row =+ (const uint8_t *)view->weights + row * packed_words * 4;+ double result = 0.0;++ for (size_t column = 0; column < view->input_dim; column++) {+ const uint32_t word =+ coli_affine_load_le32(weight_row + (column / per_word) * 4);+ const uint32_t quantized =+ (word >> (bits * (unsigned)(column % per_word))) & mask;+ const size_t group = row * groups + column / view->group_size;+ const float scale = coli_affine_load_scalar(+ view->scales, group, view->scalar_format);+ const float bias = coli_affine_load_scalar(+ view->biases, group, view->scalar_format);+ const float x = input[sample * view->input_dim + column];+ result += (double)(scale * (float)quantized + bias) * (double)x;+ }+ return (float)result;+}++static int run_parity(ColiAffineFormat format,+ ColiAffineScalarFormat scalar_format) {+ const unsigned bits = coli_affine_bits(format);+ const unsigned per_word = 32u / bits;+ const size_t packed_words = INPUT_DIM / per_word;+ const size_t scalar_count = OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE);+ const size_t scalar_size = coli_affine_scalar_size(scalar_format);+ uint8_t weights[OUTPUT_DIM * INPUT_DIM];+ uint8_t scales[OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE) * 4];+ uint8_t biases[OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE) * 4];+ float input[BATCH * INPUT_DIM], output[BATCH * OUTPUT_DIM];++ memset(weights, 0, sizeof(weights));+ memset(scales, 0, sizeof(scales));+ memset(biases, 0, sizeof(biases));+ for (size_t row = 0; row < OUTPUT_DIM; row++) {+ for (size_t word_index = 0; word_index < packed_words; word_index++) {+ uint32_t word = 0;+ for (unsigned lane = 0; lane < per_word; lane++) {+ const size_t column = word_index * per_word + lane;+ word |= packed_value(row, column, bits) << (lane * bits);+ }+ store_le32(weights + (row * packed_words + word_index) * 4, word);+ }+ }+ for (size_t index = 0; index < scalar_count; index++) {+ const float scale = (float)((int)(index % 5) - 2) * 0.125f;+ const float bias = (float)((int)(index % 3) - 1) * 0.0625f;+ store_scalar(scales, index, scalar_format, scale);+ store_scalar(biases, index, scalar_format, bias);+ }+ for (size_t index = 0; index < BATCH * INPUT_DIM; index++)+ input[index] = (float)((int)(index % 13) - 6) * 0.125f;++ ColiAffineQuantizedView view = {+ weights, scales, biases,+ OUTPUT_DIM * packed_words * 4, scalar_count * scalar_size,+ scalar_count * scalar_size,+ OUTPUT_DIM, INPUT_DIM, GROUP_SIZE, format, scalar_format+ };+ ColiAffineStatus status = coli_affine_validate(&view);+ if (status != COLI_AFFINE_OK) {+ fprintf(stderr, "affine validation failed bits=%u scalar=%d: %s\n",+ bits, (int)scalar_format, coli_affine_status_string(status));+ return 1;+ }+ status = coli_affine_matmul_ref(output, input, BATCH, &view);+ if (status != COLI_AFFINE_OK) {+ fprintf(stderr, "affine matmul failed bits=%u scalar=%d: %s\n",+ bits, (int)scalar_format, coli_affine_status_string(status));+ return 1;+ }++ for (size_t sample = 0; sample < BATCH; sample++) {+ for (size_t row = 0; row < OUTPUT_DIM; row++) {+ const float expected = direct_reference(&view, input, sample, row);+ const float actual = output[sample * OUTPUT_DIM + row];+ const float tolerance = 2e-6f * fmaxf(1.0f, fabsf(expected));+ if (fabsf(actual - expected) > tolerance) {+ fprintf(stderr,+ "affine parity failed bits=%u scalar=%d sample=%zu row=%zu: "+ "got %.9g want %.9g\n",+ bits, (int)scalar_format, sample, row, actual, expected);+ return 1;+ }+ }+ }+ return 0;+}++static int test_validation(void) {+ uint32_t weight = 0;+ float scales = 1.0f, bias = 0.0f;+ float input[8] = {0}, output = 123.0f;+ ColiAffineQuantizedView view = {+ &weight, &scales, &bias, sizeof(weight), sizeof(scales), sizeof(bias),+ 1, 8, 8, COLI_AFFINE_MLX_Q4, COLI_AFFINE_SCALAR_F32+ };++#define EXPECT_STATUS(expression, expected) do { \+ ColiAffineStatus got = (expression); \+ if (got != (expected)) { \+ fprintf(stderr, "validation line %d: got %s, want %s\n", __LINE__, \+ coli_affine_status_string(got), \+ coli_affine_status_string(expected)); \+ return 1; \+ } \+ } while (0)++ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_OK);+ view.biases = NULL;+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_NULL);+ view.biases = &bias;+ view.format = (ColiAffineFormat)99;+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_BAD_FORMAT);+ view.format = COLI_AFFINE_MLX_Q4;+ view.input_dim = 7;+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_BAD_SHAPE);+ view.input_dim = 8;+ view.weight_bytes--;+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);+ view.weight_bytes++;+ view.scale_bytes--;+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);+ view.scale_bytes++;+ view.bias_bytes--;+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);+ view.bias_bytes++;+ view.output_dim = SIZE_MAX;+ EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_OVERFLOW);+ view.output_dim = 1;+ EXPECT_STATUS(coli_affine_matmul_ref(&output, input, 0, &view),+ COLI_AFFINE_BAD_SHAPE);+ EXPECT_STATUS(coli_affine_matmul_ref(&output, input, SIZE_MAX, &view),+ COLI_AFFINE_OVERFLOW);+#undef EXPECT_STATUS+ if (output != 123.0f) return 1;+ return 0;+}++int main(void) {+ for (int format = COLI_AFFINE_MLX_Q4;+ format <= COLI_AFFINE_MLX_Q8; format++) {+ for (int scalar = COLI_AFFINE_SCALAR_F32;+ scalar <= COLI_AFFINE_SCALAR_BF16; scalar++) {+ if (run_parity((ColiAffineFormat)format,+ (ColiAffineScalarFormat)scalar))+ return 1;+ }+ }+ if (test_validation()) return 1;+ puts("affine quant tests: ok");+ return 0;+}