Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
9.9 KB
Raw Blame
#include "../backend_metal.h"

#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>

enum { OUTPUT_DIM = 3, INPUT_DIM = 32, GROUP_SIZE = 16, BATCH = 2 };

static void store_le16(uint8_t *destination, uint16_t value) {
    destination[0] = (uint8_t)value;
    destination[1] = (uint8_t)(value >> 8);
}

static void store_le32(uint8_t *destination, uint32_t value) {
    destination[0] = (uint8_t)value;
    destination[1] = (uint8_t)(value >> 8);
    destination[2] = (uint8_t)(value >> 16);
    destination[3] = (uint8_t)(value >> 24);
}

static uint16_t f32_to_f16(float value) {
    uint32_t bits;
    uint32_t sign, mantissa;
    int exponent;
    memcpy(&bits, &value, sizeof(bits));
    sign = (bits >> 16) & 0x8000u;
    exponent = (int)((bits >> 23) & 0xffu) - 127 + 15;
    mantissa = bits & 0x7fffffu;
    if (exponent <= 0) return (uint16_t)sign;
    if (exponent >= 31) return (uint16_t)(sign | 0x7c00u);
    return (uint16_t)(sign | ((uint32_t)exponent << 10) | (mantissa >> 13));
}

static void store_scalar(uint8_t *destination, size_t index,
                         ColiAffineScalarFormat format, float value) {
    uint32_t bits;
    if (format == COLI_AFFINE_SCALAR_F32) {
        memcpy(&bits, &value, sizeof(bits));
        store_le32(destination + index * 4, bits);
    } else if (format == COLI_AFFINE_SCALAR_F16) {
        store_le16(destination + index * 2, f32_to_f16(value));
    } else {
        memcpy(&bits, &value, sizeof(bits));
        store_le16(destination + index * 2, (uint16_t)(bits >> 16));
    }
}

static uint32_t packed_value(size_t row, size_t column, unsigned bits) {
    const uint32_t mask = (UINT32_C(1) << bits) - 1u;
    return (uint32_t)(row * 17 + column * 5 + 3) & mask;
}

static float direct_reference(const ColiAffineQuantizedView *view,
                              const float *input, size_t sample, size_t row) {
    const unsigned bits = coli_affine_bits(view->format);
    const unsigned per_word = 32u / bits;
    const size_t packed_words = view->input_dim / per_word;
    const size_t groups = view->input_dim / view->group_size;
    const uint32_t mask = (UINT32_C(1) << bits) - 1u;
    const uint8_t *weight_row =
        (const uint8_t *)view->weights + row * packed_words * 4;
    double result = 0.0;

    for (size_t column = 0; column < view->input_dim; column++) {
        const uint32_t word =
            coli_affine_load_le32(weight_row + (column / per_word) * 4);
        const uint32_t quantized =
            (word >> (bits * (unsigned)(column % per_word))) & mask;
        const size_t group = row * groups + column / view->group_size;
        const float scale = coli_affine_load_scalar(
            view->scales, group, view->scalar_format);
        const float bias = coli_affine_load_scalar(
            view->biases, group, view->scalar_format);
        const float x = input[sample * view->input_dim + column];
        result += (double)(scale * (float)quantized + bias) * (double)x;
    }
    return (float)result;
}

static int run_parity(ColiAffineFormat format,
                      ColiAffineScalarFormat scalar_format) {
    const unsigned bits = coli_affine_bits(format);
    const unsigned per_word = 32u / bits;
    const size_t packed_words = INPUT_DIM / per_word;
    const size_t scalar_count = OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE);
    const size_t scalar_size = coli_affine_scalar_size(scalar_format);
    uint8_t weights[OUTPUT_DIM * INPUT_DIM];
    uint8_t scales[OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE) * 4];
    uint8_t biases[OUTPUT_DIM * (INPUT_DIM / GROUP_SIZE) * 4];
    float input[BATCH * INPUT_DIM], output[BATCH * OUTPUT_DIM];

    memset(weights, 0, sizeof(weights));
    memset(scales, 0, sizeof(scales));
    memset(biases, 0, sizeof(biases));
    for (size_t row = 0; row < OUTPUT_DIM; row++) {
        for (size_t word_index = 0; word_index < packed_words; word_index++) {
            uint32_t word = 0;
            for (unsigned lane = 0; lane < per_word; lane++) {
                const size_t column = word_index * per_word + lane;
                word |= packed_value(row, column, bits) << (lane * bits);
            }
            store_le32(weights + (row * packed_words + word_index) * 4, word);
        }
    }
    for (size_t index = 0; index < scalar_count; index++) {
        const float scale = (float)((int)(index % 5) - 2) * 0.125f;
        const float bias = (float)((int)(index % 3) - 1) * 0.0625f;
        store_scalar(scales, index, scalar_format, scale);
        store_scalar(biases, index, scalar_format, bias);
    }
    for (size_t index = 0; index < BATCH * INPUT_DIM; index++)
        input[index] = (float)((int)(index % 13) - 6) * 0.125f;

    ColiAffineQuantizedView view = {
        weights, scales, biases,
        OUTPUT_DIM * packed_words * 4, scalar_count * scalar_size,
        scalar_count * scalar_size,
        OUTPUT_DIM, INPUT_DIM, GROUP_SIZE, format, scalar_format
    };
    ColiAffineStatus status = coli_affine_validate(&view);
    if (status != COLI_AFFINE_OK) {
        fprintf(stderr, "affine validation failed bits=%u scalar=%d: %s\n",
                bits, (int)scalar_format, coli_affine_status_string(status));
        return 1;
    }
    if (!coli_metal_affine_dispatch_supported(&view, BATCH)) {
        fprintf(stderr, "valid Metal dispatch rejected bits=%u scalar=%d\n",
                bits, (int)scalar_format);
        return 1;
    }
    status = coli_affine_matmul_ref(output, input, BATCH, &view);
    if (status != COLI_AFFINE_OK) {
        fprintf(stderr, "affine matmul failed bits=%u scalar=%d: %s\n",
                bits, (int)scalar_format, coli_affine_status_string(status));
        return 1;
    }

    for (size_t sample = 0; sample < BATCH; sample++) {
        for (size_t row = 0; row < OUTPUT_DIM; row++) {
            const float expected = direct_reference(&view, input, sample, row);
            const float actual = output[sample * OUTPUT_DIM + row];
            const float tolerance = 2e-6f * fmaxf(1.0f, fabsf(expected));
            if (fabsf(actual - expected) > tolerance) {
                fprintf(stderr,
                        "affine parity failed bits=%u scalar=%d sample=%zu row=%zu: "
                        "got %.9g want %.9g\n",
                        bits, (int)scalar_format, sample, row, actual, expected);
                return 1;
            }
        }
    }
    return 0;
}

static int test_validation(void) {
    uint32_t weight = 0;
    float scales = 1.0f, bias = 0.0f;
    float input[8] = {0}, output = 123.0f;
    ColiAffineQuantizedView view = {
        &weight, &scales, &bias, sizeof(weight), sizeof(scales), sizeof(bias),
        1, 8, 8, COLI_AFFINE_MLX_Q4, COLI_AFFINE_SCALAR_F32
    };

#define EXPECT_STATUS(expression, expected) do {                                  \
        ColiAffineStatus got = (expression);                                      \
        if (got != (expected)) {                                                  \
            fprintf(stderr, "validation line %d: got %s, want %s\n", __LINE__,  \
                    coli_affine_status_string(got),                               \
                    coli_affine_status_string(expected));                         \
            return 1;                                                             \
        }                                                                         \
    } while (0)

    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_OK);
    view.biases = NULL;
    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_NULL);
    view.biases = &bias;
    view.format = (ColiAffineFormat)99;
    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_BAD_FORMAT);
    view.format = COLI_AFFINE_MLX_Q4;
    view.input_dim = 7;
    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_BAD_SHAPE);
    view.input_dim = 8;
    view.weight_bytes--;
    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);
    view.weight_bytes++;
    view.scale_bytes--;
    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);
    view.scale_bytes++;
    view.bias_bytes--;
    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_TRUNCATED);
    view.bias_bytes++;
    view.output_dim = SIZE_MAX;
    EXPECT_STATUS(coli_affine_validate(&view), COLI_AFFINE_OVERFLOW);
    view.output_dim = 1;
    EXPECT_STATUS(coli_affine_matmul_ref(&output, input, 0, &view),
                  COLI_AFFINE_BAD_SHAPE);
    EXPECT_STATUS(coli_affine_matmul_ref(&output, input, SIZE_MAX, &view),
                  COLI_AFFINE_OVERFLOW);
#undef EXPECT_STATUS
    if (output != 123.0f) return 1;

    if (!coli_metal_affine_dispatch_supported(&view, 1) ||
        coli_metal_affine_dispatch_supported(&view, 0) ||
        coli_metal_affine_dispatch_supported(&view, -1))
        return 1;
    view.format = (ColiAffineFormat)99;
    if (coli_metal_affine_dispatch_supported(&view, 1)) return 1;
    view.format = COLI_AFFINE_MLX_Q4;
#if SIZE_MAX > UINT32_MAX
    view.output_dim = (size_t)UINT32_MAX + 1u;
    view.weight_bytes = view.scale_bytes = view.bias_bytes = SIZE_MAX;
    if (coli_affine_validate(&view) != COLI_AFFINE_OK ||
        coli_metal_affine_dispatch_supported(&view, 1))
        return 1;
    view.output_dim = UINT32_MAX;
    if (coli_affine_validate(&view) != COLI_AFFINE_OK ||
        coli_metal_affine_dispatch_supported(&view, 2))
        return 1;
    view.output_dim = UINT32_C(65536);
    view.input_dim = UINT32_C(1048576);
    view.group_size = 8;
    if (coli_affine_validate(&view) != COLI_AFFINE_OK ||
        coli_metal_affine_dispatch_supported(&view, 1))
        return 1;
#endif
    return 0;
}

int main(void) {
    for (int format = COLI_AFFINE_MLX_Q4;
         format <= COLI_AFFINE_MLX_Q8; format++) {
        for (int scalar = COLI_AFFINE_SCALAR_F32;
             scalar <= COLI_AFFINE_SCALAR_BF16; scalar++) {
            if (run_parity((ColiAffineFormat)format,
                           (ColiAffineScalarFormat)scalar))
                return 1;
        }
    }
    if (test_validation()) return 1;
    puts("affine quant tests: ok");
    return 0;
}