Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
11.1 KB
Raw Blame
/* affine_quant.h -- portable MLX affine-quantization contract and CPU oracle.
 *
 * This format is deliberately separate from QT fmt=4.  Colibri's fmt=4 stores
 * signed int4 values with multiplicative group scales; MLX stores unsigned
 * uint32-packed values and reconstructs each group as scale*q + bias.
 */
#ifndef COLI_AFFINE_QUANT_H
#define COLI_AFFINE_QUANT_H

#include <stddef.h>
#include <stdint.h>
#include <string.h>

#ifdef __cplusplus
extern "C" {
#endif

typedef enum {
    COLI_AFFINE_MLX_Q4 = 0,
    COLI_AFFINE_MLX_Q8 = 1
} ColiAffineFormat;

typedef enum {
    COLI_AFFINE_SCALAR_F32 = 0,
    COLI_AFFINE_SCALAR_F16 = 1,
    COLI_AFFINE_SCALAR_BF16 = 2
} ColiAffineScalarFormat;

typedef enum {
    COLI_AFFINE_OK = 0,
    COLI_AFFINE_NULL = -1,
    COLI_AFFINE_BAD_FORMAT = -2,
    COLI_AFFINE_BAD_SHAPE = -3,
    COLI_AFFINE_TRUNCATED = -4,
    COLI_AFFINE_OVERFLOW = -5
} ColiAffineStatus;

/* MLX affine weights are little-endian uint32 words, packed along input_dim.
 * Scales and biases have identical scalar types and [output_dim, groups]
 * layout, where groups = input_dim/group_size.  Byte lengths are mandatory:
 * qpack sections and resident tensors must be checked before GPU registration.
 */
typedef struct {
    const void *weights;
    const void *scales;
    const void *biases;
    size_t weight_bytes;
    size_t scale_bytes;
    size_t bias_bytes;
    size_t output_dim;
    size_t input_dim;
    size_t group_size;
    ColiAffineFormat format;
    ColiAffineScalarFormat scalar_format;
} ColiAffineQuantizedView;

static inline const char *coli_affine_status_string(ColiAffineStatus status) {
    switch (status) {
        case COLI_AFFINE_OK: return "ok";
        case COLI_AFFINE_NULL: return "null affine buffer";
        case COLI_AFFINE_BAD_FORMAT: return "unsupported affine format";
        case COLI_AFFINE_BAD_SHAPE: return "invalid affine shape";
        case COLI_AFFINE_TRUNCATED: return "truncated affine buffer";
        case COLI_AFFINE_OVERFLOW: return "affine size overflow";
    }
    return "unknown affine status";
}

static inline unsigned coli_affine_bits(ColiAffineFormat format) {
    return format == COLI_AFFINE_MLX_Q4 ? 4u :
           format == COLI_AFFINE_MLX_Q8 ? 8u : 0u;
}

static inline size_t coli_affine_scalar_size(ColiAffineScalarFormat format) {
    return format == COLI_AFFINE_SCALAR_F32 ? 4u :
           (format == COLI_AFFINE_SCALAR_F16 ||
            format == COLI_AFFINE_SCALAR_BF16) ? 2u : 0u;
}

static inline int coli_affine_size_mul(size_t left, size_t right, size_t *out) {
    if (left && right > SIZE_MAX / left) return 0;
    *out = left * right;
    return 1;
}

static inline ColiAffineStatus
coli_affine_validate(const ColiAffineQuantizedView *view) {
    size_t packed_words, group_count, need_weights, need_scalars;
    unsigned bits, per_word;
    size_t scalar_size;

    if (!view || !view->weights || !view->scales || !view->biases)
        return COLI_AFFINE_NULL;
    bits = coli_affine_bits(view->format);
    scalar_size = coli_affine_scalar_size(view->scalar_format);
    if (!bits || !scalar_size) return COLI_AFFINE_BAD_FORMAT;
    if (!view->output_dim || !view->input_dim || !view->group_size)
        return COLI_AFFINE_BAD_SHAPE;

    per_word = 32u / bits;
    if (view->input_dim % view->group_size ||
        view->input_dim % per_word || view->group_size % per_word)
        return COLI_AFFINE_BAD_SHAPE;

    packed_words = view->input_dim / per_word;
    group_count = view->input_dim / view->group_size;
    if (!coli_affine_size_mul(view->output_dim, packed_words, &need_weights) ||
        !coli_affine_size_mul(need_weights, sizeof(uint32_t), &need_weights) ||
        !coli_affine_size_mul(view->output_dim, group_count, &need_scalars) ||
        !coli_affine_size_mul(need_scalars, scalar_size, &need_scalars))
        return COLI_AFFINE_OVERFLOW;
    if (view->weight_bytes < need_weights ||
        view->scale_bytes < need_scalars || view->bias_bytes < need_scalars)
        return COLI_AFFINE_TRUNCATED;
    return COLI_AFFINE_OK;
}

static inline uint16_t coli_affine_load_le16(const uint8_t *source) {
    return (uint16_t)source[0] | (uint16_t)((uint16_t)source[1] << 8);
}

static inline uint32_t coli_affine_load_le32(const uint8_t *source) {
    return (uint32_t)source[0] | ((uint32_t)source[1] << 8) |
           ((uint32_t)source[2] << 16) | ((uint32_t)source[3] << 24);
}

static inline float coli_affine_f32_from_bits(uint32_t bits) {
    float value;
    memcpy(&value, &bits, sizeof(value));
    return value;
}

static inline float coli_affine_f16_to_f32(uint16_t value) {
    uint32_t sign = ((uint32_t)value & 0x8000u) << 16;
    int exponent = (int)(((uint32_t)value >> 10) & 0x1fu);
    uint32_t mantissa = (uint32_t)value & 0x3ffu;

    if (exponent == 0) {
        if (!mantissa) return coli_affine_f32_from_bits(sign);
        exponent = 1;
        while (!(mantissa & 0x400u)) {
            mantissa <<= 1;
            exponent--;
        }
        mantissa &= 0x3ffu;
        exponent += 127 - 15;
    } else if (exponent == 31) {
        exponent = 255;
    } else {
        exponent += 127 - 15;
    }
    return coli_affine_f32_from_bits(sign | ((uint32_t)exponent << 23) |
                                     (mantissa << 13));
}

static inline float coli_affine_load_scalar(const void *values, size_t index,
                                             ColiAffineScalarFormat format) {
    const uint8_t *bytes = (const uint8_t *)values;
    if (format == COLI_AFFINE_SCALAR_F32)
        return coli_affine_f32_from_bits(coli_affine_load_le32(bytes + index * 4));
    if (format == COLI_AFFINE_SCALAR_F16)
        return coli_affine_f16_to_f32(coli_affine_load_le16(bytes + index * 2));
    return coli_affine_f32_from_bits(
        (uint32_t)coli_affine_load_le16(bytes + index * 2) << 16);
}

/* Correctness-first y[S,O] = x[S,I] @ dequant(W[O,I])^T.  Accumulation is
 * grouped exactly like Swiftlet's generic Metal kernel: two f32 reductions
 * (q*x and x), followed by scale*qdot + bias*xsum for each group.
 */
static inline ColiAffineStatus
coli_affine_matmul_ref(float *output, const float *input, size_t batch,
                       const ColiAffineQuantizedView *view) {
    ColiAffineStatus status = coli_affine_validate(view);
    unsigned bits, per_word;
    uint32_t mask;
    size_t packed_words, groups, unused;
    const uint8_t *weights;

    if (status != COLI_AFFINE_OK) return status;
    if (!output || !input) return COLI_AFFINE_NULL;
    if (!batch) return COLI_AFFINE_BAD_SHAPE;
    if (!coli_affine_size_mul(batch, view->input_dim, &unused) ||
        !coli_affine_size_mul(batch, view->output_dim, &unused))
        return COLI_AFFINE_OVERFLOW;

    bits = coli_affine_bits(view->format);
    per_word = 32u / bits;
    mask = (UINT32_C(1) << bits) - 1u;
    packed_words = view->input_dim / per_word;
    groups = view->input_dim / view->group_size;
    weights = (const uint8_t *)view->weights;

    for (size_t row = 0; row < view->output_dim; row++) {
        const uint8_t *weight_row = weights + row * packed_words * 4;
        for (size_t sample = 0; sample < batch; sample++) {
            const float *input_row = input + sample * view->input_dim;
            float sum = 0.0f;
            for (size_t group = 0; group < groups; group++) {
                const size_t word_base = group * (view->group_size / per_word);
                const size_t input_base = group * view->group_size;
                float quantized_dot = 0.0f, input_sum = 0.0f;

                for (size_t local = 0; local < view->group_size; local++) {
                    const size_t column = input_base + local;
                    const size_t word_index = word_base + local / per_word;
                    const uint32_t word =
                        coli_affine_load_le32(weight_row + word_index * 4);
                    const unsigned shift = bits * (unsigned)(local % per_word);
                    const float x = input_row[column];
                    quantized_dot += (float)((word >> shift) & mask) * x;
                    input_sum += x;
                }
                const size_t scalar_index = row * groups + group;
                const float scale = coli_affine_load_scalar(
                    view->scales, scalar_index, view->scalar_format);
                const float bias = coli_affine_load_scalar(
                    view->biases, scalar_index, view->scalar_format);
                sum += scale * quantized_dot + bias * input_sum;
            }
            output[sample * view->output_dim + row] = sum;
        }
    }
    return COLI_AFFINE_OK;
}

/* Row-major logical expansion out[output_dim, input_dim] = dequant(view):
 * out[row*I + col] = scale[row, col/gs] * q[row, col] + bias[row, col/gs],
 * with the SAME bit conventions as coli_affine_matmul_ref above (little-endian
 * uint32 words, lowest bits = lowest logical column).  This is the loader-side
 * oracle: matmul_ref folds the dequant into its accumulation and never
 * materialises the weights, but a dense tensor the ENGINE keeps resident
 * (qwen36's attention/DeltaNet/shared-expert set from a Swiftlet container's
 * model.safetensors) needs the expanded f32 rows exactly once, at load time.
 * Kept next to matmul_ref so the two bit-level readings of the packed words
 * cannot drift apart without this file changing. */
static inline ColiAffineStatus
coli_affine_dequant_ref(const ColiAffineQuantizedView *view, float *output) {
    ColiAffineStatus status = coli_affine_validate(view);
    unsigned bits, per_word;
    uint32_t mask;
    size_t groups, unused;
    const uint8_t *weights;

    if (status != COLI_AFFINE_OK) return status;
    if (!output) return COLI_AFFINE_NULL;
    if (!coli_affine_size_mul(view->output_dim, view->input_dim, &unused))
        return COLI_AFFINE_OVERFLOW;

    bits = coli_affine_bits(view->format);
    per_word = 32u / bits;
    mask = (UINT32_C(1) << bits) - 1u;
    groups = view->input_dim / view->group_size;
    weights = (const uint8_t *)view->weights;

    for (size_t row = 0; row < view->output_dim; row++) {
        const uint8_t *weight_row =
            weights + row * (view->input_dim / per_word) * 4;
        float *out_row = output + row * view->input_dim;
        for (size_t group = 0; group < groups; group++) {
            const size_t word_base = group * (view->group_size / per_word);
            const size_t input_base = group * view->group_size;
            const size_t scalar_index = row * groups + group;
            const float scale = coli_affine_load_scalar(
                view->scales, scalar_index, view->scalar_format);
            const float bias = coli_affine_load_scalar(
                view->biases, scalar_index, view->scalar_format);
            for (size_t local = 0; local < view->group_size; local++) {
                const size_t word_index = word_base + local / per_word;
                const uint32_t word =
                    coli_affine_load_le32(weight_row + word_index * 4);
                const unsigned shift = bits * (unsigned)(local % per_word);
                out_row[input_base + local] =
                    scale * (float)((word >> shift) & mask) + bias;
            }
        }
    }
    return COLI_AFFINE_OK;
}

#ifdef __cplusplus
}
#endif

#endif