Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust
17.2 KB
Raw Blame
//! The GPU shrink (Captain's design, 8 Oct): an expert arriving from the file at Q3_K or Q5_K is
//! shrunk to Q2_K on the GPU the moment its bytes land, so the bank holds Q2 copies and parks
//! more experts in the same memory. One thread per 256-weight super-block: dequantise with the
//! same code the matvec kernels use, then fit 16 sub-blocks of 16 to 2 bits by min and max.
//! Its own command queue, so reader threads can run it without touching Candle's queue.

use candle::{MetalDevice, Result};
use candle_metal_kernels::metal::{Buffer, ComputePipeline, Device};
use objc2::rc::Retained;
use objc2::runtime::ProtocolObject;
use objc2_metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLComputeCommandEncoder, MTLDevice, MTLSize};
use std::time::Instant;

pub const QK_K: usize = 256;
pub const Q2K_BLOCK_BYTES: usize = 84;
pub const Q3K_BLOCK_BYTES: usize = 110;
pub const Q5K_BLOCK_BYTES: usize = 176;

const SOURCE: &str = r#"
#include <metal_stdlib>
using namespace metal;
#define QK_K 256
#define K_SCALE_SIZE 12

typedef struct {
    uint8_t scales[QK_K/16];
    uint8_t qs[QK_K/4];
    half d;
    half dmin;
} block_q2_K;

typedef struct {
    uint8_t hmask[QK_K/8];
    uint8_t qs[QK_K/4];
    uint8_t scales[12];
    half d;
} block_q3_K;

typedef struct {
    half d;
    half dmin;
    uint8_t scales[K_SCALE_SIZE];
    uint8_t qh[QK_K/8];
    uint8_t qs[QK_K/2];
} block_q5_K;

static inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) {
    return j < 4 ? uchar2{uchar(q[j+0+k] & 63), uchar(q[j+4+k] & 63)}
                 : uchar2{uchar((q[j+4+k] & 0xF) | ((q[j-4+k] & 0xc0) >> 2)), uchar((q[j+4+k] >> 4) | ((q[j-0+k] & 0xc0) >> 2))};
}

// 16 values of a Q3_K block (il = 0..15), as the matvec kernels read them.
void dq_q3(device const block_q3_K *xb, short il, thread float *out) {
    const half d_all = xb->d;
    device const uint8_t * q = (device const uint8_t *)xb->qs;
    device const uint8_t * h = (device const uint8_t *)xb->hmask;
    device const int8_t * scales = (device const int8_t *)xb->scales;
    q = q + 32 * (il/8) + 16 * (il&1);
    h = h + 16 * (il&1);
    uint8_t m = 1 << (il/2);
    uint16_t kmask1 = (il/4)>1 ? ((il/4)>2 ? 192 : 48) : ((il/4)>0 ? 12 : 3);
    uint16_t kmask2 = il/8 ? 0xF0 : 0x0F;
    uint16_t scale_2 = scales[il%8], scale_1 = scales[8 + il%4];
    int16_t dl_int = (il/4)&1 ? (scale_2&kmask2) | ((scale_1&kmask1) << 2) : (scale_2&kmask2) | ((scale_1&kmask1) << 4);
    float dl = il<8 ? d_all * (dl_int - 32.f) : d_all * (dl_int / 16.f - 32.f);
    const float ml = 4.f * dl;
    short il2 = (il/2) & 3;
    const half coef = il2>1 ? (il2>2 ? 1/64.h : 1/16.h) : (il2>0 ? 1/4.h : 1.h);
    const uint8_t mask = il2>1 ? (il2>2 ? 192 : 48) : (il2>0 ? 12 : 3);
    dl *= coef;
    for (int i = 0; i < 16; ++i) out[i] = dl * (q[i] & mask) - (h[i] & m ? 0 : ml);
}

typedef struct {
    half d;
    half dmin;
    uint8_t scales[K_SCALE_SIZE];
    uint8_t qs[QK_K/2];
} block_q4_K;

void dq_q4(device const block_q4_K *xb, short il, thread float *out) {
    device const uchar * q = xb->qs;
    short is = (il/4) * 2;
    q = q + (il/4) * 32 + 16 * (il&1);
    short il2 = il & 3;
    const uchar2 sc = get_scale_min_k4_just2(is, il2/2, xb->scales);
    const float d   = il2 < 2 ? xb->d : xb->d / 16.h;
    const float min = xb->dmin;
    const float dl = d * sc[0];
    const float ml = min * sc[1];
    const ushort mask = il2<2 ? 0x0F : 0xF0;
    for (int i = 0; i < 16; ++i) out[i] = dl * (q[i] & mask) - ml;
}

void dq_q5(device const block_q5_K *xb, short il, thread float *out) {
    device const uint8_t * q  = xb->qs;
    device const uint8_t * qh = xb->qh;
    short is = (il/4) * 2;
    q  = q + 32 * (il/4) + 16 * (il&1);
    qh = qh + 16 * (il&1);
    uint8_t ul = 1 << (il/2);
    short il2 = il & 3;
    const uchar2 sc = get_scale_min_k4_just2(is, il2/2, xb->scales);
    const float d = il2 < 2 ? xb->d : xb->d / 16.f;
    const float min = xb->dmin;
    const float dl = d * sc[0];
    const float ml = min * sc[1];
    const ushort mask = il2<2 ? 0x0F : 0xF0;
    const float qh_val = il2<2 ? 16.f : 256.f;
    for (int i = 0; i < 16; ++i) out[i] = dl * ((q[i] & mask) + (qh[i] & ul ? qh_val : 0)) - ml;
}

// Fit 256 values to a Q2_K block: per sub-block of 16, scale = (max-min)/3 and min; the 16 scales
// and mins go to 4 bits against the block's largest; quants are 2 bits.
// Weighted fit of one sub-block of 16: given a trial (scale, min), the quants are fixed by rounding,
// then scale and min are re-solved by weighted least squares on those quants. Two rounds.
static inline void fit16(thread const float *x, thread const float *w, thread float &scale, thread float &mn) {
    for (int round = 0; round < 2; ++round) {
        float sw = 0.f, swl = 0.f, swll = 0.f, swx = 0.f, swlx = 0.f;
        for (int i = 0; i < 16; ++i) {
            int l = 0;
            if (scale > 0.f) { l = (int)((x[i] + mn) / scale + 0.5f); l = clamp(l, 0, 3); }
            const float wi = w[i];
            sw += wi; swl += wi * l; swll += wi * l * l; swx += wi * x[i]; swlx += wi * l * x[i];
        }
        // x ~ scale*l - mn : solve for (scale, mn) weighted
        const float det = sw * swll - swl * swl;
        if (det > 1e-12f) {
            const float a = (sw * swlx - swl * swx) / det;      // scale
            const float b = (swl * swlx - swll * swx) / det;    // = mn
            if (a > 0.f && b >= 0.f) { scale = a; mn = b; }
        }
    }
}

void q2k_encode_w(thread const float *x, device const float *w, uint w_offset, device block_q2_K *y) {
    float scales[16], mins[16];
    float max_scale = 0.f, max_min = 0.f;
    for (int j = 0; j < 16; ++j) {
        float lo = x[16*j], hi = x[16*j];
        for (int i = 1; i < 16; ++i) { lo = min(lo, x[16*j+i]); hi = max(hi, x[16*j+i]); }
        if (lo > 0.f) lo = 0.f;
        float sc = (hi - lo) / 3.f;
        float mn = -lo;
        float wl[16];
        for (int i = 0; i < 16; ++i) wl[i] = sqrt(max(w[w_offset + 16*j + i], 0.f)) + 1e-6f;
        fit16(x + 16*j, wl, sc, mn);
        scales[j] = sc;
        mins[j] = mn;
        max_scale = max(max_scale, scales[j]);
        max_min = max(max_min, mins[j]);
    }
    float inv_scale = max_scale > 0.f ? 15.f / max_scale : 0.f;
    float inv_min = max_min > 0.f ? 15.f / max_min : 0.f;
    uint8_t ls[16], lm[16];
    for (int j = 0; j < 16; ++j) {
        ls[j] = (uint8_t)min(15, (int)(scales[j] * inv_scale + 0.5f));
        lm[j] = (uint8_t)min(15, (int)(mins[j] * inv_min + 0.5f));
        y->scales[j] = ls[j] | (lm[j] << 4);
    }
    y->d = half(max_scale / 15.f);
    y->dmin = half(max_min / 15.f);
    const float d = float(y->d), dmin = float(y->dmin);
    uint8_t L[256];
    for (int j = 0; j < 16; ++j) {
        const float dj = d * ls[j];
        const float mj = dmin * lm[j];
        for (int i = 0; i < 16; ++i) {
            int l = 0;
            if (dj > 0.f) { l = (int)((x[16*j+i] + mj) / dj + 0.5f); l = clamp(l, 0, 3); }
            L[16*j+i] = (uint8_t)l;
        }
    }
    for (int j = 0; j < QK_K; j += 128) {
        for (int l = 0; l < 32; ++l) {
            y->qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);
        }
    }
}

void q2k_encode(thread const float *x, device block_q2_K *y) {
    float scales[16], mins[16];
    float max_scale = 0.f, max_min = 0.f;
    const float ones[16] = {1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f,1.f};
    for (int j = 0; j < 16; ++j) {
        float lo = x[16*j], hi = x[16*j];
        for (int i = 1; i < 16; ++i) { lo = min(lo, x[16*j+i]); hi = max(hi, x[16*j+i]); }
        if (lo > 0.f) lo = 0.f;
        float sc = (hi - lo) / 3.f;
        float mn = -lo;
        fit16(x + 16*j, ones, sc, mn);
        scales[j] = sc;
        mins[j] = mn;
        max_scale = max(max_scale, scales[j]);
        max_min = max(max_min, mins[j]);
    }
    float inv_scale = max_scale > 0.f ? 15.f / max_scale : 0.f;
    float inv_min = max_min > 0.f ? 15.f / max_min : 0.f;
    uint8_t ls[16], lm[16];
    for (int j = 0; j < 16; ++j) {
        ls[j] = (uint8_t)min(15, (int)(scales[j] * inv_scale + 0.5f));
        lm[j] = (uint8_t)min(15, (int)(mins[j] * inv_min + 0.5f));
        y->scales[j] = ls[j] | (lm[j] << 4);
    }
    y->d = half(max_scale / 15.f);
    y->dmin = half(max_min / 15.f);
    const float d = float(y->d), dmin = float(y->dmin);
    uint8_t L[256];
    for (int j = 0; j < 16; ++j) {
        const float dj = d * ls[j];
        const float mj = dmin * lm[j];
        for (int i = 0; i < 16; ++i) {
            int l = 0;
            if (dj > 0.f) {
                l = (int)((x[16*j+i] + mj) / dj + 0.5f);
                l = clamp(l, 0, 3);
            }
            L[16*j+i] = (uint8_t)l;
        }
    }
    for (int j = 0; j < QK_K; j += 128) {
        for (int l = 0; l < 32; ++l) {
            y->qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6);
        }
    }
}

kernel void coachwhip_requant_q3k_to_q2k(
    device const block_q3_K *src [[buffer(0)]],
    device block_q2_K *dst       [[buffer(1)]],
    constant uint& n_blocks      [[buffer(2)]],
    uint tid [[thread_position_in_grid]])
{
    if (tid >= n_blocks) return;
    float x[256];
    for (short il = 0; il < 16; ++il) dq_q3(src + tid, il, x + 16*il);
    q2k_encode(x, dst + tid);
}

kernel void coachwhip_requant_q3k_to_q2k_w(
    device const block_q3_K *src [[buffer(0)]],
    device block_q2_K *dst       [[buffer(1)]],
    constant uint& n_blocks      [[buffer(2)]],
    device const float *shape    [[buffer(3)]],
    constant uint& blocks_per_row [[buffer(4)]],
    uint tid [[thread_position_in_grid]])
{
    if (tid >= n_blocks) return;
    float x[256];
    for (short il = 0; il < 16; ++il) dq_q3(src + tid, il, x + 16*il);
    q2k_encode_w(x, shape, (tid % blocks_per_row) * 256, dst + tid);
}

kernel void coachwhip_requant_q4k_to_q2k(
    device const block_q4_K *src [[buffer(0)]],
    device block_q2_K *dst       [[buffer(1)]],
    constant uint& n_blocks      [[buffer(2)]],
    uint tid [[thread_position_in_grid]])
{
    if (tid >= n_blocks) return;
    float x[256];
    for (short il = 0; il < 16; ++il) dq_q4(src + tid, il, x + 16*il);
    q2k_encode(x, dst + tid);
}

kernel void coachwhip_requant_q4k_to_q2k_w(
    device const block_q4_K *src [[buffer(0)]],
    device block_q2_K *dst       [[buffer(1)]],
    constant uint& n_blocks      [[buffer(2)]],
    device const float *shape    [[buffer(3)]],
    constant uint& blocks_per_row [[buffer(4)]],
    uint tid [[thread_position_in_grid]])
{
    if (tid >= n_blocks) return;
    float x[256];
    for (short il = 0; il < 16; ++il) dq_q4(src + tid, il, x + 16*il);
    q2k_encode_w(x, shape, (tid % blocks_per_row) * 256, dst + tid);
}

kernel void coachwhip_requant_q5k_to_q2k(
    device const block_q5_K *src [[buffer(0)]],
    device block_q2_K *dst       [[buffer(1)]],
    constant uint& n_blocks      [[buffer(2)]],
    uint tid [[thread_position_in_grid]])
{
    if (tid >= n_blocks) return;
    float x[256];
    for (short il = 0; il < 16; ++il) dq_q5(src + tid, il, x + 16*il);
    q2k_encode(x, dst + tid);
}

kernel void coachwhip_requant_q5k_to_q2k_w(
    device const block_q5_K *src [[buffer(0)]],
    device block_q2_K *dst       [[buffer(1)]],
    constant uint& n_blocks      [[buffer(2)]],
    device const float *shape    [[buffer(3)]],
    constant uint& blocks_per_row [[buffer(4)]],
    uint tid [[thread_position_in_grid]])
{
    if (tid >= n_blocks) return;
    float x[256];
    for (short il = 0; il < 16; ++il) dq_q5(src + tid, il, x + 16*il);
    q2k_encode_w(x, shape, (tid % blocks_per_row) * 256, dst + tid);
}
"#;

/// The shrink: two pipelines and a command queue of its own.
/// Which quantisation the arriving bytes are in.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SrcKind {
    Q3K,
    Q4K,
    Q5K,
}

impl SrcKind {
    pub fn of(dtype: candle::quantized::GgmlDType) -> Option<Self> {
        use candle::quantized::GgmlDType as G;
        match dtype {
            G::Q3K => Some(Self::Q3K),
            G::Q4K => Some(Self::Q4K),
            G::Q5K => Some(Self::Q5K),
            _ => None,
        }
    }
}

pub struct Requant {
    queue: Retained<ProtocolObject<dyn MTLCommandQueue>>,
    q3: ComputePipeline,
    q4: ComputePipeline,
    q5: ComputePipeline,
    q3w: ComputePipeline,
    q4w: ComputePipeline,
    q5w: ComputePipeline,
}

unsafe impl Send for Requant {}
unsafe impl Sync for Requant {}

impl Requant {
    pub fn new(metal: &MetalDevice) -> Result<Self> {
        let device: &Device = metal.device();
        let lib = device.new_library_with_source(SOURCE, None).map_err(candle::MetalError::from)?;
        let mut get = |name: &str| -> Result<ComputePipeline> {
            let f = lib.get_function(name, None).map_err(candle::MetalError::from)?;
            Ok(device.new_compute_pipeline_state_with_function(&f).map_err(candle::MetalError::from)?)
        };
        let q3 = get("coachwhip_requant_q3k_to_q2k")?;
        let q4 = get("coachwhip_requant_q4k_to_q2k")?;
        let q5 = get("coachwhip_requant_q5k_to_q2k")?;
        let q3w = get("coachwhip_requant_q3k_to_q2k_w")?;
        let q4w = get("coachwhip_requant_q4k_to_q2k_w")?;
        let q5w = get("coachwhip_requant_q5k_to_q2k_w")?;
        let queue = device.as_ref().newCommandQueue().ok_or_else(|| candle::Error::Msg("no Metal command queue for the shrink".into()))?;
        Ok(Self { queue, q3, q4, q5, q3w, q4w, q5w })
    }

    /// Shrink `n_blocks` super-blocks from `src` (Q3_K or Q5_K bytes, from `src_offset`) into
    /// `dst` at `dst_offset` (Q2_K bytes). Waits for the GPU, so the caller may mark the slot ready.
    pub fn run(&self, kind: SrcKind, src: &Buffer, src_offset: usize, dst: &Buffer, dst_offset: usize, n_blocks: usize) -> Result<()> {
        let cb = self.queue.commandBuffer().ok_or_else(|| candle::Error::Msg("no command buffer for the shrink".into()))?;
        let enc = cb.computeCommandEncoder().ok_or_else(|| candle::Error::Msg("no encoder for the shrink".into()))?;
        let pipe = match kind { SrcKind::Q3K => &self.q3, SrcKind::Q4K => &self.q4, SrcKind::Q5K => &self.q5 };
        enc.setComputePipelineState(pipe.as_ref());
        unsafe {
            enc.setBuffer_offset_atIndex(Some(src.as_ref()), src_offset, 0);
            enc.setBuffer_offset_atIndex(Some(dst.as_ref()), dst_offset, 1);
            let n = n_blocks as u32;
            enc.setBytes_length_atIndex(std::ptr::NonNull::new(&n as *const u32 as *mut std::ffi::c_void).unwrap(), 4, 2);
        }
        let tg = 64usize;
        let groups = (n_blocks + tg - 1) / tg;
        enc.dispatchThreadgroups_threadsPerThreadgroup(MTLSize { width: groups, height: 1, depth: 1 }, MTLSize { width: tg, height: 1, depth: 1 });
        enc.endEncoding();
        cb.commit();
        unsafe { cb.waitUntilCompleted() };
        Ok(())
    }
}

impl Requant {
    /// The shaped shrink: like `run`, but each input column's error is weighted by `shape`
    /// (one float per column, `k` of them), the running importance of that column at this layer.
    pub fn run_shaped(&self, kind: SrcKind, src: &Buffer, src_offset: usize, dst: &Buffer, dst_offset: usize, n_blocks: usize, shape: &Buffer, k: usize) -> Result<()> {
        let cb = self.queue.commandBuffer().ok_or_else(|| candle::Error::Msg("no command buffer for the shrink".into()))?;
        let enc = cb.computeCommandEncoder().ok_or_else(|| candle::Error::Msg("no encoder for the shrink".into()))?;
        let pipe = match kind { SrcKind::Q3K => &self.q3w, SrcKind::Q4K => &self.q4w, SrcKind::Q5K => &self.q5w };
        enc.setComputePipelineState(pipe.as_ref());
        unsafe {
            enc.setBuffer_offset_atIndex(Some(src.as_ref()), src_offset, 0);
            enc.setBuffer_offset_atIndex(Some(dst.as_ref()), dst_offset, 1);
            let n = n_blocks as u32;
            enc.setBytes_length_atIndex(std::ptr::NonNull::new(&n as *const u32 as *mut std::ffi::c_void).unwrap(), 4, 2);
            enc.setBuffer_offset_atIndex(Some(shape.as_ref()), 0, 3);
            let bpr = (k / QK_K) as u32;
            enc.setBytes_length_atIndex(std::ptr::NonNull::new(&bpr as *const u32 as *mut std::ffi::c_void).unwrap(), 4, 4);
        }
        let tg = 64usize;
        let groups = (n_blocks + tg - 1) / tg;
        enc.dispatchThreadgroups_threadsPerThreadgroup(MTLSize { width: groups, height: 1, depth: 1 }, MTLSize { width: tg, height: 1, depth: 1 });
        enc.endEncoding();
        cb.commit();
        unsafe { cb.waitUntilCompleted() };
        Ok(())
    }
}

/// Lab timing, run by the `gpu_shrink` test: one expert's three matrices through the shrink.
pub fn time_one_expert(metal: &MetalDevice, parts: &[(SrcKind, Vec<u8>, usize)]) -> Result<(f64, usize, usize)> {
    let rq = Requant::new(metal)?;
    let mut bytes_in = 0;
    let mut bytes_out = 0;
    let started = Instant::now();
    for (kind, data, n_blocks) in parts {
        let src = metal.new_buffer_builder().with_size(data.len()).with_label("shrink_src").build()?;
        unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), src.contents() as *mut u8, data.len()) };
        let dst = metal.new_buffer_builder().with_size(n_blocks * Q2K_BLOCK_BYTES).with_label("shrink_dst").build()?;
        rq.run(*kind, &src, 0, &dst, 0, *n_blocks)?;
        bytes_in += data.len();
        bytes_out += n_blocks * Q2K_BLOCK_BYTES;
    }
    Ok((started.elapsed().as_secs_f64() * 1e3, bytes_in, bytes_out))
}