Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust
15.1 KB
Raw Blame
//! One Metal dispatch for every selected expert of every token: ggml's `kernel_mul_mv_id`,
//! which ships inside Candle's Metal kernel library but has no Rust entry point there.

use candle_metal_kernels::metal::{Buffer, Device};
use candle_metal_kernels::source::Source;
use candle_metal_kernels::utils::EncoderProvider;
use candle_metal_kernels::{set_params, GgmlDType, Kernels, MetalKernelError, Output};
use objc2_metal::MTLSize;

/// For every token and each of its `nei0` selected experts, multiplies one row of `src1` by that
/// expert's quantised matrix.
/// - `src0s` holds `ne02` expert matrices of `n` rows x `k` columns, `nb02` bytes apart.
/// - `ids` holds `nei1 x nei0` int32 expert indices into `src0s`.
/// - `src1` rows are f32; row `i11 = slot % ne11` of token `i12` sits at `i11*nb11 + i12*nb12` bytes.
/// - `dst` gets f32 `[nei1][nei0][n]`.
#[allow(clippy::too_many_arguments)]
pub fn matvec_id(
    device: &Device,
    ep: impl EncoderProvider,
    kernels: &Kernels,
    dtype: GgmlDType,
    (n, k, ne02, nb02): (usize, usize, usize, usize),
    src0s: &Buffer,
    src1: &Buffer,
    src1_offset: usize,
    (ne11, nb11, nb12): (usize, usize, usize),
    ids: &Buffer,
    ids_offset: usize,
    (nei0, nei1): (usize, usize),
    dst: &Buffer,
) -> Result<(), MetalKernelError> {
    let (nth0, nth1, align) = match dtype {
        GgmlDType::Q4_0 | GgmlDType::Q4_1 | GgmlDType::Q5_0 | GgmlDType::Q5_1 | GgmlDType::Q8_0 | GgmlDType::Q8_1 | GgmlDType::Mxfp4 => (8, 8, 8),
        GgmlDType::Q2K => (2, 32, 8),
        GgmlDType::Q4K => (4, 8, 4),
        GgmlDType::Q3K | GgmlDType::Q5K => (2, 32, 4),
        GgmlDType::Q6K => (2, 32, 2),
        GgmlDType::F16 | GgmlDType::BF16 | GgmlDType::Q8K | GgmlDType::F32 => (32, 1, 8),
        GgmlDType::Iq2Xxs | GgmlDType::Iq3Xxs => (4, 16, 8),
        GgmlDType::Iq4Xs => (4, 16, 4),
    };
    let name = match dtype {
        GgmlDType::Q4_0 => "kernel_mul_mv_id_q4_0_f32",
        GgmlDType::Q4_1 => "kernel_mul_mv_id_q4_1_f32",
        GgmlDType::Q5_0 => "kernel_mul_mv_id_q5_0_f32",
        GgmlDType::Q5_1 => "kernel_mul_mv_id_q5_1_f32",
        GgmlDType::Q8_0 => "kernel_mul_mv_id_q8_0_f32",
        GgmlDType::Mxfp4 => "kernel_mul_mv_id_mxfp4_f32",
        GgmlDType::Iq2Xxs => "kernel_mul_mv_id_iq2_xxs_f32",
        GgmlDType::Iq3Xxs => "kernel_mul_mv_id_iq3_xxs_f32",
        GgmlDType::Iq4Xs => "kernel_mul_mv_id_iq4_xs_f32",
        GgmlDType::Q2K => "kernel_mul_mv_id_q2_K_f32",
        GgmlDType::Q3K => "kernel_mul_mv_id_q3_K_f32",
        GgmlDType::Q4K => "kernel_mul_mv_id_q4_K_f32",
        GgmlDType::Q5K => "kernel_mul_mv_id_q5_K_f32",
        GgmlDType::Q6K => "kernel_mul_mv_id_q6_K_f32",
        GgmlDType::F16 => "kernel_mul_mv_id_f16_f32",
        GgmlDType::F32 => "kernel_mul_mv_id_f32_f32",
        other => return Err(MetalKernelError::LoadLibraryError(format!("no mul_mv_id kernel for {other:?}"))),
    };
    let pipeline = kernels.load_pipeline(device, Source::Quantized, name)?;
    let encoder = ep.encoder();
    let encoder = encoder.as_ref();
    encoder.set_compute_pipeline_state(&pipeline);
    // IQ kernels keep ggml's grids and signs in threadgroup memory.
    let iq_shmem: usize = match dtype { GgmlDType::Iq2Xxs => 256 * 8 + 128, GgmlDType::Iq3Xxs => 256 * 4 + 128, GgmlDType::Iq4Xs => 32 * 4, _ => 0 };
    if iq_shmem > 0 {
        encoder.set_threadgroup_memory_length(0, iq_shmem);
    }

    let ne00 = k as i64;
    let ne01 = n as i64;
    let ne02 = ne02 as i64;
    let nb00 = 0u64;
    let nb01 = 0u64;
    let nb02 = nb02 as u64;
    let ne10 = k as i64;
    let ne11 = ne11 as i64;
    let ne12 = nei1 as i64;
    let ne13 = 1i64;
    let nb10 = 4u64;
    let nb11 = nb11 as u64;
    let nb12 = nb12 as u64;
    let ne0 = n as i64;
    let ne1 = nei0 as i64;
    let nb1 = (n * 4) as u64;
    let nei0 = nei0 as i64;
    let nei1 = nei1 as i64;
    let nbi1 = (nei0 * 4) as u64;

    set_params!(
        encoder,
        (
            src0s,
            (src1, src1_offset),
            Output::with_offset(dst, 0),
            (ids, ids_offset),
            nei0,
            nei1,
            nbi1,
            ne00,
            ne01,
            ne02,
            nb00,
            nb01,
            nb02,
            ne10,
            ne11,
            ne12,
            ne13,
            nb10,
            nb11,
            nb12,
            ne0,
            ne1,
            nb1
        )
    );
    let thread_groups_count = MTLSize { width: n.div_ceil(align), height: 1, depth: (nei0 * nei1) as usize };
    let threads_per_threadgroup = MTLSize { width: nth0, height: nth1, depth: 1 };
    encoder.dispatch_thread_groups(thread_groups_count, threads_per_threadgroup);
    Ok(())
}


/// Lab switch: ggml's current mul_mv_id kernels for the IQ expert types.
pub fn new_on() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_NEWMV").map(|v| v == "1").unwrap_or(false))
}

#[repr(C)]
struct KargsMvId {
    nei0: i32, nei1: i32, nbi1: u64,
    ne00: i32, ne01: i32, ne02: i32, nb00: u64, nb01: u64, nb02: u64,
    ne10: i32, ne11: i32, ne12: i32, ne13: i32, nb10: u64, nb11: u64, nb12: u64,
    ne0: i32, ne1: i32, nb1: u64, nr0: i32,
}

fn new_pipe(device: &Device, name: &str, split: bool) -> Result<std::sync::Arc<candle_metal_kernels::metal::ComputePipeline>, MetalKernelError> {
    use std::collections::HashMap;
    use std::sync::{Arc, Mutex, OnceLock};
    static P: OnceLock<Mutex<HashMap<(String, bool), Arc<candle_metal_kernels::metal::ComputePipeline>>>> = OnceLock::new();
    let mut g = P.get_or_init(|| Mutex::new(HashMap::new())).lock().unwrap();
    if let Some(p) = g.get(&(name.to_string(), split)) {
        return Ok(p.clone());
    }
    let src = if split { include_str!("ggml_mv_split.metal") } else { include_str!("ggml_mv_plain.metal") };
    let lib = device.new_library_with_source(src, None)?;
    let f = lib.get_function(name, None)?;
    let p = Arc::new(device.new_compute_pipeline_state_with_function(&f)?);
    g.insert((name.to_string(), split), p.clone());
    Ok(p)
}

/// Same contract as [`matvec_id`], IQ types only; returns Ok(false) for any other type.
#[allow(clippy::too_many_arguments)]
pub fn matvec_id_new(
    device: &Device,
    ep: impl EncoderProvider,
    dtype: GgmlDType,
    (n, k, ne02, nb02): (usize, usize, usize, usize),
    src0s: &Buffer,
    src1: &Buffer,
    src1_offset: usize,
    (ne11, nb11, nb12): (usize, usize, usize),
    ids: &Buffer,
    ids_offset: usize,
    (nei0, nei1): (usize, usize),
    dst: &Buffer,
) -> Result<bool, MetalKernelError> {
    let nb32 = k / 32;
    let short_row = nb32 < 32 && 32 % nb32 == 0;
    let (name, block, nr0, split, smem) = match dtype {
        GgmlDType::Iq2Xxs => ("kernel_mul_mv_id_iq2_xxs_f32", 66usize, if short_row { 8 } else { 4 }, short_row, 256 * 8 + 128),
        GgmlDType::Iq3Xxs => ("kernel_mul_mv_id_iq3_xxs_f32", 98, if short_row { 8 } else { 4 }, short_row, 256 * 4 + 128),
        GgmlDType::Iq4Xs => ("kernel_mul_mv_id_iq4_xs_f32", 136, 2, false, 32 * 4),
        _ => return Ok(false),
    };
    let nsg = 2usize;
    let pipeline = new_pipe(device, name, split)?;
    let encoder = ep.encoder();
    let encoder = encoder.as_ref();
    encoder.set_compute_pipeline_state(&pipeline);
    encoder.set_threadgroup_memory_length(0, smem);
    let args = KargsMvId {
        nei0: nei0 as i32, nei1: nei1 as i32, nbi1: (nei0 * 4) as u64,
        ne00: k as i32, ne01: n as i32, ne02: ne02 as i32,
        nb00: block as u64, nb01: (k / 256 * block) as u64, nb02: nb02 as u64,
        ne10: k as i32, ne11: ne11 as i32, ne12: nei1 as i32, ne13: 1,
        nb10: 4, nb11: nb11 as u64, nb12: nb12 as u64,
        ne0: n as i32, ne1: nei0 as i32, nb1: (n * 4) as u64, nr0: nr0 as i32,
    };
    encoder.set_bytes(0, &args);
    encoder.set_input_buffer(1, Some(src0s), 0);
    encoder.set_input_buffer(2, Some(src1), src1_offset);
    encoder.set_output_buffer(3, Some(dst), 0);
    encoder.set_input_buffer(4, Some(ids), ids_offset);
    let groups = MTLSize { width: n.div_ceil(nr0 * nsg), height: 1, depth: nei0 * nei1 };
    encoder.dispatch_thread_groups(groups, MTLSize { width: 32, height: nsg, depth: 1 });
    Ok(true)
}


pub fn group_on() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_GROUP").map(|v| v == "1").unwrap_or(false))
}

/// Same contract as [`matvec_id`], for a batch of words: one threadgroup column per expert gathers the
/// words that picked it and multiplies them as one small matrix. Returns Ok(false) when not supported.
#[allow(clippy::too_many_arguments)]
pub fn matmul_id_grouped(
    device: &Device,
    ep: impl EncoderProvider,
    kernels: &Kernels,
    dtype: GgmlDType,
    (n, k, ne02, nb02): (usize, usize, usize, usize),
    src0s: &Buffer,
    src1: &Buffer,
    src1_offset: usize,
    (ne11, nb11, nb12): (usize, usize, usize),
    ids: &Buffer,
    ids_offset: usize,
    (nei0, nei1): (usize, usize),
    dst: &Buffer,
) -> Result<bool, MetalKernelError> {
    let (name, block) = match dtype {
        GgmlDType::Iq2Xxs => ("kernel_mul_mm_id_iq2_xxs_f32", 66usize),
        GgmlDType::Iq3Xxs => ("kernel_mul_mm_id_iq3_xxs_f32", 98),
        GgmlDType::Iq4Xs => ("kernel_mul_mm_id_iq4_xs_f32", 136),
        _ => return Ok(false),
    };
    let smem = (8192 + 4 * nei0 * nei1 + 15) / 16 * 16;
    if smem > 32768 || k % 256 != 0 {
        return Ok(false);
    }
    let pipeline = kernels.load_pipeline(device, Source::Quantized, name)?;
    let encoder = ep.encoder();
    let encoder = encoder.as_ref();
    encoder.set_compute_pipeline_state(&pipeline);
    encoder.set_threadgroup_memory_length(0, smem);
    let (nei0_, nei1_) = (nei0 as i64, nei1 as i64);
    let nbi1 = (nei0 * 4) as u64;
    let ne00 = k as i64;
    let ne02_ = ne02 as i64;
    let nb01 = (k / 256 * block) as u64;
    let nb02_ = nb02 as u64;
    let ne11_ = ne11 as i64;
    let ne12 = nei1 as i64;
    let ne13 = 1i64;
    let nb10 = 4u64;
    let nb11_ = nb11 as u64;
    let nb12_ = nb12 as u64;
    let ne0 = n as i64;
    let ne1 = nei0 as i64;
    let nb1 = (n * 4) as u64;
    set_params!(
        encoder,
        (
            src0s,
            (src1, src1_offset),
            Output::with_offset(dst, 0),
            (ids, ids_offset),
            nei0_, nei1_, nbi1, ne00, ne02_, nb01, nb02_, ne11_, ne12, ne13, nb10, nb11_, nb12_, ne0, ne1, nb1
        )
    );
    let groups = MTLSize { width: nei1.div_ceil(32), height: n.div_ceil(64), depth: ne02 };
    encoder.dispatch_thread_groups(groups, MTLSize { width: 128, height: 1, depth: 1 });
    Ok(true)
}


pub fn group2_on() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_GROUP").map(|v| v == "2").unwrap_or(false))
}

/// Grouped multiply from piles sorted on the CPU: `piles` (slot, word) u16 pairs, `heads` (expert,
/// first, count) u32 triples, `used` experts, `most` = the largest pile.
#[allow(clippy::too_many_arguments)]
pub fn matmul_id_piles(
    device: &Device,
    ep: impl EncoderProvider,
    kernels: &Kernels,
    dtype: GgmlDType,
    (n, k, ne02, nb02): (usize, usize, usize, usize),
    src0s: &Buffer,
    src1: &Buffer,
    src1_offset: usize,
    (ne11, nb11, nb12): (usize, usize, usize),
    piles: &Buffer,
    heads: &Buffer,
    (used, most, nei0): (usize, usize, usize),
    dst: &Buffer,
    min_cnt: u32,
) -> Result<bool, MetalKernelError> {
    let (name, block) = match dtype {
        GgmlDType::Iq2Xxs => ("kernel_mul_mm_idg_iq2_xxs_f32", 66usize),
        GgmlDType::Iq3Xxs => ("kernel_mul_mm_idg_iq3_xxs_f32", 98),
        GgmlDType::Iq4Xs => ("kernel_mul_mm_idg_iq4_xs_f32", 136),
        _ => return Ok(false),
    };
    let smem = (8192 + 4 * most + 15) / 16 * 16;
    if smem > 32768 || k % 256 != 0 {
        return Ok(false);
    }
    let pipeline = kernels.load_pipeline(device, Source::Quantized, name)?;
    let encoder = ep.encoder();
    let encoder = encoder.as_ref();
    encoder.set_compute_pipeline_state(&pipeline);
    encoder.set_threadgroup_memory_length(0, smem);
    let ne00 = k as i64;
    let ne02_ = ne02 as i64;
    let nb01 = (k / 256 * block) as u64;
    let nb02_ = nb02 as u64;
    let ne11_ = ne11 as i64;
    let ne12 = 1i64;
    let nb10 = 4u64;
    let nb11_ = nb11 as u64;
    let nb12_ = nb12 as u64;
    let ne0 = n as i64;
    let ne1 = nei0 as i64;
    set_params!(
        encoder,
        (src0s, (src1, src1_offset), Output::with_offset(dst, 0), piles, heads, ne00, ne02_, nb01, nb02_, ne11_, ne12, nb10, nb11_, nb12_, ne0, ne1, min_cnt)
    );
    let groups = MTLSize { width: most.div_ceil(32), height: n.div_ceil(64), depth: used };
    encoder.dispatch_thread_groups(groups, MTLSize { width: 128, height: 1, depth: 1 });
    Ok(true)
}


/// The one-word expert multiply over a list of (slot, word) pairs; `npairs` holds the list length.
#[allow(clippy::too_many_arguments)]
pub fn matvec_id_list(
    device: &Device,
    ep: impl EncoderProvider,
    dtype: GgmlDType,
    (n, k, ne02, nb02): (usize, usize, usize, usize),
    src0s: &Buffer,
    src1: &Buffer,
    src1_offset: usize,
    (ne11, nb11, nb12): (usize, usize, usize),
    ids: &Buffer,
    ids_offset: usize,
    (nei0, nei1): (usize, usize),
    pairs: &Buffer,
    npairs: &Buffer,
    max_pairs: usize,
    dst: &Buffer,
) -> Result<bool, MetalKernelError> {
    let nb32 = k / 32;
    let short_row = nb32 < 32 && 32 % nb32 == 0;
    let (name, block, nr0, split, smem) = match dtype {
        GgmlDType::Iq2Xxs => ("kernel_mul_mv_idl_iq2_xxs_f32", 66usize, if short_row { 8 } else { 4 }, short_row, 256 * 8 + 128),
        GgmlDType::Iq3Xxs => ("kernel_mul_mv_idl_iq3_xxs_f32", 98, if short_row { 8 } else { 4 }, short_row, 256 * 4 + 128),
        GgmlDType::Iq4Xs => ("kernel_mul_mv_idl_iq4_xs_f32", 136, 2, false, 32 * 4),
        _ => return Ok(false),
    };
    let nsg = 2usize;
    let pipeline = new_pipe(device, name, split)?;
    let encoder = ep.encoder();
    let encoder = encoder.as_ref();
    encoder.set_compute_pipeline_state(&pipeline);
    encoder.set_threadgroup_memory_length(0, smem);
    let args = KargsMvId {
        nei0: nei0 as i32, nei1: nei1 as i32, nbi1: (nei0 * 4) as u64,
        ne00: k as i32, ne01: n as i32, ne02: ne02 as i32,
        nb00: block as u64, nb01: (k / 256 * block) as u64, nb02: nb02 as u64,
        ne10: k as i32, ne11: ne11 as i32, ne12: nei1 as i32, ne13: 1,
        nb10: 4, nb11: nb11 as u64, nb12: nb12 as u64,
        ne0: n as i32, ne1: nei0 as i32, nb1: (n * 4) as u64, nr0: nr0 as i32,
    };
    encoder.set_bytes(0, &args);
    encoder.set_input_buffer(1, Some(src0s), 0);
    encoder.set_input_buffer(2, Some(src1), src1_offset);
    encoder.set_output_buffer(3, Some(dst), 0);
    encoder.set_input_buffer(4, Some(ids), ids_offset);
    encoder.set_input_buffer(5, Some(pairs), 0);
    encoder.set_input_buffer(6, Some(npairs), 0);
    encoder.dispatch_thread_groups(MTLSize { width: n.div_ceil(nr0 * nsg), height: 1, depth: max_pairs }, MTLSize { width: 32, height: nsg, depth: 1 });
    Ok(true)
}

pub fn pile_min() -> u32 {
    static N: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
    *N.get_or_init(|| std::env::var("COACHWHIP_PILE_MIN").ok().and_then(|v| v.parse().ok()).unwrap_or(0))
}