Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust
31.6 KB
Raw Blame
//! Lab (`COACHWHIP_FUSE=1`): fewer GPU jobs per layer on the resident expert path.
//! - `route`: softmax over the router's scores, top-k picks (first index wins ties), weights
//!   renormalised over the picks when asked: one dispatch instead of about seven.
//! - `silu_mul`: hidden = silu(gate) * up: one dispatch instead of two.
//! - `weighted_sum`: out[t][j] = sum over the k picks of w[t][e] * down[t][e][j]: one instead of three.

use candle::backend::BackendStorage;
use candle::{CustomOp1, CustomOp2, DType, Layout, MetalStorage, Result, Shape, Tensor};
use candle_metal_kernels::metal::ComputePipeline;
use candle_metal_kernels::{set_params, Output};
use objc2_metal::MTLSize;
use std::sync::{Arc, OnceLock};

const SOURCE: &str = r#"
#include <metal_stdlib>
using namespace metal;

// One threadgroup per token, one thread per expert (E <= 256), k <= 16. All threads take part in
// each pick: a max-reduction over (probability, lowest index) — the order a stable descending sort gives.
kernel void cw_route(
    device const float* logits [[buffer(0)]],
    device uint* outp          [[buffer(1)]],
    constant uint& E           [[buffer(2)]],
    constant uint& K           [[buffer(3)]],
    constant uint& norm        [[buffer(4)]],
    constant uint& which       [[buffer(5)]],
    uint t    [[threadgroup_position_in_grid]],
    uint tid  [[thread_position_in_threadgroup]],
    uint tgs  [[threads_per_threadgroup]],
    uint lane [[thread_index_in_simdgroup]],
    uint sg   [[simdgroup_index_in_threadgroup]])
{
    threadgroup float rv[8];
    threadgroup uint ri[8];
    threadgroup float picked_w[16];
    threadgroup uint picked_i[16];
    device const float* x = logits + t * E;
    float v = tid < E ? x[tid] : -INFINITY;
    // softmax pieces: max and sum over the experts
    float m = simd_max(v);
    if (lane == 0) rv[sg] = m;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    float mm = rv[0]; for (uint q = 1; q < (tgs + 31) / 32; q++) mm = max(mm, rv[q]);
    threadgroup_barrier(mem_flags::mem_threadgroup);
    float e = tid < E ? exp(v - mm) : 0.0f;
    float s = simd_sum(e);
    if (lane == 0) rv[sg] = s;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    float ss = 0.0f; for (uint q = 0; q < (tgs + 31) / 32; q++) ss += rv[q];
    threadgroup_barrier(mem_flags::mem_threadgroup);
    float pr = tid < E ? e / ss : -1.0f;
    bool taken = tid >= E;
    for (uint j = 0; j < K; j++) {
        float bv = taken ? -2.0f : pr;
        uint bi = tid;
        for (uint off = 16; off > 0; off >>= 1) {
            float ov = simd_shuffle_down(bv, off);
            uint oi = simd_shuffle_down(bi, off);
            if (ov > bv || (ov == bv && oi < bi)) { bv = ov; bi = oi; }
        }
        if (lane == 0) { rv[sg] = bv; ri[sg] = bi; }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        if (tid == 0) {
            float b = rv[0]; uint bix = ri[0];
            for (uint q = 1; q < (tgs + 31) / 32; q++) { if (rv[q] > b || (rv[q] == b && ri[q] < bix)) { b = rv[q]; bix = ri[q]; } }
            picked_w[j] = b; picked_i[j] = bix;
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        if (tid == picked_i[j]) taken = true;
    }
    if (tid == 0) {
        float tot = 0.0f; for (uint j = 0; j < K; j++) tot += picked_w[j];
        for (uint j = 0; j < K; j++) outp[t * K + j] = which == 0 ? picked_i[j] : as_type<uint>(norm ? picked_w[j] / tot : picked_w[j]);
    }
}

kernel void cw_silu_mul(
    device const float* g [[buffer(0)]],
    device const float* u [[buffer(1)]],
    device float* o       [[buffer(2)]],
    constant uint& n      [[buffer(3)]],
    uint i [[thread_position_in_grid]])
{
    if (i < n) { float x = g[i]; o[i] = x / (1.0f + exp(-x)) * u[i]; }
}

// gu: rows of [gate (I) | up (I)]; out: rows of I. out[r][j] = silu(gu[r][j]) * gu[r][I + j].
kernel void cw_silu_mul_gu(
    device const float* gu [[buffer(0)]],
    device float* o        [[buffer(1)]],
    constant uint& I       [[buffer(2)]],
    constant uint& n       [[buffer(3)]],
    uint i [[thread_position_in_grid]])
{
    if (i < n) { uint r = i / I, j = i % I; float x = gu[r * 2 * I + j]; o[i] = x / (1.0f + exp(-x)) * gu[r * 2 * I + I + j]; }
}

// First index of the largest value (what the CPU loop `if v > best` picks). One threadgroup, 1024 threads.
kernel void cw_argmax(
    device const float* x [[buffer(0)]],
    device uint* out      [[buffer(1)]],
    constant uint& n      [[buffer(2)]],
    uint tid [[thread_position_in_threadgroup]],
    uint lane [[thread_index_in_simdgroup]],
    uint sg   [[simdgroup_index_in_threadgroup]])
{
    threadgroup float bv_s[32];
    threadgroup uint bi_s[32];
    float bv = -INFINITY; uint bi = 0xFFFFFFFFu;
    for (uint i = tid; i < n; i += 1024) { float v = x[i]; if (v > bv || bi == 0xFFFFFFFFu) { bv = v; bi = i; } }
    for (uint off = 16; off > 0; off >>= 1) {
        float ov = simd_shuffle_down(bv, off); uint oi = simd_shuffle_down(bi, off);
        if (ov > bv || (ov == bv && oi < bi)) { bv = ov; bi = oi; }
    }
    if (lane == 0) { bv_s[sg] = bv; bi_s[sg] = bi; }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (tid == 0) {
        float b = bv_s[0]; uint ix = bi_s[0];
        for (uint q = 1; q < 32; q++) { if (bv_s[q] > b || (bv_s[q] == b && bi_s[q] < ix)) { b = bv_s[q]; ix = bi_s[q]; } }
        out[0] = ix;
    }
}

// One row of a Q5_K table, unpacked exactly as Candle's CPU to_float does, then rounded through half.
kernel void cw_embed_q5k(
    device const uchar* table [[buffer(0)]],
    device const uint* ids    [[buffer(1)]],
    device float* out         [[buffer(2)]],
    constant uint& hidden     [[buffer(3)]],
    constant uint& row_bytes  [[buffer(4)]],
    uint2 g [[thread_position_in_grid]])
{
    #pragma clang fp contract(off)
    uint e = g.x, t = g.y;
    if (e >= hidden) return;
    device const uchar* blk = table + (ulong)ids[t] * row_bytes + (e / 256) * 176;
    uint pos = e % 256;
    float d = float(as_type<half>(*(device const ushort*)(blk)));
    float mn = float(as_type<half>(*(device const ushort*)(blk + 2)));
    device const uchar* sc = blk + 4;
    device const uchar* qh = blk + 16;
    device const uchar* ql = blk + 48;
    uint chunk = pos / 64, hi = (pos % 64) / 32, l = pos % 32;
    uint j = 2 * chunk + hi;
    uint s, m;
    if (j < 4) { s = sc[j] & 63; m = sc[j + 4] & 63; }
    else { s = (sc[j + 4] & 0xF) | ((sc[j - 4] >> 6) << 4); m = (sc[j + 4] >> 4) | ((sc[j] >> 6) << 4); }
    float dd = d * float(s);
    float mm = mn * float(m);
    uchar q = ql[chunk * 32 + l];
    uint u = (hi ? 2u : 1u) << (2 * chunk);
    float add = (qh[l] & u) ? 16.0f : 0.0f;
    float qv = float(hi ? (q >> 4) : (q & 0xF)) + add;
    float prod = dd * qv;
    float v = prod - mm;
    out[t * hidden + e] = float(half(v));
}

// y[r] = dot(w[r], x), w rows f16, x f32; one simdgroup per row, 4 rows per threadgroup.
kernel void cw_mv_f16(
    device const half* w  [[buffer(0)]],
    device const float* x [[buffer(1)]],
    device float* y       [[buffer(2)]],
    constant uint& k      [[buffer(3)]],
    constant uint& rows   [[buffer(4)]],
    uint gid  [[threadgroup_position_in_grid]],
    uint lane [[thread_index_in_simdgroup]],
    uint sg   [[simdgroup_index_in_threadgroup]])
{
    uint r = gid * 4 + sg;
    if (r >= rows) return;
    device const half4* w4 = (device const half4*)(w + (ulong)r * k);
    device const float4* x4 = (device const float4*)x;
    float acc = 0.0f;
    for (uint i = lane; i < k / 4; i += 32) acc += dot(float4(w4[i]), x4[i]);
    acc = simd_sum(acc);
    if (lane == 0) y[r] = acc;
}

// Piles: thread e counts the (slot, word) picks of expert e, finds where its pile starts, writes it.
// heads[3e] = e, heads[3e+1] = first pair, heads[3e+2] = count. Pairs keep the words' order.
kernel void cw_pile(
    device const uint* ids  [[buffer(0)]],
    device ushort2* piles   [[buffer(1)]],
    device uint* heads      [[buffer(2)]],
    constant uint& n_ids    [[buffer(3)]],
    constant uint& nei0     [[buffer(4)]],
    constant uint& experts  [[buffer(5)]],
    uint e [[thread_position_in_threadgroup]])
{
    threadgroup uint counts[256];
    uint c = 0;
    if (e < experts) { for (uint i = 0; i < n_ids; i++) if (ids[i] == e) c++; }
    counts[e] = c;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (e >= experts) return;
    uint off = 0;
    for (uint j = 0; j < e; j++) off += counts[j];
    heads[3 * e] = e; heads[3 * e + 1] = off; heads[3 * e + 2] = c;
    uint w = off;
    for (uint i = 0; i < n_ids; i++) if (ids[i] == e) { piles[w] = ushort2(i % nei0, i / nei0); w++; }
}

// The small piles' words: thread e copies its pile when it has fewer than min_cnt words.
kernel void cw_small(
    device const ushort2* piles [[buffer(0)]],
    device const uint* heads    [[buffer(1)]],
    device ushort2* outp        [[buffer(2)]],
    device uint* npairs         [[buffer(3)]],
    constant uint& experts      [[buffer(4)]],
    constant uint& min_cnt      [[buffer(5)]],
    uint e [[thread_position_in_threadgroup]])
{
    threadgroup uint counts[256];
    uint c = 0, first = 0;
    if (e < experts) { c = heads[3 * e + 2]; first = heads[3 * e + 1]; if (c >= min_cnt) c = 0; }
    counts[e] = c;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    uint off = 0;
    for (uint j = 0; j < e; j++) off += counts[j];
    for (uint i = 0; i < c; i++) outp[off + i] = piles[first + i];
    if (e == 255) { uint tot = off + c; npairs[0] = tot; }
}

// Top-k, one pass: group g takes elements [g*4096, (g+1)*4096), 256 threads x 16 values in registers,
// and writes its k best (value, index) in order. has_ids: the input carries its own indices (pass 2).
kernel void cw_topk(
    device const float* xv [[buffer(0)]],
    device const uint* xi  [[buffer(1)]],
    device float* ov       [[buffer(2)]],
    device uint* oi        [[buffer(3)]],
    constant uint& n       [[buffer(4)]],
    constant uint& k       [[buffer(5)]],
    constant uint& has_ids [[buffer(6)]],
    uint g    [[threadgroup_position_in_grid]],
    uint tid  [[thread_position_in_threadgroup]],
    uint lane [[thread_index_in_simdgroup]],
    uint sg   [[simdgroup_index_in_threadgroup]])
{
    const uint T = 256;
    const uint start = g * T * 16;
    float v[16]; uint id[16];
    for (uint j = 0; j < 16; j++) {
        uint i = start + j * T + tid;
        if (i < n) { v[j] = xv[i]; id[j] = has_ids ? xi[i] : i; } else { v[j] = -INFINITY; id[j] = 0xFFFFFFFFu; }
    }
    threadgroup float bv[8];
    threadgroup uint bi[8];
    threadgroup uint win;
    for (uint r = 0; r < k; r++) {
        float best = -INFINITY; uint bid = 0xFFFFFFFFu; uint bj = 99;
        for (uint j = 0; j < 16; j++) {
            if (id[j] != 0xFFFFFFFFu && (bj == 99 || v[j] > best || (v[j] == best && id[j] < bid))) { best = v[j]; bid = id[j]; bj = j; }
        }
        float sb = best; uint si = bid;
        for (uint off = 16; off > 0; off >>= 1) {
            float o_v = simd_shuffle_down(sb, off); uint o_i = simd_shuffle_down(si, off);
            if (o_i != 0xFFFFFFFFu && (si == 0xFFFFFFFFu || o_v > sb || (o_v == sb && o_i < si))) { sb = o_v; si = o_i; }
        }
        if (lane == 0) { bv[sg] = sb; bi[sg] = si; }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        if (tid == 0) {
            float b = bv[0]; uint ix = bi[0];
            for (uint q = 1; q < 8; q++) {
                if (bi[q] != 0xFFFFFFFFu && (ix == 0xFFFFFFFFu || bv[q] > b || (bv[q] == b && bi[q] < ix))) { b = bv[q]; ix = bi[q]; }
            }
            ov[g * k + r] = b; oi[g * k + r] = ix; win = ix;
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const uint w = win;
        if (bj < 16 && bid == w) { v[bj] = -INFINITY; id[bj] = 0xFFFFFFFFu; }
        threadgroup_barrier(mem_flags::mem_threadgroup);
    }
}

// down: (T, K, H) rows; w: (T, K); out: (T, H). One thread per (t, j).
kernel void cw_weighted_sum(
    device const float* down [[buffer(0)]],
    device const float* w    [[buffer(1)]],
    device float* o          [[buffer(2)]],
    constant uint& T         [[buffer(3)]],
    constant uint& K         [[buffer(4)]],
    constant uint& H         [[buffer(5)]],
    uint i [[thread_position_in_grid]])
{
    if (i >= T * H) return;
    uint t = i / H, j = i % H;
    float acc = 0.0f;
    for (uint e = 0; e < K; e++) acc += down[(t * K + e) * H + j] * w[t * K + e];
    o[i] = acc;
}
"#;

fn pipe(dev: &candle::MetalDevice, name: &str) -> Result<Arc<ComputePipeline>> {
    static LIB: OnceLock<std::sync::Mutex<std::collections::HashMap<String, Arc<ComputePipeline>>>> = OnceLock::new();
    let lib = LIB.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()));
    let mut g = lib.lock().unwrap();
    if let Some(p) = g.get(name) {
        return Ok(p.clone());
    }
    let l = dev.device().new_library_with_source(SOURCE, None).map_err(candle::MetalError::from)?;
    let f = l.get_function(name, None).map_err(candle::MetalError::from)?;
    let p = Arc::new(dev.device().new_compute_pipeline_state_with_function(&f).map_err(candle::MetalError::from)?);
    g.insert(name.to_string(), p.clone());
    Ok(p)
}

pub fn on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_FUSE").map(|v| v == "1").unwrap_or(false))
}

fn need(st: &MetalStorage, l: &Layout, what: &str) -> Result<()> {
    if st.dtype() != DType::F32 || l.contiguous_offsets().is_none() {
        candle::bail!("cw fuse: {what} must be contiguous f32")
    }
    Ok(())
}

/// logits (T, E) -> ids (T, K) u32 (`which` 0) or weights (T, K) f32 (`which` 1). Two dispatches
/// replace about seven; the picks never leave the GPU.
struct Route { k: usize, norm: bool, which: u32 }
impl CustomOp1 for Route {
    fn name(&self) -> &'static str { "cw-route" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, x: &MetalStorage, l: &Layout) -> Result<(MetalStorage, Shape)> {
        need(x, l, "logits")?;
        let (t, e) = l.shape().dims2()?;
        if e > 256 || self.k > 16 { candle::bail!("cw-route: {e} experts / k {}", self.k) }
        let dev = x.device().clone();
        let n = t * self.k;
        let dt = if self.which == 0 { DType::U32 } else { DType::F32 };
        let dst = crate::arena::take_for(&dev, n, dt, "cw-route")?;
        let p = pipe(&dev, "cw_route")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let (e_u, k_u, nm, wh) = (e as u32, self.k as u32, self.norm as u32, self.which);
            set_params!(enc, ((x.buffer(), l.start_offset() * 4), Output::with_offset(&dst, 0), e_u, k_u, nm, wh));
            enc.dispatch_thread_groups(MTLSize { width: t, height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, n, dt), Shape::from((t, self.k))))
    }
}

pub fn route(logits: &Tensor, k: usize, norm: bool) -> Result<(Tensor, Tensor)> {
    let lg = logits.to_dtype(DType::F32)?.contiguous()?;
    let ids = lg.apply_op1_no_bwd(&Route { k, norm, which: 0 })?;
    let w = lg.apply_op1_no_bwd(&Route { k, norm, which: 1 })?;
    Ok((ids, w))
}

struct SiluMul;
impl CustomOp2 for SiluMul {
    fn name(&self) -> &'static str { "cw-silu-mul" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, g: &MetalStorage, gl: &Layout, u: &MetalStorage, ul: &Layout) -> Result<(MetalStorage, Shape)> {
        need(g, gl, "gate")?; need(u, ul, "up")?;
        let n = gl.shape().elem_count();
        let dev = g.device().clone();
        let dst = crate::arena::take_for(&dev, n, DType::F32, "cw-silu-mul")?;
        let p = pipe(&dev, "cw_silu_mul")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let n_u = n as u32;
            set_params!(enc, ((g.buffer(), gl.start_offset() * 4), (u.buffer(), ul.start_offset() * 4), Output::with_offset(&dst, 0), n_u));
            enc.dispatch_thread_groups(MTLSize { width: n.div_ceil(256), height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, n, DType::F32), gl.shape().clone()))
    }
}
pub fn silu_mul(g: &Tensor, u: &Tensor) -> Result<Tensor> { g.contiguous()?.apply_op2_no_bwd(&u.contiguous()?, &SiluMul) }

struct WSum;
impl CustomOp2 for WSum {
    fn name(&self) -> &'static str { "cw-wsum" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, d: &MetalStorage, dl: &Layout, w: &MetalStorage, wl: &Layout) -> Result<(MetalStorage, Shape)> {
        need(d, dl, "down")?; need(w, wl, "weights")?;
        let (t, k, h) = dl.shape().dims3()?;
        let dev = d.device().clone();
        let dst = crate::arena::take_for(&dev, t * h, DType::F32, "cw-wsum")?;
        let p = pipe(&dev, "cw_weighted_sum")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let (t_u, k_u, h_u) = (t as u32, k as u32, h as u32);
            set_params!(enc, ((d.buffer(), dl.start_offset() * 4), (w.buffer(), wl.start_offset() * 4), Output::with_offset(&dst, 0), t_u, k_u, h_u));
            enc.dispatch_thread_groups(MTLSize { width: (t * h).div_ceil(256), height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, t * h, DType::F32), Shape::from((t, h))))
    }
}
pub fn weighted_sum(down: &Tensor, w: &Tensor) -> Result<Tensor> { down.contiguous()?.apply_op2_no_bwd(&w.contiguous()?, &WSum) }

struct SiluMulGu;
impl CustomOp1 for SiluMulGu {
    fn name(&self) -> &'static str { "cw-silu-mul-gu" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, g: &MetalStorage, gl: &Layout) -> Result<(MetalStorage, Shape)> {
        need(g, gl, "gate|up")?;
        let mut dims = gl.shape().dims().to_vec();
        let two_i = *dims.last().unwrap();
        let i = two_i / 2;
        *dims.last_mut().unwrap() = i;
        let n = gl.shape().elem_count() / 2;
        let dev = g.device().clone();
        let dst = crate::arena::take_for(&dev, n, DType::F32, "cw-silu-mul-gu")?;
        let p = pipe(&dev, "cw_silu_mul_gu")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let (i_u, n_u) = (i as u32, n as u32);
            set_params!(enc, ((g.buffer(), gl.start_offset() * 4), Output::with_offset(&dst, 0), i_u, n_u));
            enc.dispatch_thread_groups(MTLSize { width: n.div_ceil(256), height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, n, DType::F32), Shape::from(dims)))
    }
}
/// gu (.., 2I) with gate in the first half of each row and up in the second -> silu(gate) * up (.., I).
pub fn silu_mul_gu(gu: &Tensor) -> Result<Tensor> { gu.contiguous()?.apply_op1_no_bwd(&SiluMulGu) }

pub fn addnorm_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| on() && std::env::var("COACHWHIP_ADDNORM").map(|v| v == "1").unwrap_or(false))
}

/// (a + b (+ c), rms_norm(a + b (+ c)) * alpha) in one dispatch, Candle's rms_norm block size.
pub fn add_norm(a: &Tensor, b: &Tensor, c: Option<&Tensor>, alpha: &Tensor, eps: f32) -> Result<(Tensor, Tensor)> {
    use candle::Storage;
    let a = a.contiguous()?;
    let b = b.contiguous()?;
    let c = match c { Some(c) => Some(c.contiguous()?), None => None };
    let alpha = alpha.contiguous()?;
    let shape = a.shape().clone();
    let h = *shape.dims().last().unwrap();
    let numel = shape.elem_count();
    let (sa, la) = a.storage_and_layout();
    let (sb, lb) = b.storage_and_layout();
    let (sw, lw) = alpha.storage_and_layout();
    let cs = c.as_ref().map(|c| c.storage_and_layout());
    let (Storage::Metal(ma), Storage::Metal(mb), Storage::Metal(mw)) = (&*sa, &*sb, &*sw) else { candle::bail!("add_norm: metal only") };
    if ma.dtype() != DType::F32 || mb.dtype() != DType::F32 || mw.dtype() != DType::F32 { candle::bail!("add_norm: f32 only") }
    let (mc, lc_off) = match &cs {
        Some((sc, lc)) => match &**sc { Storage::Metal(m) => (m.buffer().clone(), lc.start_offset() * 4), _ => candle::bail!("add_norm: metal only") },
        None => (ma.buffer().clone(), la.start_offset() * 4),
    };
    let dev = ma.device().clone();
    let sum = dev.new_buffer_builder().with_size_for(numel, DType::F32).with_label("cw-add").build()?;
    let dst = dev.new_buffer_builder().with_size_for(numel, DType::F32).with_label("cw-add-norm").build()?;
    let p = dev.kernels().load_pipeline(dev.device(), candle_metal_kernels::source::Source::Reduce, "add_rms_norm_f32").map_err(candle::MetalError::from)?;
    let rp = dev.kernels().load_pipeline(dev.device(), candle_metal_kernels::source::Source::Reduce, "rmsnorm_f32").map_err(candle::MetalError::from)?;
    let width = p.max_total_threads_per_threadgroup().min((h / 2).next_power_of_two());
    let rwidth = rp.max_total_threads_per_threadgroup().min((h / 2).next_power_of_two());
    if width != rwidth { candle::bail!("add_norm: block {width} differs from rms_norm's {rwidth}") }
    {
        let enc = dev.command_encoder()?;
        let enc = enc.as_ref();
        enc.set_compute_pipeline_state(&p);
        let (n_u, h_u, has_c) = (numel as u32, h as u32, cs.is_some() as u32);
        set_params!(enc, (n_u, h_u, (ma.buffer(), la.start_offset() * 4), (mb.buffer(), lb.start_offset() * 4), (&mc, lc_off), has_c,
            Output::with_offset(&sum, 0), Output::with_offset(&dst, 0), (mw.buffer(), lw.start_offset() * 4), eps));
        enc.dispatch_thread_groups(MTLSize { width: numel / h, height: 1, depth: 1 }, MTLSize { width, height: 1, depth: 1 });
    }
    drop((sa, sb, sw, cs));
    let st = |buf| candle::Storage::Metal(MetalStorage::new(buf, dev.clone(), numel, DType::F32));
    let s_t = Tensor::from_storage(st(sum), shape.clone(), candle::op::BackpropOp::none(), false);
    let d_t = Tensor::from_storage(st(dst), shape, candle::op::BackpropOp::none(), false);
    Ok((s_t, d_t))
}

pub fn gpupick_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_GPUPICK").map(|v| v == "1").unwrap_or(false))
}

struct Argmax;
impl CustomOp1 for Argmax {
    fn name(&self) -> &'static str { "cw-argmax" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, x: &MetalStorage, l: &Layout) -> Result<(MetalStorage, Shape)> {
        need(x, l, "logits")?;
        let n = l.shape().elem_count();
        let dev = x.device().clone();
        let dst = dev.new_buffer_builder().with_size_for(1, DType::U32).with_label("cw-argmax").build()?;
        let p = pipe(&dev, "cw_argmax")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let n_u = n as u32;
            set_params!(enc, ((x.buffer(), l.start_offset() * 4), Output::with_offset(&dst, 0), n_u));
            enc.dispatch_thread_groups(MTLSize { width: 1, height: 1, depth: 1 }, MTLSize { width: 1024, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, 1, DType::U32), Shape::from((1, 1))))
    }
}
/// The first index of the largest logit, as a (1, 1) u32 tensor that stays on the GPU.
pub fn argmax(logits: &Tensor) -> Result<Tensor> { logits.flatten_all()?.to_dtype(DType::F32)?.contiguous()?.apply_op1_no_bwd(&Argmax) }

struct EmbedQ5k { hidden: usize, row_bytes: usize }
impl CustomOp2 for EmbedQ5k {
    fn name(&self) -> &'static str { "cw-embed-q5k" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, t: &MetalStorage, tl: &Layout, ids: &MetalStorage, il: &Layout) -> Result<(MetalStorage, Shape)> {
        let n = il.shape().elem_count();
        let dev = t.device().clone();
        let elems = n * self.hidden;
        let dst = dev.new_buffer_builder().with_size_for(elems, DType::F32).with_label("cw-embed").build()?;
        let p = pipe(&dev, "cw_embed_q5k")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let (h_u, rb_u) = (self.hidden as u32, self.row_bytes as u32);
            set_params!(enc, ((t.buffer(), tl.start_offset()), (ids.buffer(), il.start_offset() * 4), Output::with_offset(&dst, 0), h_u, rb_u));
            enc.dispatch_thread_groups(MTLSize { width: self.hidden.div_ceil(256), height: n, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, elems, DType::F32), Shape::from((n, self.hidden))))
    }
}
/// Rows `ids` (u32, on the GPU) of a packed Q5_K word table held on the GPU as bytes.
pub fn embed_q5k(table: &Tensor, ids: &Tensor, hidden: usize, row_bytes: usize) -> Result<Tensor> {
    table.apply_op2_no_bwd(&ids.flatten_all()?.contiguous()?, &EmbedQ5k { hidden, row_bytes })
}

pub fn router16_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_ROUTER16").map(|v| v == "1").unwrap_or(false))
}

struct MvF16;
impl CustomOp2 for MvF16 {
    fn name(&self) -> &'static str { "cw-mv-f16" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, w: &MetalStorage, wl: &Layout, x: &MetalStorage, xl: &Layout) -> Result<(MetalStorage, Shape)> {
        let (rows, k) = wl.shape().dims2()?;
        if w.dtype() != DType::F16 || x.dtype() != DType::F32 || k % 4 != 0 || xl.shape().elem_count() != k { candle::bail!("cw-mv-f16: shapes") }
        let dev = w.device().clone();
        let dst = dev.new_buffer_builder().with_size_for(rows, DType::F32).with_label("cw-mv-f16").build()?;
        let p = pipe(&dev, "cw_mv_f16")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let (k_u, r_u) = (k as u32, rows as u32);
            set_params!(enc, ((w.buffer(), wl.start_offset() * 2), (x.buffer(), xl.start_offset() * 4), Output::with_offset(&dst, 0), k_u, r_u));
            enc.dispatch_thread_groups(MTLSize { width: rows.div_ceil(4), height: 1, depth: 1 }, MTLSize { width: 32, height: 4, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, rows, DType::F32), Shape::from((1, rows))))
    }
}
/// (1, k) f32 times an f16 (rows, k) table -> (1, rows) f32.
pub fn mv_f16(w16: &Tensor, x: &Tensor) -> Result<Tensor> { w16.apply_op2_no_bwd(&x.to_dtype(DType::F32)?.contiguous()?, &MvF16) }

pub fn group3_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_GROUP").map(|v| v == "3").unwrap_or(false))
}

/// The piles for a batch of picks, made on the GPU: (piles, heads) buffers.
pub fn piles(dev: &candle::MetalDevice, ids: &candle_metal_kernels::metal::Buffer, ids_offset: usize, n_ids: usize, nei0: usize, experts: usize)
    -> Result<(Arc<candle_metal_kernels::metal::Buffer>, Arc<candle_metal_kernels::metal::Buffer>)> {
    if experts > 256 { candle::bail!("cw-pile: {experts} experts") }
    let pb = crate::arena::take_for(&dev, n_ids, DType::U32, "cw-piles")?;
    let hb = crate::arena::take_for(&dev, 3 * experts, DType::U32, "cw-heads")?;
    let p = pipe(dev, "cw_pile")?;
    {
        let enc = dev.command_encoder()?;
        let enc = enc.as_ref();
        enc.set_compute_pipeline_state(&p);
        let (n_u, k_u, e_u) = (n_ids as u32, nei0 as u32, experts as u32);
        set_params!(enc, ((ids, ids_offset), Output::with_offset(&pb, 0), Output::with_offset(&hb, 0), n_u, k_u, e_u));
        enc.dispatch_thread_groups(MTLSize { width: 1, height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
    }
    Ok((pb, hb))
}

struct TopK { k: usize }
impl CustomOp1 for TopK {
    fn name(&self) -> &'static str { "cw-topk" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> { candle::bail!("metal only") }
    fn metal_fwd(&self, x: &MetalStorage, l: &Layout) -> Result<(MetalStorage, Shape)> {
        need(x, l, "scores")?;
        let n = l.shape().elem_count();
        let k = self.k;
        if k == 0 || k > 64 { candle::bail!("cw-topk: k {k}") }
        let groups = n.div_ceil(4096);
        let n2 = groups * k;
        if n2 > 4096 { candle::bail!("cw-topk: {n} scores too many for two passes") }
        let dev = x.device().clone();
        let v1 = dev.new_buffer_builder().with_size_for(n2, DType::F32).with_label("cw-topk-v1").build()?;
        let i1 = dev.new_buffer_builder().with_size_for(n2, DType::U32).with_label("cw-topk-i1").build()?;
        let dst = dev.new_buffer_builder().with_size_for(2 * k, DType::U32).with_label("cw-topk").build()?;
        let p = pipe(&dev, "cw_topk")?;
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let (n_u, k_u, no) = (n as u32, k as u32, 0u32);
            set_params!(enc, ((x.buffer(), l.start_offset() * 4), (x.buffer(), l.start_offset() * 4), Output::with_offset(&v1, 0), Output::with_offset(&i1, 0), n_u, k_u, no));
            enc.dispatch_thread_groups(MTLSize { width: groups, height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
        }
        {
            let enc = dev.command_encoder()?;
            let enc = enc.as_ref();
            enc.set_compute_pipeline_state(&p);
            let (n_u, k_u, yes) = (n2 as u32, k as u32, 1u32);
            set_params!(enc, ((&*v1, 0usize), (&*i1, 0usize), Output::with_offset(&dst, 0), Output::with_offset(&dst, k * 4), n_u, k_u, yes));
            enc.dispatch_thread_groups(MTLSize { width: 1, height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, dev, 2 * k, DType::U32), Shape::from(2 * k)))
    }
}
/// The k best scores and their word ids, best first (first id on equal scores); only 2k numbers come back.
pub fn topk(logits: &Tensor, k: usize) -> Result<(Vec<f32>, Vec<u32>)> {
    let x = logits.flatten_all()?.to_dtype(DType::F32)?.contiguous()?;
    let out: Vec<u32> = x.apply_op1_no_bwd(&TopK { k })?.to_vec1()?;
    let vals = out[..k].iter().map(|b| f32::from_bits(*b)).collect();
    Ok((vals, out[k..].to_vec()))
}

/// From the piles: the list of words in piles smaller than `min_cnt`, and its length (on the GPU).
pub fn small_pairs(dev: &candle::MetalDevice, piles: &candle_metal_kernels::metal::Buffer, heads: &candle_metal_kernels::metal::Buffer, n_pairs: usize, experts: usize, min_cnt: u32)
    -> Result<(Arc<candle_metal_kernels::metal::Buffer>, Arc<candle_metal_kernels::metal::Buffer>)> {
    let ob = dev.new_buffer_builder().with_size_for(n_pairs.max(1), DType::U32).with_label("cw-small").build()?;
    let nb = dev.new_buffer_builder().with_size_for(1, DType::U32).with_label("cw-small-n").build()?;
    let p = pipe(dev, "cw_small")?;
    {
        let enc = dev.command_encoder()?;
        let enc = enc.as_ref();
        enc.set_compute_pipeline_state(&p);
        let (e_u, m_u) = (experts as u32, min_cnt);
        set_params!(enc, ((piles, 0usize), (heads, 0usize), Output::with_offset(&ob, 0), Output::with_offset(&nb, 0), e_u, m_u));
        enc.dispatch_thread_groups(MTLSize { width: 1, height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
    }
    Ok((ob, nb))
}