Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust
41.0 KB
Raw Blame
//! Lab (`COACHWHIP_ATTN_KERNEL=1`): the whole middle of a Gated DeltaNet layer for ONE word in one Metal
//! kernel: causal conv taps + silu, the two L2 norms, the key-head widening and query scale, the delta
//! rule over the memory, the output RMS norm and the silu gate. Only the two big projections stay as
//! their own matmul kernels. Same arithmetic as the op-by-op path in `model_next.rs`.
//! One threadgroup per value head, one thread per value column.

use candle::backend::BackendStorage;
use candle::{CustomOp3, DType, Layout, MetalStorage, Result, Shape, Storage, Tensor, D};
use candle_metal_kernels::metal::ComputePipeline;
use candle_metal_kernels::{set_params, Output};
use objc2_metal::MTLSize;
use std::sync::{Arc, OnceLock};

const SOURCE: &str = r#"
#include <metal_stdlib>
using namespace metal;

inline float silu_f(float x) { return x / (1.0f + exp(-x)); }

// qkv: (channels) this word's projection: q heads (n_k*dk), k heads (n_k*dk), v heads (n_v*dv).
// z: (n_v*dv) gate projection. ba: beta (n_v) then alpha (n_v), raw.
// conv_state: (taps-1, channels) past inputs. conv_w: (channels, taps). dt_bias, a: (n_v). norm_w: (dv).
// s: (n_v, dk, dv) memory. out: o (n_v*dv) | s' (n_v*dk*dv) | conv_state' ((taps-1)*channels).
kernel void coachwhip_gdn_word(
    device const float* qkv        [[buffer(0)]],
    device const float* z          [[buffer(1)]],
    device const float* ba         [[buffer(2)]],
    device const float* conv_state [[buffer(3)]],
    device const float* conv_w     [[buffer(4)]],
    device const float* dt_bias    [[buffer(5)]],
    device const float* a_vec      [[buffer(6)]],
    device const float* norm_w     [[buffer(7)]],
    device const float* s          [[buffer(8)]],
    device float* out              [[buffer(9)]],
    constant uint& n_k             [[buffer(10)]],
    constant uint& n_v             [[buffer(11)]],
    constant uint& dk              [[buffer(12)]],
    constant uint& dv              [[buffer(13)]],
    constant uint& taps            [[buffer(14)]],
    constant uint& cycle           [[buffer(15)]],
    constant float& eps            [[buffer(16)]],
    uint h   [[threadgroup_position_in_grid]],
    uint tid [[thread_position_in_threadgroup]],
    uint tgs [[threads_per_threadgroup]])
{
    threadgroup float q_sh[512];
    threadgroup float k_sh[512];
    threadgroup float red[2];
    const uint channels = 2 * n_k * dk + n_v * dv;
    const uint per_group = n_v / n_k;
    const uint kh = cycle ? (h % n_k) : (h / per_group);
    device float* o_out = out;
    device float* s_out = out + n_v * dv;
    device float* c_out = s_out + n_v * dk * dv;

    // 1. conv + silu for this head's q, k channels (into shared memory) and its v channels (registers).
    for (uint i = tid; i < dk; i += tgs) {
        uint cq = kh * dk + i;
        uint ck = n_k * dk + kh * dk + i;
        float xq = 0.0f, xk = 0.0f;
        for (uint t = 0; t + 1 < taps; t++) {
            xq += conv_w[cq * taps + t] * conv_state[t * channels + cq];
            xk += conv_w[ck * taps + t] * conv_state[t * channels + ck];
        }
        xq += conv_w[cq * taps + taps - 1] * qkv[cq];
        xk += conv_w[ck * taps + taps - 1] * qkv[ck];
        q_sh[i] = silu_f(xq);
        k_sh[i] = silu_f(xk);
        // shift this channel's conv window: rows 1..taps-1 move up, the new word is the last row.
        for (uint t = 0; t + 2 < taps; t++) {
            c_out[t * channels + cq] = conv_state[(t + 1) * channels + cq];
            c_out[t * channels + ck] = conv_state[(t + 1) * channels + ck];
        }
        c_out[(taps - 2) * channels + cq] = qkv[cq];
        c_out[(taps - 2) * channels + ck] = qkv[ck];
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    // 2. L2 norms of q and k (one thread sums; dk is small), query scale 1/sqrt(dk).
    if (tid == 0) {
        float sq = 0.0f, sk = 0.0f;
        for (uint i = 0; i < dk; i++) { sq += q_sh[i] * q_sh[i]; sk += k_sh[i] * k_sh[i]; }
        red[0] = sqrt(sq + eps);
        red[1] = sqrt(sk + eps);
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    const float qscale = 1.0f / (red[0] * sqrt((float)dk));
    const float kscale = 1.0f / red[1];
    for (uint i = tid; i < dk; i += tgs) { q_sh[i] *= qscale; k_sh[i] *= kscale; }
    threadgroup_barrier(mem_flags::mem_threadgroup);

    // 3. gates for this head.
    const float beta = 1.0f / (1.0f + exp(-ba[h]));
    const float alpha = ba[n_v + h];
    const float g = exp(log(exp(alpha + dt_bias[h]) + 1.0f) * a_vec[h]);

    // 4. the delta rule, one column per thread; 5. output RMS norm and silu gate.
    device const float* sh = s + h * dk * dv;
    device float* so = s_out + h * dk * dv;
    for (uint j = tid; j < dv; j += tgs) {
        uint cv = 2 * n_k * dk + h * dv + j;
        float xv = 0.0f;
        for (uint t = 0; t + 1 < taps; t++) xv += conv_w[cv * taps + t] * conv_state[t * channels + cv];
        xv += conv_w[cv * taps + taps - 1] * qkv[cv];
        const float v = silu_f(xv);
        for (uint t = 0; t + 2 < taps; t++) c_out[t * channels + cv] = conv_state[(t + 1) * channels + cv];
        c_out[(taps - 2) * channels + cv] = qkv[cv];

        float skv = 0.0f;
        for (uint i = 0; i < dk; i++) skv += k_sh[i] * sh[i * dv + j];
        skv *= g;
        const float d = (v - skv) * beta;
        float acc = 0.0f;
        for (uint i = 0; i < dk; i++) {
            const float ns = g * sh[i * dv + j] + k_sh[i] * d;
            so[i * dv + j] = ns;
            acc += q_sh[i] * ns;
        }
        o_out[h * dv + j] = acc;
    }
    threadgroup_barrier(mem_flags::mem_device);
    if (tid == 0) {
        float ss = 0.0f;
        for (uint j = 0; j < dv; j++) { float x = o_out[h * dv + j]; ss += x * x; }
        red[0] = rsqrt(ss / (float)dv + eps);
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    const float rms = red[0];
    for (uint j = tid; j < dv; j += tgs) {
        float x = o_out[h * dv + j] * rms * norm_w[j];
        o_out[h * dv + j] = x * silu_f(z[h * dv + j]);
    }
}
"#;


const SOURCE2: &str = r#"
#include <metal_stdlib>
using namespace metal;

inline float silu_f(float x) { return x / (1.0f + exp(-x)); }

// Same contract as coachwhip_gdn_word; requires dv == 128, dk == 128, 512 threads per head.
kernel void coachwhip_gdn_word4(
    device const float* qkv        [[buffer(0)]],
    device const float* z          [[buffer(1)]],
    device const float* ba         [[buffer(2)]],
    device const float* conv_state [[buffer(3)]],
    device const float* conv_w     [[buffer(4)]],
    device const float* dt_bias    [[buffer(5)]],
    device const float* a_vec      [[buffer(6)]],
    device const float* norm_w     [[buffer(7)]],
    device const float* s          [[buffer(8)]],
    device float* out              [[buffer(9)]],
    constant uint& n_k             [[buffer(10)]],
    constant uint& n_v             [[buffer(11)]],
    constant uint& dk              [[buffer(12)]],
    constant uint& dv              [[buffer(13)]],
    constant uint& taps            [[buffer(14)]],
    constant uint& cycle           [[buffer(15)]],
    constant float& eps            [[buffer(16)]],
    uint h   [[threadgroup_position_in_grid]],
    uint tid [[thread_position_in_threadgroup]],
    uint tgs [[threads_per_threadgroup]])
{
    threadgroup float q_sh[128];
    threadgroup float k_sh[128];
    threadgroup float part[4][128];
    threadgroup float red[2];
    const uint channels = 2 * n_k * dk + n_v * dv;
    const uint per_group = n_v / n_k;
    const uint kh = cycle ? (h % n_k) : (h / per_group);
    device float* o_out = out;
    device float* s_out = out + n_v * dv;
    device float* c_out = s_out + n_v * dk * dv;

    for (uint i = tid; i < dk; i += tgs) {
        uint cq = kh * dk + i;
        uint ck = n_k * dk + kh * dk + i;
        float xq = 0.0f, xk = 0.0f;
        for (uint t = 0; t + 1 < taps; t++) {
            xq += conv_w[cq * taps + t] * conv_state[t * channels + cq];
            xk += conv_w[ck * taps + t] * conv_state[t * channels + ck];
        }
        xq += conv_w[cq * taps + taps - 1] * qkv[cq];
        xk += conv_w[ck * taps + taps - 1] * qkv[ck];
        q_sh[i] = silu_f(xq);
        k_sh[i] = silu_f(xk);
        for (uint t = 0; t + 2 < taps; t++) {
            c_out[t * channels + cq] = conv_state[(t + 1) * channels + cq];
            c_out[t * channels + ck] = conv_state[(t + 1) * channels + ck];
        }
        c_out[(taps - 2) * channels + cq] = qkv[cq];
        c_out[(taps - 2) * channels + ck] = qkv[ck];
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (tid == 0) {
        float sq = 0.0f, sk = 0.0f;
        for (uint i = 0; i < dk; i++) { sq += q_sh[i] * q_sh[i]; sk += k_sh[i] * k_sh[i]; }
        red[0] = sqrt(sq + eps);
        red[1] = sqrt(sk + eps);
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    const float qscale = 1.0f / (red[0] * sqrt((float)dk));
    const float kscale = 1.0f / red[1];
    for (uint i = tid; i < dk; i += tgs) { q_sh[i] *= qscale; k_sh[i] *= kscale; }
    threadgroup_barrier(mem_flags::mem_threadgroup);

    const float beta = 1.0f / (1.0f + exp(-ba[h]));
    const float alpha = ba[n_v + h];
    const float g = exp(log(exp(alpha + dt_bias[h]) + 1.0f) * a_vec[h]);

    const uint j = tid % 128;
    const uint r = tid / 128;
    const uint i0 = r * 32;
    device const float* sh = s + h * dk * dv;
    device float* so = s_out + h * dk * dv;
    const uint cv = 2 * n_k * dk + h * dv + j;
    float xv = 0.0f;
    for (uint t = 0; t + 1 < taps; t++) xv += conv_w[cv * taps + t] * conv_state[t * channels + cv];
    xv += conv_w[cv * taps + taps - 1] * qkv[cv];
    const float v = silu_f(xv);
    if (r == 0) {
        for (uint t = 0; t + 2 < taps; t++) c_out[t * channels + cv] = conv_state[(t + 1) * channels + cv];
        c_out[(taps - 2) * channels + cv] = qkv[cv];
    }
    float col[32];
    float skv = 0.0f;
    for (uint ii = 0; ii < 32; ii++) { col[ii] = sh[(i0 + ii) * dv + j]; skv += k_sh[i0 + ii] * col[ii]; }
    part[r][j] = skv;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    const float tot = ((part[0][j] + part[1][j]) + part[2][j]) + part[3][j];
    const float d = (v - tot * g) * beta;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    float acc = 0.0f;
    for (uint ii = 0; ii < 32; ii++) {
        const float ns = g * col[ii] + k_sh[i0 + ii] * d;
        so[(i0 + ii) * dv + j] = ns;
        acc += q_sh[i0 + ii] * ns;
    }
    part[r][j] = acc;
    threadgroup_barrier(mem_flags::mem_threadgroup);
    if (r == 0) o_out[h * dv + j] = ((part[0][j] + part[1][j]) + part[2][j]) + part[3][j];
    threadgroup_barrier(mem_flags::mem_device);
    if (tid == 0) {
        float ss = 0.0f;
        for (uint jj = 0; jj < dv; jj++) { float x = o_out[h * dv + jj]; ss += x * x; }
        red[0] = rsqrt(ss / (float)dv + eps);
    }
    threadgroup_barrier(mem_flags::mem_threadgroup);
    const float rms = red[0];
    if (r == 0) {
        float x = o_out[h * dv + j] * rms * norm_w[j];
        o_out[h * dv + j] = x * silu_f(z[h * dv + j]);
    }
}
"#;

fn pipeline4(device: &candle::MetalDevice) -> Result<Arc<ComputePipeline>> {
    static PIPE: OnceLock<Arc<ComputePipeline>> = OnceLock::new();
    if let Some(p) = PIPE.get() {
        return Ok(p.clone());
    }
    let lib = device.device().new_library_with_source(SOURCE2, None).map_err(candle::MetalError::from)?;
    let f = lib.get_function("coachwhip_gdn_word4", None).map_err(candle::MetalError::from)?;
    let p = Arc::new(device.device().new_compute_pipeline_state_with_function(&f).map_err(candle::MetalError::from)?);
    let _ = PIPE.set(p.clone());
    Ok(p)
}

pub fn gdn4_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_GDN2").map(|v| v == "1").unwrap_or(false))
}


const SOURCE_PIECE: &str = r#"
#include <metal_stdlib>
using namespace metal;

inline float silu_f(float x) { return x / (1.0f + exp(-x)); }

// qkv: (L, channels). z: (L, n_v*dv). ba: beta (L*n_v) then alpha (L*n_v). conv_state: (taps-1, channels).
// out: o (L, n_v*dv) | s' (n_v, dk, dv) | conv_state' ((taps-1), channels). dk == dv == 128, 512 threads.
kernel void coachwhip_gdn_piece4(
    device const float* qkv        [[buffer(0)]],
    device const float* z          [[buffer(1)]],
    device const float* ba         [[buffer(2)]],
    device const float* conv_state [[buffer(3)]],
    device const float* conv_w     [[buffer(4)]],
    device const float* dt_bias    [[buffer(5)]],
    device const float* a_vec      [[buffer(6)]],
    device const float* norm_w     [[buffer(7)]],
    device const float* s          [[buffer(8)]],
    device float* out              [[buffer(9)]],
    constant uint& n_k             [[buffer(10)]],
    constant uint& n_v             [[buffer(11)]],
    constant uint& dk              [[buffer(12)]],
    constant uint& dv              [[buffer(13)]],
    constant uint& taps            [[buffer(14)]],
    constant uint& cycle           [[buffer(15)]],
    constant float& eps            [[buffer(16)]],
    constant uint& L               [[buffer(17)]],
    uint h   [[threadgroup_position_in_grid]],
    uint tid [[thread_position_in_threadgroup]],
    uint tgs [[threads_per_threadgroup]])
{
    threadgroup float q_sh[128];
    threadgroup float k_sh[128];
    threadgroup float part[4][128];
    threadgroup float red[2];
    const uint channels = 2 * n_k * dk + n_v * dv;
    const uint per_group = n_v / n_k;
    const uint kh = cycle ? (h % n_k) : (h / per_group);
    const uint od = n_v * dv;
    device float* s_out = out + L * od;
    device float* c_out = s_out + n_v * dk * dv;

    const uint j = tid % 128;
    const uint r = tid / 128;
    const uint i0 = r * 32;
    device const float* sh = s + h * dk * dv;
    device float* so = s_out + h * dk * dv;
    float col[32];
    for (uint ii = 0; ii < 32; ii++) col[ii] = sh[(i0 + ii) * dv + j];
    const uint cv = 2 * n_k * dk + h * dv + j;
    const float dtb = dt_bias[h], av = a_vec[h];

    for (uint t = 0; t < L; t++) {
        // input row u of the conv window: past rows from conv_state, then this piece's rows
        #define ROWV(u, c) ((int)(u) < 0 ? conv_state[((int)(u) + (int)taps - 1) * channels + (c)] : qkv[(uint)(u) * channels + (c)])
        const int base = (int)t - (int)(taps - 1);
        for (uint i = tid; i < dk; i += tgs) {
            uint cq = kh * dk + i;
            uint ck = n_k * dk + kh * dk + i;
            float xq = 0.0f, xk = 0.0f;
            for (uint tt = 0; tt + 1 < taps; tt++) {
                xq += conv_w[cq * taps + tt] * ROWV(base + (int)tt, cq);
                xk += conv_w[ck * taps + tt] * ROWV(base + (int)tt, ck);
            }
            xq += conv_w[cq * taps + taps - 1] * qkv[t * channels + cq];
            xk += conv_w[ck * taps + taps - 1] * qkv[t * channels + ck];
            q_sh[i] = silu_f(xq);
            k_sh[i] = silu_f(xk);
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        if (tid == 0) {
            float sq = 0.0f, sk = 0.0f;
            for (uint i = 0; i < dk; i++) { sq += q_sh[i] * q_sh[i]; sk += k_sh[i] * k_sh[i]; }
            red[0] = sqrt(sq + eps);
            red[1] = sqrt(sk + eps);
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const float qscale = 1.0f / (red[0] * sqrt((float)dk));
        const float kscale = 1.0f / red[1];
        threadgroup_barrier(mem_flags::mem_threadgroup);
        for (uint i = tid; i < dk; i += tgs) { q_sh[i] *= qscale; k_sh[i] *= kscale; }
        threadgroup_barrier(mem_flags::mem_threadgroup);

        const float beta = 1.0f / (1.0f + exp(-ba[t * n_v + h]));
        const float alpha = ba[L * n_v + t * n_v + h];
        const float g = exp(log(exp(alpha + dtb) + 1.0f) * av);

        float xv = 0.0f;
        for (uint tt = 0; tt + 1 < taps; tt++) xv += conv_w[cv * taps + tt] * ROWV(base + (int)tt, cv);
        xv += conv_w[cv * taps + taps - 1] * qkv[t * channels + cv];
        const float v = silu_f(xv);

        float skv = 0.0f;
        for (uint ii = 0; ii < 32; ii++) skv += k_sh[i0 + ii] * col[ii];
        part[r][j] = skv;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const float tot = ((part[0][j] + part[1][j]) + part[2][j]) + part[3][j];
        const float d = (v - tot * g) * beta;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        float acc = 0.0f;
        for (uint ii = 0; ii < 32; ii++) {
            const float ns = g * col[ii] + k_sh[i0 + ii] * d;
            col[ii] = ns;
            acc += q_sh[i0 + ii] * ns;
        }
        part[r][j] = acc;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        device float* o_out = out + t * od;
        if (r == 0) o_out[h * dv + j] = ((part[0][j] + part[1][j]) + part[2][j]) + part[3][j];
        threadgroup_barrier(mem_flags::mem_device);
        if (tid == 0) {
            float ss = 0.0f;
            for (uint jj = 0; jj < dv; jj++) { float x = o_out[h * dv + jj]; ss += x * x; }
            red[0] = rsqrt(ss / (float)dv + eps);
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const float rms = red[0];
        if (r == 0) {
            float x = o_out[h * dv + j] * rms * norm_w[j];
            o_out[h * dv + j] = x * silu_f(z[t * od + h * dv + j]);
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
    }
    for (uint ii = 0; ii < 32; ii++) so[(i0 + ii) * dv + j] = col[ii];
    // the conv window after the piece: its last taps-1 input rows
    const int last = (int)L - (int)(taps - 1);
    for (uint i = tid; i < dk; i += tgs) {
        uint cq = kh * dk + i;
        uint ck = n_k * dk + kh * dk + i;
        for (uint tt = 0; tt + 1 < taps; tt++) {
            c_out[tt * channels + cq] = ROWV(last + (int)tt, cq);
            c_out[tt * channels + ck] = ROWV(last + (int)tt, ck);
        }
    }
    if (r == 0) for (uint tt = 0; tt + 1 < taps; tt++) c_out[tt * channels + cv] = ROWV(last + (int)tt, cv);
}
"#;

fn pipeline_piece(device: &candle::MetalDevice) -> Result<Arc<ComputePipeline>> {
    static PIPE: OnceLock<Arc<ComputePipeline>> = OnceLock::new();
    if let Some(p) = PIPE.get() {
        return Ok(p.clone());
    }
    let lib = device.device().new_library_with_source(SOURCE_PIECE, None).map_err(candle::MetalError::from)?;
    let f = lib.get_function("coachwhip_gdn_piece4", None).map_err(candle::MetalError::from)?;
    let p = Arc::new(device.device().new_compute_pipeline_state_with_function(&f).map_err(candle::MetalError::from)?);
    let _ = PIPE.set(p.clone());
    Ok(p)
}


const SOURCE_PIECE_COLS: &str = r#"
#include <metal_stdlib>
using namespace metal;

inline float silu_f(float x) { return x / (1.0f + exp(-x)); }

// Group g: head h = g / 4, column block cb = g % 4 (columns cb*32 .. cb*32+31). 128 threads:
// jj = tid % 32 (column in block), r = tid / 32 (row quarter, 32 rows). Writes the RAW output (before
// the RMS norm and gate) to out[t*od + h*dv + j]; the norm+gate pass finishes it.
kernel void coachwhip_gdn_piece_cols(
    device const float* qkv        [[buffer(0)]],
    device const float* z          [[buffer(1)]],
    device const float* ba         [[buffer(2)]],
    device const float* conv_state [[buffer(3)]],
    device const float* conv_w     [[buffer(4)]],
    device const float* dt_bias    [[buffer(5)]],
    device const float* a_vec      [[buffer(6)]],
    device const float* norm_w     [[buffer(7)]],
    device const float* s          [[buffer(8)]],
    device float* out              [[buffer(9)]],
    constant uint& n_k             [[buffer(10)]],
    constant uint& n_v             [[buffer(11)]],
    constant uint& dk              [[buffer(12)]],
    constant uint& dv              [[buffer(13)]],
    constant uint& taps            [[buffer(14)]],
    constant uint& cycle           [[buffer(15)]],
    constant float& eps            [[buffer(16)]],
    constant uint& L               [[buffer(17)]],
    device const float* pre        [[buffer(18)]],
    constant uint& use_pre         [[buffer(19)]],
    uint g   [[threadgroup_position_in_grid]],
    uint tid [[thread_position_in_threadgroup]],
    uint tgs [[threads_per_threadgroup]])
{
    threadgroup float q_sh[128];
    threadgroup float k_sh[128];
    threadgroup float part[4][32];
    threadgroup float red[2];
    const uint h = g / 4, cb = g % 4;
    const uint channels = 2 * n_k * dk + n_v * dv;
    const uint per_group = n_v / n_k;
    const uint kh = cycle ? (h % n_k) : (h / per_group);
    const uint od = n_v * dv;
    device float* s_out = out + L * od;
    device float* c_out = s_out + n_v * dk * dv;
    const uint jj = tid % 32, r = tid / 32;
    const uint j = cb * 32 + jj;
    const uint i0 = r * 32;
    device const float* sh = s + h * dk * dv;
    device float* so = s_out + h * dk * dv;
    float col[32];
    for (uint ii = 0; ii < 32; ii++) col[ii] = sh[(i0 + ii) * dv + j];
    const uint cv = 2 * n_k * dk + h * dv + j;
    const float dtb = dt_bias[h], av = a_vec[h];
    #define ROWV(u, c) ((int)(u) < 0 ? conv_state[((int)(u) + (int)taps - 1) * channels + (c)] : qkv[(uint)(u) * channels + (c)])
    for (uint t = 0; t < L; t++) {
        const int base = (int)t - (int)(taps - 1);
        float v;
        if (use_pre) {
            device const float* pr = pre + t * channels;
            for (uint i = tid; i < dk; i += tgs) { q_sh[i] = pr[kh * dk + i]; k_sh[i] = pr[n_k * dk + kh * dk + i]; }
            v = pr[cv];
            threadgroup_barrier(mem_flags::mem_threadgroup);
        } else {
        for (uint i = tid; i < dk; i += tgs) {
            uint cq = kh * dk + i;
            uint ck = n_k * dk + kh * dk + i;
            float xq = 0.0f, xk = 0.0f;
            for (uint tt = 0; tt + 1 < taps; tt++) {
                xq += conv_w[cq * taps + tt] * ROWV(base + (int)tt, cq);
                xk += conv_w[ck * taps + tt] * ROWV(base + (int)tt, ck);
            }
            xq += conv_w[cq * taps + taps - 1] * qkv[t * channels + cq];
            xk += conv_w[ck * taps + taps - 1] * qkv[t * channels + ck];
            q_sh[i] = silu_f(xq);
            k_sh[i] = silu_f(xk);
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        if (tid == 0) {
            float sq = 0.0f, sk = 0.0f;
            for (uint i = 0; i < dk; i++) { sq += q_sh[i] * q_sh[i]; sk += k_sh[i] * k_sh[i]; }
            red[0] = sqrt(sq + eps);
            red[1] = sqrt(sk + eps);
        }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const float qscale = 1.0f / (red[0] * sqrt((float)dk));
        const float kscale = 1.0f / red[1];
        threadgroup_barrier(mem_flags::mem_threadgroup);
        for (uint i = tid; i < dk; i += tgs) { q_sh[i] *= qscale; k_sh[i] *= kscale; }
        threadgroup_barrier(mem_flags::mem_threadgroup);
        float xv = 0.0f;
        for (uint tt = 0; tt + 1 < taps; tt++) xv += conv_w[cv * taps + tt] * ROWV(base + (int)tt, cv);
        xv += conv_w[cv * taps + taps - 1] * qkv[t * channels + cv];
        v = silu_f(xv);
        }

        const float beta = 1.0f / (1.0f + exp(-ba[t * n_v + h]));
        const float alpha = ba[L * n_v + t * n_v + h];
        const float gg = exp(log(exp(alpha + dtb) + 1.0f) * av);

        float skv = 0.0f;
        for (uint ii = 0; ii < 32; ii++) skv += k_sh[i0 + ii] * col[ii];
        part[r][jj] = skv;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        const float tot = ((part[0][jj] + part[1][jj]) + part[2][jj]) + part[3][jj];
        const float d = (v - tot * gg) * beta;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        float acc = 0.0f;
        for (uint ii = 0; ii < 32; ii++) {
            const float ns = gg * col[ii] + k_sh[i0 + ii] * d;
            col[ii] = ns;
            acc += q_sh[i0 + ii] * ns;
        }
        part[r][jj] = acc;
        threadgroup_barrier(mem_flags::mem_threadgroup);
        if (r == 0) out[t * od + h * dv + j] = ((part[0][jj] + part[1][jj]) + part[2][jj]) + part[3][jj];
        threadgroup_barrier(mem_flags::mem_threadgroup);
    }
    for (uint ii = 0; ii < 32; ii++) so[(i0 + ii) * dv + j] = col[ii];
    const int last = (int)L - (int)(taps - 1);
    if (cb == 0) {
        for (uint i = tid; i < dk; i += tgs) {
            uint cq = kh * dk + i;
            uint ck = n_k * dk + kh * dk + i;
            for (uint tt = 0; tt + 1 < taps; tt++) {
                c_out[tt * channels + cq] = ROWV(last + (int)tt, cq);
                c_out[tt * channels + ck] = ROWV(last + (int)tt, ck);
            }
        }
    }
    if (r == 0) for (uint tt = 0; tt + 1 < taps; tt++) c_out[tt * channels + cv] = ROWV(last + (int)tt, cv);
}

// One thread per (word, channel): the conv over the taps and the silu, once per piece.
kernel void coachwhip_gdn_prep(
    device const float* qkv        [[buffer(0)]],
    device const float* conv_state [[buffer(1)]],
    device const float* conv_w     [[buffer(2)]],
    device float* pre              [[buffer(3)]],
    constant uint& channels        [[buffer(4)]],
    constant uint& taps            [[buffer(5)]],
    constant uint& L               [[buffer(6)]],
    uint i [[thread_position_in_grid]])
{
    if (i >= L * channels) return;
    const uint t = i / channels, c = i % channels;
    const int base = (int)t - (int)(taps - 1);
    float x = 0.0f;
    for (uint tt = 0; tt + 1 < taps; tt++) x += conv_w[c * taps + tt] * ROWV(base + (int)tt, c);
    x += conv_w[c * taps + taps - 1] * qkv[t * channels + c];
    pre[i] = silu_f(x);
}

// One thread per (word, k-head): the q and k norms, same serial sums as the walk had.
kernel void coachwhip_gdn_normqk(
    device float* pre              [[buffer(0)]],
    constant uint& channels        [[buffer(1)]],
    constant uint& n_k             [[buffer(2)]],
    constant uint& dk              [[buffer(3)]],
    constant float& eps            [[buffer(4)]],
    constant uint& L               [[buffer(5)]],
    uint i [[thread_position_in_grid]])
{
    if (i >= L * n_k) return;
    const uint t = i / n_k, kh = i % n_k;
    device float* q = pre + t * channels + kh * dk;
    device float* k = pre + t * channels + n_k * dk + kh * dk;
    float sq = 0.0f, sk = 0.0f;
    for (uint j = 0; j < dk; j++) { sq += q[j] * q[j]; sk += k[j] * k[j]; }
    const float r0 = sqrt(sq + eps), r1 = sqrt(sk + eps);
    const float qscale = 1.0f / (r0 * sqrt((float)dk));
    const float kscale = 1.0f / r1;
    for (uint j = 0; j < dk; j++) { q[j] *= qscale; k[j] *= kscale; }
}

// One thread per (word, head): the output RMS norm over the head's dv values, then the silu gate,
// in the same serial order as the one-group kernel.
kernel void coachwhip_gdn_norm_gate(
    device float* out           [[buffer(0)]],
    device const float* z       [[buffer(1)]],
    device const float* norm_w  [[buffer(2)]],
    constant uint& rows         [[buffer(3)]],
    constant uint& dv           [[buffer(4)]],
    constant float& eps         [[buffer(5)]],
    uint i [[thread_position_in_grid]])
{
    if (i >= rows) return;
    device float* o = out + i * dv;
    device const float* zz = z + i * dv;
    float ss = 0.0f;
    for (uint jj = 0; jj < dv; jj++) { float x = o[jj]; ss += x * x; }
    const float rms = rsqrt(ss / (float)dv + eps);
    for (uint jj = 0; jj < dv; jj++) {
        float x = o[jj] * rms * norm_w[jj];
        o[jj] = x * silu_f(zz[jj]);
    }
}
"#;

fn pipelines_cols(device: &candle::MetalDevice) -> Result<(Arc<ComputePipeline>, Arc<ComputePipeline>)> {
    static PIPE: OnceLock<(Arc<ComputePipeline>, Arc<ComputePipeline>)> = OnceLock::new();
    if let Some(p) = PIPE.get() {
        return Ok(p.clone());
    }
    let lib = device.device().new_library_with_source(SOURCE_PIECE_COLS, None).map_err(candle::MetalError::from)?;
    let mk = |name: &str| -> Result<Arc<ComputePipeline>> {
        let f = lib.get_function(name, None).map_err(candle::MetalError::from)?;
        Ok(Arc::new(device.device().new_compute_pipeline_state_with_function(&f).map_err(candle::MetalError::from)?))
    };
    let p = (mk("coachwhip_gdn_piece_cols")?, mk("coachwhip_gdn_norm_gate")?);
    let _ = PIPE.set(p.clone());
    Ok(p)
}

fn pipelines_prep(device: &candle::MetalDevice) -> Result<(Arc<ComputePipeline>, Arc<ComputePipeline>)> {
    static PIPE: OnceLock<(Arc<ComputePipeline>, Arc<ComputePipeline>)> = OnceLock::new();
    if let Some(p) = PIPE.get() {
        return Ok(p.clone());
    }
    let lib = device.device().new_library_with_source(SOURCE_PIECE_COLS, None).map_err(candle::MetalError::from)?;
    let mk = |name: &str| -> Result<Arc<ComputePipeline>> {
        let f = lib.get_function(name, None).map_err(candle::MetalError::from)?;
        Ok(Arc::new(device.device().new_compute_pipeline_state_with_function(&f).map_err(candle::MetalError::from)?))
    };
    let p = (mk("coachwhip_gdn_prep")?, mk("coachwhip_gdn_normqk")?);
    let _ = PIPE.set(p.clone());
    Ok(p)
}

fn piece_prep_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_PIECE_PREP").map(|v| v != "0").unwrap_or(true))
}

fn piece_cols_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_PIECE_COLS").map(|v| v != "0").unwrap_or(true))
}

pub fn piece_on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_PIECE").map(|v| v == "1").unwrap_or(false))
}

/// A whole piece of L words. Same layer constants as [`GdnWord`].
pub struct GdnPiece { pub w: GdnWord, pub l: usize }

impl CustomOp3 for GdnPiece {
    fn name(&self) -> &'static str { "coachwhip-gdn-piece" }
    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> {
        candle::bail!("coachwhip-gdn-piece runs on Metal only")
    }
    fn metal_fwd(&self, qkv: &MetalStorage, ql: &Layout, z: &MetalStorage, zl: &Layout, ba: &MetalStorage, bl: &Layout) -> Result<(MetalStorage, Shape)> {
        let w = &self.w;
        let (n_k, n_v, dk, dv, taps, l) = (w.n_k, w.n_v, w.dk, w.dv, w.taps, self.l);
        let channels = 2 * n_k * dk + n_v * dv;
        let n_out = l * n_v * dv + n_v * dk * dv + (taps - 1) * channels;
        let device = qkv.device().clone();
        let dst = crate::arena::take_for(&device, n_out, DType::F32, "gdn-piece")?;
        let pre = crate::arena::take_for(&device, l * channels, DType::F32, "gdn-pre")?;
        let use_pre = piece_cols_on() && piece_prep_on();
        let pipe = pipeline_piece(&device)?;
        let cs = w.conv_state.storage_and_layout();
        let cw = w.conv_w.storage_and_layout();
        let db = w.dt_bias.storage_and_layout();
        let av = w.a.storage_and_layout();
        let nw = w.norm_w.storage_and_layout();
        let ss = w.s.storage_and_layout();
        fn m<'a>(g: &'a Storage) -> Result<&'a MetalStorage> {
            match g { Storage::Metal(m) => Ok(m), _ => candle::bail!("coachwhip-gdn-piece: not on Metal") }
        }
        let (cs_m, cw_m, db_m, av_m, nw_m, ss_m) = (m(&cs.0)?, m(&cw.0)?, m(&db.0)?, m(&av.0)?, m(&nw.0)?, m(&ss.0)?);
        if use_pre {
            let (pp, pn) = pipelines_prep(&device)?;
            let encoder = device.command_encoder()?;
            let encoder = encoder.as_ref();
            let (ch_u, taps_u, l_u, n_k_u, dk_u) = (channels as u32, taps as u32, l as u32, n_k as u32, dk as u32);
            encoder.set_compute_pipeline_state(&pp);
            set_params!(encoder, ((qkv.buffer(), ql.start_offset() * 4), (cs_m.buffer(), cs.1.start_offset() * 4), (cw_m.buffer(), cw.1.start_offset() * 4), Output::with_offset(&pre, 0), ch_u, taps_u, l_u));
            encoder.dispatch_thread_groups(MTLSize { width: (l * channels).div_ceil(256), height: 1, depth: 1 }, MTLSize { width: 256, height: 1, depth: 1 });
            encoder.set_compute_pipeline_state(&pn);
            set_params!(encoder, (Output::with_offset(&pre, 0), ch_u, n_k_u, dk_u, w.eps, l_u));
            encoder.dispatch_thread_groups(MTLSize { width: (l * n_k).div_ceil(64), height: 1, depth: 1 }, MTLSize { width: 64, height: 1, depth: 1 });
        }
        {
            let encoder = device.command_encoder()?;
            let encoder = encoder.as_ref();
            encoder.set_compute_pipeline_state(&pipe);
            let (n_k_u, n_v_u, dk_u, dv_u, taps_u, cycle_u, l_u) = (n_k as u32, n_v as u32, dk as u32, dv as u32, taps as u32, w.cycle as u32, l as u32);
            let use_pre_u = use_pre as u32;
            set_params!(encoder, (
                (qkv.buffer(), ql.start_offset() * 4), (z.buffer(), zl.start_offset() * 4), (ba.buffer(), bl.start_offset() * 4),
                (cs_m.buffer(), cs.1.start_offset() * 4), (cw_m.buffer(), cw.1.start_offset() * 4), (db_m.buffer(), db.1.start_offset() * 4),
                (av_m.buffer(), av.1.start_offset() * 4), (nw_m.buffer(), nw.1.start_offset() * 4), (ss_m.buffer(), ss.1.start_offset() * 4),
                Output::with_offset(&dst, 0), n_k_u, n_v_u, dk_u, dv_u, taps_u, cycle_u, w.eps, l_u, Output::with_offset(&pre, 0), use_pre_u
            ));
            if piece_cols_on() {
                let (pc, _) = pipelines_cols(&device)?;
                encoder.set_compute_pipeline_state(&pc);
                encoder.dispatch_thread_groups(MTLSize { width: n_v * 4, height: 1, depth: 1 }, MTLSize { width: 128, height: 1, depth: 1 });
            } else {
                encoder.dispatch_thread_groups(MTLSize { width: n_v, height: 1, depth: 1 }, MTLSize { width: 512, height: 1, depth: 1 });
            }
        }
        if piece_cols_on() {
            let (_, ng) = pipelines_cols(&device)?;
            let encoder = device.command_encoder()?;
            let encoder = encoder.as_ref();
            encoder.set_compute_pipeline_state(&ng);
            let rows = l * n_v;
            let (rows_u, dv_u) = (rows as u32, dv as u32);
            set_params!(encoder, (Output::with_offset(&dst, 0), (z.buffer(), zl.start_offset() * 4), (nw_m.buffer(), nw.1.start_offset() * 4), rows_u, dv_u, w.eps));
            encoder.dispatch_thread_groups(MTLSize { width: rows.div_ceil(64), height: 1, depth: 1 }, MTLSize { width: 64, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, device, n_out, DType::F32), Shape::from(n_out)))
    }
}

/// A piece of L words through the middle of the layer: `(o (L, n_v*dv), s', conv_state')`.
pub fn gdn_piece(op: GdnWord, qkv: &Tensor, z: &Tensor, beta: &Tensor, alpha: &Tensor, l: usize) -> Result<(Tensor, Tensor, Tensor)> {
    let (n_k, n_v, dk, dv, taps) = (op.n_k, op.n_v, op.dk, op.dv, op.taps);
    let channels = 2 * n_k * dk + n_v * dv;
    for (t, w) in [(&op.conv_state, "conv_state"), (&op.conv_w, "conv_w"), (&op.dt_bias, "dt_bias"), (&op.a, "a"), (&op.norm_w, "norm_w"), (&op.s, "s")] {
        check(t, w)?;
    }
    let f = |t: &Tensor| -> Result<Tensor> { t.flatten_all()?.to_dtype(DType::F32)?.contiguous() };
    let qkv = f(qkv)?;
    let z = f(z)?;
    let ba = Tensor::cat(&[&f(beta)?, &f(alpha)?], 0)?.contiguous()?;
    let out = qkv.apply_op3_no_bwd(&z, &ba, &GdnPiece { w: op, l })?;
    let o = out.narrow(0, 0, l * n_v * dv)?.reshape((l, n_v * dv))?;
    let s_new = out.narrow(0, l * n_v * dv, n_v * dk * dv)?.affine(1.0, 0.0)?.reshape((n_v, dk, dv))?;
    let conv_new = out.narrow(0, l * n_v * dv + n_v * dk * dv, (taps - 1) * channels)?.affine(1.0, 0.0)?.reshape((taps - 1, channels))?;
    Ok((o, s_new, conv_new))
}

fn pipeline(device: &candle::MetalDevice) -> Result<Arc<ComputePipeline>> {
    static PIPE: OnceLock<Arc<ComputePipeline>> = OnceLock::new();
    if let Some(p) = PIPE.get() {
        return Ok(p.clone());
    }
    let lib = device.device().new_library_with_source(SOURCE, None).map_err(candle::MetalError::from)?;
    let f = lib.get_function("coachwhip_gdn_word", None).map_err(candle::MetalError::from)?;
    let p = Arc::new(device.device().new_compute_pipeline_state_with_function(&f).map_err(candle::MetalError::from)?);
    let _ = PIPE.set(p.clone());
    Ok(p)
}

pub fn on() -> bool {
    static ON: OnceLock<bool> = OnceLock::new();
    *ON.get_or_init(|| std::env::var("COACHWHIP_ATTN_KERNEL").map(|v| v != "0").unwrap_or(true))
}

/// The layer's constants and this word's state, read straight from their tensors' Metal buffers.
pub struct GdnWord {
    pub conv_state: Tensor,
    pub conv_w: Tensor,
    pub dt_bias: Tensor,
    pub a: Tensor,
    pub norm_w: Tensor,
    pub s: Tensor,
    pub n_k: usize,
    pub n_v: usize,
    pub dk: usize,
    pub dv: usize,
    pub taps: usize,
    pub cycle: bool,
    pub eps: f32,
}

fn check(t: &Tensor, what: &str) -> Result<()> {
    if t.dtype() != DType::F32 {
        candle::bail!("coachwhip-gdn-word: {what} must be f32, got {:?}", t.dtype())
    }
    if !t.is_contiguous() {
        candle::bail!("coachwhip-gdn-word: {what} must be contiguous")
    }
    Ok(())
}

impl CustomOp3 for GdnWord {
    fn name(&self) -> &'static str {
        "coachwhip-gdn-word"
    }

    fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> {
        candle::bail!("coachwhip-gdn-word runs on Metal only")
    }

    fn metal_fwd(&self, qkv: &MetalStorage, ql: &Layout, z: &MetalStorage, zl: &Layout, ba: &MetalStorage, bl: &Layout) -> Result<(MetalStorage, Shape)> {
        for (st, l, what) in [(qkv, ql, "qkv"), (z, zl, "z"), (ba, bl, "ba")] {
            if st.dtype() != DType::F32 {
                candle::bail!("coachwhip-gdn-word: {what} must be f32")
            }
            if l.contiguous_offsets().is_none() {
                candle::bail!("coachwhip-gdn-word: {what} must be contiguous")
            }
        }
        let (n_k, n_v, dk, dv, taps) = (self.n_k, self.n_v, self.dk, self.dv, self.taps);
        let channels = 2 * n_k * dk + n_v * dv;
        let n_out = n_v * dv + n_v * dk * dv + (taps - 1) * channels;
        let device = qkv.device().clone();
        let dst = device.new_buffer_builder().with_size_for(n_out, DType::F32).with_label("gdn-word").build()?;
        let four = gdn4_on() && dk == 128 && dv == 128;
        let pipe = if four { pipeline4(&device)? } else { pipeline(&device)? };
        { static ONCE: std::sync::Once = std::sync::Once::new(); ONCE.call_once(|| eprintln!("coachwhip: gdn word kernel, four threads per column = {four}")); }
        // Hold the read guards of the layer's own tensors for the encode.
        let cs = self.conv_state.storage_and_layout();
        let cw = self.conv_w.storage_and_layout();
        let db = self.dt_bias.storage_and_layout();
        let av = self.a.storage_and_layout();
        let nw = self.norm_w.storage_and_layout();
        let ss = self.s.storage_and_layout();
        fn m<'a>(g: &'a Storage, name: &str) -> Result<&'a MetalStorage> {
            match g {
                Storage::Metal(m) => Ok(m),
                _ => candle::bail!("coachwhip-gdn-word: {name} is not on Metal"),
            }
        }
        let (cs_m, cw_m, db_m, av_m, nw_m, ss_m) = (m(&cs.0, "conv_state")?, m(&cw.0, "conv_w")?, m(&db.0, "dt_bias")?, m(&av.0, "a")?, m(&nw.0, "norm_w")?, m(&ss.0, "s")?);
        {
            let encoder = device.command_encoder()?;
            let encoder = encoder.as_ref();
            encoder.set_compute_pipeline_state(&pipe);
            let (n_k_u, n_v_u, dk_u, dv_u, taps_u, cycle_u) = (n_k as u32, n_v as u32, dk as u32, dv as u32, taps as u32, self.cycle as u32);
            set_params!(
                encoder,
                (
                    (qkv.buffer(), ql.start_offset() * 4),
                    (z.buffer(), zl.start_offset() * 4),
                    (ba.buffer(), bl.start_offset() * 4),
                    (cs_m.buffer(), cs.1.start_offset() * 4),
                    (cw_m.buffer(), cw.1.start_offset() * 4),
                    (db_m.buffer(), db.1.start_offset() * 4),
                    (av_m.buffer(), av.1.start_offset() * 4),
                    (nw_m.buffer(), nw.1.start_offset() * 4),
                    (ss_m.buffer(), ss.1.start_offset() * 4),
                    Output::with_offset(&dst, 0),
                    n_k_u,
                    n_v_u,
                    dk_u,
                    dv_u,
                    taps_u,
                    cycle_u,
                    self.eps
                )
            );
            let threads = if four { 512 } else { dv.min(256) };
            encoder.dispatch_thread_groups(MTLSize { width: n_v, height: 1, depth: 1 }, MTLSize { width: threads, height: 1, depth: 1 });
        }
        Ok((MetalStorage::new(dst, device, n_out, DType::F32), Shape::from(n_out)))
    }
}

/// One word through the middle of the layer. Returns `(o_gated (1, n_v*dv), s' (n_v, dk, dv), conv_state' (taps-1, channels))`.
pub fn gdn_word(op: GdnWord, qkv: &Tensor, z: &Tensor, beta: &Tensor, alpha: &Tensor) -> Result<(Tensor, Tensor, Tensor)> {
    let (n_k, n_v, dk, dv, taps) = (op.n_k, op.n_v, op.dk, op.dv, op.taps);
    let channels = 2 * n_k * dk + n_v * dv;
    for (t, w) in [(&op.conv_state, "conv_state"), (&op.conv_w, "conv_w"), (&op.dt_bias, "dt_bias"), (&op.a, "a"), (&op.norm_w, "norm_w"), (&op.s, "s")] {
        check(t, w)?;
    }
    let f = |t: &Tensor| -> Result<Tensor> { t.flatten_all()?.to_dtype(DType::F32)?.contiguous() };
    let qkv = f(qkv)?;
    let z = f(z)?;
    let ba = Tensor::cat(&[&f(beta)?, &f(alpha)?], D::Minus1)?.contiguous()?;
    let out = qkv.apply_op3_no_bwd(&z, &ba, &op)?;
    let o = out.narrow(0, 0, n_v * dv)?.reshape((1, n_v * dv))?;
    let s_new = out.narrow(0, n_v * dv, n_v * dk * dv)?.reshape((n_v, dk, dv))?;
    let conv_new = out.narrow(0, n_v * dv + n_v * dk * dv, (taps - 1) * channels)?.reshape((taps - 1, channels))?;
    Ok((o, s_new, conv_new))
}