Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust
17.7 KB
Raw Blame
//! gpt-oss (OpenAI's mixture of experts). The family owns attention (sliding windows and sinks),
//! the biases and YaRN; the experts stream through the core like every other family, with the
//! family's clamped activation plugged into the core's hook. Experts are MXFP4.
use crate::experts::{ExpertAct, ExpertStore, Settings, StreamedMoe};
use crate::model::{Progress, ThinkOpen};
use candle::quantized::{ggml_file::qtensor_from_ggml, gguf_file, GgmlDType};
use candle::{DType, Device, Module, Result, Tensor};
use candle_nn::Linear;
use candle_transformers::models::quantized_qwen3::Gguf;
use candle_transformers::models::with_tracing::QMatMul;
use candle_transformers::quantized_nn::RmsNorm;
use std::os::unix::fs::FileExt;
use std::path::Path;
use std::sync::Arc;

const SWIGLU_LIMIT: f32 = 7.0;
const SWIGLU_ALPHA: f64 = 1.702;

struct Layer {
    attn_norm: RmsNorm,
    post_norm: RmsNorm,
    wq: QMatMul,
    bq: Tensor,
    wk: QMatMul,
    bk: Tensor,
    wv: QMatMul,
    bv: Tensor,
    wo: QMatMul,
    bo: Tensor,
    sinks: Tensor,
    moe: StreamedMoe,
    sliding: bool,
    k_cache: Option<Tensor>,
    v_cache: Option<Tensor>,
    /// The position of the first key the cache still holds (a sliding layer drops old ones).
    kv_start: usize,
}

/// The word table, packed as the file stores it (Q8_0), on the CPU; rows are unpacked when asked for.
struct Table {
    bytes: Vec<u8>,
    dtype: GgmlDType,
    row_bytes: usize,
    hidden: usize,
}

impl Table {
    fn rows(&self, ids: &[u32]) -> Result<Tensor> {
        let mut buf = Vec::with_capacity(ids.len() * self.row_bytes);
        for &id in ids {
            let at = id as usize * self.row_bytes;
            buf.extend_from_slice(&self.bytes[at..at + self.row_bytes]);
        }
        qtensor_from_ggml(self.dtype, &buf, vec![ids.len(), self.hidden], &Device::Cpu)?.dequantize(&Device::Cpu)
    }
}

pub struct GptOss {
    table: Table,
    layers: Vec<Layer>,
    norm: RmsNorm,
    output: QMatMul,
    store: Arc<ExpertStore>,
    device: Device,
    heads: usize,
    kv_heads: usize,
    head_dim: usize,
    window: usize,
    /// The experts' input width in the file (wider than the model when squeezed and padded).
    xpad: usize,
    cos: Tensor,
    sin: Tensor,
    pub think: ThinkOpen,
    pub progress: Option<Progress>,
}

/// YaRN rotary frequencies, as the reference computes them (truncate = false), with the
/// attention factor folded into cos and sin.
fn yarn_tables(dim: usize, base: f64, factor: f64, orig: f64, beta_fast: f64, beta_slow: f64, max_pos: usize, device: &Device) -> Result<(Tensor, Tensor)> {
    let half = dim / 2;
    let corr = |rot: f64| (dim as f64 * (orig / (rot * 2.0 * std::f64::consts::PI)).ln()) / (2.0 * base.ln());
    let low = corr(beta_fast).max(0.0);
    let mut high = corr(beta_slow).min(dim as f64 - 1.0);
    if (high - low).abs() < 1e-9 {
        high += 0.001;
    }
    let mscale = 0.1 * factor.ln() + 1.0;
    let mut inv = Vec::with_capacity(half);
    for i in 0..half {
        let pos_freq = base.powf((2 * i) as f64 / dim as f64);
        let extrap = 1.0 / pos_freq;
        let interp = 1.0 / (factor * pos_freq);
        let ramp = ((i as f64 - low) / (high - low)).clamp(0.0, 1.0);
        let extrap_factor = 1.0 - ramp;
        inv.push(interp * (1.0 - extrap_factor) + extrap * extrap_factor);
    }
    let mut c = Vec::with_capacity(max_pos * half);
    let mut s = Vec::with_capacity(max_pos * half);
    for p in 0..max_pos {
        for f in &inv {
            let a = p as f64 * f;
            c.push((a.cos() * mscale) as f32);
            s.push((a.sin() * mscale) as f32);
        }
    }
    Ok((Tensor::from_vec(c, (max_pos, half), device)?, Tensor::from_vec(s, (max_pos, half), device)?))
}


impl GptOss {
    pub fn load(path: &Path, device: &Device, settings: &Settings) -> Result<Self> {
        let mut file = std::fs::File::open(path)?;
        let ct = gguf_file::Content::read(&mut file).map_err(|e| e.with_path(path))?;
        let md_u = |k: &str| -> Result<usize> { ct.metadata.get(k).ok_or_else(|| candle::Error::Msg(format!("missing {k}")))?.to_u32().map(|v| v as usize) };
        let md_f = |k: &str| -> Result<f64> { ct.metadata.get(k).ok_or_else(|| candle::Error::Msg(format!("missing {k}")))?.to_f32().map(|v| v as f64) };
        let block_count = md_u("gpt-oss.block_count")?;
        let heads = md_u("gpt-oss.attention.head_count")?;
        let kv_heads = md_u("gpt-oss.attention.head_count_kv")?;
        let head_dim = md_u("gpt-oss.attention.key_length")?;
        let window = md_u("gpt-oss.attention.sliding_window")?;
        // Fast mode (`--experts n`, `COACHWHIP_TOPK=n`): fewer experts per word than the model was trained with.
        let trained_k = md_u("gpt-oss.expert_used_count")?;
        let top_k = match std::env::var("COACHWHIP_TOPK").ok().and_then(|v| v.parse::<usize>().ok()) {
            Some(k) if k >= 1 && k < trained_k => k,
            _ => trained_k,
        };
        let eps = md_f("gpt-oss.attention.layer_norm_rms_epsilon")?;
        let (cos, sin) = yarn_tables(
            head_dim,
            md_f("gpt-oss.rope.freq_base")?,
            md_f("gpt-oss.rope.scaling.factor")?,
            md_u("gpt-oss.rope.scaling.original_context_length")? as f64,
            md_f("gpt-oss.rope.scaling.yarn_beta_fast")?,
            md_f("gpt-oss.rope.scaling.yarn_beta_slow")?,
            16384,
            device,
        )?;
        let template = ct.metadata.get("tokenizer.chat_template").and_then(|t| t.to_string().ok().cloned());
        let think = ThinkOpen::from_template(template.as_deref());

        // A squeezed file pads expert rows to whole 256-weight blocks; the input to the experts
        // (and to the router, which shares it) gets the same zeros.
        let cols = |m: &str| ct.tensor_infos.get(&format!("blk.0.ffn_{m}_exps.weight")).map(|i| i.shape.dims()[2]).unwrap_or(0);
        let (xpad, hpad) = (cols("gate"), cols("down"));
        let store = ExpertStore::new(&ct, device, block_count, path, settings)?;
        let table = {
            let q = ct.tensor(&mut file, "token_embd.weight", &Device::Cpu)?;
            let hidden = *q.shape().dims().last().unwrap();
            let dtype = q.dtype();
            let row_bytes = hidden / dtype.block_size() * dtype.type_size();
            Table { bytes: q.data()?.into_owned(), dtype, row_bytes, hidden }
        };
        let mut gg = Gguf::new(ct, &mut file, device.clone());
        let dense = |gg: &mut Gguf<&mut std::fs::File>, name: &str| -> Result<Tensor> { gg.tensor(name)?.dequantize(device)?.to_dtype(DType::F32) };
        let mut layers = Vec::with_capacity(block_count);
        for i in 0..block_count {
            let p = format!("blk.{i}");
            let moe = StreamedMoe {
                act: ExpertAct::ClampedSwiglu {
                    alpha: SWIGLU_ALPHA,
                    limit: SWIGLU_LIMIT,
                    gate_b: dense(&mut gg, &format!("{p}.ffn_gate_exps.bias"))?,
                    up_b: dense(&mut gg, &format!("{p}.ffn_up_exps.bias"))?,
                    down_b: dense(&mut gg, &format!("{p}.ffn_down_exps.bias"))?,
                    pad: hpad,
                },
                layer: i,
                gate: Linear::new(
                    {
                        let w = dense(&mut gg, &format!("{p}.ffn_gate_inp.weight"))?;
                        let width = w.dim(1)?;
                        if xpad > width { w.pad_with_zeros(1, 0, xpad - width)? } else { w }
                    },
                    Some(dense(&mut gg, &format!("{p}.ffn_gate_inp.bias"))?),
                ),
                next_gate: None,
                gate1: None,
                ahead: settings.ahead.max(1),
                store: store.clone(),
                top_k,
                norm_topk_prob: true,
                prefetch_k: settings.prefetch,
            };
            layers.push(Layer {
                attn_norm: gg.rms_norm(&format!("{p}.attn_norm.weight"), eps)?,
                post_norm: gg.rms_norm(&format!("{p}.post_attention_norm.weight"), eps)?,
                wq: gg.qmatmul(&format!("{p}.attn_q.weight"))?,
                bq: dense(&mut gg, &format!("{p}.attn_q.bias"))?,
                wk: gg.qmatmul(&format!("{p}.attn_k.weight"))?,
                bk: dense(&mut gg, &format!("{p}.attn_k.bias"))?,
                wv: gg.qmatmul(&format!("{p}.attn_v.weight"))?,
                bv: dense(&mut gg, &format!("{p}.attn_v.bias"))?,
                wo: gg.qmatmul(&format!("{p}.attn_output.weight"))?,
                bo: dense(&mut gg, &format!("{p}.attn_output.bias"))?,
                sinks: dense(&mut gg, &format!("{p}.attn_sinks.weight"))?,
                moe,
                // The reference's layer_types start with sliding_attention and alternate.
                sliding: i % 2 == 0,
                k_cache: None,
                v_cache: None,
                kv_start: 0,
            });
        }
        // The guess for a later layer comes from that layer's own router, run early.
        let ahead = settings.ahead.max(1);
        for i in ahead..layers.len() {
            layers[i - ahead].moe.next_gate = Some(layers[i].moe.gate.clone());
            if ahead >= 2 {
                layers[i - ahead].moe.gate1 = Some(layers[i - ahead + 1].moe.gate.clone());
            }
        }
        let norm = gg.rms_norm("output_norm.weight", eps)?;
        let output = gg.qmatmul("output.weight")?;
        eprintln!("coachwhip: gpt-oss, {block_count} layers, {heads} heads over {kv_heads}, sliding window {window}, {top_k} experts per word");
        let m = Self { table, layers, norm, output, store, device: device.clone(), heads, kv_heads, head_dim, window, xpad, cos, sin, think, progress: None };
        if std::env::var("COACHWHIP_GPTOSS_CHECK").is_ok() {
            check_experts(path, 0, device)?;
        }
        Ok(m)
    }

    pub fn forward(&mut self, x: &Tensor, offset: usize) -> Result<Tensor> {
        let ids: Vec<u32> = x.flatten_all()?.to_vec1()?;
        if !self.store.drafting() {
            self.store.note_tokens(ids.clone());
        }
        let n = ids.len();
        let mut xs = self.table.rows(&ids)?.to_dtype(DType::F32)?.to_device(&self.device)?;
        let n_layers = self.layers.len();
        let (heads, kv_heads, head_dim, window) = (self.heads, self.kv_heads, self.head_dim, self.window);
        let group = heads / kv_heads;
        let scale = 1.0 / (head_dim as f64).sqrt();
        let cos = self.cos.narrow(0, offset, n)?;
        let sin = self.sin.narrow(0, offset, n)?;
        // Lab timing switches (text is junk with either on): skip the experts, or skip attention.
        let no_moe = std::env::var("COACHWHIP_GPTOSS_NOMOE").is_ok();
        let no_attn = std::env::var("COACHWHIP_GPTOSS_NOATTN").is_ok();
        for l in 0..n_layers {
            let layer = &mut self.layers[l];
            if !no_attn {
            // Attention with sinks; sliding layers see the last `window` tokens only.
            let h = layer.attn_norm.forward(&xs)?;
            // Heads first, (1, heads, n, d), so the rotation is candle's one fused kernel and the
            // cache grows along its own axis without a transpose per word.
            let q = layer.wq.forward(&h)?.broadcast_add(&layer.bq)?.reshape((1, n, heads, head_dim))?.transpose(1, 2)?.contiguous()?;
            let k = layer.wk.forward(&h)?.broadcast_add(&layer.bk)?.reshape((1, n, kv_heads, head_dim))?.transpose(1, 2)?.contiguous()?;
            let v = layer.wv.forward(&h)?.broadcast_add(&layer.bv)?.reshape((1, n, kv_heads, head_dim))?.transpose(1, 2)?.contiguous()?;
            let q = candle_nn::rotary_emb::rope(&q, &cos, &sin)?;
            let k = candle_nn::rotary_emb::rope(&k, &cos, &sin)?;
            // The cache holds keys and values in f16, as GGML does: half the memory, which the
            // Wrangler hands to the bank. They are widened back to f32 for the maths.
            let (k, v) = (k.to_dtype(DType::F16)?, v.to_dtype(DType::F16)?);
            let k = match &layer.k_cache { Some(c) => Tensor::cat(&[c, &k], 2)?, None => k };
            let v = match &layer.v_cache { Some(c) => Tensor::cat(&[c, &v], 2)?, None => v };
            // A sliding layer never looks further back than its window, so it keeps no more: one new
            // word needs exactly the last `window` keys (itself included), and then no mask at all.
            let keep = if n == 1 { window } else { window + n };
            let (k, v, first) = if layer.sliding && k.dim(2)? > keep {
                let drop = k.dim(2)? - keep;
                (k.narrow(2, drop, keep)?.contiguous()?, v.narrow(2, drop, keep)?.contiguous()?, layer.kv_start + drop)
            } else {
                (k, v, layer.kv_start)
            };
            layer.kv_start = first;
            layer.k_cache = Some(k.clone());
            layer.v_cache = Some(v.clone());
            let t = k.dim(2)?;
            // Grouped heads share their keys: q as (kv_heads, group * rows, d) against k as (kv_heads, t, d),
            // so nothing is copied out to all 64 heads.
            let kt = k.squeeze(0)?.to_dtype(DType::F32)?.t()?.contiguous()?;
            let vv = v.squeeze(0)?.to_dtype(DType::F32)?;
            // A long prompt is attended a block of rows at a time: its score table (heads x rows x
            // keys) then stays a few hundred MB instead of several GB on a 16 GB Mac.
            const ROWS: usize = 64;
            let mut parts = Vec::with_capacity(n.div_ceil(ROWS));
            let mut i0 = 0;
            while i0 < n {
                let rows = ROWS.min(n - i0);
                let qg = q.narrow(2, i0, rows)?.contiguous()?.reshape((kv_heads, group * rows, head_dim))?;
                let scores = (qg.matmul(&kt)? * scale)?.reshape((heads, rows, t))?;
                let scores = if n == 1 {
                    scores
                } else {
                    let mut mask = vec![0f32; rows * t];
                    for i in 0..rows {
                        let pos = offset + i0 + i;
                        for j in 0..t {
                            let at = first + j;
                            if at > pos || (layer.sliding && at + window <= pos) {
                                mask[i * t + j] = f32::NEG_INFINITY;
                            }
                        }
                    }
                    scores.broadcast_add(&Tensor::from_vec(mask, (1, rows, t), &self.device)?)?
                };
                let sinks = layer.sinks.reshape((heads, 1, 1))?.broadcast_as((heads, rows, 1))?.contiguous()?;
                let all = Tensor::cat(&[&scores, &sinks], 2)?;
                let probs = candle_nn::ops::softmax_last_dim(&all)?.narrow(2, 0, t)?.contiguous()?.reshape((kv_heads, group * rows, t))?;
                parts.push(probs.matmul(&vv)?.reshape((heads, rows, head_dim))?.transpose(0, 1)?.reshape((rows, heads * head_dim))?);
                i0 += rows;
            }
            let ctx = if parts.len() == 1 { parts.pop().unwrap() } else { Tensor::cat(&parts, 0)? };
            let o = layer.wo.forward(&ctx)?.broadcast_add(&layer.bo)?;
            xs = (xs + o)?;
            }
            if no_moe {
                continue;
            }
            // The experts, through the core.
            let post = layer.post_norm.forward(&xs)?;
            let width = post.dim(1)?;
            let post = if self.xpad > width { post.pad_with_zeros(1, 0, self.xpad - width)? } else { post };
            let post = post.unsqueeze(0)?;
            let routed = layer.moe.forward(&post)?.squeeze(0)?;
            xs = (xs + routed)?;
            if n > 1 {
                if let Some(p) = &mut self.progress {
                    p(crate::model::Step::Layer(l + 1, n_layers));
                }
            }
        }
        let last = self.norm.forward(&xs.narrow(0, n - 1, 1)?)?;
        self.output.forward(&last)?.to_dtype(DType::F32)
    }

    pub fn clear_kv_cache(&mut self) {
        for l in &mut self.layers {
            l.k_cache = None;
            l.v_cache = None;
            l.kv_start = 0;
        }
    }

    pub fn report(&self) {
        self.store.report_and_reset();
    }
}

/// Lab, step 4 gate: every expert matrix of one layer, multiplied on `gpu`, against the same
/// matrix decoded to f32 and multiplied exactly on the CPU. Prints the worst relative error.
pub fn check_experts(path: &Path, layer: usize, gpu: &Device) -> Result<()> {
    let mut file = std::fs::File::open(path)?;
    let ct = gguf_file::Content::read(&mut file).map_err(|e| e.with_path(path))?;
    let mut worst = (0f32, 0usize, "gate");
    let mut count = 0;
    for name in ["gate", "up", "down"] {
        let info = &ct.tensor_infos[&format!("blk.{layer}.ffn_{name}_exps.weight")];
        let dims = info.shape.dims().to_vec();
        let (n, rows, cols) = (dims[0], dims[1], dims[2]);
        let bytes = rows * cols / info.ggml_dtype.block_size() * info.ggml_dtype.type_size();
        let x = Tensor::randn(0f32, 1f32, (3, cols), &Device::Cpu)?;
        count = n;
        for e in 0..n {
            let mut buf = vec![0u8; bytes];
            file.read_exact_at(&mut buf, ct.tensor_data_offset + info.offset + (e * bytes) as u64)?;
            let w = qtensor_from_ggml(info.ggml_dtype, &buf, vec![rows, cols], &Device::Cpu)?.dequantize(&Device::Cpu)?;
            let exact = x.matmul(&w.t()?)?;
            let q = QMatMul::from_weights(Arc::new(qtensor_from_ggml(info.ggml_dtype, &buf, vec![rows, cols], gpu)?))?;
            let got = q.forward(&x.to_device(gpu)?)?.to_device(&Device::Cpu)?;
            let err = (got - &exact)?.abs()?.max_all()?.to_scalar::<f32>()? / exact.abs()?.max_all()?.to_scalar::<f32>()?;
            if err > worst.0 {
                worst = (err, e, name);
            }
        }
    }
    eprintln!("coachwhip: expert check, layer {layer}, {count} experts x 3 matrices on {gpu:?}: worst relative error {:.2e} (expert {} {})", worst.0, worst.1, worst.2);
    Ok(())
}