The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
//! Qwen3-Next, read from a GGUF file. The same expert streaming as Qwen3 MoE; what is new is the
//! attention. Three layers in four keep a fixed-size running memory per head (Gated DeltaNet)
//! instead of a cache that grows with every word; every fourth layer is classic attention with a
//! gate on its output. Each layer also has a shared expert that every word uses.
//!
//! Follows llama.cpp's qwen3next graph for the tensor layout and the maths.
use crate::experts::{ExpertStore, Settings, StreamedMoe};
use candle::quantized::{ggml_file::qtensor_from_ggml, gguf_file, GgmlDType, QTensor};
use candle::{DType, Device, Module, Result, Tensor, D};
/// Inputs this long or shorter run the delta rule word by word (exact); longer ones block-wise.
pub const DELTA_STEPS_MAX: usize = 8;
/// The delta rule one word at a time over the first `upto` positions: the memory decays, is
/// corrected by each key and value, and is read out with the query. Returns the outputs
/// `(upto, n_v, dv)` and the memory after them.
pub fn delta_steps(qc: &Tensor, kc: &Tensor, vc: &Tensor, beta: &Tensor, log_decay: &Tensor, mut s: Tensor, upto: usize) -> Result<(Tensor, Tensor)> {
let n_v = kc.dim(1)?;
let decay = log_decay.exp()?;
if upto == 1 && crate::delta_kernel::on() {
let (o, s) = crate::delta_kernel::delta_word(&qc.get(0)?, &kc.get(0)?, &vc.get(0)?, &beta.get(0)?, &decay.get(0)?, &s)?;
return Ok((o.unsqueeze(0)?, s));
}
let mut outs = Vec::with_capacity(upto);
for t in 0..upto {
let kt = kc.get(t)?.unsqueeze(1)?.contiguous()?; // (n_v, 1, dk)
let qt = qc.get(t)?.unsqueeze(1)?.contiguous()?;
let vt = vc.get(t)?.unsqueeze(1)?; // (n_v, 1, dv)
let bt = beta.get(t)?.reshape((n_v, 1, 1))?;
let gt = decay.get(t)?.reshape((n_v, 1, 1))?;
s = s.broadcast_mul(>)?;
let sk = kt.matmul(&s)?; // (n_v, 1, dv)
let d = (vt - sk)?.broadcast_mul(&bt)?;
s = (s + kt.transpose(1, 2)?.contiguous()?.matmul(&d)?)?;
outs.push(qt.matmul(&s)?.squeeze(1)?); // (n_v, dv)
}
Ok((Tensor::stack(&outs, 0)?, s))
}
/// What a recorded forward leaves behind, so the layer's memory can be rewound to any prefix of
/// it: the memory before, the convolution window before, and every word's inputs.
struct Trail {
s0: Tensor,
conv_before: Tensor,
padded: Tensor,
qc: Tensor,
kc: Tensor,
vc: Tensor,
beta: Tensor,
log_decay: Tensor,
}
/// Keys and values kept between words, like candle's KvCache, plus `truncate` so a speculative
/// draft can be taken back.
/// The reserved conversation space in words, when the wrangler has set one.
fn conv_tokens() -> Option<usize> {
static N: std::sync::OnceLock<Option<usize>> = std::sync::OnceLock::new();
*N.get_or_init(|| std::env::var("COACHWHIP_CONV_TOKENS").ok().and_then(|v| v.parse().ok()).filter(|&n: &usize| n > 0))
}
struct KvBuf {
k: Option<Tensor>,
v: Option<Tensor>,
len: usize,
cap: usize,
}
impl KvBuf {
fn new(cap: usize) -> Self {
Self { k: None, v: None, len: 0, cap }
}
/// Appends along the sequence axis (dim 2) and returns everything kept so far.
fn append(&mut self, k: &Tensor, v: &Tensor) -> Result<(Tensor, Tensor)> {
let l = k.dim(2)?;
if let Some(fixed) = conv_tokens() {
if l > fixed / 2 {
candle::bail!("a piece of {l} words is over half the {fixed}-word conversation space")
}
if self.len + l > fixed {
// Roll: keep the opening and the newest, let the oldest middle go.
let sink = 256.min(self.len);
// Roll a quarter of the space at once, so the shift is rare, not once per word.
let drop = (self.len + l - fixed).max(fixed / 4).min(self.len.saturating_sub(256.min(self.len)));
let keep_from = (sink + drop).min(self.len);
if let (Some(kb), Some(vb)) = (&self.k, &self.v) {
if keep_from < self.len {
let tail_k = kb.narrow(2, keep_from, self.len - keep_from)?.contiguous()?;
let tail_v = vb.narrow(2, keep_from, self.len - keep_from)?.contiguous()?;
kb.slice_set(&tail_k, 2, sink)?;
vb.slice_set(&tail_v, 2, sink)?;
}
}
self.len = sink + (self.len - keep_from);
}
if self.k.is_none() || self.cap != fixed {
let (b, h, _, d) = k.dims4()?;
let len = self.len;
let keep = |old: &Option<Tensor>, t: &Tensor| -> Result<Tensor> {
let fresh = Tensor::zeros((b, h, fixed, d), t.dtype(), t.device())?;
if let (Some(o), true) = (old, len > 0) {
fresh.slice_set(&o.narrow(2, 0, len)?.contiguous()?, 2, 0)?;
}
Ok(fresh)
};
self.k = Some(keep(&self.k, k)?);
self.v = Some(keep(&self.v, v)?);
self.cap = fixed;
}
} else if self.k.is_none() || self.len + l > self.cap {
let (b, h, _, d) = k.dims4()?;
let cap = (self.len + l).max(self.cap) + 1024;
let len = self.len;
let grow = |old: &Option<Tensor>, t: &Tensor| -> Result<Tensor> {
let fresh = Tensor::zeros((b, h, cap, d), t.dtype(), t.device())?;
if let (Some(o), true) = (old, len > 0) {
fresh.slice_set(&o.narrow(2, 0, len)?.contiguous()?, 2, 0)?;
}
Ok(fresh)
};
self.k = Some(grow(&self.k, k)?);
self.v = Some(grow(&self.v, v)?);
self.cap = cap;
}
let kb = self.k.as_ref().unwrap();
let vb = self.v.as_ref().unwrap();
kb.slice_set(k, 2, self.len)?;
vb.slice_set(v, 2, self.len)?;
self.len += l;
Ok((kb.narrow(2, 0, self.len)?, vb.narrow(2, 0, self.len)?))
}
fn truncate(&mut self, len: usize) {
self.len = self.len.min(len);
}
fn reset(&mut self) {
self.len = 0;
}
}
use candle_nn::Linear;
use candle_transformers::models::quantized_qwen3::Gguf;
use candle_transformers::models::with_tracing::QMatMul;
use candle_transformers::quantized_nn::RmsNorm;
use std::path::Path;
use std::sync::Arc;
/// x / sqrt(sum(x^2) + eps) over the last dimension.
fn l2_norm(x: &Tensor, eps: f64) -> Result<Tensor> {
let n = (x.sqr()?.sum_keepdim(D::Minus1)? + eps)?.sqrt()?;
x.broadcast_div(&n)
}
/// Give each layer the router of the layer `ahead` on, so it can guess that layer's experts early.
/// The last `ahead` layers have no later router and fall back to the learned route table.
fn wire_lookahead(layers: &mut [Layer], ahead: usize) {
let ahead = ahead.max(1);
for i in ahead..layers.len() {
layers[i - ahead].moe.next_gate = Some(layers[i].moe.gate.clone());
if ahead >= 2 {
layers[i - ahead].moe.gate1 = Some(layers[i - ahead + 1].moe.gate.clone());
}
}
}
/// Lab (`COACHWHIP_PREDICTOR=file`): fitted expert predictors in place of the borrowed routers.
/// File: u32 layers, hidden, experts, ahead; then per layer L in 0..layers-ahead: f32 W[experts x
/// hidden] (row-major), f32 b[experts]. W maps layer L's MoE input to layer L+ahead's scores.
fn load_predictor(layers: &mut [Layer], path: &str, ahead: usize, device: &Device) -> Result<()> {
let bytes = std::fs::read(path).map_err(|e| candle::Error::Msg(format!("predictor {path}: {e}")))?;
let u32_at = |i: usize| u32::from_le_bytes(bytes[i..i + 4].try_into().unwrap()) as usize;
let (n_layers, hidden, experts, f_ahead) = (u32_at(0), u32_at(4), u32_at(8), u32_at(12));
if n_layers != layers.len() || f_ahead != ahead {
candle::bail!("predictor {path}: {n_layers} layers, ahead {f_ahead}; model has {} layers, ahead {ahead}", layers.len())
}
let mut pos = 16;
let floats = |pos: usize, n: usize| -> Vec<f32> { bytes[pos..pos + 4 * n].chunks_exact(4).map(|c| f32::from_le_bytes(c.try_into().unwrap())).collect() };
for l in 0..n_layers - ahead {
let w = Tensor::from_vec(floats(pos, experts * hidden), (experts, hidden), device)?;
pos += 4 * experts * hidden;
let b = Tensor::from_vec(floats(pos, experts), experts, device)?;
pos += 4 * experts;
layers[l].moe.next_gate = Some(Linear::new(w, Some(b)));
}
eprintln!("coachwhip: fitted expert predictor loaded from {path} ({} layers, {ahead} ahead)", n_layers - ahead);
Ok(())
}
fn softplus(x: &Tensor) -> Result<Tensor> {
(x.exp()? + 1.0)?.log()
}
/// Words per block when a running-memory layer reads a prompt.
pub const DELTA_CHUNK: usize = 64;
/// The delta rule over a whole prompt, a block of words at a time (Gated DeltaNet, Yang, Kautz and
/// Hatamizadeh, ICLR 2025, section 3). Gives the same answer as stepping word by word, but each
/// block is a few large matrix products instead of a dozen small GPU jobs per word. Inside a block,
/// every word's correction depends on the corrections before it; that chain is a small triangular
/// system, solved once per block on the CPU. Decays are kept as logarithms so long blocks never
/// divide by a vanishing number.
/// q, k: (l, n_v, dk); v: (l, n_v, dv); beta, log_decay: (l, n_v); s: (n_v, dk, dv).
pub fn delta_chunked(q: &Tensor, k: &Tensor, v: &Tensor, beta: &Tensor, log_decay: &Tensor, mut s: Tensor) -> Result<(Tensor, Tensor)> {
let (l, n_v, dk) = q.dims3()?;
let dv = v.dim(2)?;
let c = DELTA_CHUNK;
let nc = l.div_ceil(c);
let pad = nc * c - l;
let device = q.device();
// Padding words have no key, no value, no write strength and no decay, so they change nothing.
let blocks = |t: &Tensor, width: usize| -> Result<Tensor> {
let t = if pad > 0 { t.pad_with_zeros(0, 0, pad)? } else { t.clone() };
t.reshape((nc, c, n_v, width))?.permute((2, 0, 1, 3))?.contiguous()?.reshape((n_v * nc, c, width))
};
let (q, k, v) = (blocks(q, dk)?, blocks(k, dk)?, blocks(v, dv)?);
let beta = blocks(&beta.unsqueeze(2)?, 1)?; // (n, c, 1)
let cum = blocks(&log_decay.unsqueeze(2)?, 1)?.squeeze(2)?.cumsum(1)?; // (n, c): decay since block start, inclusive
let mask = |strict: bool| -> Result<Tensor> {
let m: Vec<f32> = (0..c * c).map(|x| if x % c < x / c || (!strict && x % c == x / c) { 0.0 } else { -1e30 }).collect();
Tensor::from_vec(m, (c, c), device)
};
let diff = cum.unsqueeze(2)?.broadcast_sub(&cum.unsqueeze(1)?)?; // [t, i] = decay from i to t
let fade_strict = diff.broadcast_add(&mask(true)?)?.exp()?;
let fade_incl = diff.broadcast_add(&mask(false)?)?.exp()?;
let kt = k.transpose(1, 2)?.contiguous()?;
let a = (k.matmul(&kt)? * fade_strict)?.broadcast_mul(&beta)?;
let p = (q.matmul(&kt)? * fade_incl)?;
let gam = cum.exp()?.unsqueeze(2)?; // (n, c, 1)
let last = cum.narrow(1, c - 1, 1)?; // (n, 1)
let gam_last = last.exp()?.unsqueeze(2)?; // (n, 1, 1)
let k_to_end = k.broadcast_mul(&last.broadcast_sub(&cum)?.exp()?.unsqueeze(2)?)?;
let q_gam = q.broadcast_mul(&gam)?;
// T = (I + A)^-1, A strictly lower triangular: forward substitution, one small matrix per block and head.
let a_rows = a.to_device(&Device::Cpu)?.flatten_all()?.to_vec1::<f32>()?;
let mut t_rows = vec![0f32; a_rows.len()];
use rayon::prelude::*;
t_rows.par_chunks_mut(c * c).zip(a_rows.par_chunks(c * c)).for_each(|(t, a)| {
for r in 0..c {
t[r * c + r] = 1.0;
for i in 0..r {
let air = a[r * c + i];
if air != 0.0 {
for j in 0..=i {
t[r * c + j] -= air * t[i * c + j];
}
}
}
}
});
let tri = Tensor::from_vec(t_rows, (n_v * nc, c, c), device)?;
let w = tri.matmul(&k.broadcast_mul(&(beta.broadcast_mul(&gam))?)?)?; // (n, c, dk)
let u = tri.matmul(&v.broadcast_mul(&beta)?)?; // (n, c, dv)
let per_block = |t: Tensor, width: usize| t.reshape((n_v, nc, c, width));
let (w, u, p, q_gam) = (per_block(w, dk)?, per_block(u, dv)?, per_block(p, c)?, per_block(q_gam, dk)?);
let k_to_end_t = per_block(k_to_end, dk)?.transpose(2, 3)?.contiguous()?; // (n_v, nc, dk, c)
let gam_last = gam_last.reshape((n_v, nc, 1, 1))?;
let mut outs = Vec::with_capacity(nc);
for b in 0..nc {
let at = |t: &Tensor| t.narrow(1, b, 1)?.squeeze(1);
let d = (at(&u)? - at(&w)?.matmul(&s)?)?; // (n_v, c, dv): each word's correction
outs.push((at(&q_gam)?.matmul(&s)? + at(&p)?.contiguous()?.matmul(&d)?)?);
s = (s.broadcast_mul(&at(&gam_last)?)? + at(&k_to_end_t)?.matmul(&d)?)?;
}
let o = Tensor::cat(&outs, 1)?.narrow(1, 0, l)?.transpose(0, 1)?.contiguous()?; // (l, n_v, dv)
Ok((o, s))
}
/// The linear-attention input projections, as the two Qwen generations store them.
enum Proj {
/// Qwen3-Next: one matrix holding, per key head, q, k, its values and their gates; and one
/// holding, per key head, the betas then the alphas of its value heads.
Fused { qkvz: QMatMul, ba: QMatMul },
/// Qwen3.5 and 3.6: q, k and v in one matrix (all heads of each in turn), and the gate, beta
/// and alpha each in their own.
Split { qkv: QMatMul, z: QMatMul, beta: QMatMul, alpha: QMatMul },
}
#[cfg(test)]
mod proj_tests {
//! The two tensor layouts must be the same layer: a Qwen3.5-style split set of weights and
//! the Qwen3-Next fused arrangement of the very same numbers give the same output.
use super::*;
use candle::quantized::{GgmlDType, QTensor};
struct Lcg(u64);
impl Lcg {
fn tensor(&mut self, shape: &[usize], scale: f32) -> Tensor {
let n: usize = shape.iter().product();
let v: Vec<f32> = (0..n)
.map(|_| {
self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
(((self.0 >> 33) as f32 / (1u64 << 31) as f32) * 2.0 - 1.0) * scale
})
.collect();
Tensor::from_vec(v, shape, &Device::Cpu).unwrap()
}
}
fn qm(w: &Tensor) -> QMatMul {
QMatMul::from_weights(Arc::new(QTensor::quantize(w, GgmlDType::F32).unwrap())).unwrap()
}
fn layer(proj: Proj, g: &mut Lcg, n_k: usize, n_v: usize, dk: usize, dv: usize, h: usize) -> LinearAttn {
let channels = 2 * n_k * dk + n_v * dv;
LinearAttn {
proj,
conv_w: g.tensor(&[channels, 4], 0.5),
dt_bias: g.tensor(&[n_v], 0.5),
a: (g.tensor(&[n_v], 0.5).abs().unwrap() * -1.0).unwrap(),
norm: RmsNorm::from_qtensor(QTensor::quantize(&Tensor::ones(dv, DType::F32, &Device::Cpu).unwrap(), GgmlDType::F32).unwrap(), 1e-6).unwrap(),
norm_w: Tensor::ones(dv, DType::F32, &Device::Cpu).unwrap(),
out: qm(&g.tensor(&[h, n_v * dv], 0.3)),
n_k,
n_v,
dk,
dv,
eps: 1e-6,
conv_state: None,
state: None,
snapshot: None,
record: false,
trail: None,
}
}
#[test]
fn split_and_fused_projections_are_the_same_layer() {
let (n_k, n_v, dk, dv, h, l) = (2usize, 4usize, 4usize, 4usize, 8usize, 3usize);
let per_group = n_v / n_k;
let mut g = Lcg(7);
let q: Vec<Tensor> = (0..n_k).map(|_| g.tensor(&[dk, h], 0.5)).collect();
let k: Vec<Tensor> = (0..n_k).map(|_| g.tensor(&[dk, h], 0.5)).collect();
let v: Vec<Tensor> = (0..n_v).map(|_| g.tensor(&[dv, h], 0.5)).collect();
let z: Vec<Tensor> = (0..n_v).map(|_| g.tensor(&[dv, h], 0.5)).collect();
let b: Vec<Tensor> = (0..n_v).map(|_| g.tensor(&[1, h], 0.5)).collect();
let a: Vec<Tensor> = (0..n_v).map(|_| g.tensor(&[1, h], 0.5)).collect();
// Qwen3-Next: per key head, q, k, its value heads, their gates; betas then alphas per key head.
let mut fused_rows = Vec::new();
let mut ba_rows = Vec::new();
for kh in 0..n_k {
fused_rows.push(q[kh].clone());
fused_rows.push(k[kh].clone());
fused_rows.extend(v[kh * per_group..(kh + 1) * per_group].iter().cloned());
fused_rows.extend(z[kh * per_group..(kh + 1) * per_group].iter().cloned());
ba_rows.extend(b[kh * per_group..(kh + 1) * per_group].iter().cloned());
ba_rows.extend(a[kh * per_group..(kh + 1) * per_group].iter().cloned());
}
let fused = Proj::Fused { qkvz: qm(&Tensor::cat(&fused_rows, 0).unwrap()), ba: qm(&Tensor::cat(&ba_rows, 0).unwrap()) };
// Qwen3.5: all q, all k, all v in one; the gate, betas and alphas on their own. Its value
// head j pairs with key head j % n_k, so the same per-head weights sit in a different order.
let order: Vec<usize> = (0..n_v).map(|j| (j % n_k) * per_group + j / n_k).collect();
let pick = |t: &[Tensor]| -> Vec<Tensor> { order.iter().map(|&j| t[j].clone()).collect() };
let qkv: Vec<Tensor> = q.iter().chain(k.iter()).cloned().chain(pick(&v)).collect();
let split = Proj::Split {
qkv: qm(&Tensor::cat(&qkv, 0).unwrap()),
z: qm(&Tensor::cat(&pick(&z), 0).unwrap()),
beta: qm(&Tensor::cat(&pick(&b), 0).unwrap()),
alpha: qm(&Tensor::cat(&pick(&a), 0).unwrap()),
};
let mut shared = Lcg(99);
let mut lf = layer(fused, &mut shared, n_k, n_v, dk, dv, h);
let mut shared = Lcg(99);
let mut ls = layer(split, &mut shared, n_k, n_v, dk, dv, h);
// The per-value-head pieces follow the same order.
let idx = Tensor::new(order.iter().map(|&j| j as u32).collect::<Vec<_>>(), &Device::Cpu).unwrap();
ls.dt_bias = lf.dt_bias.index_select(&idx, 0).unwrap();
ls.a = lf.a.index_select(&idx, 0).unwrap();
let qk_rows = 2 * n_k * dk;
let conv_v = lf.conv_w.narrow(0, qk_rows, n_v * dv).unwrap().reshape((n_v, dv, 4)).unwrap().index_select(&idx, 0).unwrap().reshape((n_v * dv, 4)).unwrap();
ls.conv_w = Tensor::cat(&[&lf.conv_w.narrow(0, 0, qk_rows).unwrap(), &conv_v], 0).unwrap();
let mut shared = Lcg(99);
let _ = (shared.tensor(&[qk_rows + n_v * dv, 4], 0.5), shared.tensor(&[n_v], 0.5), shared.tensor(&[n_v], 0.5));
let out_w = shared.tensor(&[h, n_v * dv], 0.3).reshape((h, n_v, dv)).unwrap().index_select(&idx, 1).unwrap().reshape((h, n_v * dv)).unwrap();
ls.out = qm(&out_w);
let x = Lcg(5).tensor(&[1, l, h], 1.0);
let yf = lf.forward(&x).unwrap();
let ys = ls.forward(&x).unwrap();
let diff = (yf - ys).unwrap().abs().unwrap().flatten_all().unwrap().max(0).unwrap().to_scalar::<f32>().unwrap();
assert!(diff < 1e-5, "fused and split layouts differ by {diff}");
// And a second word, so the carried state and the convolution window agree too.
let x2 = Lcg(6).tensor(&[1, 1, h], 1.0);
let d2 = (lf.forward(&x2).unwrap() - ls.forward(&x2).unwrap()).unwrap().abs().unwrap().flatten_all().unwrap().max(0).unwrap().to_scalar::<f32>().unwrap();
assert!(d2 < 1e-5, "second word differs by {d2}");
}
}
/// A running-memory layer: per value head a 128 x 128 matrix that decays, is corrected by each
/// new key and value (the delta rule), and is read out with the query.
struct LinearAttn {
proj: Proj,
conv_w: Tensor,
dt_bias: Tensor,
a: Tensor,
norm: RmsNorm,
/// The output norm's weight as a plain tensor, for the fused kernel.
norm_w: Tensor,
out: QMatMul,
n_k: usize,
n_v: usize,
dk: usize,
dv: usize,
eps: f64,
conv_state: Option<Tensor>,
state: Option<Tensor>,
/// The memories as they were before a speculative draft, to go back to.
snapshot: Option<(Option<Tensor>, Option<Tensor>)>,
/// Keep a trail of the next forward, so it can be rewound.
record: bool,
trail: Option<Trail>,
}
impl LinearAttn {
fn reset(&mut self) {
self.conv_state = None;
self.state = None;
self.snapshot = None;
}
fn take_snapshot(&mut self) {
self.snapshot = Some((self.conv_state.clone(), self.state.clone()));
}
fn restore_snapshot(&mut self) {
if let Some((c, s)) = self.snapshot.take() {
self.conv_state = c;
self.state = s;
}
}
/// Rewind the last recorded forward to its first `m` words: the memory is replayed from
/// before it over those words, the convolution window set as it was after them.
fn rewind(&mut self, m: usize) -> Result<()> {
let Some(t) = self.trail.take() else { return Ok(()) };
let taps = self.conv_w.dim(1)?;
self.state = Some(if m == 0 { t.s0 } else { delta_steps(&t.qc, &t.kc, &t.vc, &t.beta, &t.log_decay, t.s0, m)?.1 });
self.conv_state = Some(if m == 0 { t.conv_before } else { t.padded.narrow(0, m, taps - 1)?.contiguous()? });
Ok(())
}
/// The projections of one input, in one layout: q, k and v side by side as `(l, channels)`,
/// the gate as `(l, n_v, dv)`, and the raw beta and alpha as `(l, n_v)`.
fn project(&self, x2: &Tensor, l: usize) -> Result<(Tensor, Tensor, Tensor, Tensor)> {
let (n_k, n_v, dk, dv) = (self.n_k, self.n_v, self.dk, self.dv);
let per_group = n_v / n_k;
match &self.proj {
Proj::Fused { qkvz, ba } => {
// Per key head: q, k, then the values and gates of its value heads.
let mixed = qkvz.forward(x2)?.reshape((l, n_k, 2 * dk + 2 * dv * per_group))?;
let q = mixed.narrow(2, 0, dk)?.reshape((l, n_k * dk))?;
let k = mixed.narrow(2, dk, dk)?.reshape((l, n_k * dk))?;
let v = mixed.narrow(2, 2 * dk, dv * per_group)?.reshape((l, n_v * dv))?;
let z = mixed.narrow(2, 2 * dk + dv * per_group, dv * per_group)?.reshape((l, n_v, dv))?;
let ba = ba.forward(x2)?.reshape((l, n_k, 2 * per_group))?;
let beta = ba.narrow(2, 0, per_group)?.reshape((l, n_v))?;
let alpha = ba.narrow(2, per_group, per_group)?.reshape((l, n_v))?;
Ok((Tensor::cat(&[&q, &k, &v], 1)?, z, beta, alpha))
}
Proj::Split { qkv, z, beta, alpha } => Ok((
qkv.forward(x2)?, // already q (all heads), k (all heads), v (all heads)
z.forward(x2)?.reshape((l, n_v, dv))?,
if l == 1 && skip("ab") { Tensor::zeros((1, n_v), DType::F32, x2.device())? } else { beta.forward(x2)? },
if l == 1 && skip("ab") { Tensor::zeros((1, n_v), DType::F32, x2.device())? } else { alpha.forward(x2)? },
)),
}
}
fn forward(&mut self, x: &Tensor) -> Result<Tensor> {
let (b, l, h) = x.dims3()?;
let device = x.device();
let (n_k, n_v, dk, dv) = (self.n_k, self.n_v, self.dk, self.dv);
let per_group = n_v / n_k;
let x2 = x.reshape((l, h))?;
let mut t = std::time::Instant::now();
let timed = sec_l(l) && sections_on();
if timed { x.device().synchronize()?; t = std::time::Instant::now(); }
let (qkv, z, beta, alpha) = if l == 1 && skip("laproj") {
let c = 2 * n_k * dk + n_v * dv;
(Tensor::zeros((1, c), DType::F32, device)?, Tensor::zeros((1, n_v, dv), DType::F32, device)?, Tensor::zeros((1, n_v), DType::F32, device)?, Tensor::zeros((1, n_v), DType::F32, device)?)
} else {
self.project(&x2, l)?
};
if timed { split_tick(x.device(), &mut t, 9)?; }
let channels = qkv.dim(1)?;
if l <= short_max() && !self.record && crate::attn_kernel::on() {
let taps = self.conv_w.dim(1)?;
if l > 1 && crate::attn_kernel::piece_on() && dk == 128 && dv == 128 && l >= taps {
let conv_state = match &self.conv_state {
Some(s) => s.to_dtype(DType::F32)?.contiguous()?,
None => Tensor::zeros((taps - 1, channels), DType::F32, device)?,
};
let st = match &self.state {
Some(s) => s.to_dtype(DType::F32)?.contiguous()?,
None => Tensor::zeros((n_v, dk, dv), DType::F32, device)?,
};
let op = crate::attn_kernel::GdnWord {
conv_state,
conv_w: self.conv_w.to_dtype(DType::F32)?.contiguous()?,
dt_bias: self.dt_bias.to_dtype(DType::F32)?.contiguous()?,
a: self.a.to_dtype(DType::F32)?.contiguous()?,
norm_w: self.norm_w.clone(),
s: st,
n_k, n_v, dk, dv, taps,
cycle: matches!(self.proj, Proj::Split { .. }),
eps: self.eps as f32,
};
let (o, s_new, conv_new) = crate::attn_kernel::gdn_piece(op, &qkv, &z, &beta, &alpha, l)?;
self.state = Some(s_new);
self.conv_state = Some(conv_new);
return self.out.forward(&o)?.reshape((b, l, h));
}
if l > 1 {
let conv_w = self.conv_w.to_dtype(DType::F32)?.contiguous()?;
let dt_bias = self.dt_bias.to_dtype(DType::F32)?.contiguous()?;
let a = self.a.to_dtype(DType::F32)?.contiguous()?;
let mut outs = Vec::with_capacity(l);
for i in 0..l {
let conv_state = match &self.conv_state {
Some(s) => s.to_dtype(DType::F32)?.contiguous()?,
None => Tensor::zeros((taps - 1, channels), DType::F32, device)?,
};
let st = match &self.state {
Some(s) => s.to_dtype(DType::F32)?.contiguous()?,
None => Tensor::zeros((n_v, dk, dv), DType::F32, device)?,
};
let op = crate::attn_kernel::GdnWord {
conv_state,
conv_w: conv_w.clone(),
dt_bias: dt_bias.clone(),
a: a.clone(),
norm_w: self.norm_w.clone(),
s: st,
n_k, n_v, dk, dv, taps,
cycle: matches!(self.proj, Proj::Split { .. }),
eps: self.eps as f32,
};
let row = |t: &Tensor| -> Result<Tensor> { t.narrow(0, i, 1)?.contiguous() };
let (o, s_new, conv_new) = crate::attn_kernel::gdn_word(op, &row(&qkv)?, &row(&z)?, &row(&beta)?, &row(&alpha)?)?;
self.state = Some(s_new);
self.conv_state = Some(conv_new);
outs.push(o);
}
let o = Tensor::cat(&outs, 0)?;
return self.out.forward(&o)?.reshape((b, l, h));
}
let conv_state = match &self.conv_state {
Some(s) => s.to_dtype(DType::F32)?.contiguous()?,
None => Tensor::zeros((taps - 1, channels), DType::F32, device)?,
};
let s = match &self.state {
Some(s) => s.to_dtype(DType::F32)?.contiguous()?,
None => Tensor::zeros((n_v, dk, dv), DType::F32, device)?,
};
let op = crate::attn_kernel::GdnWord {
conv_state,
conv_w: self.conv_w.to_dtype(DType::F32)?.contiguous()?,
dt_bias: self.dt_bias.to_dtype(DType::F32)?.contiguous()?,
a: self.a.to_dtype(DType::F32)?.contiguous()?,
norm_w: self.norm_w.clone(),
s,
n_k, n_v, dk, dv, taps,
cycle: matches!(self.proj, Proj::Split { .. }),
eps: self.eps as f32,
};
let (o, s_new, conv_new) = if skip("lamid") {
(Tensor::zeros((1, n_v * dv), DType::F32, device)?, op.s.clone(), op.conv_state.clone())
} else {
crate::attn_kernel::gdn_word(op, &qkv, &z, &beta, &alpha)?
};
self.state = Some(s_new);
self.conv_state = Some(conv_new);
if timed { split_tick(x.device(), &mut t, 10)?; }
let y = if skip("laout") { Tensor::zeros((b, l, h), DType::F32, device)? } else { self.out.forward(&o)?.reshape((b, l, h))? };
if timed { split_tick(x.device(), &mut t, 11)?; }
return Ok(y);
}
let beta = candle_nn::ops::sigmoid(&beta)?;
let log_decay = softplus(&alpha.broadcast_add(&self.dt_bias)?)?.broadcast_mul(&self.a)?; // (l, n_v)
// Causal depthwise convolution over the last `taps` inputs, carried across calls.
let taps = self.conv_w.dim(1)?;
let past = match &self.conv_state {
Some(s) => s.clone(),
None => Tensor::zeros((taps - 1, channels), DType::F32, device)?,
};
let padded = Tensor::cat(&[&past, &qkv], 0)?; // (taps - 1 + l, channels)
let mut conv = padded.narrow(0, 0, l)?.broadcast_mul(&self.conv_w.narrow(1, 0, 1)?.t()?)?;
for w in 1..taps {
conv = (conv + padded.narrow(0, w, l)?.broadcast_mul(&self.conv_w.narrow(1, w, 1)?.t()?)?)?;
}
self.conv_state = Some(padded.narrow(0, l, taps - 1)?.contiguous()?);
let conv = candle_nn::ops::silu(&conv)?;
let qc = l2_norm(&conv.narrow(1, 0, n_k * dk)?.reshape((l, n_k, dk))?, self.eps)?;
let kc = l2_norm(&conv.narrow(1, n_k * dk, n_k * dk)?.reshape((l, n_k, dk))?, self.eps)?;
let vc = conv.narrow(1, 2 * n_k * dk, n_v * dv)?.reshape((l, n_v, dv))?;
// Each key head serves `per_group` value heads. Qwen3-Next lists a key head's value heads
// side by side (value head j uses key head j / per_group); Qwen3.5 and 3.6 cycle through the
// key heads (value head j uses key head j % n_k).
let widen = |t: Tensor| -> Result<Tensor> {
match &self.proj {
Proj::Fused { .. } => t.unsqueeze(2)?.broadcast_as((l, n_k, per_group, dk))?.reshape((l, n_v, dk)),
Proj::Split { .. } => t.unsqueeze(1)?.broadcast_as((l, per_group, n_k, dk))?.reshape((l, n_v, dk)),
}
};
let qc = (widen(qc)? * (1.0 / (dk as f64).sqrt()))?;
let kc = widen(kc)?;
let mut s = match &self.state {
Some(s) => s.clone(),
None => Tensor::zeros((n_v, dk, dv), DType::F32, device)?,
};
// Long inputs take the block-at-a-time path; short ones (one word, or a speculative
// verify of a few) go word by word, the exact arithmetic plain decoding uses.
let s0 = s.clone();
let (o, s) = if l > DELTA_STEPS_MAX {
delta_chunked(&qc, &kc, &vc, &beta, &log_decay, s)?
} else {
delta_steps(&qc, &kc, &vc, &beta, &log_decay, s, l)?
};
self.state = Some(s);
if self.record {
self.trail = Some(Trail { s0, conv_before: past, padded: padded.clone(), qc: qc.clone(), kc: kc.clone(), vc: vc.clone(), beta: beta.clone(), log_decay: log_decay.clone() });
}
let o = (self.norm.forward(&o)? * candle_nn::ops::silu(&z)?)?;
self.out.forward(&o.reshape((l, n_v * dv))?)?.reshape((b, l, h))
}
}
/// Classic attention with a sigmoid gate on its output and rotary position on a quarter of each head.
struct FullAttn {
wq: QMatMul,
wk: QMatMul,
wv: QMatMul,
wo: QMatMul,
q_norm: RmsNorm,
k_norm: RmsNorm,
n_head: usize,
n_kv: usize,
hd: usize,
rot: usize,
freq_base: f64,
kv_cache: KvBuf,
}
impl FullAttn {
fn rope(&self, x: &Tensor, offset: usize) -> Result<Tensor> {
let (_b, _h, l, _d) = x.dims4()?;
let half = self.rot / 2;
let inv: Vec<f32> = (0..half).map(|i| (1.0 / self.freq_base.powf(2.0 * i as f64 / self.rot as f64)) as f32).collect();
let pos: Vec<f32> = (offset..offset + l).map(|p| p as f32).collect();
let inv = Tensor::from_vec(inv, (1, half), x.device())?;
let pos = Tensor::from_vec(pos, (l, 1), x.device())?;
let freqs = pos.broadcast_mul(&inv)?; // (l, half)
let (cos, sin) = (freqs.cos()?, freqs.sin()?);
let rotated = candle_nn::rotary_emb::rope(&x.narrow(3, 0, self.rot)?.contiguous()?, &cos, &sin)?;
Tensor::cat(&[&rotated, &x.narrow(3, self.rot, self.hd - self.rot)?], 3)
}
fn forward(&mut self, x: &Tensor, mask: Option<&Tensor>, offset: usize) -> Result<Tensor> {
let (b, l, _) = x.dims3()?;
let full = self.wq.forward(x)?.reshape((l, self.n_head, 2 * self.hd))?;
let q = self.q_norm.forward(&full.narrow(2, 0, self.hd)?.contiguous()?)?;
let gate = full.narrow(2, self.hd, self.hd)?.reshape((l, self.n_head * self.hd))?;
let k = self.k_norm.forward(&self.wk.forward(x)?.reshape((l, self.n_kv, self.hd))?)?;
let v = self.wv.forward(x)?.reshape((l, self.n_kv, self.hd))?;
let heads = |t: Tensor| -> Result<Tensor> { t.transpose(0, 1)?.unsqueeze(0)?.contiguous() }; // (1, H, l, D)
let q = self.rope(&heads(q)?, offset)?;
let k = self.rope(&heads(k)?, offset)?;
let v = heads(v)?;
let (k, v) = self.kv_cache.append(&k.contiguous()?, &v.contiguous()?)?;
let s_len = k.dim(2)?;
let g = self.n_head / self.n_kv;
// After the space has rolled, the mask is rebuilt for the words actually kept.
let rebuilt;
let mask = match mask {
Some(m) if m.dim(D::Minus1)? != s_len => {
rebuilt = candle_transformers::utils::build_additive_causal_mask(l, s_len - l, None, q.device(), DType::F32)?;
Some(&rebuilt)
}
other => other,
};
let slice = attn_slice();
let kt = k.transpose(2, 3)?;
let mut parts = Vec::new();
let mut a = 0usize;
while a < l {
let ls = if l > slice { slice.min(l - a) } else { l };
let qs = if ls == l { q.clone() } else { q.narrow(2, a, ls)? };
let qs = qs.contiguous()?.reshape((1, self.n_kv, g * ls, self.hd))?;
let mut scores = (qs.matmul(&kt)? * (1.0 / (self.hd as f64).sqrt()))?;
if let Some(m) = mask {
let ms = if ls == l { m.clone() } else { let r = m.rank(); m.narrow(r - 2, a, ls)? };
scores = scores
.reshape((1, self.n_kv, g, ls, s_len))?
.broadcast_add(&ms.to_dtype(scores.dtype())?.unsqueeze(1)?)?
.reshape((1, self.n_kv, g * ls, s_len))?;
}
let probs = candle_nn::ops::softmax_last_dim(&scores)?;
parts.push(probs.matmul(&v)?.reshape((1, self.n_head, ls, self.hd))?);
a += ls;
}
let ctx = if parts.len() == 1 { parts.pop().unwrap() } else { Tensor::cat(&parts, 2)? };
let ctx = ctx.transpose(1, 2)?.reshape((l, self.n_head * self.hd))?;
let ctx = (ctx * candle_nn::ops::sigmoid(&gate)?)?;
self.wo.forward(&ctx)?.reshape((b, l, ()))
}
}
enum Mixer {
Linear(LinearAttn),
Full(FullAttn),
}
/// The expert every word uses, with its own sigmoid gate.
struct SharedExpert {
gate_inp: Tensor,
gate: QMatMul,
up: QMatMul,
down: QMatMul,
}
impl SharedExpert {
fn forward(&self, x: &Tensor) -> Result<Tensor> {
let hidden = if crate::fuse_kernel::on() && x.dtype() == DType::F32 {
crate::fuse_kernel::silu_mul(&self.gate.forward(x)?, &self.up.forward(x)?)?
} else {
(candle_nn::ops::silu(&self.gate.forward(x)?)? * self.up.forward(x)?)?
};
let y = self.down.forward(&hidden)?;
let g = candle_nn::ops::sigmoid(&x.broadcast_mul(&self.gate_inp)?.sum_keepdim(D::Minus1)?)?;
y.broadcast_mul(&g)
}
}
struct Layer {
mixer: Mixer,
attn_norm: RmsNorm,
post_norm: RmsNorm,
attn_norm_w: Tensor,
post_norm_w: Tensor,
norm_eps: f32,
moe: StreamedMoe,
shared: SharedExpert,
}
static SECTIONS: std::sync::Mutex<([f64; 12], u64)> = std::sync::Mutex::new(([0.0; 12], 0));
const SECTION_NAMES: [&str; 12] = ["embed", "attn_norm", "linear_attn", "full_attn", "post_norm", "moe", "shared", "residual", "head", "la_proj", "la_mid", "la_out"];
fn sections_on() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_SECTIONS").map(|v| v == "1").unwrap_or(false))
}
fn tick(dev: &Device, t: &mut std::time::Instant, which: usize) -> Result<()> {
if !sections_on() {
return Ok(());
}
dev.synchronize()?;
let now = std::time::Instant::now();
SECTIONS.lock().unwrap().0[which] += (now - *t).as_secs_f64();
*t = now;
Ok(())
}
fn split_tick(dev: &Device, t: &mut std::time::Instant, which: usize) -> Result<()> {
dev.synchronize()?;
let now = std::time::Instant::now();
SECTIONS.lock().unwrap().0[which] += (now - *t).as_secs_f64();
*t = now;
Ok(())
}
/// Passes up to this many words take the one-word kernels (guess-and-check); longer ones the
/// prompt path. `COACHWHIP_SHORT=n`, default 1 (off) while in the lab.
fn short_max() -> usize {
static N: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*N.get_or_init(|| std::env::var("COACHWHIP_SHORT").ok().and_then(|v| v.parse().ok()).unwrap_or(1))
}
/// Query rows per slice when full attention reads a long piece (`COACHWHIP_ATTN_SLICE`, default 128).
fn attn_slice() -> usize {
static N: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*N.get_or_init(|| std::env::var("COACHWHIP_ATTN_SLICE").ok().and_then(|v| v.parse().ok()).filter(|&n: &usize| n > 0).unwrap_or(128))
}
fn piece_sync() -> usize {
static N: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*N.get_or_init(|| std::env::var("COACHWHIP_PIECE_SYNC").ok().and_then(|v| v.parse().ok()).unwrap_or(4))
}
fn skip(what: &str) -> bool {
static S: std::sync::OnceLock<Vec<String>> = std::sync::OnceLock::new();
S.get_or_init(|| std::env::var("COACHWHIP_SKIP").map(|v| v.split(',').map(|x| x.trim().to_string()).collect()).unwrap_or_default()).iter().any(|x| x == what)
}
fn prefill_sections() -> bool {
static P: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*P.get_or_init(|| std::env::var("COACHWHIP_SECTIONS_PREFILL").is_ok())
}
fn sec_l(l: usize) -> bool { if prefill_sections() { l > 1 } else { l <= 8 } }
fn word_done() {
if !sections_on() {
return;
}
let mut g = SECTIONS.lock().unwrap();
g.1 += 1;
if g.1 % (if prefill_sections() { 10 } else { 100 }) == 0 {
let w = g.1 as f64;
let line: Vec<String> = SECTION_NAMES.iter().zip(g.0.iter()).map(|(n, v)| format!("{n} {:.1}", 1000.0 * v / w)).collect();
let total: f64 = g.0[..9].iter().sum();
eprintln!("coachwhip sections, ms per word over {} words: {} | total {:.1}", g.1, line.join(" · "), 1000.0 * total / w);
}
}
/// The word table, packed as the file stores it; rows are unpacked when a forward asks for them.
struct PackedTable {
bytes: Vec<u8>,
dtype: GgmlDType,
row_bytes: usize,
hidden: usize,
}
impl PackedTable {
/// `ids` (any shape) -> (..., hidden) f32 on the CPU, rounded through f16 like the old table.
fn rows(&self, ids: &Tensor) -> Result<Tensor> {
let flat: Vec<u32> = ids.flatten_all()?.to_vec1()?;
let mut buf = Vec::with_capacity(flat.len() * self.row_bytes);
for &id in &flat {
let at = id as usize * self.row_bytes;
if at + self.row_bytes > self.bytes.len() {
candle::bail!("word {id} is outside the word table")
}
buf.extend_from_slice(&self.bytes[at..at + self.row_bytes]);
}
let q = qtensor_from_ggml(self.dtype, &buf, vec![flat.len(), self.hidden], &Device::Cpu)?;
let t = q.dequantize(&Device::Cpu)?.to_dtype(DType::F16)?.to_dtype(DType::F32)?;
let mut dims = ids.dims().to_vec();
dims.push(self.hidden);
t.reshape(dims)
}
}
pub struct Qwen3Next {
embeddings: PackedTable,
/// Lab (`COACHWHIP_GPUPICK=1`): the packed word table on the GPU, for ids that never leave it.
gpu_table: Option<Tensor>,
layers: Vec<Layer>,
norm: RmsNorm,
output: QMatMul,
store: Arc<ExpertStore>,
device: Device,
pub think: crate::model::ThinkOpen,
pub progress: Option<crate::model::Progress>,
}
impl Qwen3Next {
/// `arch` is the GGUF architecture name: `qwen3next` (Qwen3-Next, fused projections) or
/// `qwen35moe` (Qwen3.5 and 3.6, split projections). Same maths, different tensor layout.
pub fn load(path: &Path, device: &Device, settings: &Settings, arch: &str) -> Result<Self> {
let mut file = std::fs::File::open(path)?;
let ct = gguf_file::Content::read(&mut file).map_err(|e| e.with_path(path))?;
let md = |s: &str| match ct.metadata.get(s) {
None => candle::bail!("cannot find {s} in the GGUF metadata"),
Some(v) => Ok(v.clone()),
};
let md_u = |s: &str| -> Result<usize> { Ok(md(&format!("{arch}.{s}"))?.to_u32()? as usize) };
let split = ct.tensor_infos.contains_key("blk.0.attn_qkv.weight");
let n_head = md_u("attention.head_count")?;
let n_kv = md_u("attention.head_count_kv")?;
let hd = md_u("attention.key_length")?;
let rot = md_u("rope.dimension_count")?;
let block_count = md_u("block_count")?;
let top_k = md_u("expert_used_count")?;
// Fast mode (`--experts n`, `COACHWHIP_TOPK=n`): fewer experts per word than the model was trained with.
// Fewer experts = fewer bytes per word; the text changes, so quality is measured alongside.
let top_k = match std::env::var("COACHWHIP_TOPK").ok().and_then(|v| v.parse::<usize>().ok()) {
Some(k) if k >= 1 && k < top_k => {
eprintln!("coachwhip: fast mode: {k} experts per word (the model uses {top_k}); answers can differ from the model's own");
k
}
_ => top_k,
};
let embedding_length = md_u("embedding_length")?;
let d_state = md_u("ssm.state_size")?;
let n_group = md_u("ssm.group_count")?;
let dt_rank = md_u("ssm.time_step_rank")?;
let inner = md_u("ssm.inner_size")?;
let interval = md_u("full_attention_interval").unwrap_or(4);
let eps = md(&format!("{arch}.attention.layer_norm_rms_epsilon"))?.to_f32()? as f64;
// Some conversions leave the rope base out; this family's is ten million.
let freq_base = md(&format!("{arch}.rope.freq_base")).ok().and_then(|v| v.to_f32().ok()).map_or(1e7, |f| f as f64);
let template = md("tokenizer.chat_template").ok().and_then(|t| t.to_string().ok().cloned());
let think = crate::model::ThinkOpen::from_template(template.as_deref());
let store = ExpertStore::new(&ct, device, block_count, path, settings)?;
// Wrangler: the word table is read packed, onto the CPU only, before the rest loads.
let table = {
let q = ct.tensor(&mut file, "token_embd.weight", &Device::Cpu)?;
let dims = q.shape().dims().to_vec();
let hidden = *dims.last().unwrap();
let dtype = q.dtype();
let row_bytes = hidden / dtype.block_size() * dtype.type_size();
let bytes = q.data()?.into_owned();
PackedTable { bytes, dtype, row_bytes, hidden }
};
let mut gg = Gguf::new(ct, &mut file, device.clone());
let dense = |gg: &mut Gguf<&mut std::fs::File>, name: &str| -> Result<Tensor> { gg.tensor(name)?.dequantize(device)?.to_dtype(DType::F32) };
// Lab (`COACHWHIP_DENSE=q2k|q3k|q4k`): the dense weights are shrunk once at load, so the
// memory they give back can hold more expert slots. The answers change with them.
let dense_q: Option<GgmlDType> = match std::env::var("COACHWHIP_DENSE").ok().as_deref() {
Some("q2k") => Some(GgmlDType::Q2K),
Some("q3k") => Some(GgmlDType::Q3K),
Some("q4k") => Some(GgmlDType::Q4K),
_ => None,
};
let shrunk = std::cell::Cell::new(0usize);
let saved = std::cell::Cell::new(0usize);
let qm = |gg: &mut Gguf<&mut std::fs::File>, name: &str| -> Result<QMatMul> {
let t = gg.tensor(name)?;
let Some(dt) = dense_q else { return QMatMul::from_weights(Arc::new(t)) };
// Only shrink what is bigger than the target and quantised to begin with.
if t.dtype().block_size() == 1 || t.dtype().type_size() * dt.block_size() <= dt.type_size() * t.dtype().block_size() {
return QMatMul::from_weights(Arc::new(t));
}
let dims = t.shape().dims().to_vec();
if dims.last().map_or(true, |&k| k % dt.block_size() != 0) {
return QMatMul::from_weights(Arc::new(t));
}
let before = t.storage_size_in_bytes();
let f = t.dequantize(&Device::Cpu)?.to_dtype(DType::F32)?;
let q = QTensor::quantize(&f, dt)?;
let bytes = q.data()?;
let on_device = qtensor_from_ggml(dt, &bytes, dims, device)?;
shrunk.set(shrunk.get() + 1);
saved.set(saved.get() + before.saturating_sub(on_device.storage_size_in_bytes()));
QMatMul::from_weights(Arc::new(on_device))
};
let norm = gg.rms_norm("output_norm.weight", eps)?;
// The output head goes through the shrink in slices (conveyor belt): dequantise 8,192 rows,
// shrink them, stack the bytes, so the load never holds more than one slice in f32 on the CPU.
let head_q: Option<GgmlDType> = match std::env::var("COACHWHIP_DENSE_HEAD").ok().as_deref() {
Some("q2k") => Some(GgmlDType::Q2K),
Some("q3k") => Some(GgmlDType::Q3K),
Some("q4k") => Some(GgmlDType::Q4K),
Some("off") => None,
_ => dense_q,
};
let head_name = if gg.qmatmul("output.weight").is_ok() { "output.weight" } else { "token_embd.weight" };
let output = match head_q {
Some(dt) => {
let t = gg.tensor(head_name)?;
let dims = t.shape().dims().to_vec();
let before = t.storage_size_in_bytes();
if t.dtype().block_size() == 1 || dims.len() != 2 || dims[1] % dt.block_size() != 0 || t.dtype().type_size() * dt.block_size() <= dt.type_size() * t.dtype().block_size() {
QMatMul::from_weights(Arc::new(t))?
} else {
let f = t.dequantize(device)?.to_dtype(DType::F32)?; // on the GPU, once
drop(t);
let rows = dims[0];
let step = 8192usize;
let mut bytes: Vec<u8> = Vec::with_capacity(rows / dt.block_size() * dims[1] * dt.type_size() / dims[1].max(1) * dims[1] / dt.block_size() * dt.type_size() / 1);
let mut r = 0;
while r < rows {
let n = step.min(rows - r);
let slice = f.narrow(0, r, n)?.to_device(&Device::Cpu)?;
let q = QTensor::quantize(&slice, dt)?;
bytes.extend_from_slice(&q.data()?);
r += n;
}
drop(f);
let on_device = qtensor_from_ggml(dt, &bytes, dims, device)?;
shrunk.set(shrunk.get() + 1);
saved.set(saved.get() + before.saturating_sub(on_device.storage_size_in_bytes()));
eprintln!("coachwhip: output head shrunk in slices: {:.2} GB -> {:.2} GB", before as f64 / 1e9, on_device.storage_size_in_bytes() as f64 / 1e9);
QMatMul::from_weights(Arc::new(on_device))?
}
}
None => match gg.qmatmul("output.weight") {
Ok(v) => v,
_ => gg.qmatmul("token_embd.weight")?,
},
};
let mut layers = Vec::with_capacity(block_count);
for layer in 0..block_count {
let p = format!("blk.{layer}");
let mixer = if (layer + 1) % interval == 0 {
Mixer::Full(FullAttn {
wq: qm(&mut gg, &format!("{p}.attn_q.weight"))?,
wk: qm(&mut gg, &format!("{p}.attn_k.weight"))?,
wv: qm(&mut gg, &format!("{p}.attn_v.weight"))?,
wo: qm(&mut gg, &format!("{p}.attn_output.weight"))?,
q_norm: gg.rms_norm(&format!("{p}.attn_q_norm.weight"), eps)?,
k_norm: gg.rms_norm(&format!("{p}.attn_k_norm.weight"), eps)?,
n_head,
n_kv,
hd,
rot,
freq_base,
kv_cache: KvBuf::new(4096),
})
} else {
let conv_w = dense(&mut gg, &format!("{p}.ssm_conv1d.weight"))?; // (channels, taps)
let proj = if split {
Proj::Split {
qkv: qm(&mut gg, &format!("{p}.attn_qkv.weight"))?,
z: qm(&mut gg, &format!("{p}.attn_gate.weight"))?,
beta: qm(&mut gg, &format!("{p}.ssm_beta.weight"))?,
alpha: qm(&mut gg, &format!("{p}.ssm_alpha.weight"))?,
}
} else {
Proj::Fused { qkvz: qm(&mut gg, &format!("{p}.ssm_in.weight"))?, ba: qm(&mut gg, &format!("{p}.ssm_ba.weight"))? }
};
Mixer::Linear(LinearAttn {
proj,
conv_w,
dt_bias: dense(&mut gg, &format!("{p}.ssm_dt.bias"))?,
a: dense(&mut gg, &format!("{p}.ssm_a"))?,
norm: gg.rms_norm(&format!("{p}.ssm_norm.weight"), eps)?,
norm_w: dense(&mut gg, &format!("{p}.ssm_norm.weight"))?.to_dtype(DType::F32)?,
out: qm(&mut gg, &format!("{p}.ssm_out.weight"))?,
n_k: n_group,
n_v: dt_rank,
dk: d_state,
dv: inner / dt_rank,
eps,
conv_state: None,
state: None,
snapshot: None,
record: false,
trail: None,
})
};
let gate = dense(&mut gg, &format!("{p}.ffn_gate_inp.weight"))?;
let moe = StreamedMoe {
act: crate::experts::ExpertAct::Silu,
layer,
gate: Linear::new(gate, None),
next_gate: None,
gate1: None,
ahead: settings.ahead.max(1),
store: store.clone(),
top_k,
norm_topk_prob: true,
prefetch_k: settings.prefetch,
};
let shared = SharedExpert {
gate_inp: dense(&mut gg, &format!("{p}.ffn_gate_inp_shexp.weight"))?,
gate: qm(&mut gg, &format!("{p}.ffn_gate_shexp.weight"))?,
up: qm(&mut gg, &format!("{p}.ffn_up_shexp.weight"))?,
down: qm(&mut gg, &format!("{p}.ffn_down_shexp.weight"))?,
};
layers.push(Layer {
mixer,
attn_norm: gg.rms_norm(&format!("{p}.attn_norm.weight"), eps)?,
post_norm: gg.rms_norm(&format!("{p}.post_attention_norm.weight"), eps)?,
attn_norm_w: dense(&mut gg, &format!("{p}.attn_norm.weight"))?,
post_norm_w: dense(&mut gg, &format!("{p}.post_attention_norm.weight"))?,
norm_eps: eps as f32,
moe,
shared,
});
}
if dense_q.is_some() {
eprintln!("coachwhip: dense weights shrunk at load: {} tensors, {:.2} GB given back to the bank", shrunk.get(), saved.get() as f64 / 1e9);
}
wire_lookahead(&mut layers, settings.ahead);
if let Ok(path) = std::env::var("COACHWHIP_PREDICTOR") {
load_predictor(&mut layers, &path, settings.ahead.max(1), device)?;
}
let gpu_table = if crate::fuse_kernel::gpupick_on() && device.is_metal() && table.dtype == GgmlDType::Q5K {
let t = Tensor::from_slice(&table.bytes, table.bytes.len(), device)?;
eprintln!("coachwhip: word table on the GPU too, {:.2} GB", table.bytes.len() as f64 / 1e9);
Some(t)
} else {
None
};
Ok(Self { embeddings: table, gpu_table, layers, norm, output, store, device: device.clone(), think, progress: None })
}
pub fn report(&self) {
self.store.report_and_reset();
}
/// Forget the conversation: the classic layers' caches and the running memories together.
pub fn clear_kv_cache(&mut self) {
for layer in self.layers.iter_mut() {
match &mut layer.mixer {
Mixer::Full(a) => a.kv_cache.reset(),
Mixer::Linear(a) => a.reset(),
}
}
}
/// The conversation's whole state, layer by layer, as (shape, f32 values); an empty shape for a
/// memory not yet started. Waits for the GPU once.
pub fn save_state(&self) -> Result<Vec<(Vec<usize>, Vec<f32>)>> {
let part = |t: Option<&Tensor>| -> Result<(Vec<usize>, Vec<f32>)> {
match t {
Some(t) => Ok((t.dims().to_vec(), t.to_dtype(DType::F32)?.flatten_all()?.to_vec1::<f32>()?)),
None => Ok((vec![], vec![])),
}
};
let mut out = Vec::new();
for layer in self.layers.iter() {
match &layer.mixer {
Mixer::Linear(a) => {
out.push(part(a.conv_state.as_ref())?);
out.push(part(a.state.as_ref())?);
}
Mixer::Full(a) => {
let kv = &a.kv_cache;
match (&kv.k, &kv.v) {
(Some(k), Some(v)) if kv.len > 0 => {
out.push(part(Some(&k.narrow(2, 0, kv.len)?.contiguous()?))?);
out.push(part(Some(&v.narrow(2, 0, kv.len)?.contiguous()?))?);
}
_ => { out.push((vec![], vec![])); out.push((vec![], vec![])); }
}
}
}
}
Ok(out)
}
/// Puts back a state from [`Self::save_state`]; the model then stands where that conversation ended.
pub fn load_state(&mut self, parts: &[(Vec<usize>, Vec<f32>)]) -> Result<()> {
if parts.len() != 2 * self.layers.len() {
candle::bail!("snapshot holds {} parts, this model needs {}", parts.len(), 2 * self.layers.len())
}
self.clear_kv_cache();
let device = self.device.clone();
let t = |p: &(Vec<usize>, Vec<f32>)| -> Result<Option<Tensor>> {
if p.0.is_empty() { Ok(None) } else { Ok(Some(Tensor::from_slice(&p.1, p.0.as_slice(), &device)?)) }
};
for (i, layer) in self.layers.iter_mut().enumerate() {
let (a0, a1) = (t(&parts[2 * i])?, t(&parts[2 * i + 1])?);
match &mut layer.mixer {
Mixer::Linear(a) => { a.conv_state = a0; a.state = a1; }
Mixer::Full(a) => {
if let (Some(k), Some(v)) = (a0, a1) {
a.kv_cache.append(&k, &v)?;
}
}
}
}
Ok(())
}
/// S6 measurement. Drafts `k` words after `last` with routing confined to the bank (no reads),
/// takes the memories back, then writes the same `k` words the plain way and counts how many
/// drafted words the plain path agrees with, in order. Returns the plain words (the model has
/// read all but the last, exactly as after `k` plain steps) and the accepted count.
pub fn speculate_measure(&mut self, last: u32, offset: usize, k: usize) -> Result<(Vec<u32>, usize)> {
let argmax = |logits: Tensor| -> Result<u32> { Ok(logits.argmax(D::Minus1)?.to_vec1::<u32>()?[0]) };
// A cold bank (right after a prompt read outside it) has nothing to draft from: write the
// words the plain way and count nothing; the next block drafts.
let cold = self.layers.iter().any(|l| self.store.resident_ready(l.moe.layer).map_or(true, |r| r.len() < l.moe.top_k));
if cold {
let mut plain = Vec::with_capacity(k);
let mut t = last;
for i in 0..k {
let x = Tensor::new(&[t], &self.device)?.unsqueeze(0)?;
t = argmax(self.forward(&x, offset + i)?)?;
plain.push(t);
}
return Ok((plain, 0));
}
let kv_len = self.layers.iter().find_map(|l| match &l.mixer {
Mixer::Full(a) => Some(a.kv_cache.len),
_ => None,
});
for layer in self.layers.iter_mut() {
if let Mixer::Linear(a) = &mut layer.mixer {
a.take_snapshot();
}
}
self.store.set_draft(true);
let mut draft = Vec::with_capacity(k);
let mut t = last;
let drafted: Result<()> = (|| {
for i in 0..k {
let x = Tensor::new(&[t], &self.device)?.unsqueeze(0)?;
t = argmax(self.forward(&x, offset + i)?)?;
draft.push(t);
}
Ok(())
})();
self.store.set_draft(false);
for layer in self.layers.iter_mut() {
match &mut layer.mixer {
Mixer::Linear(a) => a.restore_snapshot(),
Mixer::Full(a) => a.kv_cache.truncate(kv_len.unwrap_or(0)),
}
}
drafted?;
// The plain way, word by word: the truth the draft is scored against.
let mut plain = Vec::with_capacity(k);
let mut t = last;
for i in 0..k {
let x = Tensor::new(&[t], &self.device)?.unsqueeze(0)?;
t = argmax(self.forward(&x, offset + i)?)?;
plain.push(t);
}
let accepted = draft.iter().zip(plain.iter()).take_while(|(d, p)| d == p).count();
self.store.note_speculation(k, accepted);
Ok((plain, accepted))
}
/// S6: draft `k` words from the bank, check them all in one plain pass, keep the ones the
/// model agrees with and the corrected word after them, and rewind the memories to match.
/// Returns None when speculation cannot run (cold bank); otherwise the words to emit, the
/// last of which the model has not read yet, exactly as after that many plain steps.
pub fn speculate(&mut self, last: u32, offset: usize, k: usize) -> Result<Option<Vec<u32>>> {
let argmax = |logits: Tensor| -> Result<u32> { Ok(logits.argmax(D::Minus1)?.to_vec1::<u32>()?[0]) };
if k == 0 || self.layers.iter().any(|l| self.store.resident_ready(l.moe.layer).map_or(true, |r| r.len() < l.moe.top_k)) {
return Ok(None);
}
let kv_len = self.layers.iter().find_map(|l| match &l.mixer {
Mixer::Full(a) => Some(a.kv_cache.len),
_ => None,
}).unwrap_or(0);
for layer in self.layers.iter_mut() {
if let Mixer::Linear(a) = &mut layer.mixer {
a.take_snapshot();
}
}
// Draft.
self.store.set_draft(true);
let mut draft = Vec::with_capacity(k);
let mut t = last;
let drafted: Result<()> = (|| {
for i in 0..k {
let x = Tensor::new(&[t], &self.device)?.unsqueeze(0)?;
t = argmax(self.forward(&x, offset + i)?)?;
draft.push(t);
}
Ok(())
})();
self.store.set_draft(false);
for layer in self.layers.iter_mut() {
match &mut layer.mixer {
Mixer::Linear(a) => {
a.restore_snapshot();
a.record = true;
}
Mixer::Full(a) => a.kv_cache.truncate(kv_len),
}
}
drafted?;
// Load: the last word and the draft in one pass. Its only job is to bring every expert
// those words need into the bank in one batched read; its arithmetic is thrown away, so
// the words below come from the exact same path plain decoding takes.
let mut words = Vec::with_capacity(k + 1);
words.push(last);
words.extend_from_slice(&draft);
let x = Tensor::new(words.as_slice(), &self.device)?.unsqueeze(0)?;
let loaded = self.forward_inner(&x, offset, false);
for layer in self.layers.iter_mut() {
match &mut layer.mixer {
Mixer::Linear(a) => {
a.record = false;
a.rewind(0)?;
}
Mixer::Full(a) => a.kv_cache.truncate(kv_len),
}
}
loaded?;
// Check, the plain way, word by word, with the bank hot: keep a drafted word only while
// the plain path says the same; the first disagreement is the plain word, and ends it.
let mut seq = Vec::with_capacity(k + 1);
let mut t = last;
for i in 0..=k {
let x = Tensor::new(&[t], &self.device)?.unsqueeze(0)?;
let w = argmax(self.forward(&x, offset + i)?)?;
seq.push(w);
if i < k && w == draft[i] {
t = w;
} else {
break;
}
}
let accepted = seq.len() - 1;
self.store.note_speculation(k, accepted);
Ok(Some(seq))
}
/// Foresight: one bank-only draft pass of `last`, whose only product is the reads it starts for
/// the real pass that follows. Nothing it computes is kept. False when the bank is cold.
pub fn foresee(&mut self, last: u32, offset: usize, steps: usize) -> Result<bool> {
let argmax = |logits: Tensor| -> Result<u32> { Ok(logits.argmax(D::Minus1)?.to_vec1::<u32>()?[0]) };
if self.layers.iter().any(|l| self.store.resident_ready(l.moe.layer).map_or(true, |r| r.len() < l.moe.top_k)) {
return Ok(false);
}
let kv_len = self.layers.iter().find_map(|l| match &l.mixer {
Mixer::Full(a) => Some(a.kv_cache.len),
_ => None,
}).unwrap_or(0);
for layer in self.layers.iter_mut() {
if let Mixer::Linear(a) = &mut layer.mixer {
a.take_snapshot();
}
}
self.store.set_draft(true);
let mut t = last;
let drafted: Result<()> = (|| {
let steps = steps.max(1);
for i in 0..steps {
self.store.set_foresee(true, i as u64);
let x = Tensor::new(&[t], &self.device)?.unsqueeze(0)?;
if i + 1 == steps {
// The last step's word is never needed, only the reads its layers started.
self.body(&x, offset + i)?;
} else {
t = argmax(self.forward(&x, offset + i)?)?;
}
let top_k = self.layers[0].moe.top_k;
self.store.flush_foresight(top_k)?;
}
Ok(())
})();
self.store.set_foresee(false, 0);
self.store.set_draft(false);
for layer in self.layers.iter_mut() {
match &mut layer.mixer {
Mixer::Linear(a) => a.restore_snapshot(),
Mixer::Full(a) => a.kv_cache.truncate(kv_len),
}
}
drafted?;
Ok(true)
}
pub fn forward(&mut self, x: &Tensor, offset: usize) -> Result<Tensor> {
self.forward_inner(x, offset, false)
}
/// `all`: scores for every position, not only the last.
fn forward_inner(&mut self, x: &Tensor, offset: usize, all: bool) -> Result<Tensor> {
let enc_t = std::time::Instant::now();
let (xs, l) = self.body(x, offset)?;
if l == 1 && std::env::var("COACHWHIP_ENCODE_TIME").is_ok() {
static E: std::sync::Mutex<(f64, u64)> = std::sync::Mutex::new((0.0, 0));
let mut g = E.lock().unwrap();
g.0 += enc_t.elapsed().as_secs_f64();
g.1 += 1;
if g.1 % 100 == 0 {
use std::sync::atomic::Ordering as O;
let m = candle_metal_kernels::metal::commands::N_DISPATCH.load(O::Relaxed);
let e = candle_metal_kernels::metal::commands::N_ENC.load(O::Relaxed);
let b = candle_metal_kernels::metal::commands::N_BLIT.load(O::Relaxed);
let br = candle_metal_kernels::metal::commands::N_BARRIER.load(O::Relaxed);
let c = candle_metal_kernels::metal::commands::N_COMMIT.load(O::Relaxed);
let gns = candle_metal_kernels::metal::commands::GPU_NS.load(O::Relaxed);
let nw = candle_metal_kernels::metal::commands::N_WAIT.load(O::Relaxed);
let wn = candle_metal_kernels::metal::commands::WAIT_NS.load(O::Relaxed);
let gap = candle_metal_kernels::metal::commands::GAP_NS.load(O::Relaxed);
let bg = candle_metal_kernels::metal::commands::BIG_GAPS.load(O::Relaxed);
let bgn = candle_metal_kernels::metal::commands::BIG_GAP_NS.load(O::Relaxed);
eprintln!("coachwhip barriers: {br} barriers, {c} commits, GPU busy {:.3} s, idle between buffers {:.3} s ({bg} gaps over 0.5 ms = {:.3} s), {nw} CPU waits taking {:.3} s, since start", gns as f64 / 1e9, gap as f64 / 1e9, bgn as f64 / 1e9, wn as f64 / 1e9);
eprintln!("coachwhip encode: {:.1} ms of CPU per token to queue the layers (no GPU wait), over {} tokens; since start: {m} dispatches, {e} compute encoders, {b} blit encoders", 1000.0 * g.0 / g.1 as f64, g.1);
}
}
if sec_l(l) && !all && sections_on() {
let mut t = std::time::Instant::now();
let xs = self.norm.forward(&xs.narrow(1, l - 1, 1)?)?;
let out = self.output.forward(&xs)?.to_dtype(DType::F32)?.squeeze(1)?;
tick(&self.device, &mut t, 8)?;
word_done();
return Ok(out);
}
if all {
let xs = self.norm.forward(&xs)?;
return self.output.forward(&xs)?.to_dtype(DType::F32)?.squeeze(0);
}
let xs = self.norm.forward(&xs.narrow(1, l - 1, 1)?)?;
if l == 1 && skip("head") {
return Tensor::zeros((1, 248320), DType::F32, xs.device());
}
self.output.forward(&xs)?.to_dtype(DType::F32)?.squeeze(1)
}
/// The layers only, no output head: the hidden state and the number of positions.
fn body(&mut self, x: &Tensor, offset: usize) -> Result<(Tensor, usize)> {
let on_gpu = x.device().is_metal() && self.gpu_table.is_some();
if !self.store.drafting() && !on_gpu {
self.store.note_tokens(x.flatten_all()?.to_vec1()?);
}
let mut t = std::time::Instant::now();
let mut xs = if on_gpu {
let (_, l0) = x.dims2()?;
let h = self.embeddings.hidden;
crate::fuse_kernel::embed_q5k(self.gpu_table.as_ref().unwrap(), x, h, self.embeddings.row_bytes)?.reshape((1, l0, h))?
} else {
self.embeddings.rows(&x.to_device(&Device::Cpu)?)?.to_device(&self.device)?
};
let (_b, l) = x.dims2()?;
let timed = sec_l(l) && sections_on();
if timed { tick(&self.device, &mut t, 0)?; }
let mask = if l == 1 {
None
} else {
Some(candle_transformers::utils::build_additive_causal_mask(l, offset, None, &self.device, DType::F32)?)
};
let n = self.layers.len();
let fuse_norm = crate::fuse_kernel::addnorm_on() && xs.dtype() == DType::F32;
let next_w: Vec<Tensor> = if fuse_norm { self.layers.iter().map(|l| l.attn_norm_w.clone()).collect() } else { Vec::new() };
let mut carried: Option<Tensor> = None;
for (i, layer) in self.layers.iter_mut().enumerate() {
let normed = if let Some(nm) = carried.take() { nm } else if l == 1 && skip("norms") { xs.clone() } else { layer.attn_norm.forward(&xs)? };
if timed { tick(&self.device, &mut t, 1)?; }
let (mixed, which) = match &mut layer.mixer {
Mixer::Linear(a) => (a.forward(&normed)?, 2),
Mixer::Full(_) if l == 1 && skip("fullattn") => (normed.zeros_like()?, 3),
Mixer::Full(a) => (a.forward(&normed, mask.as_ref(), offset)?, 3),
};
if timed { tick(&self.device, &mut t, which)?; }
let (h, post) = if fuse_norm {
crate::fuse_kernel::add_norm(&mixed, &xs, None, &layer.post_norm_w, layer.norm_eps)?
} else {
let h = (mixed + &xs)?;
let post = if l == 1 && skip("norms") { h.clone() } else { layer.post_norm.forward(&h)? };
(h, post)
};
if timed { tick(&self.device, &mut t, 4)?; }
let routed = if l == 1 && skip("moe") { post.zeros_like()? } else { layer.moe.forward(&post)? };
if timed { tick(&self.device, &mut t, 5)?; }
let shared = if l == 1 && skip("shared") { post.zeros_like()? } else { layer.shared.forward(&post)? };
if timed { tick(&self.device, &mut t, 6)?; }
if fuse_norm && i + 1 < n {
let (x2, nn) = crate::fuse_kernel::add_norm(&routed, &shared, Some(&h), &next_w[i + 1], layer.norm_eps)?;
xs = x2;
carried = Some(nn);
} else {
xs = ((routed + shared)? + h)?;
}
if timed { tick(&self.device, &mut t, 7)?; }
// Reading a prompt piece: hand scratch memory back every few layers, so a big piece never
// holds 40 layers of scratch at once (`COACHWHIP_PIECE_SYNC`, layers, default 4; 0 = off).
if l > 1 {
let every = piece_sync();
if every > 0 && (i + 1) % every == 0 {
self.device.synchronize()?;
}
}
if l > 1 {
if let Some(p) = &mut self.progress {
p(crate::model::Step::Layer(i + 1, n));
}
}
}
Ok((xs, l))
}
}