The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
//! Qwen3 mixture-of-experts, read from a GGUF file. Everything except the experts is loaded onto
//! the GPU at start-up; the experts stay in the file and stream through the ExpertStore.
//!
//! Derived from Candle's `quantized_qwen3_moe.rs` (Apache-2.0 / MIT). Changes: the experts are
//! streamed; the KV cache is one preallocated buffer per layer; grouped-query attention runs
//! without copying K and V per head; the token embedding table lives on the CPU.
use crate::experts::{ExpertStore, Settings, StreamedMoe};
use candle::quantized::gguf_file;
use candle::{DType, Device, Module, Result, Tensor};
use candle_nn::kv_cache::KvCache;
use candle_nn::{Embedding, Linear};
use candle_transformers::models::quantized_qwen3::{Gguf, RotaryEmbedding};
use candle_transformers::models::with_tracing::QMatMul;
use candle_transformers::quantized_nn::RmsNorm;
use std::path::Path;
use std::sync::Arc;
struct Attention {
wq: QMatMul,
wk: QMatMul,
wv: QMatMul,
wo: QMatMul,
q_norm: RmsNorm,
k_norm: RmsNorm,
n_head: usize,
n_kv_head: usize,
head_dim: usize,
rotary_emb: Arc<RotaryEmbedding>,
dtype: DType,
kv_cache: KvCache,
}
impl Attention {
fn forward(&mut self, x: &Tensor, mask: Option<&Tensor>, input_pos: usize) -> Result<Tensor> {
let (b, seq_len, _) = x.dims3()?;
let in_dtype = x.dtype();
let heads = |t: Tensor, n: usize| -> Result<Tensor> { t.reshape((1, seq_len, n, self.head_dim))?.transpose(1, 2)?.contiguous() };
let q = heads(self.wq.forward(x)?, self.n_head)?;
let k = heads(self.wk.forward(x)?, self.n_kv_head)?;
let v = heads(self.wv.forward(x)?, self.n_kv_head)?;
// Per-head RMSNorm, as Qwen3 does it.
let q = self.q_norm.forward(&q.flatten(0, 2)?)?.reshape((1, self.n_head, seq_len, self.head_dim))?;
let k = self.k_norm.forward(&k.flatten(0, 2)?)?.reshape((1, self.n_kv_head, seq_len, self.head_dim))?;
let (q, k, v) = (q.to_dtype(self.dtype)?, k.to_dtype(self.dtype)?, v.to_dtype(self.dtype)?);
let (q, k) = self.rotary_emb.apply(&q, &k, input_pos)?;
// One preallocated buffer per layer: append writes the new rows in place and hands back
// views, so nothing is copied per word.
let (k, v) = self.kv_cache.append(&k, &v)?;
let s_len = k.dim(2)?;
// Grouped-query attention without repeating K and V: the G query heads that share one kv
// head are stacked as G*L rows of one batched matmul.
let g = self.n_head / self.n_kv_head;
let q = q.contiguous()?.reshape((1, self.n_kv_head, g * seq_len, self.head_dim))?;
let scale = 1.0 / (self.head_dim as f64).sqrt();
let mut scores = (q.matmul(&k.transpose(2, 3)?)? * scale)?; // (1, KVH, G*L, S)
if let Some(m) = mask {
let mask = m.to_dtype(scores.dtype())?.unsqueeze(1)?; // (1, 1, 1, L, S)
scores = scores
.reshape((1, self.n_kv_head, g, seq_len, s_len))?
.broadcast_add(&mask)?
.reshape((1, self.n_kv_head, g * seq_len, s_len))?;
}
let probs = candle_nn::ops::softmax_last_dim(&scores)?;
let ctx = probs.matmul(&v)?.reshape((1, self.n_head, seq_len, self.head_dim))?;
let ctx = ctx.transpose(1, 2)?.reshape((b, seq_len, self.n_head * self.head_dim))?;
self.wo.forward(&ctx.to_dtype(in_dtype)?)
}
}
struct Layer {
attn: Attention,
attn_norm: RmsNorm,
moe: StreamedMoe,
ffn_norm: RmsNorm,
}
/// What the chat template puts right after `<|im_start|>assistant\n`. A thinking model's template
/// opens the turn with an empty think block when thinking is off, so the model answers straight
/// away; Qwen3.5 and later open a think block for it when thinking is on. Both empty for a model
/// with no thinking mode.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ThinkOpen {
pub off: &'static str,
pub on: &'static str,
}
impl ThinkOpen {
pub fn from_template(template: Option<&str>) -> Self {
let t = template.unwrap_or("");
if !t.contains("enable_thinking") {
return Self { off: "", on: "" };
}
Self {
off: if t.contains(r"'<think>\n\n</think>\n\n'") { "<think>\n\n</think>\n\n" } else { "" },
on: if t.contains(r"{{- '<think>\n' }}") { "<think>\n" } else { "" },
}
}
pub fn thinks(&self) -> bool {
!self.off.is_empty()
}
}
pub struct Qwen3Moe {
embeddings: Embedding,
layers: Vec<Layer>,
norm: RmsNorm,
output: QMatMul,
store: Arc<ExpertStore>,
dtype: DType,
device: Device,
pub think: ThinkOpen,
pub progress: Option<Progress>,
}
/// Told what the model is doing while the page would otherwise be silent.
pub type Progress = Box<dyn FnMut(Step)>;
pub enum Step {
/// (layers done, layers in all) while a prompt is being read.
Layer(usize, usize),
/// Words written so far of a summary that makes room in a long chat.
Summary(usize),
}
impl Qwen3Moe {
pub fn load(path: &Path, device: &Device, dtype: DType, settings: &Settings) -> Result<Self> {
let mut file = std::fs::File::open(path)?;
let ct = gguf_file::Content::read(&mut file).map_err(|e| e.with_path(path))?;
let md = |s: &str| match ct.metadata.get(s) {
None => candle::bail!("cannot find {s} in the GGUF metadata"),
Some(v) => Ok(v.clone()),
};
let arch = md("general.architecture")?.to_string()?.clone();
let md_u = |s: &str| -> Result<usize> { Ok(md(&format!("{arch}.{s}"))?.to_u32()? as usize) };
let head_count = md_u("attention.head_count")?;
let head_count_kv = md_u("attention.head_count_kv")?;
let embedding_length = md_u("embedding_length")?;
let head_dim = md_u("attention.key_length").unwrap_or(embedding_length / head_count);
let context_length = md_u("context_length")?;
let block_count = md_u("block_count")?;
let top_k = md_u("expert_used_count")?;
let rms_norm_eps = md(&format!("{arch}.attention.layer_norm_rms_epsilon"))?.to_f32()? as f64;
let rope_freq_base = md(&format!("{arch}.rope.freq_base")).and_then(|m| m.to_f32()).unwrap_or(10000f32);
let shared = md_u("expert_shared_feed_forward_length").unwrap_or(0);
let norm_topk_prob = shared == 0;
let template = md("tokenizer.chat_template").ok().and_then(|t| t.to_string().ok().cloned());
let think = ThinkOpen::from_template(template.as_deref());
let store = ExpertStore::new(&ct, device, block_count, path, settings)?;
let mut gg = Gguf::new(ct, &mut file, device.clone());
let embeddings = gg.tensor("token_embd.weight")?.dequantize(device)?.to_dtype(DType::F16)?.to_device(&Device::Cpu)?;
let norm = gg.rms_norm("output_norm.weight", rms_norm_eps)?;
let output = match gg.qmatmul("output.weight") {
Ok(v) => v,
_ => gg.qmatmul("token_embd.weight")?,
};
let rotary_emb = Arc::new(RotaryEmbedding::new(dtype, head_dim, context_length, rope_freq_base as f64, device)?);
let mut layers = Vec::with_capacity(block_count);
for layer in 0..block_count {
let p = format!("blk.{layer}");
let gate = gg.tensor(&format!("{p}.ffn_gate_inp.weight"))?.dequantize(device)?.to_dtype(DType::F32)?;
let moe = StreamedMoe {
act: crate::experts::ExpertAct::Silu,
layer,
gate: Linear::new(gate, None),
next_gate: None,
gate1: None,
ahead: settings.ahead.max(1),
store: store.clone(),
top_k,
norm_topk_prob,
prefetch_k: settings.prefetch,
};
let attn = Attention {
wq: gg.qmatmul(&format!("{p}.attn_q.weight"))?,
wk: gg.qmatmul(&format!("{p}.attn_k.weight"))?,
wv: gg.qmatmul(&format!("{p}.attn_v.weight"))?,
wo: gg.qmatmul(&format!("{p}.attn_output.weight"))?,
q_norm: gg.rms_norm(&format!("{p}.attn_q_norm.weight"), rms_norm_eps)?,
k_norm: gg.rms_norm(&format!("{p}.attn_k_norm.weight"), rms_norm_eps)?,
n_head: head_count,
n_kv_head: head_count_kv,
head_dim,
rotary_emb: rotary_emb.clone(),
dtype,
kv_cache: KvCache::new(2, 4096),
};
layers.push(Layer {
attn,
attn_norm: gg.rms_norm(&format!("{p}.attn_norm.weight"), rms_norm_eps)?,
moe,
ffn_norm: gg.rms_norm(&format!("{p}.ffn_norm.weight"), rms_norm_eps)?,
});
}
// Each layer gets the router of the layer `ahead` on, to guess its experts early; the last
// `ahead` layers have no later router and use the learned route table.
let ahead = settings.ahead.max(1);
for i in ahead..layers.len() {
layers[i - ahead].moe.next_gate = Some(layers[i].moe.gate.clone());
}
Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, dtype, device: device.clone(), think, progress: None })
}
/// Print where each written word's time went since the last call, then reset the counters.
pub fn report(&self) {
self.store.report_and_reset();
}
/// Forget the conversation so the next prompt starts at position 0.
pub fn clear_kv_cache(&mut self) {
for layer in self.layers.iter_mut() {
layer.attn.kv_cache.reset();
}
}
pub fn forward(&mut self, x: &Tensor, offset: usize) -> Result<Tensor> {
let mut xs = self.embeddings.forward(&x.to_device(&Device::Cpu)?)?.to_device(&self.device)?.to_dtype(DType::F32)?;
let (_b, l) = x.dims2()?;
let mask = if l == 1 {
None
} else {
Some(candle_transformers::utils::build_additive_causal_mask(l, offset, None, &self.device, self.dtype)?)
};
let n = self.layers.len();
for (i, layer) in self.layers.iter_mut().enumerate() {
let residual = &xs;
let h = (layer.attn.forward(&layer.attn_norm.forward(residual)?, mask.as_ref(), offset)? + residual)?;
let out = (layer.moe.forward(&layer.ffn_norm.forward(&h)?)? + &h)?;
xs = out;
if l > 1 {
if let Some(p) = &mut self.progress {
p(Step::Layer(i + 1, n));
}
}
}
let xs = self.norm.forward(&xs.narrow(1, l - 1, 1)?)?;
self.output.forward(&xs)?.to_dtype(DType::F32)?.squeeze(1)
}
}
/// Every model family Coachwhip runs, picked from the GGUF file's architecture.
pub enum Model {
Qwen3Moe(Qwen3Moe),
Qwen3Next(crate::model_next::Qwen3Next),
GptOss(crate::model_gptoss::GptOss),
}
impl Model {
pub fn load(path: &Path, device: &Device, settings: &Settings) -> Result<Self> {
let mut file = std::fs::File::open(path)?;
let ct = gguf_file::Content::read(&mut file).map_err(|e| e.with_path(path))?;
let arch = match ct.metadata.get("general.architecture") {
Some(v) => v.to_string()?.clone(),
None => candle::bail!("cannot find general.architecture in the GGUF metadata"),
};
drop(ct);
match arch.as_str() {
// f16 for attention: bf16 garbles long answers in this model.
"qwen3moe" => Ok(Self::Qwen3Moe(Qwen3Moe::load(path, device, DType::F16, settings)?)),
"qwen3next" | "qwen35moe" => Ok(Self::Qwen3Next(crate::model_next::Qwen3Next::load(path, device, settings, arch.as_str())?)),
"gpt-oss" => Ok(Self::GptOss(crate::model_gptoss::GptOss::load(path, device, settings)?)),
other => candle::bail!(
"this model's architecture is \"{other}\", which Coachwhip does not support yet (today: qwen3moe, qwen3next). \
Open an issue at https://github.com/1picassoai/coachwhip/issues so it moves up the list."
),
}
}
pub fn forward(&mut self, x: &Tensor, offset: usize) -> Result<Tensor> {
match self {
Self::Qwen3Moe(m) => m.forward(x, offset),
Self::Qwen3Next(m) => m.forward(x, offset),
Self::GptOss(m) => m.forward(x, offset),
}
}
pub fn clear_kv_cache(&mut self) {
match self {
Self::Qwen3Moe(m) => m.clear_kv_cache(),
Self::Qwen3Next(m) => m.clear_kv_cache(),
Self::GptOss(m) => m.clear_kv_cache(),
}
}
pub fn report(&self) {
match self {
Self::Qwen3Moe(m) => m.report(),
Self::Qwen3Next(m) => m.report(),
Self::GptOss(m) => m.report(),
}
}
/// The conversation's state, for models that can hand it over (None otherwise).
pub fn save_state(&self) -> Result<Option<Vec<(Vec<usize>, Vec<f32>)>>> {
match self {
Self::Qwen3Next(m) => m.save_state().map(Some),
_ => Ok(None),
}
}
/// Puts a saved conversation state back; false when this model cannot.
pub fn load_state(&mut self, parts: &[(Vec<usize>, Vec<f32>)]) -> Result<bool> {
match self {
Self::Qwen3Next(m) => m.load_state(parts).map(|_| true),
_ => Ok(false),
}
}
/// True for a model whose chat format is OpenAI's harmony (gpt-oss) rather than ChatML.
pub fn harmony(&self) -> bool {
matches!(self, Self::GptOss(_))
}
pub fn thinks(&self) -> bool {
self.harmony() || self.think().thinks()
}
/// S6 measurement (Qwen3-Next family only): see `Qwen3Next::speculate_measure`.
pub fn speculate_measure(&mut self, last: u32, offset: usize, k: usize) -> Result<Option<(Vec<u32>, usize)>> {
match self {
Self::Qwen3Moe(_) => Ok(None),
Self::Qwen3Next(m) => m.speculate_measure(last, offset, k).map(Some),
Self::GptOss(_) => Ok(None),
}
}
/// Foresight (Qwen3-Next family only): see `Qwen3Next::foresee`.
pub fn foresee(&mut self, last: u32, offset: usize, steps: usize) -> Result<bool> {
match self {
Self::Qwen3Moe(_) => Ok(false),
Self::Qwen3Next(m) => m.foresee(last, offset, steps),
Self::GptOss(_) => Ok(false),
}
}
/// S6 (Qwen3-Next family only): see `Qwen3Next::speculate`.
pub fn speculate(&mut self, last: u32, offset: usize, k: usize) -> Result<Option<Vec<u32>>> {
match self {
Self::Qwen3Moe(_) => Ok(None),
Self::Qwen3Next(m) => m.speculate(last, offset, k),
Self::GptOss(_) => Ok(None),
}
}
/// The text that opens the assistant's turn, with thinking on or off.
pub fn assistant_open(&self, think: bool) -> &'static str {
let t = self.think();
if think { t.on } else { t.off }
}
fn think(&self) -> ThinkOpen {
match self {
Self::Qwen3Moe(m) => m.think,
Self::Qwen3Next(m) => m.think,
Self::GptOss(m) => m.think,
}
}
/// Who to tell, layer by layer, while a prompt is being read; `None` to stop telling.
pub fn set_progress(&mut self, progress: Option<Progress>) {
match self {
Self::Qwen3Moe(m) => m.progress = progress,
Self::Qwen3Next(m) => m.progress = progress,
Self::GptOss(m) => m.progress = progress,
}
}
}