The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
//! Coachwhip: run mixture-of-experts models that are bigger than the memory, on a Mac. A 122B
//! model runs on 16 GB: the experts stream from the SSD into a small bank on the GPU, and the
//! ones the next layer will probably need are fetched while the current layer computes.
use coachwhip_engine::{experts, model};
use coachwhip_server::chat;
use anyhow::Result;
use candle::Device;
use candle_transformers::generation::Sampling;
use clap::Parser;
use std::io::Write;
use std::path::PathBuf;
use tokenizers::Tokenizer;
#[derive(Parser, Debug)]
#[command(version, about = "Run mixture-of-experts models bigger than your memory on a Mac: a 122B model on 16 GB")]
struct Args {
/// The model, as a GGUF file: Qwen3 MoE or Qwen3-Next (Q4_K_M tested).
#[arg(long)]
model: PathBuf,
/// The model's tokenizer.json.
#[arg(long)]
tokenizer: PathBuf,
/// Serve the chat page on 127.0.0.1 at this port.
#[arg(long, conflicts_with = "prompt")]
chat: Option<u16>,
/// Lab: read the prompt as raw text, no chat template, and decode greedily.
#[arg(long, hide = true)]
raw: bool,
/// Answer one prompt and exit.
#[arg(long)]
prompt: Option<String>,
/// Let the model think before it answers (slower).
#[arg(long)]
think: bool,
/// Longest answer, in tokens.
#[arg(long, default_value_t = 4000)]
max_tokens: usize,
/// Sampling temperature; 0 always picks the likeliest token.
#[arg(long, default_value_t = 0.7)]
temperature: f64,
/// Nucleus sampling: keep the likeliest tokens up to this probability mass.
#[arg(long, default_value_t = 0.8)]
top_p: f64,
/// Expert slots kept on the GPU per layer.
#[arg(long, default_value_t = 44)]
bank: usize,
/// Experts guessed and fetched ahead per layer; 0 turns prefetch off. Default 10, or 6 for
/// the 122B, the settings each model's numbers were measured with.
#[arg(long)]
prefetch: Option<usize>,
/// How many layers ahead the guess looks (each layer is about 2 ms of GPU work to read in).
/// Default 4, or 2 for the 122B.
#[arg(long)]
ahead: Option<usize>,
/// Reader threads for the prefetch.
#[arg(long, default_value_t = 3)]
readers: usize,
/// Folder of starting route logs, one sub-folder per model, named after the model file.
#[arg(long, default_value = "data/routes")]
routes: PathBuf,
/// Folder where the routes of every answer are saved, one sub-folder per model, named after
/// the model file, so each model's hot path keeps learning on its own.
#[arg(long, default_value = "learned")]
learned: PathBuf,
/// Print where each word's time went after every answer.
#[arg(long)]
profile: bool,
/// Ask the router for this many experts per word instead of the model's own number (Qwen3.5-122B:
/// 4 instead of 8). Fewer experts means fewer bytes per word and faster writing; the answers
/// change, so check them for your use.
#[arg(long)]
experts: Option<usize>,
/// The parallel path: the GPU waits on a shared event for its experts instead of the CPU
/// re-queuing each layer. Faster on big models; the answers do not change.
#[arg(long)]
parallel: bool,
/// The register: learns, as it runs, what each expert adds to the running state, and guesses
/// the next layers' experts from the state it predicts. Fewer reads; the answers do not change.
#[arg(long)]
register: bool,
/// Stand-ins (gpt-oss): a light pick that is not in the bank is replaced by the best expert
/// that is, by the router's own scores. The first pick is always read for real, and the second
/// whenever the router gives it at least 0.3 of the weight, and a substitute is used only when its
/// router score is close to the real pick's. Fewer reads; the answers change, so check them for
/// your use.
#[arg(long)]
stand_ins: bool,
/// Flash reads: experts are read straight from the SSD, past the page cache, which has nothing
/// to give back on a 16 GB Mac. The answers do not change.
#[arg(long)]
flash_reads: bool,
}
fn main() -> Result<()> {
let args = Args::parse();
// The engine reads these as switches; the flags are their front door.
if let Some(k) = args.experts {
std::env::set_var("COACHWHIP_TOPK", k.to_string());
}
if args.parallel {
std::env::set_var("COACHWHIP_PARALLEL", "1");
}
// The parallel path commits GPU work itself, layer by layer, once the CPU has written what
// that work reads; Candle must not commit on its own count in between.
if coachwhip_engine::experts::parallel() && std::env::var("CANDLE_METAL_COMPUTE_PER_BUFFER").is_err() {
std::env::set_var("CANDLE_METAL_COMPUTE_PER_BUFFER", "1000000");
}
if args.register {
std::env::set_var("COACHWHIP_REGISTER", "1");
}
if args.stand_ins {
std::env::set_var("COACHWHIP_STANDIN", "1");
if std::env::var("COACHWHIP_STANDIN_W").is_err() {
std::env::set_var("COACHWHIP_STANDIN_W", "0.3");
}
// A substitute more than this far below its real pick (router logits) is not used; the
// real expert is read. Three exam runs at 1.0 scored 5.0, 3.8 and 4.8; without it 4.0, 4.0, 0.8.
if std::env::var("COACHWHIP_STANDIN_GAP").is_err() {
std::env::set_var("COACHWHIP_STANDIN_GAP", "1.0");
}
}
if args.flash_reads {
std::env::set_var("COACHWHIP_NOCACHE", "1");
}
let device = Device::new_metal(0)?;
// Wrangler: when the whole file fits in the GPU with a gigabyte to spare for the chat, every
// expert is kept resident and the fast path is used. Any switch set by hand wins.
if std::env::var("COACHWHIP_RESIDENT").is_err() {
let arch = std::fs::File::open(&args.model).ok().and_then(|mut f| candle::quantized::gguf_file::Content::read(&mut f).ok())
.and_then(|c| c.metadata.get("general.architecture").and_then(|v| v.to_string().ok().cloned()));
let file_len = std::fs::metadata(&args.model).map(|m| m.len()).unwrap_or(u64::MAX);
let max = coachwhip_engine::gpu_sync::gpu_memory(&device).map(|(_, m)| m).unwrap_or(0);
let fits = file_len.saturating_add(1_000_000_000) < max;
if fits && matches!(arch.as_deref(), Some("qwen35moe") | Some("qwen3next")) {
for (k, v) in [("COACHWHIP_RESIDENT", "1"), ("COACHWHIP_FUSE", "1"), ("COACHWHIP_NEWMV", "1"), ("COACHWHIP_ADDNORM", "1"),
("COACHWHIP_ROUTER16", "1"), ("COACHWHIP_GDN2", "1"), ("COACHWHIP_PIECE", "1"), ("COACHWHIP_GROUP", "3"),
("COACHWHIP_PROMPT_CHUNK", "512"), ("COACHWHIP_SHORT", "512"), ("COACHWHIP_PIECE_SYNC", "1"), ("COACHWHIP_FAST_CHAT", "1"), ("COACHWHIP_CONV_TOKENS", "8192")] {
if std::env::var(k).is_err() {
std::env::set_var(k, v);
}
}
eprintln!("coachwhip: the whole model fits in the GPU ({:.1} of {:.1} GB): every expert stays resident", file_len as f64 / 1e9, max as f64 / 1e9);
}
}
// A hot path belongs to one model: its routes live in a folder named after the model file.
let model_name = args.model.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_else(|| "model".into());
let routes = args.routes.join(&model_name);
let learned = args.learned.join(&model_name);
std::fs::create_dir_all(&learned)?;
// The 122B (48-layer qwen35moe) guesses best at 6 ahead by 2, gpt-oss at 2 by 2 (its 13 MB
// experts make a wrong guess dear); the others at 10 by 4.
let (big, oss) = {
let mut f = std::fs::File::open(&args.model)?;
let ct = candle::quantized::gguf_file::Content::read(&mut f)?;
let arch = ct.metadata.get("general.architecture").and_then(|v| v.to_string().ok().cloned()).unwrap_or_default();
let layers = ct.metadata.get(&format!("{arch}.block_count")).and_then(|v| v.to_u32().ok()).unwrap_or(0);
(arch == "qwen35moe" && layers >= 48, arch == "gpt-oss")
};
// gpt-oss guesses about half the experts it uses per word (13 MB each makes a wrong guess dear).
// With stand-ins most picks are already in the bank; one guess a layer keeps the guessed reads
// out of the way of the ones a word needs (1,000 words: 8.4 tok/s at 2 guesses, 10.9 at 1).
let oss_guess = if args.stand_ins { 1 } else { args.experts.map_or(2, |k| (k / 2).max(1)) };
let (prefetch, ahead) = match (args.prefetch, args.ahead) {
(Some(p), Some(a)) => (p, a),
(Some(p), None) => (p, if big || oss { 2 } else { 4 }),
(None, Some(a)) => (if oss { oss_guess } else if big { 6 } else { 10 }, a),
(None, None) => if oss { (oss_guess, 2) } else if big { (6, 2) } else { (10, 4) },
};
let settings = experts::Settings {
bank: args.bank,
prefetch,
ahead: ahead.max(1),
readers: args.readers,
routes: routes.is_dir().then_some(routes),
learned: Some(learned.clone()),
profile: args.profile,
};
eprintln!("coachwhip: hot path for {model_name} learns in {}", learned.display());
let started = std::time::Instant::now();
let mut model = model::Model::load(&args.model, &device, &settings)?;
if let Some((used, max)) = coachwhip_engine::gpu_sync::gpu_memory(&device) {
eprintln!("coachwhip: memlog after load: {:.2} GB of {:.2} GB", used as f64 / 1e9, max as f64 / 1e9);
}
eprintln!("coachwhip: model ready in {:.1} s", started.elapsed().as_secs_f64());
let tokenizer = Tokenizer::from_file(&args.tokenizer).map_err(anyhow::Error::msg)?;
let sampling = if args.temperature <= 0.0 {
Sampling::ArgMax
} else {
Sampling::TopP { p: args.top_p, temperature: args.temperature }
};
if let Some(port) = args.chat {
return chat::serve(port, &mut model, &model_name, tokenizer, &device, args.max_tokens, sampling);
}
let Some(prompt) = args.prompt else {
anyhow::bail!("give --chat <port> or --prompt <text>")
};
if args.raw {
let ids = tokenizer.encode(prompt.as_str(), false).map_err(anyhow::Error::msg)?.get_ids().to_vec();
eprintln!("coachwhip: raw prompt, {} tokens", ids.len());
model.clear_kv_cache();
// Read the prompt in 512-token pieces, as the chat does; the last piece goes through the loop.
let mut offset = 0usize;
let last = ids.len() - (ids.len() - 1) % 512 - 1;
for piece in ids[..last].chunks(512) {
model.forward(&candle::Tensor::new(piece, &device)?.unsqueeze(0)?, offset)?;
offset += piece.len();
}
let mut input = candle::Tensor::new(&ids[last..], &candle::Device::Cpu)?.unsqueeze(0)?;
let mut outs = Vec::new();
// Lab (`COACHWHIP_BALLAST_MB=n`): hold n MB of GPU memory, to price running near the limit.
let _ballast = match std::env::var("COACHWHIP_BALLAST_MB").ok().and_then(|v| v.parse::<usize>().ok()) {
Some(mb) => Some(candle::Tensor::ones(mb * 1024 * 256, candle::DType::F32, &device)?),
None => None,
};
let t0 = std::time::Instant::now();
if coachwhip_engine::fuse_kernel::gpupick_on() && device.is_metal() {
// The GPU picks each word and feeds it straight back in; the CPU reads the words back
// every 16, so it is never waiting for the GPU between two words.
let mut pending: Vec<candle::Tensor> = Vec::new();
for step in 0..args.max_tokens {
let logits = model.forward(&input, offset)?;
offset += input.dim(1)?;
let next = coachwhip_engine::fuse_kernel::argmax(&logits)?;
pending.push(next.clone());
input = next;
if step == 0 || pending.len() == 16 || step + 1 == args.max_tokens {
for t in pending.drain(..) {
outs.push(t.flatten_all()?.to_vec1::<u32>()?[0]);
}
if step == 0 {
eprintln!("coachwhip: first token after {:.1} s", t0.elapsed().as_secs_f64());
}
}
}
println!("{:?}", tokenizer.decode(&outs, false).map_err(anyhow::Error::msg)?);
eprintln!("coachwhip: {} tokens in {:.1} s", outs.len(), t0.elapsed().as_secs_f64());
return Ok(());
}
for step in 0..args.max_tokens {
let logits: Vec<f32> = model.forward(&input, offset)?.flatten_all()?.to_dtype(candle::DType::F32)?.to_vec1()?;
offset += input.dim(1)?;
// The likeliest token, first of equals (what a stable descending sort put first), in one pass.
let mut best = 0usize;
for (i, &v) in logits.iter().enumerate() {
if v > logits[best] {
best = i;
}
}
if step == 0 {
let m = logits[best];
let lse = m + logits.iter().map(|v| (v - m).exp()).sum::<f32>().ln();
let mut idx: Vec<usize> = (0..logits.len()).collect();
idx.sort_by(|&a, &b| logits[b].partial_cmp(&logits[a]).unwrap());
let top: Vec<(String, f32)> = idx[..3].iter().map(|&i| (tokenizer.decode(&[i as u32], false).unwrap_or_default(), logits[i] - lse)).collect();
eprintln!("coachwhip: first token top-3 {top:?} after {:.1} s", t0.elapsed().as_secs_f64());
debug_assert_eq!(idx[0], best);
}
if std::env::var("COACHWHIP_POOLLOG").is_ok() && step % 25 == 0 {
if let candle::Device::Metal(m) = &device {
let (pooled, in_use, count) = m.pool_stats();
eprintln!("coachwhip: pool at word {step}: {:.2} GB pooled in {count} buffers, {:.2} GB in use", pooled as f64 / 1e9, in_use as f64 / 1e9);
}
}
let next = best as u32;
outs.push(next);
let bk: usize = std::env::var("COACHWHIP_BATCH_TIME").ok().and_then(|v| v.parse().ok()).unwrap_or(1);
input = candle::Tensor::new(vec![next; bk].as_slice(), &candle::Device::Cpu)?.unsqueeze(0)?;
}
println!("{:?}", tokenizer.decode(&outs, false).map_err(anyhow::Error::msg)?);
eprintln!("coachwhip: {} tokens in {:.1} s", outs.len(), t0.elapsed().as_secs_f64());
return Ok(());
}
let mut out = std::io::stdout();
let mut emit = |t: &str| -> Result<()> {
out.write_all(t.as_bytes())?;
out.flush()?;
Ok(())
};
let a = chat::generate(&mut model, &tokenizer, &device, &mut chat::Session::default(), true, &prompt, args.think, args.max_tokens, &sampling, None, &mut emit)?;
if let Some((used, max)) = coachwhip_engine::gpu_sync::gpu_memory(&device) {
eprintln!("coachwhip: memlog after answer: {:.2} GB of {:.2} GB", used as f64 / 1e9, max as f64 / 1e9);
}
eprintln!(
"\n\ncoachwhip: read {} tokens at {:.1} tok/s, wrote {} at {:.1} tok/s",
a.prompt_tokens, a.read_tps, a.written, a.write_tps
);
model.report();
Ok(())
}