The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
bank at its exact size: candle pool rounds buffers to a power of two (1.15 GB lost at bank 32 on the squeezed 122B); plus a GPU memory print
Merlinc3eb493965c2
diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rsindex 8381a6b..cecd02f 100644--- a/crates/cli/src/main.rs+++ b/crates/cli/src/main.rs@@ -146,6 +146,9 @@ fn main() -> Result<()> {let started = std::time::Instant::now();let mut model = model::Model::load(&args.model, &device, &settings)?;+ if let Some((used, max)) = coachwhip_engine::gpu_sync::gpu_memory(&device) {+ eprintln!("coachwhip: memlog after load: {:.2} GB of {:.2} GB", used as f64 / 1e9, max as f64 / 1e9);+ }eprintln!("coachwhip: model ready in {:.1} s", started.elapsed().as_secs_f64());let tokenizer = Tokenizer::from_file(&args.tokenizer).map_err(anyhow::Error::msg)?;let sampling = if args.temperature <= 0.0 {@@ -167,6 +170,9 @@ fn main() -> Result<()> {Ok(())};let a = chat::generate(&mut model, &tokenizer, &device, &mut chat::Session::default(), true, &prompt, args.think, args.max_tokens, &sampling, None, &mut emit)?;+ if let Some((used, max)) = coachwhip_engine::gpu_sync::gpu_memory(&device) {+ eprintln!("coachwhip: memlog after answer: {:.2} GB of {:.2} GB", used as f64 / 1e9, max as f64 / 1e9);+ }eprintln!("\n\ncoachwhip: read {} tokens at {:.1} tok/s, wrote {} at {:.1} tok/s",a.prompt_tokens, a.read_tps, a.written, a.write_tpsdiff --git a/crates/engine/src/experts.rs b/crates/engine/src/experts.rsindex 2e71d0a..63c112d 100644--- a/crates/engine/src/experts.rs+++ b/crates/engine/src/experts.rs@@ -93,6 +93,28 @@ pub struct Slots {pub prefetched: HashSet<u32>,}+/// A zeroed quantised buffer of exactly `elems` weights, outside candle's power-of-two rounding.+fn exact_zeros(metal: &MetalDevice, elems: usize, dtype: GgmlDType) -> Result<QMetalStorage> {+ use candle::quantized::k_quants::{BlockQ2K, BlockQ3K, BlockQ4K, BlockQ5K, BlockQ6K, GgmlType};+ use candle::quantized::metal::load_quantized;+ if std::env::var("COACHWHIP_POOLED_BANK").map(|v| v == "1").unwrap_or(false) {+ return QMetalStorage::zeros(metal, elems, dtype);+ }+ let n = elems / dtype.block_size();+ let q = match dtype {+ GgmlDType::Q2K => load_quantized(metal, &vec![BlockQ2K::zeros(); n])?,+ GgmlDType::Q3K => load_quantized(metal, &vec![BlockQ3K::zeros(); n])?,+ GgmlDType::Q4K => load_quantized(metal, &vec![BlockQ4K::zeros(); n])?,+ GgmlDType::Q5K => load_quantized(metal, &vec![BlockQ5K::zeros(); n])?,+ GgmlDType::Q6K => load_quantized(metal, &vec![BlockQ6K::zeros(); n])?,+ _ => return QMetalStorage::zeros(metal, elems, dtype),+ };+ match q {+ candle::quantized::QStorage::Metal(s) => Ok(s),+ _ => candle::bail!("exact bank: not a Metal buffer"),+ }+}+/// What one forward needs from a layer's bank: the experts already there and the ones to read in.#[derive(Debug, Default, PartialEq)]pub struct Plan {@@ -1207,7 +1229,7 @@ impl ExpertStore {} else {(s.dtype, s.bytes)};- let storage = QMetalStorage::zeros(&self.metal, elems * self.per_layer, dtype)?;+ let storage = exact_zeros(&self.metal, elems * self.per_layer, dtype)?;let base = storage.buffer().contents() as usize;if base == 0 {candle::bail!("Metal gave a buffer the CPU cannot write to");diff --git a/crates/engine/src/gpu_sync.rs b/crates/engine/src/gpu_sync.rsindex f8a9b16..d2f594c 100644--- a/crates/engine/src/gpu_sync.rs+++ b/crates/engine/src/gpu_sync.rs@@ -190,6 +190,13 @@ impl GpuEvent {}}+/// GPU memory in use and the working-set limit macOS recommends, in bytes.+pub fn gpu_memory(device: &candle::Device) -> Option<(u64, u64)> {+ let candle::Device::Metal(m) = device else { return None };+ let d = m.device().as_ref();+ Some((d.currentAllocatedSize() as u64, d.recommendedMaxWorkingSetSize()))+}+/// Commit the work encoded so far, so it runs and completes without waiting for anything later.pub fn flush() -> Result<()> {let commands = Commands::current().ok_or_else(|| candle::Error::Msg("no Metal command queue in use yet".into()))?;