Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust

bank at its exact size: candle pool rounds buffers to a power of two (1.15 GB lost at bank 32 on the squeezed 122B); plus a GPU memory print

Merlinc3eb493965c2
diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs
index 8381a6b..cecd02f 100644
--- a/crates/cli/src/main.rs
+++ b/crates/cli/src/main.rs
@@ -146,6 +146,9 @@ fn main() -> Result<()> {
let started = std::time::Instant::now();
let mut model = model::Model::load(&args.model, &device, &settings)?;
+ if let Some((used, max)) = coachwhip_engine::gpu_sync::gpu_memory(&device) {
+ eprintln!("coachwhip: memlog after load: {:.2} GB of {:.2} GB", used as f64 / 1e9, max as f64 / 1e9);
+ }
eprintln!("coachwhip: model ready in {:.1} s", started.elapsed().as_secs_f64());
let tokenizer = Tokenizer::from_file(&args.tokenizer).map_err(anyhow::Error::msg)?;
let sampling = if args.temperature <= 0.0 {
@@ -167,6 +170,9 @@ fn main() -> Result<()> {
Ok(())
};
let a = chat::generate(&mut model, &tokenizer, &device, &mut chat::Session::default(), true, &prompt, args.think, args.max_tokens, &sampling, None, &mut emit)?;
+ if let Some((used, max)) = coachwhip_engine::gpu_sync::gpu_memory(&device) {
+ eprintln!("coachwhip: memlog after answer: {:.2} GB of {:.2} GB", used as f64 / 1e9, max as f64 / 1e9);
+ }
eprintln!(
"\n\ncoachwhip: read {} tokens at {:.1} tok/s, wrote {} at {:.1} tok/s",
a.prompt_tokens, a.read_tps, a.written, a.write_tps
diff --git a/crates/engine/src/experts.rs b/crates/engine/src/experts.rs
index 2e71d0a..63c112d 100644
--- a/crates/engine/src/experts.rs
+++ b/crates/engine/src/experts.rs
@@ -93,6 +93,28 @@ pub struct Slots {
pub prefetched: HashSet<u32>,
}
+/// A zeroed quantised buffer of exactly `elems` weights, outside candle's power-of-two rounding.
+fn exact_zeros(metal: &MetalDevice, elems: usize, dtype: GgmlDType) -> Result<QMetalStorage> {
+ use candle::quantized::k_quants::{BlockQ2K, BlockQ3K, BlockQ4K, BlockQ5K, BlockQ6K, GgmlType};
+ use candle::quantized::metal::load_quantized;
+ if std::env::var("COACHWHIP_POOLED_BANK").map(|v| v == "1").unwrap_or(false) {
+ return QMetalStorage::zeros(metal, elems, dtype);
+ }
+ let n = elems / dtype.block_size();
+ let q = match dtype {
+ GgmlDType::Q2K => load_quantized(metal, &vec![BlockQ2K::zeros(); n])?,
+ GgmlDType::Q3K => load_quantized(metal, &vec![BlockQ3K::zeros(); n])?,
+ GgmlDType::Q4K => load_quantized(metal, &vec![BlockQ4K::zeros(); n])?,
+ GgmlDType::Q5K => load_quantized(metal, &vec![BlockQ5K::zeros(); n])?,
+ GgmlDType::Q6K => load_quantized(metal, &vec![BlockQ6K::zeros(); n])?,
+ _ => return QMetalStorage::zeros(metal, elems, dtype),
+ };
+ match q {
+ candle::quantized::QStorage::Metal(s) => Ok(s),
+ _ => candle::bail!("exact bank: not a Metal buffer"),
+ }
+}
+
/// What one forward needs from a layer's bank: the experts already there and the ones to read in.
#[derive(Debug, Default, PartialEq)]
pub struct Plan {
@@ -1207,7 +1229,7 @@ impl ExpertStore {
} else {
(s.dtype, s.bytes)
};
- let storage = QMetalStorage::zeros(&self.metal, elems * self.per_layer, dtype)?;
+ let storage = exact_zeros(&self.metal, elems * self.per_layer, dtype)?;
let base = storage.buffer().contents() as usize;
if base == 0 {
candle::bail!("Metal gave a buffer the CPU cannot write to");
diff --git a/crates/engine/src/gpu_sync.rs b/crates/engine/src/gpu_sync.rs
index f8a9b16..d2f594c 100644
--- a/crates/engine/src/gpu_sync.rs
+++ b/crates/engine/src/gpu_sync.rs
@@ -190,6 +190,13 @@ impl GpuEvent {
}
}
+/// GPU memory in use and the working-set limit macOS recommends, in bytes.
+pub fn gpu_memory(device: &candle::Device) -> Option<(u64, u64)> {
+ let candle::Device::Metal(m) = device else { return None };
+ let d = m.device().as_ref();
+ Some((d.currentAllocatedSize() as u64, d.recommendedMaxWorkingSetSize()))
+}
+
/// Commit the work encoded so far, so it runs and completes without waiting for anything later.
pub fn flush() -> Result<()> {
let commands = Commands::current().ok_or_else(|| candle::Error::Msg("no Metal command queue in use yet".into()))?;