The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
[package]
name = "coachwhip"
description = "Run mixture-of-experts models bigger than your memory on a Mac: a 122B model on 16 GB. The experts stream from the SSD into a small GPU bank."
version.workspace = true
edition.workspace = true
license.workspace = true
publish.workspace = true
[[bin]]
name = "coachwhip"
path = "src/main.rs"
[dependencies]
coachwhip-engine.workspace = true
coachwhip-server.workspace = true
candle.workspace = true
candle-transformers.workspace = true
tokenizers.workspace = true
clap.workspace = true
anyhow.workspace = true