The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
[workspace]
members = ["crates/engine", "crates/server", "crates/cli", "crates/prepare"]
resolver = "2"
[workspace.package]
version = "0.5.3"
edition = "2021"
license = "Apache-2.0"
publish = false
[workspace.dependencies]
candle = { package = "candle-core", git = "https://github.com/huggingface/candle", rev = "5ba5d5b468b5b1df40e82dd3d556987bedeea041", features = ["metal"] }
candle-nn = { git = "https://github.com/huggingface/candle", rev = "5ba5d5b468b5b1df40e82dd3d556987bedeea041", features = ["metal"] }
candle-transformers = { git = "https://github.com/huggingface/candle", rev = "5ba5d5b468b5b1df40e82dd3d556987bedeea041", features = ["metal"] }
candle-metal-kernels = { git = "https://github.com/huggingface/candle", rev = "5ba5d5b468b5b1df40e82dd3d556987bedeea041" }
objc2-metal = "0.3.1"
objc2 = "0.6"
tokenizers = { version = "0.23.1", default-features = false, features = ["fancy-regex"] }
clap = { version = "4.2.4", features = ["derive"] }
anyhow = "1"
serde_json = "1.0.99"
rayon = "1"
coachwhip-engine = { path = "crates/engine" }
coachwhip-server = { path = "crates/server" }
[profile.release]
debug = false
[patch."https://github.com/huggingface/candle"]
candle-metal-kernels = { path = "vendor/candle-metal-kernels" }
candle-core = { path = "vendor/candle-core" }