The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
//! The expert store: run a mixture-of-experts model whose experts do not fit in memory.
//!
//! Apple silicon has one memory for the CPU and the GPU. Every expert is a slice of bytes inside
//! the GGUF file, and the OS page cache is the shelf. Each layer owns a bank: for gate, up and
//! down one shared Metal buffer holding `bank` expert slots side by side, allocated once and
//! overwritten in place, so a load is one read from the file straight into the memory the GPU
//! uses. A layer's maths is three dispatches of ggml's `mul_mv_id` kernel (every selected expert
//! of every token at once) plus the activation and the weighted sum.
//!
//! While the GPU works on one layer, reader threads fetch the experts the next layer will
//! probably need, predicted from a table of routes the model took before (the hot path). The
//! table keeps learning: every answer's routes are added to it at once and saved, so the
//! predictions follow what this machine is actually asked.
use candle::quantized::metal::QMetalStorage;
use candle::quantized::{ggml_file::qtensor_from_ggml, gguf_file, GgmlDType, QMatMul, QTensor};
use candle::{CustomOp2, DType, Device, Layout, MetalDevice, MetalStorage, Module, Result, Shape, Tensor, D};
use candle::backend::BackendStorage;
use candle_nn::Linear;
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::os::unix::fs::FileExt;
use rayon::prelude::*;
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{mpsc, Arc, Mutex, Weak};
use std::time::Instant;
/// Where one matrix of one expert lives in the file.
#[derive(Clone)]
struct Slice {
dtype: GgmlDType,
offset: u64,
bytes: usize,
dims: Vec<usize>,
}
/// One matrix of a layer's bank: `per_layer` expert slots in one shared buffer.
struct BankMat {
storage: Arc<QMetalStorage>,
base: usize,
bytes: usize,
dtype: GgmlDType,
n: usize,
k: usize,
}
/// Two kinds of slot (`COACHWHIP_SHARP=<n>`, Captain's design): the experts the model leans on
/// keep the file's own bytes in `n` sharp slots per layer; the rest go through the GPU shrink
/// into the small (Q2) bank. A sharp slot's index carries `SHARP_FLAG`.
pub const SHARP_FLAG: usize = 1 << 30;
/// A prefetch job's expert id with this bit set goes to the sharp bank (decided by weight).
pub const SHARP_TAG: u32 = 1 << 31;
struct SharpBank {
mats: [BankMat; 3],
slots: Slots,
ready: Vec<Arc<AtomicBool>>,
}
struct Bank {
mats: [BankMat; 3],
slots: Slots,
sharp: Option<SharpBank>,
/// False while the prefetch thread is still reading an expert into the slot.
ready: Vec<Arc<AtomicBool>>,
/// The GPU's own copy of which slot holds which expert, and which slots are ready (S4).
gpu: Arc<crate::gpu_sync::GpuBank>,
/// S4: slots the GPU is about to read for the word in flight; a prefetch may not take them.
pinned: Vec<AtomicBool>,
}
impl Bank {
/// Rewrite the GPU's table from the CPU's: every expert's slot, or none.
fn sync_table(&self) {
for e in 0..self.gpu.experts as u32 {
match self.slots.slot_of.get(&e) {
Some(&s) => self.gpu.set_slot(e, s),
None => self.gpu.clear(e),
}
}
}
}
/// A bank's bookkeeping, apart from the GPU memory it describes: which expert sits in which
/// slot, and the order slots were last used in. Pure, so it can be tested without a GPU.
#[derive(Clone, Debug)]
pub struct Slots {
pub holder: Vec<Option<u32>>,
pub slot_of: HashMap<u32, usize>,
/// Slots from least to most recently used.
pub lru: VecDeque<usize>,
/// Experts a prefetch brought in that no token has asked for yet.
pub prefetched: HashSet<u32>,
}
/// A zeroed quantised buffer of exactly `elems` weights, outside candle's power-of-two rounding.
/// A Metal buffer of exactly these bytes, as quantised storage of `dtype` (no power-of-two rounding).
fn exact_load(metal: &MetalDevice, dtype: GgmlDType, raw: &[u8]) -> Result<QMetalStorage> {
use candle::quantized::k_quants::*;
use candle::quantized::metal::load_quantized;
fn cast<T>(raw: &[u8]) -> &[T] {
unsafe { std::slice::from_raw_parts(raw.as_ptr() as *const T, raw.len() / std::mem::size_of::<T>()) }
}
let q = match dtype {
GgmlDType::Q2K => load_quantized(metal, cast::<BlockQ2K>(raw))?,
GgmlDType::Q3K => load_quantized(metal, cast::<BlockQ3K>(raw))?,
GgmlDType::Q4K => load_quantized(metal, cast::<BlockQ4K>(raw))?,
GgmlDType::Q5K => load_quantized(metal, cast::<BlockQ5K>(raw))?,
GgmlDType::Q6K => load_quantized(metal, cast::<BlockQ6K>(raw))?,
GgmlDType::Iq2Xxs => load_quantized(metal, cast::<BlockIq2Xxs>(raw))?,
GgmlDType::Iq3Xxs => load_quantized(metal, cast::<BlockIq3Xxs>(raw))?,
GgmlDType::Iq4Xs => load_quantized(metal, cast::<BlockIq4Xs>(raw))?,
other => candle::bail!("resident: no exact load for {other:?}"),
};
match q {
candle::quantized::QStorage::Metal(s) => Ok(s),
_ => candle::bail!("resident: not a Metal buffer"),
}
}
fn exact_zeros(metal: &MetalDevice, elems: usize, dtype: GgmlDType) -> Result<QMetalStorage> {
use candle::quantized::k_quants::{BlockQ2K, BlockQ3K, BlockQ4K, BlockQ5K, BlockQ6K, GgmlType};
use candle::quantized::metal::load_quantized;
if std::env::var("COACHWHIP_POOLED_BANK").map(|v| v == "1").unwrap_or(false) {
return QMetalStorage::zeros(metal, elems, dtype);
}
let n = elems / dtype.block_size();
let q = match dtype {
GgmlDType::Q2K => load_quantized(metal, &vec![BlockQ2K::zeros(); n])?,
GgmlDType::Q3K => load_quantized(metal, &vec![BlockQ3K::zeros(); n])?,
GgmlDType::Q4K => load_quantized(metal, &vec![BlockQ4K::zeros(); n])?,
GgmlDType::Q5K => load_quantized(metal, &vec![BlockQ5K::zeros(); n])?,
GgmlDType::Q6K => load_quantized(metal, &vec![BlockQ6K::zeros(); n])?,
_ => return QMetalStorage::zeros(metal, elems, dtype),
};
match q {
candle::quantized::QStorage::Metal(s) => Ok(s),
_ => candle::bail!("exact bank: not a Metal buffer"),
}
}
/// The chat's longest conversation, in tokens (the server's limit); the governor keeps room for it.
const WRANGLER_CONTEXT: f64 = 16384.0;
/// Kept free under macOS's GPU working-set limit, and the working memory a forward needs on top of
/// the weights, banks and keys. Measured 9 Oct 2026 on the squeezed 122B, 16 GB Mac mini: bank 40
/// with a 15.7K prompt peaked at 11.34 GB of 12.71 GB and ran; bank 44 stalled.
const WRANGLER_MARGIN: f64 = 1.3e9;
const WRANGLER_WORKING: f64 = 1.0e9;
/// Wrangler: the most bank slots per layer that fit a full chat under the GPU limit, or None when
/// switched off (`COACHWHIP_WRANGLER=0`) or not measurable.
fn wrangler_fit(ct: &gguf_file::Content, metal: &MetalDevice, layers: usize, expert_bytes: f64) -> Option<usize> {
use objc2_metal::MTLDevice as _;
if std::env::var("COACHWHIP_WRANGLER").map(|v| v == "0").unwrap_or(false) {
return None;
}
let limit = metal.device().as_ref().recommendedMaxWorkingSetSize() as f64;
let md_u = |k: &str| ct.metadata.get(k).and_then(|v| v.to_u32().ok()).map(|v| v as f64);
let arch = ct.metadata.get("general.architecture").and_then(|v| v.to_string().ok().cloned()).unwrap_or_default();
// The weights that live on the GPU: everything but the experts and the packed word table.
let dense: f64 = ct
.tensor_infos
.iter()
.filter(|(n, _)| !n.contains("_exps") && n.as_str() != "token_embd.weight")
.map(|(_, i)| (i.shape.elem_count() / i.ggml_dtype.block_size() * i.ggml_dtype.type_size()) as f64)
.sum();
// Keys and values for a full chat, f32, on the layers that keep them.
// gpt-oss alternates full and 128-token sliding layers; only the full ones keep a whole chat.
let interval = if arch == "gpt-oss" { 2.0 } else { md_u(&format!("{arch}.full_attention_interval")).unwrap_or(1.0).max(1.0) };
let n_kv = md_u(&format!("{arch}.attention.head_count_kv")).unwrap_or(1.0);
let hd = md_u(&format!("{arch}.attention.key_length")).unwrap_or(128.0);
// gpt-oss keeps its cache in f16 (2 bytes a value); the others in f32.
let bytes = if arch == "gpt-oss" { 2.0 } else { 4.0 };
let kv = (layers as f64 / interval).floor() * 2.0 * n_kv * hd * bytes * WRANGLER_CONTEXT;
let room = limit - WRANGLER_MARGIN - WRANGLER_WORKING - dense - kv;
let fit = (room / (layers as f64 * expert_bytes)).floor().max(8.0) as usize;
eprintln!(
"coachwhip: wrangler: GPU limit {:.2} GB; weights {:.2} GB, keys for a {}-token chat {:.2} GB, working {:.1} GB, margin {:.1} GB; room for {fit} slots per layer",
limit / 1e9, dense / 1e9, WRANGLER_CONTEXT as usize, kv / 1e9, WRANGLER_WORKING / 1e9, WRANGLER_MARGIN / 1e9
);
Some(fit)
}
/// macOS memory pressure: 1 normal, 2 warning, 4 critical (0 when it cannot be read).
fn memory_pressure() -> i32 {
let mut level: libc::c_int = 0;
let mut size = std::mem::size_of::<libc::c_int>();
let name = b"kern.memorystatus_vm_pressure_level\0";
let r = unsafe { libc::sysctlbyname(name.as_ptr() as *const libc::c_char, &mut level as *mut _ as *mut libc::c_void, &mut size, std::ptr::null_mut(), 0) };
if r == 0 { level } else { 0 }
}
/// What one forward needs from a layer's bank: the experts already there and the ones to read in.
#[derive(Debug, Default, PartialEq)]
pub struct Plan {
pub hits: Vec<(u32, usize)>,
pub misses: Vec<(u32, usize)>,
/// Hits that a prefetch had brought in.
pub prefetch_hits: u64,
}
impl Slots {
pub fn new(n: usize) -> Self {
Self { holder: vec![None; n], slot_of: HashMap::new(), lru: (0..n).collect(), prefetched: HashSet::new() }
}
/// The least recently used slot that is not being filled (`ready`) and does not hold one of `keep`.
fn victim(&self, ready: &impl Fn(usize) -> bool, keep: &[u32]) -> Option<usize> {
self.lru.iter().position(|&slot| ready(slot) && self.holder[slot].map_or(true, |h| !keep.contains(&h)))
}
fn put(&mut self, pos: usize, e: u32) -> usize {
let slot = self.lru.remove(pos).unwrap();
if let Some(old) = self.holder[slot] {
self.slot_of.remove(&old);
self.prefetched.remove(&old);
}
self.holder[slot] = Some(e);
self.slot_of.insert(e, slot);
self.lru.push_back(slot);
slot
}
/// Give every expert in `needed` a slot: hits move to most recently used, misses take the least
/// recently used slot that is ready and holds nothing this forward needs. None, and nothing
/// changed, when too few such slots are free (the caller then reads the experts fresh).
pub fn assign(&mut self, needed: &[u32], ready: impl Fn(usize) -> bool) -> Option<Plan> {
let missing = needed.iter().filter(|e| !self.slot_of.contains_key(e)).count();
let free = self.lru.iter().filter(|&&slot| ready(slot) && self.holder[slot].map_or(true, |h| !needed.contains(&h))).count();
if free < missing {
return None;
}
let mut plan = Plan::default();
for &e in needed {
if let Some(&slot) = self.slot_of.get(&e) {
self.lru.retain(|&x| x != slot);
self.lru.push_back(slot);
if self.prefetched.remove(&e) {
plan.prefetch_hits += 1;
}
plan.hits.push((e, slot));
continue;
}
let pos = self.victim(&ready, needed).expect("a bank always has a slot that this forward does not need");
plan.misses.push((e, self.put(pos, e)));
}
Some(plan)
}
/// Let an expert go: its slot is emptied and goes to the front of the eviction line.
pub fn release(&mut self, e: u32) {
if let Some(slot) = self.slot_of.remove(&e) {
self.holder[slot] = None;
self.prefetched.remove(&e);
self.lru.retain(|&x| x != slot);
self.lru.push_front(slot);
}
}
/// The toaster: once a layer has used these experts, their slots go to the front of the
/// eviction line, so the next reads take them first instead of the longest-unused slots.
pub fn pop(&mut self, used: &[u32]) {
for &e in used {
if let Some(&slot) = self.slot_of.get(&e) {
self.lru.retain(|&x| x != slot);
self.lru.push_front(slot);
}
}
}
/// Reserve slots for the predicted experts not already held, best first, as far as free slots
/// go; never evicts another predicted expert or a slot still being filled.
pub fn reserve(&mut self, predicted: &[u32], ready: impl Fn(usize) -> bool) -> Vec<(u32, usize)> {
let mut out = Vec::new();
for &e in predicted {
if self.slot_of.contains_key(&e) {
continue;
}
let Some(pos) = self.victim(&ready, predicted) else { continue };
let slot = self.put(pos, e);
self.prefetched.insert(e);
out.push((e, slot));
}
out
}
}
pub type Key = (usize, u32);
/// Scores for layer `layer + 1`'s experts given `layer`'s picks (expert, weight), from the route
/// table: each pick hands its weight on in proportion to where it has led before. Best first;
/// equal scores in expert order, so the same table always gives the same guess.
pub fn next_scores(routes: &Counts, layer: usize, now: &[(u32, f32)]) -> Vec<(u32, f32)> {
let mut score: HashMap<u32, f32> = HashMap::new();
for &(a, w) in now {
if let Some(row) = routes.get(&(layer, a)) {
let total = row.values().sum::<u64>().max(1) as f32;
for (&b, &n) in row {
*score.entry(b).or_insert(0.0) += w * n as f32 / total;
}
}
}
let mut v: Vec<(u32, f32)> = score.into_iter().collect();
v.sort_by(|x, y| y.1.partial_cmp(&x.1).unwrap().then(x.0.cmp(&y.0)));
v
}
/// How the store behaves. `Default` is what the Mac mini M4 with 16 GB was tuned on.
#[derive(Clone, Debug)]
pub struct Settings {
/// Expert slots kept per layer (one expert each: about 3 MB for the 30B, 2 MB for the 80B, at Q4_K_M).
pub bank: usize,
/// Experts guessed and fetched ahead per layer; 0 turns prefetch off.
pub prefetch: usize,
/// How many layers ahead the guess looks: layer n runs layer n + ahead's own router on its
/// input, so the reads have that many layers of GPU work to land in.
pub ahead: usize,
/// Reader threads that fetch predicted experts.
pub readers: usize,
/// Folder of route logs the predictions start from.
pub routes: Option<std::path::PathBuf>,
/// Folder where the routes of every answer are saved, and read back at the next start.
pub learned: Option<std::path::PathBuf>,
/// Print where each word's time went after every answer.
pub profile: bool,
}
impl Default for Settings {
fn default() -> Self {
Self { bank: 44, prefetch: 10, ahead: 4, readers: 3, routes: None, learned: None, profile: false }
}
}
/// Route table: for each (layer, expert), how often each expert of the next layer followed it.
pub type Counts = HashMap<Key, HashMap<u32, u64>>;
/// Adds one layer-to-layer step to the table: every expert picked now, for every expert picked
/// in the layer before, token by token.
pub fn count_step(counts: &mut Counts, layer: usize, before: &[Vec<u32>], now: &[Vec<u32>]) {
for (a_row, b_row) in before.iter().zip(now.iter()) {
// Nothing followed (a broken log line): leave no empty row behind to look like knowledge.
if b_row.is_empty() {
continue;
}
for &a in a_row {
let row = counts.entry((layer, a)).or_default();
for &b in b_row {
*row.entry(b).or_insert(0) += 1;
}
}
}
}
/// Reads route logs from folders. A log is a "T ..." line followed by one line per layer:
/// comma-separated tokens, each the space-separated experts the router picked for it.
pub fn load_routes(dirs: &[&Path], layers: usize) -> Counts {
let mut counts = Counts::new();
for dir in dirs {
let Ok(entries) = std::fs::read_dir(dir) else { continue };
read_logs(&mut counts, entries, layers);
}
counts
}
fn read_logs(counts: &mut Counts, entries: std::fs::ReadDir, layers: usize) {
for entry in entries.flatten() {
if entry.path().extension().and_then(|x| x.to_str()) != Some("log") {
continue;
}
let Ok(text) = std::fs::read_to_string(entry.path()) else { continue };
let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
let mut i = 0;
while i + layers < lines.len() {
if !lines[i].starts_with("T ") {
i += 1;
continue;
}
let per_layer: Vec<Vec<Vec<u32>>> = (0..layers)
.map(|l| lines[i + 1 + l].split(',').map(|t| t.split_whitespace().filter_map(|e| e.parse().ok()).collect()).collect())
.collect();
for l in 0..layers - 1 {
count_step(counts, l, &per_layer[l], &per_layer[l + 1]);
}
i += 1 + layers;
}
}
}
/// What the current forward pass has picked so far, for learning and for the log.
#[derive(Default)]
struct Trail {
/// The layer before, and the experts each token picked there.
before: Option<(usize, Vec<Vec<u32>>)>,
/// One line per layer of the forward pass in progress.
lines: Vec<String>,
/// Finished forward passes, waiting to be saved.
blocks: Vec<String>,
/// The words of the forward pass in progress, for the block's T line.
tokens: Vec<u32>,
}
/// Our register: per layer, what each expert adds to the running vector (`d`), what the layer adds
/// regardless (`d0`), and the last prediction waiting to be checked against the truth.
struct Register {
d: Vec<Tensor>,
d0: Vec<Tensor>,
lr: f32,
lr0: f32,
last: Option<(usize, Tensor, Tensor, Tensor)>,
/// The input the last prediction was made from (measurement only).
last_x: Tensor,
}
impl Default for Register {
fn default() -> Self {
Self { d: Vec::new(), d0: Vec::new(), lr: 1.0, lr0: 0.05, last: None, last_x: Tensor::zeros(1, DType::F32, &Device::Cpu).unwrap() }
}
}
#[derive(Default)]
struct Stats {
/// Lab: weaker picks swapped for a resident expert (`COACHWHIP_STANDIN`).
stand_ins: u64,
hits: u64,
loads: u64,
load_seconds: f64,
bytes_loaded: u64,
fresh: u64,
fresh_read_seconds: f64,
fresh_build_seconds: f64,
prefetch_copies: u64,
prefetch_hits: u64,
/// Guessed reads thrown away because the word had moved past their layer.
stale_dropped: u64,
/// Time a forward spent waiting for a guessed expert still being read in.
prefetch_wait_seconds: f64,
/// Forwards that fit the bank but found too few free slots (guesses still landing).
crowded: u64,
words: u64,
word_started: Option<Instant>,
word_seconds: f64,
expert_seconds: f64,
load_phase_seconds: f64,
maths_seconds: f64,
/// S6 measurement: how often a bank-only route (experts already resident) would have picked
/// the same experts as the real route. Layers seen, summed agreement fraction, layers fully
/// agreeing, words where every layer agreed, and whether the current word has agreed so far.
draft_layers: u64,
draft_agree: f64,
draft_layers_full: u64,
draft_words_full: u64,
draft_word_ok: bool,
/// S6 measurement: drafts run, words drafted, words the plain path agreed with.
spec_passes: u64,
spec_drafted: u64,
spec_accepted: u64,
/// Register measurement: relative error of its predicted state against the truth, summed per
/// layer step, and the same for the plain carry-over (x alone) as the yardstick.
/// Where the CPU time around the loads goes, per layer step (profile mode, plain path):
/// waiting for the GPU to hand the picks back, placing the experts in the bank (no reads),
/// the reads themselves, the bookkeeping (learn, log, guess, prefetch requests), and the encode.
t_readback: f64,
t_place: f64,
t_books: f64,
t_encode: f64,
reg_steps: u64,
reg_err: f64,
reg_err_plain: f64,
reg_err_by_layer: Vec<f64>,
reg_plain_by_layer: Vec<f64>,
}
static PROFILE: AtomicBool = AtomicBool::new(false);
fn profiling() -> bool {
PROFILE.load(Ordering::Relaxed)
}
fn toaster() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_TOASTER").is_ok())
}
/// Lab (`COACHWHIP_STAGE=1`): file reads land in ordinary memory and are copied into the bank.
fn staged() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_STAGE").map(|v| v == "1").unwrap_or(false))
}
/// Lab (`COACHWHIP_QRULE=<p>`): prefetch every early-guess candidate whose router probability is
/// at least `p`, instead of a flat `prefetch` per layer. `COACHWHIP_QCAP` caps the width (16).
fn q_rule() -> Option<f32> {
static V: std::sync::OnceLock<Option<f32>> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_QRULE").ok().and_then(|v| v.parse().ok()))
}
fn q_cap() -> usize {
static V: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_QCAP").ok().and_then(|v| v.parse().ok()).unwrap_or(16))
}
/// Lab (`COACHWHIP_SPLIT=1`): a layer's maths starts on the resident experts while its misses
/// are read, then finishes with the misses. Same sum, two halves.
fn split_maths() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_SPLIT").map(|v| v == "1").unwrap_or(false))
}
/// The GPU shrink (`COACHWHIP_GPUQ2=1`): the bank holds Q2_K copies, shrunk on arrival.
pub fn gpuq2() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_GPUQ2").map(|v| v == "1").unwrap_or(false))
}
/// The shaped shrink (`COACHWHIP_SHAPE=1`): the shrink's fit is weighted by the token's shape.
pub fn shaped() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_SHAPE").map(|v| v == "1").unwrap_or(false))
}
/// With the GPU shrink on, squeeze the down matrix too (`COACHWHIP_Q2_DOWN=1`). Off by default:
/// down is the fat, sensitive one, and the Q2 file that read well last night kept it sharp.
fn shrink_down() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_Q2_DOWN").map(|v| v == "1").unwrap_or(false))
}
/// By weight (`COACHWHIP_SHARP_W=<w>`, Captain's board): with two kinds of slot, the bank an
/// expert goes to is chosen by its router weight for this token: at or above `w` it fetches
/// sharp, below it Q2. Guesses use their router probability the same way. Off: the hot list rules.
fn sharp_weight() -> Option<f32> {
static V: std::sync::OnceLock<Option<f32>> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_SHARP_W").ok().and_then(|v| v.parse().ok()))
}
/// Admission (`COACHWHIP_ADMIT_W=<w>`, Captain's board): with use-sharp on, a miss whose router
/// weight for this token is below `w` is used from its sharp staging and let go — no slot kept,
/// no squeeze. Above it, it earns a slot and is squeezed for reuse.
fn admit_w() -> Option<f32> {
static V: std::sync::OnceLock<Option<f32>> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_ADMIT_W").ok().and_then(|v| v.parse().ok()))
}
/// Mechanics test (`COACHWHIP_Q2_NONE=1`): with the GPU shrink on, nothing is actually shrunk —
/// the bank holds the file's own bytes and the "squeeze" is a plain copy. The text must then be
/// identical to the plain engine's; if it is not, the flow has a bug.
fn q2_none() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_Q2_NONE").map(|v| v == "1").unwrap_or(false))
}
/// Heavy reuse re-read (`COACHWHIP_REREAD_W=<w>`): with use-sharp on, a hit whose router weight for
/// this token is at least `w` is re-read sharp and computed from the fresh bytes; the Q2 copy
/// stays in the bank for the light reuses. Heavy words exact, light ones pay the Q2.
fn reread_w() -> Option<f32> {
static V: std::sync::OnceLock<Option<f32>> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_REREAD_W").ok().and_then(|v| v.parse().ok()))
}
/// Register repair (`COACHWHIP_REPAIR=1`, Captain's board): at the squeeze, the engine measures
/// what the squeeze did to that expert's output on the input it was used for (sharp minus Q2),
/// keeps the difference per expert, and adds it back whenever the Q2 copy is reused.
fn repair_on() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_REPAIR").map(|v| v == "1").unwrap_or(false))
}
/// Use sharp, then squeeze (`COACHWHIP_USE_SHARP=1`): misses compute from their fresh bytes
/// and are squeezed into the bank afterwards.
pub fn use_sharp() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_USE_SHARP").map(|v| v == "1").unwrap_or(false))
}
/// S4: the GPU never stops for the CPU (`COACHWHIP_PARALLEL=1`).
/// Lab: gate and up side by side in one buffer per layer (resident path only).
pub fn gu_on() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_GU").map(|v| v == "1").unwrap_or(false))
}
pub fn resident_on() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_RESIDENT").map(|v| v == "1").unwrap_or(false))
}
pub fn parallel() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_PARALLEL").is_ok())
}
pub struct ExpertStore {
/// Resident mode: each layer's three expert matrices, whole, loaded once.
resident: Mutex<HashMap<usize, [(Arc<QMetalStorage>, GgmlDType, usize, usize, usize); 3]>>,
file: std::fs::File,
device: Device,
metal: MetalDevice,
per_layer: usize,
ahead: usize,
slices: Vec<Vec<[Slice; 3]>>,
banks: Mutex<HashMap<usize, Bank>>,
stats: Mutex<Stats>,
layers: usize,
routes: Mutex<Counts>,
trail: Mutex<Trail>,
learned: Option<std::path::PathBuf>,
learned_file: String,
/// Prefetch jobs: the layer to fill, the word they were guessed for, the experts.
prefetch_tx: Mutex<Option<mpsc::Sender<(usize, u64, Vec<u32>)>>>,
/// The word being written (one-word steps only) and the layer it has reached.
word: AtomicU64,
now_layer: AtomicUsize,
/// S6: routing confined to the bank, nothing read or learned.
draft: AtomicBool,
/// Foresight: the draft pass also tells the bank what the real pass of this word will need.
foresee: AtomicBool,
/// Which word ahead the foresight draft is on (0 = the word about to be written).
foresee_step: AtomicU64,
foresight_lists: Mutex<Vec<(usize, Tensor)>>,
/// Lab (`COACHWHIP_DUMP=file`): every layer's MoE input and router scores, one record per
/// layer per word, to fit the expert predictor from.
dump: Mutex<Option<std::io::BufWriter<std::fs::File>>>,
/// Our register (`COACHWHIP_REGISTER=1`): what each expert adds to the running vector, per
/// layer, learned on the GPU as the words go by; the early guess reads the state it predicts.
register: Mutex<Option<Register>>,
/// S4: the resolve-and-wait kernel, when the parallel path is on (`COACHWHIP_PARALLEL=1`).
/// The parallel path (S4), when `COACHWHIP_PARALLEL` is set.
s4: Option<S4>,
/// The GPU shrink (`COACHWHIP_GPUQ2=1`): arriving experts are shrunk to Q2_K on the GPU on
/// their way into the bank, which holds Q2 copies and so parks more of them.
requant: Option<crate::requant::Requant>,
/// Staging buffers for the shrink, by size, reused across reads.
staging: Mutex<HashMap<usize, Vec<Arc<candle_metal_kernels::metal::Buffer>>>>,
/// Use-sharp-then-squeeze (`COACHWHIP_USE_SHARP=1`, Captain's board): a miss is read into a
/// sharp one-slot staging (the file's own dtype), this token computes from it, and once the
/// GPU has drained at the next layer it is squeezed into its Q2 slot. The stagings, pooled by
/// (dtype, elements); the misses waiting for their squeeze; and this forward's fresh experts.
sharp_pool: Mutex<HashMap<(GgmlDType, usize), Vec<Arc<QMetalStorage>>>>,
pending_squeeze: Mutex<Vec<(usize, u32, usize, [Arc<QMetalStorage>; 3], f32, bool)>>,
fresh: Mutex<HashMap<(usize, u32), [Arc<QMetalStorage>; 3]>>,
/// Register repair: the input each layer last saw (1, h), and per (layer, expert) the
/// measured difference sharp minus Q2 of that expert's output (h,).
last_x: Mutex<HashMap<usize, Tensor>>,
repair: Mutex<HashMap<(usize, u32), Tensor>>,
/// Sharp slots per layer (0 = one kind of slot) and the hot experts per layer that get them,
/// from the route logs at load: the most-used experts of each layer.
sharp_n: usize,
hot: Vec<Vec<u32>>,
/// The token's shape per layer (`COACHWHIP_SHAPE=1`): running mean of x² over the hidden
/// size (for gate and up) and of the expert's hidden² over the intermediate size (for down),
/// in shared buffers the shaped shrink reads. Count of updates alongside.
shape: Mutex<HashMap<usize, (Arc<candle_metal_kernels::metal::Buffer>, Arc<candle_metal_kernels::metal::Buffer>, u64)>>,
}
/// The parallel path's fixed parts: the two kernels, the shared event, the sequence of waits and
/// the service thread's queue.
struct S4 {
pipes: crate::gpu_sync::Pipelines,
event: crate::gpu_sync::GpuEvent,
seq: AtomicU64,
tx: Mutex<Option<mpsc::Sender<S4Job>>>,
}
/// One layer's picks for the service thread: read them, place the experts, signal the GPU.
struct S4Job {
layer: usize,
seq: u64,
n_ids: usize,
n_guess: usize,
prefetch_k: usize,
ahead: usize,
}
/// Expert ids ordered by a router's scores, best first. Equal scores keep expert order, so the
/// same scores always give the same guess.
pub fn rank(scores: &[f32]) -> Vec<u32> {
let mut ids: Vec<u32> = (0..scores.len() as u32).collect();
ids.sort_by(|&a, &b| scores[b as usize].partial_cmp(&scores[a as usize]).unwrap_or(std::cmp::Ordering::Equal).then(a.cmp(&b)));
ids
}
/// A guessed read is stale once its word is over, or the word has reached its layer: from then
/// on that layer reads its own misses, and a late guess would only evict experts still to be used.
pub fn is_stale(job_layer: usize, job_word: u64, now_layer: usize, now_word: u64) -> bool {
job_word < now_word || (job_word == now_word && job_layer <= now_layer)
}
impl ExpertStore {
/// Reads the file's table of contents only; experts are read when a token first asks for them.
pub fn new(ct: &gguf_file::Content, device: &Device, layers: usize, path: &Path, settings: &Settings) -> Result<Arc<Self>> {
let Device::Metal(metal) = device else {
candle::bail!("Coachwhip needs a Metal device (Apple silicon)")
};
PROFILE.store(settings.profile, Ordering::Relaxed);
let mut per_layer = settings.bank.max(1);
let mut slices = Vec::with_capacity(layers);
for layer in 0..layers {
let parts = ["gate", "up", "down"].map(|m| {
let info = &ct.tensor_infos[&format!("blk.{layer}.ffn_{m}_exps.weight")];
let dims = info.shape.dims().to_vec();
let per_expert_elems: usize = dims[1..].iter().product();
let bytes = per_expert_elems / info.ggml_dtype.block_size() * info.ggml_dtype.type_size();
(info.ggml_dtype, ct.tensor_data_offset + info.offset, bytes, dims[1..].to_vec(), dims[0])
});
let experts = parts[0].4;
let per_expert: Vec<[Slice; 3]> = (0..experts)
.map(|e| {
parts.clone().map(|(dtype, base, bytes, dims, _)| Slice {
dtype,
offset: base + (e * bytes) as u64,
bytes,
dims,
})
})
.collect();
slices.push(per_expert);
}
let file = std::fs::File::open(path)?;
// Lab (`COACHWHIP_NOCACHE=1`): read experts straight from the flash, bypassing the page
// cache. On a 16 GB box the cache has nothing to give back and costs a copy per read.
if std::env::var("COACHWHIP_NOCACHE").map(|v| v == "1").unwrap_or(false) {
use std::os::unix::io::AsRawFd;
let r = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) };
eprintln!("coachwhip: page cache bypassed for expert reads (fcntl F_NOCACHE -> {r})");
}
let experts = slices[0].len();
let per_expert_mb = slices[0][0].iter().map(|s| s.bytes).sum::<usize>() as f64 / 1e6;
if let Some(fit) = wrangler_fit(ct, metal, layers, per_expert_mb * 1e6) {
if fit < per_layer {
eprintln!("coachwhip: wrangler: bank {per_layer} would not leave room for a full chat on this Mac; using {fit}");
per_layer = fit;
}
}
eprintln!(
"coachwhip: {layers} layers x {experts} experts, {per_expert_mb:.1} MB each; bank {per_layer} slots per layer ({:.2} GB)",
per_layer as f64 * layers as f64 * per_expert_mb / 1e3
);
let dirs: Vec<&Path> = settings.routes.iter().chain(settings.learned.iter()).map(|p| p.as_path()).collect();
let routes = load_routes(&dirs, layers);
eprintln!("coachwhip: route table for {} (layer, expert) pairs", routes.len());
let sharp_n: usize = std::env::var("COACHWHIP_SHARP").ok().and_then(|v| v.parse().ok()).unwrap_or(0);
let hot: Vec<Vec<u32>> = (0..layers)
.map(|l| {
let mut use_count: Vec<(u32, u64)> = (0..slices[l].len() as u32).map(|e| (e, routes.get(&(l, e)).map_or(0, |row| row.values().sum()))).collect();
use_count.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));
use_count.into_iter().take(sharp_n).filter(|(_, n)| *n > 0).map(|(e, _)| e).collect()
})
.collect();
if sharp_n > 0 {
let known: usize = hot.iter().map(|h| h.len()).sum();
eprintln!("coachwhip: two kinds of slot: {sharp_n} sharp per layer for the hot experts ({known} known from the route logs), {per_layer} small");
}
let started = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
let store = Arc::new(Self {
resident: Mutex::new(HashMap::new()),
file,
device: device.clone(),
metal: metal.clone(),
per_layer,
ahead: settings.ahead,
slices,
banks: Mutex::new(HashMap::new()),
stats: Mutex::new(Stats::default()),
layers,
routes: Mutex::new(routes),
trail: Mutex::new(Trail::default()),
learned: settings.learned.clone(),
learned_file: format!("{started}.log"),
prefetch_tx: Mutex::new(None),
word: AtomicU64::new(0),
now_layer: AtomicUsize::new(0),
draft: AtomicBool::new(false),
foresee: AtomicBool::new(false),
foresee_step: AtomicU64::new(0),
foresight_lists: Mutex::new(Vec::new()),
dump: Mutex::new(std::env::var("COACHWHIP_DUMP").ok().and_then(|p| std::fs::File::create(p).ok()).map(std::io::BufWriter::new)),
register: Mutex::new(if std::env::var("COACHWHIP_REGISTER").map(|v| v == "1").unwrap_or(false) { Some(Register::default()) } else { None }),
s4: if parallel() {
Some(S4 { pipes: crate::gpu_sync::pipelines(metal.device())?, event: crate::gpu_sync::GpuEvent::new(metal.device())?, seq: AtomicU64::new(0), tx: Mutex::new(None) })
} else {
None
},
requant: if gpuq2() { Some(crate::requant::Requant::new(metal)?) } else { None },
staging: Mutex::new(HashMap::new()),
shape: Mutex::new(HashMap::new()),
sharp_pool: Mutex::new(HashMap::new()),
pending_squeeze: Mutex::new(Vec::new()),
fresh: Mutex::new(HashMap::new()),
last_x: Mutex::new(HashMap::new()),
repair: Mutex::new(HashMap::new()),
sharp_n,
hot,
});
if gpuq2() {
let q2_mb = store.slices[0][0].iter().enumerate().map(|(i, s)| if !q2_none() && (i < 2 || shrink_down()) { s.dims.iter().product::<usize>() / crate::requant::QK_K * crate::requant::Q2K_BLOCK_BYTES } else { s.bytes }).sum::<usize>() as f64 / 1e6;
eprintln!("coachwhip: GPU shrink on: experts are shrunk to Q2_K on arrival, {q2_mb:.1} MB each in the bank ({:.2} GB)", per_layer as f64 * layers as f64 * q2_mb / 1e3);
}
if let Some(s4) = &store.s4 {
eprintln!("coachwhip: parallel path on: the GPU waits on a shared event for its experts; the CPU never waits for the GPU inside a word");
let (tx, rx) = mpsc::channel::<S4Job>();
*s4.tx.lock().unwrap() = Some(tx);
let weak: Weak<Self> = Arc::downgrade(&store);
std::thread::spawn(move || {
for job in rx {
let Some(st) = weak.upgrade() else { break };
st.s4_service(job);
}
});
}
// The prefetch threads read guessed experts into free slots while the GPU works on the
// current layer. They hold only a weak handle, so the store still shuts down with the model.
let (tx, rx) = mpsc::channel::<(usize, u64, Vec<u32>)>();
*store.prefetch_tx.lock().unwrap() = Some(tx);
let rx = Arc::new(Mutex::new(rx));
for _ in 0..settings.readers {
let weak: Weak<Self> = Arc::downgrade(&store);
let rx = rx.clone();
std::thread::spawn(move || loop {
let job = rx.lock().unwrap().recv();
let Ok((layer, word, experts)) = job else { break };
let Some(st) = weak.upgrade() else { break };
if is_stale(layer, word, st.now_layer.load(Ordering::Acquire), st.word.load(Ordering::Acquire)) {
st.stats.lock().unwrap().stale_dropped += experts.len() as u64;
continue;
}
if let Err(e) = st.prefetch(layer, &experts) {
eprintln!("coachwhip: prefetch failed: {e}");
}
});
}
Ok(store)
}
fn next_scores(&self, layer: usize, now: &[(u32, f32)]) -> Vec<(u32, f32)> {
next_scores(&self.routes.lock().unwrap(), layer, now)
}
/// The `k` likeliest experts of `layer` that are not in its bank yet, one layer ahead of `now`
/// (`ahead` = 1) or two (`ahead` = 2, composing the table twice).
fn predict_missing(&self, layer: usize, now: &[u32], ahead: usize, k: usize) -> Vec<u32> {
let mut scores: Vec<(u32, f32)> = now.iter().map(|&e| (e, 1.0)).collect();
for step in 0..ahead {
scores = self.next_scores(layer + step, &scores);
scores.truncate(32);
}
let banks = self.banks.lock().unwrap();
let present = banks.get(&(layer + ahead));
scores
.into_iter()
.map(|(e, _)| e)
.filter(|e| present.map_or(true, |b| !b.slots.slot_of.contains_key(e) && !b.sharp.as_ref().map_or(false, |sb| sb.slots.slot_of.contains_key(e))))
.take(k)
.collect()
}
pub fn register_on(&self) -> bool {
self.register.lock().unwrap().is_some()
}
/// One step of the register at `layer`: learn from the layer before (its prediction against
/// this layer's real input), then predict this layer's output state from the experts it chose.
/// `xs` is (1, h), `ids` the chosen experts, `top_w` their weights (1, k). Returns x̂ (1, h).
fn register_step(&self, layer: usize, xs: &Tensor, ids: &[u32], top_w: &Tensor) -> Result<Tensor> {
let mut guard = self.register.lock().unwrap();
let Some(reg) = guard.as_mut() else { return Ok(xs.clone()) };
let h = xs.dim(1)?;
let experts = self.slices[layer].len();
let device = xs.device();
if reg.d.is_empty() {
for l in 0..self.layers {
reg.d.push(Tensor::zeros((experts, h), DType::F32, device)?);
reg.d0.push(Tensor::zeros(h, DType::F32, device)?);
}
reg.lr = std::env::var("COACHWHIP_REGISTER_LR").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0);
reg.lr0 = std::env::var("COACHWHIP_REGISTER_LR0").ok().and_then(|v| v.parse().ok()).unwrap_or(0.05);
eprintln!("coachwhip: register on: {} layers x {experts} experts x {h}, lr {} / {}", self.layers, reg.lr, reg.lr0);
}
// Learn: the layer before predicted x̂; this layer's input is the truth.
if let Some((l, x_hat, ids_t, w_t)) = reg.last.take() {
if l + 1 == layer {
let err = (xs - &x_hat)?; // (1, h)
if profiling() {
let truth = xs.sqr()?.sum_all()?.to_scalar::<f32>()?.sqrt().max(1e-6);
let e = err.sqr()?.sum_all()?.to_scalar::<f32>()?.sqrt();
let plain = (xs - ®.last_x)?.sqr()?.sum_all()?.to_scalar::<f32>()?.sqrt();
let mut st = self.stats.lock().unwrap();
if st.reg_err_by_layer.len() < self.layers {
st.reg_err_by_layer = vec![0.0; self.layers];
st.reg_plain_by_layer = vec![0.0; self.layers];
}
st.reg_steps += 1;
st.reg_err += (e / truth) as f64;
st.reg_err_plain += (plain / truth) as f64;
st.reg_err_by_layer[layer] += (e / truth) as f64;
st.reg_plain_by_layer[layer] += (plain / truth) as f64;
}
let delta = w_t.broadcast_mul(&err)?.affine(reg.lr as f64, 0.0)?; // (k, h)
reg.d[l] = reg.d[l].index_add(&ids_t, &delta, 0)?;
reg.d0[l] = (®.d0[l] + err.squeeze(0)?.affine(reg.lr0 as f64, 0.0)?)?;
}
}
// Predict: x̂ = x + d0 + Σ w_e d[e].
let ids_t = Tensor::new(ids, device)?;
let w_t = top_w.reshape((ids.len(), 1))?.to_dtype(DType::F32)?;
let adds = reg.d[layer].index_select(&ids_t, 0)?; // (k, h)
let x_hat = (xs.broadcast_add(®.d0[layer])? + w_t.broadcast_mul(&adds)?.sum_keepdim(0)?)?;
reg.last = Some((layer, x_hat.clone(), ids_t, w_t));
reg.last_x = xs.clone();
Ok(x_hat)
}
/// The register step with the picks still on the GPU (the parallel path never reads them
/// back): `top_ids` (1, k) u32, `top_w` (1, k). Learns from the layer before, predicts x̂.
fn register_step_gpu(&self, layer: usize, xs: &Tensor, top_ids: &Tensor, top_w: &Tensor) -> Result<Tensor> {
let mut guard = self.register.lock().unwrap();
let Some(reg) = guard.as_mut() else { return Ok(xs.clone()) };
let h = xs.dim(1)?;
let k = top_ids.elem_count();
let experts = self.slices[layer].len();
let device = xs.device();
if reg.d.is_empty() {
for _ in 0..self.layers {
reg.d.push(Tensor::zeros((experts, h), DType::F32, device)?);
reg.d0.push(Tensor::zeros(h, DType::F32, device)?);
}
reg.lr = std::env::var("COACHWHIP_REGISTER_LR").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0);
reg.lr0 = std::env::var("COACHWHIP_REGISTER_LR0").ok().and_then(|v| v.parse().ok()).unwrap_or(0.05);
eprintln!("coachwhip: register on (GPU): {} layers x {experts} experts x {h}, lr {} / {}", self.layers, reg.lr, reg.lr0);
}
if let Some((l, x_hat, ids_t, w_t)) = reg.last.take() {
if l + 1 == layer {
let err = (xs - &x_hat)?;
let delta = w_t.broadcast_mul(&err)?.affine(reg.lr as f64, 0.0)?;
reg.d[l] = reg.d[l].index_add(&ids_t, &delta, 0)?;
reg.d0[l] = (®.d0[l] + err.squeeze(0)?.affine(reg.lr0 as f64, 0.0)?)?;
}
}
let ids_t = top_ids.flatten_all()?.contiguous()?;
let w_t = top_w.reshape((k, 1))?.to_dtype(DType::F32)?;
let adds = reg.d[layer].index_select(&ids_t, 0)?;
let x_hat = (xs.broadcast_add(®.d0[layer])? + w_t.broadcast_mul(&adds)?.sum_keepdim(0)?)?;
reg.last = Some((layer, x_hat.clone(), ids_t, w_t));
Ok(x_hat)
}
/// Chain one more layer on the GPU: route x̂ with `gate` (the next layer's router), then add
/// what those experts typically add. No read-back.
fn register_chain(&self, layer: usize, x_hat: &Tensor, gate: &Linear, k: usize) -> Result<Tensor> {
let guard = self.register.lock().unwrap();
let Some(reg) = guard.as_ref() else { return Ok(x_hat.clone()) };
if reg.d.is_empty() || layer >= reg.d.len() {
return Ok(x_hat.clone());
}
let scores = gate.forward(x_hat)?; // (1, e)
let top = scores.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, k)?.contiguous()?; // (1, k)
let p = candle_nn::ops::softmax_last_dim(&scores.gather(&top, D::Minus1)?)?; // (1, k)
let adds = reg.d[layer].index_select(&top.flatten_all()?, 0)?; // (k, h)
Ok((x_hat.broadcast_add(®.d0[layer])? + p.reshape((k, 1))?.broadcast_mul(&adds)?.sum_keepdim(0)?)?)
}
/// Lab: how many experts beyond the top k the foresight peek fetches per layer
/// (`COACHWHIP_FORESIGHT_EXTRA`, default 0).
fn foresight_extra() -> usize {
static V: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_FORESIGHT_EXTRA").ok().and_then(|v| v.parse().ok()).unwrap_or(0))
}
/// Hand a layer's guessed experts to the prefetch threads.
fn request_prefetch(&self, layer: usize, predicted: Vec<u32>) {
if predicted.is_empty() {
return;
}
if let Some(tx) = self.prefetch_tx.lock().unwrap().as_ref() {
// A draft pass runs ahead of the real pass of the same word: its reads belong to that word.
let word = self.word.load(Ordering::Acquire) + if self.drafting() { 1 + self.foresee_step.load(Ordering::Acquire) } else { 0 };
let _ = tx.send((layer, word, predicted));
}
}
/// Prefetch those of the top `k` experts in `guess` (best first) that `layer`'s bank does not
/// hold, in pairs across the reader threads so they are read side by side.
fn prefetch_guess(&self, layer: usize, guess: &[u32], k: usize) {
let missing: Vec<u32> = {
let banks = self.banks.lock().unwrap();
let present = banks.get(&layer);
guess.iter().take(k).copied().filter(|e| present.map_or(true, |b| !b.slots.slot_of.contains_key(e) && !b.sharp.as_ref().map_or(false, |sb| sb.slots.slot_of.contains_key(e)))).collect()
};
for pair in missing.chunks(2) {
self.request_prefetch(layer, pair.to_vec());
}
}
/// Mark where the word being written has got to; layer 0 starts a new word.
fn reached(&self, layer: usize) {
if layer == 0 {
self.word.fetch_add(1, Ordering::AcqRel);
}
self.now_layer.store(layer, Ordering::Release);
}
/// Prefetch thread: reserve free slots for the predicted experts, then read them in with the
/// bank unlocked, so the main thread is never held up by a read it did not ask for.
fn prefetch(&self, layer: usize, predicted_all: &[u32]) -> Result<()> {
// Two kinds: a guess tagged SHARP_TAG by the forward (its probability was above the line)
// goes to the sharp bank; untagged guesses go by the hot list when no tags are present,
// else to the small bank.
let tagged = predicted_all.iter().any(|&e| e & SHARP_TAG != 0);
let (hot, cold): (Vec<u32>, Vec<u32>) = if tagged {
let hot: Vec<u32> = predicted_all.iter().filter(|&&e| e & SHARP_TAG != 0).map(|&e| e & !SHARP_TAG).collect();
let cold: Vec<u32> = predicted_all.iter().filter(|&&e| e & SHARP_TAG == 0).copied().collect();
(hot, cold)
} else {
predicted_all.iter().partition(|&&e| self.is_hot(layer, e))
};
if !hot.is_empty() {
let mut sjobs: Vec<(u32, usize, [(usize, usize); 3], Arc<AtomicBool>)> = Vec::new();
{
let mut banks = self.banks.lock().unwrap();
if let Some(bank) = banks.get_mut(&layer) {
if let Some(sb) = bank.sharp.as_mut() {
let ready = &sb.ready;
let reserved = sb.slots.reserve(&hot, |s| ready[s].load(Ordering::Acquire));
for &(_, slot) in &reserved {
ready[slot].store(false, Ordering::Release);
}
for (e, slot) in reserved {
let ptrs = [0, 1, 2].map(|i| (sb.mats[i].base + slot * sb.mats[i].bytes, sb.mats[i].bytes));
sjobs.push((e, slot, ptrs, ready[slot].clone()));
}
}
}
}
for (e, _slot, ptrs, ready) in sjobs {
for (i, s) in self.slices[layer][e as usize].iter().enumerate() {
let dst = unsafe { std::slice::from_raw_parts_mut(ptrs[i].0 as *mut u8, s.bytes) };
self.file.read_exact_at(dst, s.offset)?;
}
ready.store(true, Ordering::Release);
self.stats.lock().unwrap().prefetch_copies += 1;
}
}
let predicted: &[u32] = &cold;
if predicted.is_empty() {
return Ok(());
}
let mut jobs: Vec<(u32, usize, [(usize, usize); 3], Arc<AtomicBool>, Arc<crate::gpu_sync::GpuBank>)> = Vec::new();
{
let mut banks = self.banks.lock().unwrap();
let Some(bank) = banks.get_mut(&layer) else { return Ok(()) };
let ready = &bank.ready;
let pinned = &bank.pinned;
let reserved = bank.slots.reserve(predicted, |s| ready[s].load(Ordering::Acquire) && !pinned[s].load(Ordering::Acquire));
for &(_, slot) in &reserved {
ready[slot].store(false, Ordering::Release);
bank.gpu.set_ready(slot, false);
}
// The GPU's table follows the CPU's: evicted experts gone, new ones present but not ready.
bank.sync_table();
let gpu = bank.gpu.clone();
for (e, slot) in reserved {
let ptrs = [0, 1, 2].map(|i| (bank.mats[i].base + slot * bank.mats[i].bytes, bank.mats[i].bytes));
jobs.push((e, slot, ptrs, ready[slot].clone(), gpu.clone()));
}
}
if self.requant.is_some() {
let mats: [(Arc<QMetalStorage>, usize); 3] = {
let banks = self.banks.lock().unwrap();
let bank = &banks[&layer];
[0, 1, 2].map(|i| (bank.mats[i].storage.clone(), bank.mats[i].bytes))
};
for (e, slot, _ptrs, ready, gpu) in jobs {
self.shrink_into(layer, e, slot, &mats)?;
ready.store(true, Ordering::Release);
gpu.set_ready(slot, true);
self.stats.lock().unwrap().prefetch_copies += 1;
}
return Ok(());
}
for (e, slot, ptrs, ready, gpu) in jobs {
for (i, s) in self.slices[layer][e as usize].iter().enumerate() {
let dst = unsafe { std::slice::from_raw_parts_mut(ptrs[i].0 as *mut u8, s.bytes) };
if staged() {
let mut tmp = vec![0u8; s.bytes];
self.file.read_exact_at(&mut tmp, s.offset)?;
dst.copy_from_slice(&tmp);
} else {
self.file.read_exact_at(dst, s.offset)?;
}
}
ready.store(true, Ordering::Release);
gpu.set_ready(slot, true);
self.stats.lock().unwrap().prefetch_copies += 1;
}
Ok(())
}
fn take_staging(&self, bytes: usize) -> Result<Arc<candle_metal_kernels::metal::Buffer>> {
if let Some(b) = self.staging.lock().unwrap().entry(bytes).or_default().pop() {
return Ok(b);
}
Ok(self.metal.new_buffer_builder().with_size(bytes).with_label("shrink_staging").build()?)
}
fn put_staging(&self, bytes: usize, b: Arc<candle_metal_kernels::metal::Buffer>) {
self.staging.lock().unwrap().entry(bytes).or_default().push(b);
}
fn take_sharp(&self, dtype: GgmlDType, elems: usize) -> Result<Arc<QMetalStorage>> {
if let Some(b) = self.sharp_pool.lock().unwrap().entry((dtype, elems)).or_default().pop() {
return Ok(b);
}
let st = QMetalStorage::zeros(&self.metal, elems, dtype)?;
self.device.synchronize()?;
Ok(Arc::new(st))
}
fn put_sharp(&self, dtype: GgmlDType, elems: usize, b: Arc<QMetalStorage>) {
self.sharp_pool.lock().unwrap().entry((dtype, elems)).or_default().push(b);
}
/// Read one miss into three sharp stagings (the file's bytes as they are) and remember it as
/// this forward's fresh expert; its Q2 slot stays not-ready until the squeeze.
fn stage_sharp(&self, layer: usize, e: u32, slot: usize, weight: f32, squeeze: bool) -> Result<()> {
let mut parts: Vec<Arc<QMetalStorage>> = Vec::with_capacity(3);
for sl in self.slices[layer][e as usize].iter() {
let elems: usize = sl.dims.iter().product();
let st = self.take_sharp(sl.dtype, elems)?;
let dst = unsafe { std::slice::from_raw_parts_mut(st.buffer().contents() as *mut u8, sl.bytes) };
self.file.read_exact_at(dst, sl.offset)?;
parts.push(st);
}
let parts: [Arc<QMetalStorage>; 3] = match parts.try_into() {
Ok(p) => p,
Err(_) => candle::bail!("an expert has three matrices"),
};
self.fresh.lock().unwrap().insert((layer, e), parts.clone());
self.pending_squeeze.lock().unwrap().push((layer, e, slot, parts, weight, squeeze));
Ok(())
}
/// After the GPU has drained: squeeze every pending miss from its sharp staging into its Q2
/// slot, mark the slot ready, return the stagings to the pool.
fn squeeze_pending(&self) -> Result<()> {
let pending: Vec<(usize, u32, usize, [Arc<QMetalStorage>; 3], f32, bool)> = std::mem::take(&mut *self.pending_squeeze.lock().unwrap());
if pending.is_empty() {
return Ok(());
}
let Some(rq) = self.requant.as_ref() else { return Ok(()) };
for (layer, e, slot, parts, weight, squeeze) in pending {
// A heavy re-read: the bank already holds its copy; just give the stagings back.
if !squeeze {
for (i, sl) in self.slices[layer][e as usize].iter().enumerate() {
self.put_sharp(sl.dtype, sl.dims.iter().product(), parts[i].clone());
}
self.fresh.lock().unwrap().remove(&(layer, e));
let _ = slot;
continue;
}
// Below the admission line: used, done, let go. No squeeze, no slot.
if admit_w().map_or(false, |line| weight < line) {
for (i, sl) in self.slices[layer][e as usize].iter().enumerate() {
self.put_sharp(sl.dtype, sl.dims.iter().product(), parts[i].clone());
}
self.fresh.lock().unwrap().remove(&(layer, e));
let mut banks = self.banks.lock().unwrap();
if let Some(bank) = banks.get_mut(&layer) {
bank.slots.release(e);
bank.ready[slot].store(true, Ordering::Release);
bank.gpu.clear(e);
bank.gpu.set_ready(slot, false);
}
self.stats.lock().unwrap().stale_dropped += 0;
continue;
}
let mats: [(Arc<QMetalStorage>, usize); 3] = {
let banks = self.banks.lock().unwrap();
let bank = &banks[&layer];
[0, 1, 2].map(|i| (bank.mats[i].storage.clone(), bank.mats[i].bytes))
};
for (i, sl) in self.slices[layer][e as usize].iter().enumerate() {
let elems: usize = sl.dims.iter().product();
if q2_none() || (i == 2 && !shrink_down()) {
let dst = unsafe { std::slice::from_raw_parts_mut((mats[i].0.buffer().contents() as usize + slot * mats[i].1) as *mut u8, sl.bytes) };
let src = unsafe { std::slice::from_raw_parts(parts[i].buffer().contents() as *const u8, sl.bytes) };
dst.copy_from_slice(src);
} else {
let Some(kind) = crate::requant::SrcKind::of(sl.dtype) else { candle::bail!("the GPU shrink takes Q3_K, Q4_K or Q5_K experts") };
let n_blocks = elems / crate::requant::QK_K;
rq.run(kind, parts[i].buffer(), 0, mats[i].0.buffer(), slot * mats[i].1, n_blocks)?;
}
}
if repair_on() {
self.measure_repair(layer, e, slot, &parts)?;
}
for (i, sl) in self.slices[layer][e as usize].iter().enumerate() {
self.put_sharp(sl.dtype, sl.dims.iter().product(), parts[i].clone());
}
self.fresh.lock().unwrap().remove(&(layer, e));
let banks = self.banks.lock().unwrap();
if let Some(bank) = banks.get(&layer) {
bank.ready[slot].store(true, Ordering::Release);
bank.gpu.set_ready(slot, true);
}
}
Ok(())
}
/// One expert's output y = down(silu(gate x) * up x) for input x (1, h), from three storages
/// at `slot` (the sharp stagings at slot 0, or the bank at the expert's slot).
fn expert_out(&self, x: &Tensor, layer: usize, ops: [MoeMatvec; 3], slot: u32) -> Result<Tensor> {
let device = x.device();
let slots = Tensor::from_vec(vec![slot], (1, 1), device)?;
let [g, u, d] = ops;
let gate = x.apply_op2_no_bwd(&slots, &g)?;
let up = x.apply_op2_no_bwd(&slots, &u)?;
let inter = gate.dim(2)?;
let hidden = (candle_nn::ops::silu(&gate)? * &up)?.reshape((1, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &d)?; // (1, 1, h)
down.flatten_all()?.to_dtype(DType::F32)
}
/// Measure the squeeze's damage to one expert on the input it was used for, and keep it.
fn measure_repair(&self, layer: usize, e: u32, slot: usize, parts: &[Arc<QMetalStorage>; 3]) -> Result<()> {
let x = { self.last_x.lock().unwrap().get(&layer).cloned() };
let Some(x) = x else { return Ok(()) };
let sharp = [0, 1, 2].map(|i| self.fresh_op(&parts[i], layer, i, 1, i == 2));
let y_sharp = self.expert_out(&x, layer, sharp, 0)?;
let q2 = [0, 1, 2].map(|i| self.bank_op(layer, i, 1, i == 2));
let y_q2 = self.expert_out(&x, layer, q2, slot as u32)?;
let d = (y_sharp - y_q2)?;
self.repair.lock().unwrap().insert((layer, e), d);
Ok(())
}
/// A one-slot `mul_mv_id` over a sharp staging (a fresh miss, the file's own dtype).
fn fresh_op(&self, st: &Arc<QMetalStorage>, layer: usize, which: usize, nei0: usize, rows_per_expert: bool) -> MoeMatvec {
let sl = &self.slices[layer][0][which];
MoeMatvec { bank: st.clone(), dtype: sl.dtype, n: sl.dims[0], k: sl.dims[1], slots: 1, bytes: sl.bytes, nei0, rows_per_expert }
}
/// Read one expert of `layer` from the file and shrink it on the GPU straight into `slot`
/// of the bank's three matrices (`mats`: storage and bytes per slot). Returns when the slot
/// holds the shrunk bytes.
fn shrink_into(&self, layer: usize, e: u32, slot: usize, mats: &[(Arc<QMetalStorage>, usize); 3]) -> Result<()> {
let Some(rq) = self.requant.as_ref() else { candle::bail!("shrink_into without the GPU shrink") };
for (i, sl) in self.slices[layer][e as usize].iter().enumerate() {
if q2_none() || (i == 2 && !shrink_down()) {
// The down matrix keeps the file's bytes: read it straight into its slot.
let dst = unsafe { std::slice::from_raw_parts_mut((mats[i].0.buffer().contents() as usize + slot * mats[i].1) as *mut u8, sl.bytes) };
self.file.read_exact_at(dst, sl.offset)?;
continue;
}
let Some(kind) = crate::requant::SrcKind::of(sl.dtype) else {
candle::bail!("the GPU shrink takes Q3_K, Q4_K or Q5_K experts, this file has {:?}", sl.dtype)
};
let staging = self.take_staging(sl.bytes)?;
let dst = unsafe { std::slice::from_raw_parts_mut(staging.contents() as *mut u8, sl.bytes) };
self.file.read_exact_at(dst, sl.offset)?;
let n_blocks = sl.dims.iter().product::<usize>() / crate::requant::QK_K;
let shape_buf = if shaped() {
let shapes = self.shape.lock().unwrap();
shapes.get(&layer).filter(|(_, _, n)| *n >= 32).map(|(xs, hs, _)| if i == 2 { hs.clone() } else { xs.clone() })
} else {
None
};
match shape_buf {
Some(sh) => rq.run_shaped(kind, &staging, 0, mats[i].0.buffer(), slot * mats[i].1, n_blocks, &sh, sl.dims[1], )?,
None => rq.run(kind, &staging, 0, mats[i].0.buffer(), slot * mats[i].1, n_blocks)?,
}
self.put_staging(sl.bytes, staging);
}
Ok(())
}
/// Fold one token's shape into `layer`'s running shape: x² over the hidden size, hidden² over
/// the intermediate size (averaged over the experts the token used). CPU side, ~20k floats.
fn note_shape(&self, layer: usize, xs: &Tensor, hidden: &Tensor) -> Result<()> {
let x: Vec<f32> = xs.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?;
let h2: Vec<Vec<f32>> = hidden.to_dtype(DType::F32)?.to_vec2()?;
let inter = h2.first().map_or(0, |r| r.len());
let mut shapes = self.shape.lock().unwrap();
let entry = match shapes.get(&layer) {
Some(_) => shapes.get_mut(&layer).unwrap(),
None => {
let xb = self.metal.new_buffer_builder().with_size(x.len() * 4).with_label("shape_x").build()?;
let hb = self.metal.new_buffer_builder().with_size(inter * 4).with_label("shape_h").build()?;
unsafe {
std::ptr::write_bytes(xb.contents() as *mut u8, 0, x.len() * 4);
std::ptr::write_bytes(hb.contents() as *mut u8, 0, inter * 4);
}
shapes.insert(layer, (xb, hb, 0));
shapes.get_mut(&layer).unwrap()
}
};
let n = entry.2 as f32;
let a = 1.0 / (n + 1.0);
let xp = entry.0.contents() as *mut f32;
let hp = entry.1.contents() as *mut f32;
unsafe {
for (i, v) in x.iter().enumerate() {
let old = *xp.add(i);
*xp.add(i) = old + a * (v * v - old);
}
for (i, row) in h2.iter().enumerate() {
let w = 1.0 / h2.len() as f32;
for (j, v) in row.iter().enumerate() {
let old = *hp.add(j);
let target = v * v;
// average over the experts used this token, then fold into the running mean
*hp.add(j) = old + a * w * (target - old) * if i == 0 { 1.0 } else { 1.0 };
}
}
}
entry.2 += 1;
Ok(())
}
/// A layer's bank: three shared buffers, each `per_layer` expert slots of one matrix.
fn make_bank(&self, layer: usize) -> Result<Bank> {
let mut mats = Vec::with_capacity(3);
for (i, s) in self.slices[layer][0].iter().enumerate() {
let elems: usize = s.dims.iter().product();
let (dtype, bytes) = if self.requant.is_some() && !q2_none() && (i < 2 || shrink_down()) {
(GgmlDType::Q2K, elems / crate::requant::QK_K * crate::requant::Q2K_BLOCK_BYTES)
} else {
(s.dtype, s.bytes)
};
let storage = exact_zeros(&self.metal, elems * self.per_layer, dtype)?;
let base = storage.buffer().contents() as usize;
if base == 0 {
candle::bail!("Metal gave a buffer the CPU cannot write to");
}
mats.push(BankMat {
storage: Arc::new(storage),
base,
bytes,
dtype,
n: s.dims[0],
k: s.dims[1],
});
}
let mats: [BankMat; 3] = match mats.try_into() {
Ok(m) => m,
Err(_) => candle::bail!("a bank has three matrices"),
};
// `zeros` queues a GPU fill; let it run now, or it would wipe the experts written in afterwards.
self.device.synchronize()?;
let gpu = Arc::new(crate::gpu_sync::GpuBank::new(&self.metal, self.slices[layer].len(), self.per_layer, 8 * 16 + 32)?);
let sharp = if self.sharp_n > 0 {
let mut smats = Vec::with_capacity(3);
for s in self.slices[layer][0].iter() {
let elems: usize = s.dims.iter().product();
let storage = QMetalStorage::zeros(&self.metal, elems * self.sharp_n, s.dtype)?;
let base = storage.buffer().contents() as usize;
if base == 0 {
candle::bail!("Metal gave a buffer the CPU cannot write to");
}
smats.push(BankMat { storage: Arc::new(storage), base, bytes: s.bytes, dtype: s.dtype, n: s.dims[0], k: s.dims[1] });
}
let smats: [BankMat; 3] = match smats.try_into() {
Ok(m) => m,
Err(_) => candle::bail!("a bank has three matrices"),
};
self.device.synchronize()?;
Some(SharpBank { mats: smats, slots: Slots::new(self.sharp_n), ready: (0..self.sharp_n).map(|_| Arc::new(AtomicBool::new(true))).collect() })
} else {
None
};
Ok(Bank {
mats,
slots: Slots::new(self.per_layer),
sharp,
ready: (0..self.per_layer).map(|_| Arc::new(AtomicBool::new(true))).collect(),
gpu,
pinned: (0..self.per_layer).map(|_| AtomicBool::new(false)).collect(),
})
}
/// Many fresh experts outside the bank, for a forward that needs more experts than a bank
/// holds (reading a prompt touches most of them). All the reads go to the SSD at once, and
/// `each` gets every expert the moment its bytes land, so the GPU computes while the rest of
/// the layer is still being read. `each` is called on this thread, in arrival order.
fn fresh_stream(&self, layer: usize, experts: &[u32], mut each: impl FnMut(usize, [Arc<QTensor>; 3]) -> Result<()>) -> Result<()> {
let started = Instant::now();
let (tx, rx) = std::sync::mpsc::channel::<(usize, Result<[Vec<u8>; 3]>)>();
let mut build_seconds = 0.0;
let mut waited = 0.0;
std::thread::scope(|scope| -> Result<()> {
scope.spawn(move || {
experts.par_iter().enumerate().for_each_with(tx, |tx, (j, &e)| {
let read = |i: usize| -> Result<Vec<u8>> {
let s = &self.slices[layer][e as usize][i];
let mut b = vec![0u8; s.bytes];
self.file.read_exact_at(&mut b, s.offset)?;
Ok(b)
};
let bytes = (|| Ok([read(0)?, read(1)?, read(2)?]))();
let _ = tx.send((j, bytes));
});
});
for _ in 0..experts.len() {
let wait = Instant::now();
let (j, bytes) = rx.recv().map_err(|e| candle::Error::Msg(format!("expert reader stopped: {e}")))?;
waited += wait.elapsed().as_secs_f64();
let bytes = bytes?;
let build = Instant::now();
let sl = &self.slices[layer][experts[j] as usize];
let [a, b, c] = [0, 1, 2].map(|i| qtensor_from_ggml(sl[i].dtype, &bytes[i], sl[i].dims.clone(), &self.device).map(Arc::new));
let mats = [a?, b?, c?];
build_seconds += build.elapsed().as_secs_f64();
each(j, mats)?;
}
Ok(())
})?;
let mut st = self.stats.lock().unwrap();
st.fresh += experts.len() as u64;
st.fresh_read_seconds += waited;
st.fresh_build_seconds += build_seconds;
st.load_seconds += started.elapsed().as_secs_f64();
Ok(())
}
/// Makes sure every needed expert of one layer sits in the bank; returns expert -> slot.
/// None when the bank is too small for this forward.
fn is_hot(&self, layer: usize, e: u32) -> bool {
self.sharp_n > 0 && self.hot[layer].contains(&e)
}
/// Where an expert goes with two kinds of slot: by its weight for this token when the weight
/// line is set (and the expert is not already parked somewhere), else by the hot list.
fn goes_sharp(&self, layer: usize, e: u32, weight: Option<f32>, bank: &Bank) -> bool {
if self.sharp_n == 0 {
return false;
}
if let Some(sb) = bank.sharp.as_ref() {
if sb.slots.slot_of.contains_key(&e) {
return true;
}
}
if bank.slots.slot_of.contains_key(&e) {
return false;
}
match (sharp_weight(), weight) {
(Some(line), Some(w)) => w >= line,
_ => self.is_hot(layer, e),
}
}
/// The sharp bank's half of a forward: give the hot experts sharp slots, read the missing
/// ones straight from the file (no shrink). Returns expert -> slot | SHARP_FLAG, or None when
/// the sharp bank cannot hold them (the caller then treats them as cold).
fn ensure_sharp(&self, layer: usize, bank: &mut Bank, hot: &[u32]) -> Result<Option<HashMap<u32, usize>>> {
let Some(sb) = bank.sharp.as_mut() else { return Ok(None) };
if hot.is_empty() {
return Ok(Some(HashMap::new()));
}
if hot.len() > self.sharp_n {
return Ok(None);
}
let ready = &sb.ready;
let Some(plan) = sb.slots.assign(hot, |s| ready[s].load(Ordering::Acquire)) else { return Ok(None) };
for &(_, slot) in &plan.hits {
while !sb.ready[slot].load(Ordering::Acquire) {
std::thread::yield_now();
}
}
self.stats.lock().unwrap().hits += plan.hits.len() as u64;
if !plan.misses.is_empty() {
let places: [(usize, usize); 3] = [0, 1, 2].map(|i| (sb.mats[i].base, sb.mats[i].bytes));
let reads: Vec<(usize, usize, u64)> = plan
.misses
.iter()
.flat_map(|&(e, slot)| self.slices[layer][e as usize].iter().enumerate().map(move |(i, s)| (places[i].0 + slot * places[i].1, s.bytes, s.offset)).collect::<Vec<_>>())
.collect();
let started = Instant::now();
reads.par_iter().try_for_each(|&(dst, len, offset)| {
let dst = unsafe { std::slice::from_raw_parts_mut(dst as *mut u8, len) };
self.file.read_exact_at(dst, offset)
})?;
let mut st = self.stats.lock().unwrap();
st.loads += plan.misses.len() as u64;
st.bytes_loaded += reads.iter().map(|r| r.1 as u64).sum::<u64>();
st.load_seconds += started.elapsed().as_secs_f64();
}
Ok(Some(hot.iter().map(|&e| (e, sb.slots.slot_of[&e] | SHARP_FLAG)).collect()))
}
fn ensure(&self, layer: usize, needed_all: &[u32]) -> Result<Option<HashMap<u32, usize>>> {
self.ensure_weighted(layer, needed_all, None)
}
fn ensure_weighted(&self, layer: usize, needed_all: &[u32], weights: Option<&HashMap<u32, f32>>) -> Result<Option<HashMap<u32, usize>>> {
// A forward that needs more experts than the banks hold (reading a prompt) goes the fresh
// way and must not make the banks here: their 8 GB would land on top of the prompt's
// transient reads and push the GPU out of memory. Found 8 Oct: the two-kinds change had
// moved this check below the bank's creation.
if needed_all.len() > self.per_layer + self.sharp_n {
return Ok(None);
}
let mut banks = self.banks.lock().unwrap();
if !banks.contains_key(&layer) {
banks.insert(layer, self.make_bank(layer)?);
}
let bank = banks.get_mut(&layer).unwrap();
// Two kinds: by weight when the line is set, else the hot experts go to the sharp bank.
let (hot, mut cold): (Vec<u32>, Vec<u32>) = needed_all.iter().partition(|&&e| self.goes_sharp(layer, e, weights.and_then(|m| m.get(&e).copied()), bank));
let sharp_map = if self.sharp_n > 0 {
match self.ensure_sharp(layer, bank, &hot)? {
Some(m) => m,
None => {
cold.extend(hot.iter().copied());
HashMap::new()
}
}
} else {
HashMap::new()
};
let needed: &[u32] = &cold;
if needed.len() > self.per_layer {
return Ok(None);
}
if needed.is_empty() {
return Ok(Some(sharp_map));
}
// A slot still being filled by a prefetch cannot be taken; if too few are free for this
// layer's misses, read them fresh instead, before the bank is touched.
let ready = &bank.ready;
let Some(plan) = bank.slots.assign(needed, |s| ready[s].load(Ordering::Acquire)) else {
self.stats.lock().unwrap().crowded += 1;
return Ok(None);
};
// S4: the GPU reads these slots later than now; keep the prefetch readers off them until
// this layer is placed again for the next word.
if self.s4.is_some() {
for p in &bank.pinned {
p.store(false, Ordering::Release);
}
for &(_, slot) in plan.hits.iter().chain(plan.misses.iter()) {
bank.pinned[slot].store(true, Ordering::Release);
}
}
// A prefetch may still be reading a hit in; wait for it rather than read it twice.
let mut waited = 0.0;
for &(_, slot) in &plan.hits {
if !bank.ready[slot].load(Ordering::Acquire) {
let wait = Instant::now();
while !bank.ready[slot].load(Ordering::Acquire) {
std::thread::yield_now();
}
waited += wait.elapsed().as_secs_f64();
}
}
{
let mut st = self.stats.lock().unwrap();
st.hits += plan.hits.len() as u64;
st.prefetch_hits += plan.prefetch_hits;
st.prefetch_wait_seconds += waited;
}
let misses = plan.misses;
// Heavy reuse: a hit this token leans on is re-read sharp; its Q2 copy stays for the light ones.
if self.requant.is_some() && use_sharp() {
if let (Some(line), Some(w)) = (reread_w(), weights) {
let heavy: Vec<(u32, usize)> = plan.hits.iter().copied().filter(|(e, _)| w.get(e).map_or(false, |&x| x >= line)).collect();
if !heavy.is_empty() {
let started = Instant::now();
heavy.par_iter().try_for_each(|&(e, slot)| self.stage_sharp(layer, e, slot, w[&e], false))?;
let mut st = self.stats.lock().unwrap();
st.loads += heavy.len() as u64;
st.load_seconds += started.elapsed().as_secs_f64();
}
}
}
// The GPU's table follows: misses are placed but not ready until their bytes land.
for &(_, slot) in &misses {
bank.gpu.set_ready(slot, false);
}
bank.sync_table();
if !misses.is_empty() && self.requant.is_some() && use_sharp() {
let started = Instant::now();
for &(_, slot) in &misses {
bank.ready[slot].store(false, Ordering::Release);
}
misses.par_iter().try_for_each(|&(e, slot)| self.stage_sharp(layer, e, slot, weights.and_then(|m| m.get(&e).copied()).unwrap_or(1.0), true))?;
let mut st = self.stats.lock().unwrap();
st.loads += misses.len() as u64;
st.bytes_loaded += misses.len() as u64 * self.slices[layer][0].iter().map(|s| s.bytes as u64).sum::<u64>();
st.load_seconds += started.elapsed().as_secs_f64();
// The slots are claimed but not ready; the squeeze at the next layer fills them.
let mut map = bank.slots.slot_of.clone();
map.extend(sharp_map);
return Ok(Some(map));
} else if !misses.is_empty() && self.requant.is_some() {
let mats: [(Arc<QMetalStorage>, usize); 3] = [0, 1, 2].map(|i| (bank.mats[i].storage.clone(), bank.mats[i].bytes));
let started = Instant::now();
misses.par_iter().try_for_each(|&(e, slot)| self.shrink_into(layer, e, slot, &mats))?;
let mut st = self.stats.lock().unwrap();
st.loads += misses.len() as u64;
st.bytes_loaded += misses.len() as u64 * self.slices[layer][0].iter().map(|s| s.bytes as u64).sum::<u64>();
st.load_seconds += started.elapsed().as_secs_f64();
} else if !misses.is_empty() {
// All of this layer's misses go to the SSD at once: it serves several requests in
// parallel far faster than one after another. Each matrix of each expert is one read.
let places: [(usize, usize); 3] = [0, 1, 2].map(|i| (bank.mats[i].base, bank.mats[i].bytes));
let reads: Vec<(usize, usize, u64)> = misses
.iter()
.flat_map(|&(e, slot)| {
self.slices[layer][e as usize]
.iter()
.enumerate()
.map(move |(i, s)| (places[i].0 + slot * places[i].1, s.bytes, s.offset))
.collect::<Vec<_>>()
})
.collect();
let started = Instant::now();
reads.par_iter().try_for_each(|&(dst, len, offset)| {
// Each read fills its own slot of a shared buffer that no GPU work in flight uses.
let dst = unsafe { std::slice::from_raw_parts_mut(dst as *mut u8, len) };
if staged() {
// Lab: read into ordinary memory first, then copy; the shared buffer may be
// slower to write straight from the file.
let mut tmp = vec![0u8; len];
self.file.read_exact_at(&mut tmp, offset)?;
dst.copy_from_slice(&tmp);
Ok(())
} else {
self.file.read_exact_at(dst, offset)
}
})?;
let mut st = self.stats.lock().unwrap();
st.loads += misses.len() as u64;
st.bytes_loaded += reads.iter().map(|r| r.1 as u64).sum::<u64>();
st.load_seconds += started.elapsed().as_secs_f64();
}
for &(_, slot) in &misses {
bank.gpu.set_ready(slot, true);
}
let mut map = bank.slots.slot_of.clone();
map.extend(sharp_map);
Ok(Some(map))
}
/// Like `ensure`, but hands the miss reads back instead of doing them, so the caller can
/// start the GPU on the hits first. Returns expert -> slot, the misses, and their reads
/// (destination, length, file offset); None when the bank is too small or crowded.
fn ensure_split(&self, layer: usize, needed: &[u32]) -> Result<Option<(HashMap<u32, usize>, Vec<(u32, usize)>, Vec<(usize, usize, u64)>)>> {
if needed.len() > self.per_layer {
return Ok(None);
}
let mut banks = self.banks.lock().unwrap();
if !banks.contains_key(&layer) {
banks.insert(layer, self.make_bank(layer)?);
}
let bank = banks.get_mut(&layer).unwrap();
let ready = &bank.ready;
let Some(plan) = bank.slots.assign(needed, |s| ready[s].load(Ordering::Acquire)) else {
self.stats.lock().unwrap().crowded += 1;
return Ok(None);
};
let mut waited = 0.0;
for &(_, slot) in &plan.hits {
if !bank.ready[slot].load(Ordering::Acquire) {
let wait = Instant::now();
while !bank.ready[slot].load(Ordering::Acquire) {
std::thread::yield_now();
}
waited += wait.elapsed().as_secs_f64();
}
}
{
let mut st = self.stats.lock().unwrap();
st.hits += plan.hits.len() as u64;
st.prefetch_hits += plan.prefetch_hits;
st.prefetch_wait_seconds += waited;
}
let misses = plan.misses;
for &(_, slot) in &misses {
bank.gpu.set_ready(slot, false);
// The prefetchers must not take a slot whose bytes are still to come.
bank.ready[slot].store(false, Ordering::Release);
}
bank.sync_table();
let places: [(usize, usize); 3] = [0, 1, 2].map(|i| (bank.mats[i].base, bank.mats[i].bytes));
let reads: Vec<(usize, usize, u64)> = misses
.iter()
.flat_map(|&(e, slot)| {
self.slices[layer][e as usize]
.iter()
.enumerate()
.map(move |(i, s)| (places[i].0 + slot * places[i].1, s.bytes, s.offset))
.collect::<Vec<_>>()
})
.collect();
Ok(Some((bank.slots.slot_of.clone(), misses, reads)))
}
/// The second half of `ensure_split`: read the misses in, then mark their slots ready.
fn read_misses(&self, layer: usize, misses: &[(u32, usize)], reads: &[(usize, usize, u64)]) -> Result<()> {
if misses.is_empty() {
return Ok(());
}
let started = Instant::now();
reads.par_iter().try_for_each(|&(dst, len, offset)| {
let dst = unsafe { std::slice::from_raw_parts_mut(dst as *mut u8, len) };
self.file.read_exact_at(dst, offset)
})?;
{
let mut st = self.stats.lock().unwrap();
st.loads += misses.len() as u64;
st.bytes_loaded += reads.iter().map(|r| r.1 as u64).sum::<u64>();
st.load_seconds += started.elapsed().as_secs_f64();
}
let banks = self.banks.lock().unwrap();
if let Some(bank) = banks.get(&layer) {
for &(_, slot) in misses {
bank.ready[slot].store(true, Ordering::Release);
bank.gpu.set_ready(slot, true);
}
}
Ok(())
}
/// S4: the GPU's view of `layer`'s bank, making the bank if the layer has none yet.
fn gpu_bank(&self, layer: usize) -> Result<Arc<crate::gpu_sync::GpuBank>> {
let mut banks = self.banks.lock().unwrap();
if !banks.contains_key(&layer) {
banks.insert(layer, self.make_bank(layer)?);
}
Ok(banks[&layer].gpu.clone())
}
/// S4: every layer's bank must exist before a word starts, so none is made mid-word while the
/// GPU queue is waiting on the service thread. Called at layer 0; cheap once they exist.
fn ensure_all_banks(&self) -> Result<()> {
let mut banks = self.banks.lock().unwrap();
if banks.len() >= self.layers {
return Ok(());
}
for layer in 0..self.layers {
if !banks.contains_key(&layer) {
let bank = self.make_bank(layer)?;
banks.insert(layer, bank);
}
}
Ok(())
}
/// S4 service: wait for the GPU to publish a layer's picks, place and read its experts, hand
/// the early guess to the readers, then let the GPU go on. Never touches the GPU queue.
fn s4_service(&self, job: S4Job) {
let Some(s4) = &self.s4 else { return };
let dbg = std::env::var("COACHWHIP_S4_DEBUG").is_ok();
let Ok(gpu) = self.gpu_bank(job.layer) else { return };
let started = Instant::now();
if dbg {
eprintln!("s4: job layer {} seq {} received", job.layer, job.seq);
}
let mut picks: Option<Vec<u32>> = None;
while picks.is_none() {
picks = gpu.picks_tagged(job.n_ids + job.n_guess, (job.seq & 0xFFFF) as u32);
if picks.is_none() && started.elapsed().as_secs() > 15 {
eprintln!("coachwhip: parallel: layer {} never published its picks; releasing the GPU", job.layer);
break;
}
std::hint::spin_loop();
}
if let Some(picks) = picks {
if dbg {
eprintln!("s4: layer {} flag seen after {:.1} ms", job.layer, started.elapsed().as_secs_f64() * 1e3);
}
self.reached(job.layer);
let ids = vec![picks[..job.n_ids].to_vec()];
self.learn(job.layer, &ids);
let mut needed: Vec<u32> = ids[0].clone();
needed.sort_unstable();
needed.dedup();
let placed = Instant::now();
loop {
match self.ensure(job.layer, &needed) {
Ok(Some(_)) => break,
Ok(None) => {
if placed.elapsed().as_secs() > 15 {
eprintln!("coachwhip: parallel: layer {} found no free slots; releasing the GPU", job.layer);
break;
}
std::thread::yield_now();
}
Err(e) => {
eprintln!("coachwhip: parallel: layer {}: {e}", job.layer);
break;
}
}
}
if dbg {
// Every needed expert must sit ready in a slot the GPU's table knows about.
let banks = self.banks.lock().unwrap();
if let Some(bank) = banks.get(&job.layer) {
for e in &needed {
match bank.slots.slot_of.get(e) {
Some(&slot) if bank.ready[slot].load(Ordering::Acquire) => {}
Some(&slot) => eprintln!("s4: MISSING layer {} expert {e}: slot {slot} not ready", job.layer),
None => eprintln!("s4: MISSING layer {} expert {e}: no slot", job.layer),
}
}
}
}
if job.n_guess > 0 && job.layer + job.ahead < self.layers {
self.prefetch_guess(job.layer + job.ahead, &picks[job.n_ids..], job.prefetch_k);
}
let mut st = self.stats.lock().unwrap();
st.load_phase_seconds += started.elapsed().as_secs_f64();
}
s4.event.signal(job.seq);
if dbg {
eprintln!("s4: layer {} signalled seq {} after {:.1} ms", job.layer, job.seq, started.elapsed().as_secs_f64() * 1e3);
}
if job.layer + 1 == self.layers {
let missing: u32 = (0..self.layers).filter_map(|l| self.banks.lock().unwrap().get(&l).map(|b| b.gpu.missing())).sum();
if missing > 0 {
eprintln!("coachwhip: parallel: {missing} expert picks so far had no slot when the GPU looked them up");
}
}
}
/// S6 measurement bookkeeping: one draft of `k` words, `accepted` of them matched the plain path.
pub fn note_speculation(&self, k: usize, accepted: usize) {
let mut st = self.stats.lock().unwrap();
st.spec_passes += 1;
st.spec_drafted += k as u64;
st.spec_accepted += accepted as u64;
}
/// S6 draft mode: route only among experts already in the bank, read nothing, learn nothing.
pub fn set_draft(&self, on: bool) {
self.draft.store(on, Ordering::Release);
}
pub fn drafting(&self) -> bool {
self.draft.load(Ordering::Acquire)
}
/// Foresight (`COACHWHIP_FORESIGHT=1`): during the draft pass, each layer's unrestricted route
/// is handed to the readers a whole word before the real pass asks for it.
pub fn set_foresee(&self, on: bool, step: u64) {
self.foresee.store(on, Ordering::Release);
self.foresee_step.store(step, Ordering::Release);
}
pub fn foreseeing(&self) -> bool {
self.foresee.load(Ordering::Acquire)
}
/// The bank's map for the GPU: a mask that shuts out every expert not sitting ready in a slot
/// (0 / -1e30, one per expert) and the slot each expert sits in (u32, one per expert).
fn peek_tables(&self, layer: usize, top_k: usize, device: &Device) -> Result<(Tensor, Tensor)> {
let (mask, table) = {
let banks = self.banks.lock().unwrap();
let Some(bank) = banks.get(&layer) else { candle::bail!("draft: layer {layer} has no bank yet") };
let experts = self.slices[layer].len();
let mut mask = vec![-1e30f32; experts];
let mut table = vec![0u32; experts];
let mut ready = 0;
for (&e, &s) in bank.slots.slot_of.iter() {
if bank.ready[s].load(Ordering::Acquire) {
mask[e as usize] = 0.0;
table[e as usize] = s as u32;
ready += 1;
}
}
if ready < top_k {
candle::bail!("draft: layer {layer} holds only {ready} ready experts")
}
(mask, table)
};
let n = mask.len();
Ok((Tensor::from_vec(mask, n, device)?, Tensor::from_vec(table, n, device)?))
}
/// A layer's foresight list, kept on the GPU until `flush_foresight`.
fn keep_foresight(&self, layer: usize, list: Tensor) {
self.foresight_lists.lock().unwrap().push((layer, list));
}
/// After a peek pass: read every layer's foresight list back (one wait) and start the reads.
pub fn flush_foresight(&self, top_k: usize) -> Result<()> {
let lists: Vec<(usize, Tensor)> = std::mem::take(&mut *self.foresight_lists.lock().unwrap());
for (layer, list) in lists {
let ids: Vec<u32> = list.to_vec1()?;
self.prefetch_guess(layer, &ids, top_k + ExpertStore::foresight_extra());
}
Ok(())
}
/// The experts of `layer` sitting in a ready slot right now, expert -> slot; None before the
/// layer has a bank at all.
pub fn resident_ready(&self, layer: usize) -> Option<HashMap<u32, usize>> {
let banks = self.banks.lock().unwrap();
let bank = banks.get(&layer)?;
Some(bank.slots.slot_of.iter().filter(|(_, &s)| bank.ready[s].load(Ordering::Acquire)).map(|(&e, &s)| (e, s)).collect())
}
/// The toaster (`COACHWHIP_TOASTER=1`): after a layer has computed, its experts' slots become
/// the first to be evicted.
fn pop_used(&self, layer: usize, used: &[u32]) {
if let Some(bank) = self.banks.lock().unwrap().get_mut(&layer) {
bank.slots.pop(used);
}
}
/// The experts of `layer` in the bank right now, before this layer's loads.
fn resident(&self, layer: usize) -> Vec<u32> {
let banks = self.banks.lock().unwrap();
banks.get(&layer).map(|b| b.slots.slot_of.keys().copied().collect()).unwrap_or_default()
}
/// S6 measurement: compare the real route with the route a draft confined to the bank would
/// take, and keep the score.
fn measure_draft(&self, layer: usize, scores: &[f32], real: &[u32], top_k: usize) {
let mut resident = self.resident(layer);
resident.sort_by(|a, b| scores[*b as usize].partial_cmp(&scores[*a as usize]).unwrap_or(std::cmp::Ordering::Equal));
let draft: Vec<u32> = resident.into_iter().take(top_k).collect();
let agree = real.iter().filter(|e| draft.contains(e)).count();
let mut st = self.stats.lock().unwrap();
if layer == 0 {
st.draft_word_ok = true;
}
st.draft_layers += 1;
st.draft_agree += agree as f64 / top_k as f64;
if agree == top_k {
st.draft_layers_full += 1;
} else {
st.draft_word_ok = false;
}
if layer + 1 == self.slices.len() && st.draft_word_ok {
st.draft_words_full += 1;
}
}
/// One `mul_mv_id` dispatch over the sharp bank's matrix.
fn bank_op_sharp(&self, layer: usize, which: usize, nei0: usize, rows_per_expert: bool) -> MoeMatvec {
let banks = self.banks.lock().unwrap();
let m = &banks[&layer].sharp.as_ref().expect("a sharp bank").mats[which];
MoeMatvec { bank: m.storage.clone(), dtype: m.dtype, n: m.n, k: m.k, slots: self.sharp_n, bytes: m.bytes, nei0, rows_per_expert }
}
/// One `mul_mv_id` dispatch over a bank matrix.
/// Resident mode: this layer's three expert matrices, all experts, read from the file once into
/// buffers of their exact size (Candle's zeros rounds up to a power of two: 230 MB would cost 400).
fn resident_op(&self, layer: usize, which: usize, nei0: usize, rows_per_expert: bool) -> Result<MoeMatvec> {
let mut res = self.resident.lock().unwrap();
if !res.contains_key(&layer) {
let experts = self.slices[layer].len();
let mut out = Vec::with_capacity(3);
let (g0, u0) = (&self.slices[layer][0][0], &self.slices[layer][0][1]);
let gu = gu_on() && g0.dtype == u0.dtype && g0.dims == u0.dims && g0.bytes == u0.bytes;
if gu {
// expert e = [its gate rows][its up rows], one after the other
let b = g0.bytes;
let mut graw = vec![0u8; b * experts];
self.file.read_exact_at(&mut graw, g0.offset)?;
let mut uraw = vec![0u8; b * experts];
self.file.read_exact_at(&mut uraw, u0.offset)?;
let mut raw = vec![0u8; 2 * b * experts];
for e in 0..experts {
raw[2 * b * e..2 * b * e + b].copy_from_slice(&graw[b * e..b * (e + 1)]);
raw[2 * b * e + b..2 * b * (e + 1)].copy_from_slice(&uraw[b * e..b * (e + 1)]);
}
drop((graw, uraw));
let st = Arc::new(exact_load(&self.metal, g0.dtype, &raw)?);
if layer == 0 { eprintln!("coachwhip: gate|up side by side, {:?}, {} rows per expert", g0.dtype, g0.dims[0] * 2); }
out.push((st.clone(), g0.dtype, g0.dims[0] * 2, g0.dims[1], 2 * b));
out.push((st, g0.dtype, g0.dims[0] * 2, g0.dims[1], 2 * b));
}
for m in (if gu { 2 } else { 0 })..3 {
let s0 = &self.slices[layer][0][m];
let mut raw = vec![0u8; s0.bytes * experts];
self.file.read_exact_at(&mut raw, s0.offset)?;
let storage = exact_load(&self.metal, s0.dtype, &raw)?;
out.push((Arc::new(storage), s0.dtype, s0.dims[0], s0.dims[1], s0.bytes));
}
let arr: [(Arc<QMetalStorage>, GgmlDType, usize, usize, usize); 3] = match out.try_into() {
Ok(a) => a,
Err(_) => candle::bail!("resident: three matrices"),
};
res.insert(layer, arr);
if let Some((used, limit)) = crate::gpu_sync::gpu_memory(&self.device) {
if layer % 10 == 0 || layer + 1 == self.layers {
eprintln!("coachwhip: resident layer {layer} loaded; GPU {:.2} of {:.2} GB", used as f64 / 1e9, limit as f64 / 1e9);
}
}
if layer + 1 == self.layers && std::env::var("COACHWHIP_HOLD_POOL").map(|v| v == "1").unwrap_or(false) {
candle::metal_backend::HOLD_POOL.store(true, std::sync::atomic::Ordering::Relaxed);
eprintln!("coachwhip: GPU memory held from here on (grow only)");
}
}
let (st, dtype, n, k, bytes) = res[&layer][which].clone();
Ok(MoeMatvec { bank: st, dtype, n, k, slots: self.slices[layer].len(), bytes, nei0, rows_per_expert })
}
fn bank_op(&self, layer: usize, which: usize, nei0: usize, rows_per_expert: bool) -> MoeMatvec {
let banks = self.banks.lock().unwrap();
let m = &banks[&layer].mats[which];
MoeMatvec {
bank: m.storage.clone(),
dtype: m.dtype,
n: m.n,
k: m.k,
slots: self.per_layer,
bytes: m.bytes,
nei0,
rows_per_expert,
}
}
/// Learn from one layer's picks: add the step from the layer before to the route table at once,
/// and keep the line for the log.
/// Lab: append one record (layer, the MoE input, the router's scores) to the dump file.
fn dump_record(&self, layer: usize, xs: &[f32], scores: &[f32]) {
use std::io::Write;
let mut d = self.dump.lock().unwrap();
if let Some(f) = d.as_mut() {
let mut buf = Vec::with_capacity(4 + 4 * (xs.len() + scores.len()));
buf.extend_from_slice(&(layer as u32).to_le_bytes());
for v in xs.iter().chain(scores.iter()) {
buf.extend_from_slice(&v.to_le_bytes());
}
let _ = f.write_all(&buf);
}
}
/// The words a forward pass is about to route, so the log's T line can name them.
pub fn note_tokens(&self, tokens: Vec<u32>) {
self.trail.lock().unwrap().tokens = tokens;
}
fn learn(&self, layer: usize, ids: &[Vec<u32>]) {
let mut trail = self.trail.lock().unwrap();
if layer == 0 {
trail.lines.clear();
} else if let Some((before_layer, before)) = trail.before.take() {
if before_layer + 1 == layer && before.len() == ids.len() {
count_step(&mut self.routes.lock().unwrap(), before_layer, &before, ids);
}
}
let line = ids.iter().map(|t| t.iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ")).collect::<Vec<_>>().join(",");
trail.lines.push(line);
if layer + 1 == self.layers {
let words = if trail.tokens.is_empty() { "-".to_string() } else { trail.tokens.iter().map(|t| t.to_string()).collect::<Vec<_>>().join(" ") };
let block = format!("T {words}
{}
", trail.lines.join("
"));
trail.blocks.push(block);
trail.lines.clear();
} else {
trail.before = Some((layer, ids.to_vec()));
}
}
/// Append the routes learned since the last save to this session's log file.
fn save_learned(&self) {
let Some(dir) = &self.learned else { return };
let blocks: Vec<String> = std::mem::take(&mut self.trail.lock().unwrap().blocks);
if blocks.is_empty() {
return;
}
let saved = std::fs::create_dir_all(dir).and_then(|_| {
use std::io::Write;
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(dir.join(&self.learned_file))?;
f.write_all(blocks.concat().as_bytes())
});
if let Err(e) = saved {
eprintln!("coachwhip: could not save learned routes: {e}");
}
}
/// Print the profile so far, save what was learned, and start counting afresh; the chat calls
/// this after each answer.
pub fn report_and_reset(&self) {
self.print_report();
self.save_learned();
*self.stats.lock().unwrap() = Stats::default();
}
fn print_report(&self) {
let s = self.stats.lock().unwrap();
if s.hits + s.loads == 0 {
return;
}
let requests = (s.hits + s.loads).max(1) as f64;
eprintln!(
"coachwhip: {} expert requests: bank {:.1}%, file {:.1}% ({} outside the bank); {:.2} GB read; {:.2} s loading; {:.2} ms per load; prefetched {} of which used {}",
requests as u64,
100.0 * s.hits as f64 / requests,
100.0 * s.loads as f64 / requests,
s.fresh,
s.bytes_loaded as f64 / 1e9,
s.load_seconds,
1000.0 * s.load_seconds / s.loads.max(1) as f64,
s.prefetch_copies,
s.prefetch_hits
);
if s.stand_ins > 0 {
eprintln!("coachwhip: stand-ins: {} weaker picks swapped for a resident expert", s.stand_ins);
}
eprintln!(
"coachwhip: waited {:.2} s on guesses still landing; dropped {} stale guessed reads; {} layers read fresh because the bank was crowded{}",
s.prefetch_wait_seconds, s.stale_dropped, s.crowded,
match memory_pressure() {
2 => "; wrangler: macOS memory pressure is at WARNING, close other apps or start with a smaller --bank",
4 => "; wrangler: macOS memory pressure is CRITICAL, start with a smaller --bank",
_ => "",
}
);
if s.fresh > 0 {
eprintln!(
"coachwhip: prompt experts: {} read outside the bank, {:.2} s reading, {:.2} s copying to the GPU",
s.fresh, s.fresh_read_seconds, s.fresh_build_seconds
);
}
if s.spec_passes > 0 {
eprintln!(
"coachwhip S6: {} drafts of {} words: {} of {} drafted words accepted ({:.1}%), {:.2} accepted per draft",
s.spec_passes,
s.spec_drafted / s.spec_passes,
s.spec_accepted,
s.spec_drafted,
100.0 * s.spec_accepted as f64 / s.spec_drafted.max(1) as f64,
s.spec_accepted as f64 / s.spec_passes as f64
);
}
if profiling() && s.draft_layers > 0 {
eprintln!(
"coachwhip S6: bank-only route agrees with the real route on {:.1}% of experts; {:.1}% of layers fully; {} of {} words fully",
100.0 * s.draft_agree / s.draft_layers as f64,
100.0 * s.draft_layers_full as f64 / s.draft_layers as f64,
s.draft_words_full,
s.words
);
}
if profiling() && s.reg_steps > 0 {
let n = s.reg_steps as f64;
eprintln!(
"coachwhip register: predicted state is {:.1}% off the truth on average (carrying the input unchanged: {:.1}% off); by layer:",
100.0 * s.reg_err / n,
100.0 * s.reg_err_plain / n
);
let per = n / s.reg_err_by_layer.len().max(1) as f64;
let line: Vec<String> = s.reg_err_by_layer.iter().zip(s.reg_plain_by_layer.iter()).enumerate().filter(|(l, _)| *l > 0).map(|(l, (e, p))| format!("{l}:{:.0}/{:.0}", 100.0 * e / per, 100.0 * p / per)).collect();
eprintln!("coachwhip register by layer (register/plain, % off): {}", line.join(" "));
}
if profiling() && s.words > 2 {
let words = (s.words - 1) as f64;
eprintln!(
"coachwhip cpu split per word: readback {:.1} ms, placing {:.1} ms, bookkeeping {:.1} ms (the rest of router+loads is the reads and the encode)",
1000.0 * s.t_readback / words,
1000.0 * s.t_place / words,
1000.0 * s.t_books / words
);
eprintln!(
"coachwhip profile: {} written words after the first, {:.1} ms per word in total, {:.1} ms in the expert layers: router+loads {:.1} (of which file reads {:.1}), expert maths {:.1}",
s.words - 1,
1000.0 * s.word_seconds / words,
1000.0 * s.expert_seconds / words,
1000.0 * s.load_phase_seconds / words,
1000.0 * s.load_seconds / words,
1000.0 * s.maths_seconds / words
);
}
}
}
impl Drop for ExpertStore {
fn drop(&mut self) {
self.print_report();
self.save_learned();
}
}
/// `x (rows, k) , ids (tokens, nei0)` -> `(tokens, nei0, n)`: for each token and each of its
/// selected slots, one row of x times that slot's quantised matrix from the bank.
/// With `rows_per_expert`, x has one row per (token, slot) instead of one per token.
struct MoeMatvec {
bank: Arc<QMetalStorage>,
dtype: GgmlDType,
n: usize,
k: usize,
slots: usize,
bytes: usize,
nei0: usize,
rows_per_expert: bool,
}
impl CustomOp2 for MoeMatvec {
fn name(&self) -> &'static str {
"coachwhip-moe-matvec"
}
fn cpu_fwd(&self, _: &candle::CpuStorage, _: &Layout, _: &candle::CpuStorage, _: &Layout) -> Result<(candle::CpuStorage, Shape)> {
candle::bail!("coachwhip-moe-matvec runs on Metal only")
}
fn metal_fwd(&self, x: &MetalStorage, xl: &Layout, ids: &MetalStorage, il: &Layout) -> Result<(MetalStorage, Shape)> {
if !xl.is_contiguous() || !il.is_contiguous() {
candle::bail!("coachwhip-moe-matvec needs contiguous inputs")
}
if x.dtype() != DType::F32 || ids.dtype() != DType::U32 {
candle::bail!("coachwhip-moe-matvec: x must be f32 and ids u32")
}
let (tokens, nei0) = il.shape().dims2()?;
if nei0 != self.nei0 {
candle::bail!("coachwhip-moe-matvec: {} ids per token, expected {}", nei0, self.nei0)
}
let (rows, k) = xl.shape().dims2()?;
if k != self.k || rows != if self.rows_per_expert { tokens * nei0 } else { tokens } {
candle::bail!("coachwhip-moe-matvec: x is {rows}x{k}, expected k={} for {tokens} tokens", self.k)
}
let (ne11, nb11, nb12) = if self.rows_per_expert { (nei0, k * 4, nei0 * k * 4) } else { (1, 0, k * 4) };
let device = x.device().clone();
let elems = tokens * nei0 * self.n;
let dst = crate::arena::take_for(&device, elems, DType::F32, "moe_mv_id")?;
{
let encoder = device.command_encoder()?;
let dt = candle_metal_kernels::GgmlDType::try_from(self.dtype)?;
if tokens > 8 && crate::fuse_kernel::group3_on() {
drop(encoder);
let (pb, hb) = crate::fuse_kernel::piles(&device, ids.buffer(), il.start_offset() * 4, tokens * nei0, nei0, self.slots)?;
let min_cnt = crate::mv_id::pile_min();
let encoder = device.command_encoder()?;
if crate::mv_id::matmul_id_piles(device.device(), &encoder, device.kernels(), dt, (self.n, self.k, self.slots, self.bytes), self.bank.buffer(), x.buffer(), xl.start_offset() * 4, (ne11, nb11, nb12), &pb, &hb, (self.slots, tokens, nei0), &dst, min_cnt).map_err(candle::MetalError::from)? {
drop(encoder);
if min_cnt > 1 {
let (sp, sn) = crate::fuse_kernel::small_pairs(&device, &pb, &hb, tokens * nei0, self.slots, min_cnt)?;
let encoder = device.command_encoder()?;
crate::mv_id::matvec_id_list(device.device(), &encoder, dt, (self.n, self.k, self.slots, self.bytes), self.bank.buffer(), x.buffer(), xl.start_offset() * 4, (ne11, nb11, nb12), ids.buffer(), il.start_offset() * 4, (nei0, tokens), &sp, &sn, tokens * nei0, &dst).map_err(candle::MetalError::from)?;
}
return Ok((MetalStorage::new(dst, device.clone(), elems, DType::F32), Shape::from((tokens, nei0, self.n))));
}
candle::bail!("grouped expert multiply: unsupported type {dt:?}")
}
if tokens > 1 && crate::mv_id::group2_on() {
// sort (slot, word) by expert once
let ids_cpu: Vec<u32> = {
drop(encoder);
let t = candle::Tensor::from_storage(candle::Storage::Metal(ids.clone()), il.shape().clone(), candle::op::BackpropOp::none(), false);
t.flatten_all()?.to_vec1::<u32>()?
};
let mut pairs: Vec<(u32, u16, u16)> = Vec::with_capacity(ids_cpu.len());
for (i, &ex) in ids_cpu.iter().enumerate() {
pairs.push((ex, (i % nei0) as u16, (i / nei0) as u16));
}
pairs.sort_by_key(|p| p.0);
let mut piles: Vec<u16> = Vec::with_capacity(pairs.len() * 2);
let mut heads: Vec<u32> = Vec::new();
let mut most = 0usize;
let mut i = 0;
while i < pairs.len() {
let ex = pairs[i].0;
let first = i;
while i < pairs.len() && pairs[i].0 == ex {
piles.push(pairs[i].1);
piles.push(pairs[i].2);
i += 1;
}
heads.extend_from_slice(&[ex, first as u32, (i - first) as u32]);
most = most.max(i - first);
}
let used = heads.len() / 3;
let pb = device.new_buffer_with_data(&piles)?;
let hb = device.new_buffer_with_data(&heads)?;
let encoder = device.command_encoder()?;
if crate::mv_id::matmul_id_piles(device.device(), &encoder, device.kernels(), dt, (self.n, self.k, self.slots, self.bytes), self.bank.buffer(), x.buffer(), xl.start_offset() * 4, (ne11, nb11, nb12), &pb, &hb, (used, most, nei0), &dst, 0).map_err(candle::MetalError::from)? {
drop(encoder);
return Ok((MetalStorage::new(dst, device.clone(), elems, DType::F32), Shape::from((tokens, nei0, self.n))));
}
candle::bail!("grouped expert multiply: unsupported type {dt:?}")
}
if tokens > 1 && crate::mv_id::group_on()
&& crate::mv_id::matmul_id_grouped(device.device(), &encoder, device.kernels(), dt, (self.n, self.k, self.slots, self.bytes), self.bank.buffer(), x.buffer(), xl.start_offset() * 4, (ne11, nb11, nb12), ids.buffer(), il.start_offset() * 4, (nei0, tokens), &dst).map_err(candle::MetalError::from)?
{
drop(encoder);
return Ok((MetalStorage::new(dst, device.clone(), elems, DType::F32), Shape::from((tokens, nei0, self.n))));
}
if crate::mv_id::new_on()
&& crate::mv_id::matvec_id_new(device.device(), &encoder, dt, (self.n, self.k, self.slots, self.bytes), self.bank.buffer(), x.buffer(), xl.start_offset() * 4, (ne11, nb11, nb12), ids.buffer(), il.start_offset() * 4, (nei0, tokens), &dst).map_err(candle::MetalError::from)?
{
drop(encoder);
return Ok((MetalStorage::new(dst, device.clone(), elems, DType::F32), Shape::from((tokens, nei0, self.n))));
}
crate::mv_id::matvec_id(
device.device(),
&encoder,
device.kernels(),
candle_metal_kernels::GgmlDType::try_from(self.dtype)?,
(self.n, self.k, self.slots, self.bytes),
self.bank.buffer(),
x.buffer(),
xl.start_offset() * 4,
(ne11, nb11, nb12),
ids.buffer(),
il.start_offset() * 4,
(nei0, tokens),
&dst,
)
.map_err(candle::MetalError::from)?;
}
Ok((MetalStorage::new(dst, device.clone(), elems, DType::F32), Shape::from((tokens, nei0, self.n))))
}
}
/// Lab (`COACHWHIP_GUESSLOG=file`): one line per word and layer, for racing guessers offline:
/// `layer ahead | picks as expert:weight | the router-ahead's top 32 as expert:probability`.
fn guess_log(path: &str, layer: usize, ahead: usize, picks: &[u32], top_w: &Tensor, scores: &[f32]) -> Result<()> {
use std::io::Write as _;
let w: Vec<f32> = top_w.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?;
let m = scores.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let z: f32 = scores.iter().map(|s| (s - m).exp()).sum();
let mut ranked: Vec<(usize, f32)> = scores.iter().enumerate().map(|(e, s)| (e, (s - m).exp() / z)).collect();
ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
let picks: Vec<String> = picks.iter().zip(w.iter()).map(|(e, w)| format!("{e}:{w:.4}")).collect();
let guess: Vec<String> = ranked.iter().take(32).map(|(e, p)| format!("{e}:{p:.5}")).collect();
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(path)?;
writeln!(f, "{layer} {ahead} | {} | {}", picks.join(" "), guess.join(" "))?;
Ok(())
}
/// Lab (`COACHWHIP_STANDIN_MAX`, default 0.3): a weaker pick is swapped for a stand-in only when
/// the router gave it less than this share of the word.
/// Lab (`COACHWHIP_STANDIN_KEEP`): how many of the top picks are always read for real; the default
/// is half the picks (at least 1).
fn stand_in_keep(k: usize) -> usize {
std::env::var("COACHWHIP_STANDIN_KEEP").ok().and_then(|v| v.parse().ok()).unwrap_or(k / 2).clamp(1, k.max(1))
}
/// Lab (`COACHWHIP_STANDIN_GAP=<g>`): a stand-in only when its router score is within g of the real
/// pick's; a pick that far ahead of every expert in the bank is read for real.
fn stand_in_gap() -> Option<f32> {
static V: std::sync::OnceLock<Option<f32>> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_STANDIN_GAP").ok().and_then(|v| v.parse().ok()))
}
/// Lab (`COACHWHIP_STANDIN_W=<w>`): with more than two picks, the second stays real when its weight is
/// at least w, else only the first does. Off: `COACHWHIP_STANDIN_KEEP` rules.
fn stand_in_w() -> Option<f32> {
static V: std::sync::OnceLock<Option<f32>> = std::sync::OnceLock::new();
*V.get_or_init(|| std::env::var("COACHWHIP_STANDIN_W").ok().and_then(|v| v.parse().ok()))
}
fn stand_in_limit() -> f32 {
std::env::var("COACHWHIP_STANDIN_MAX").ok().and_then(|v| v.parse().ok()).unwrap_or(0.3)
}
/// Router scores with every expert not `allowed` pushed far below the rest, so `route` picks
/// only among the allowed ones (S6 draft: the experts already in the bank).
pub fn restrict(logits: &Tensor, allowed: impl Fn(u32) -> bool) -> Result<Tensor> {
let n = logits.dim(D::Minus1)?;
let mask: Vec<f32> = (0..n as u32).map(|e| if allowed(e) { 0.0 } else { -1e30 }).collect();
let mask = Tensor::from_vec(mask, n, logits.device())?.to_dtype(logits.dtype())?;
logits.broadcast_add(&mask)
}
/// The router's choice for each token: the `top_k` likeliest experts from the router's scores,
/// best first, and their weights, rescaled to sum to 1 when `norm`.
pub fn route(logits: &Tensor, top_k: usize, norm: bool) -> Result<(Tensor, Tensor)> {
let probs = candle_nn::ops::softmax_last_dim(logits)?;
let top_ids = probs.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, top_k)?.contiguous()?;
let mut top_w = probs.gather(&top_ids, D::Minus1)?;
if norm {
top_w = top_w.broadcast_div(&top_w.sum_keepdim(D::Minus1)?)?;
}
Ok((top_ids, top_w))
}
/// What a family does between an expert's matrices: the activation, and any per-expert biases.
/// The core runs the matrices; the family owns this.
pub enum ExpertAct {
/// Qwen: down(silu(gate x) * up x).
Silu,
/// gpt-oss: gate and up get their biases, gate is capped at `limit`, up is clamped to
/// ±`limit`, hidden = (up + 1) * gate * sigmoid(alpha * gate), then down gets its bias.
/// Biases are (experts, width), looked up by the picked expert ids.
/// `pad`: the width down's matrix reads; a squeezed file pads rows to whole 256-weight
/// blocks, so the hidden state gets zeros up to it (0 = no padding).
ClampedSwiglu { alpha: f64, limit: f32, gate_b: Tensor, up_b: Tensor, down_b: Tensor, pad: usize },
}
impl ExpertAct {
/// gate, up: (n, k, inter); ids: the expert of each (n, k) position, asked for only when needed.
pub fn hidden(&self, gate: &Tensor, up: &Tensor, ids: impl FnOnce() -> Result<Tensor>) -> Result<Tensor> {
match self {
Self::Silu => candle_nn::ops::silu(gate)? * up,
Self::ClampedSwiglu { alpha, limit, gate_b, up_b, pad, .. } => {
let ids = ids()?;
let gate = (gate + Self::rows(gate_b, &ids, gate)?)?.clamp(f32::NEG_INFINITY, *limit)?;
let up = (up + Self::rows(up_b, &ids, up)?)?.clamp(-*limit, *limit)?;
let glu = (&gate * candle_nn::ops::sigmoid(&(&gate * *alpha)?)?)?;
let hidden = ((up + 1.0)? * glu)?;
let width = hidden.dim(candle::D::Minus1)?;
if *pad > width {
hidden.pad_with_zeros(candle::D::Minus1, 0, pad - width)
} else {
Ok(hidden)
}
}
}
}
/// down: (n, k, h), before the routing weights are applied.
pub fn finish(&self, down: Tensor, ids: impl FnOnce() -> Result<Tensor>) -> Result<Tensor> {
match self {
Self::Silu => Ok(down),
Self::ClampedSwiglu { down_b, .. } => {
let b = Self::rows(down_b, &ids()?, &down)?;
down + b
}
}
}
fn rows(bias: &Tensor, ids: &Tensor, like: &Tensor) -> Result<Tensor> {
let flat = ids.flatten_all()?.to_device(bias.device())?;
bias.index_select(&flat, 0)?.reshape(like.shape())?.to_dtype(like.dtype())?.to_device(like.device())
}
}
/// A MoE layer whose experts live in the ExpertStore instead of on the device.
pub struct StreamedMoe {
pub layer: usize,
pub gate: Linear,
/// A later layer's router (`ahead` layers on), run early on this layer's input to guess
/// that layer's experts while there is still time to read them in.
pub next_gate: Option<Linear>,
/// The very next layer's router, for the register to chain through when `ahead` is 2.
pub gate1: Option<Linear>,
pub ahead: usize,
pub store: Arc<ExpertStore>,
pub top_k: usize,
pub norm_topk_prob: bool,
pub prefetch_k: usize,
pub act: ExpertAct,
}
impl StreamedMoe {
/// Stand-ins on the parallel path, all on the GPU (2 experts a word): the first pick stays the
/// router's best; the second is the best-scoring expert in the bank other than the first,
/// which is the router's own second whenever that one is in the bank. Weights = the softmax
/// over the two picks. The bank's map goes up as a small mask; nothing is read back.
fn stand_in_gpu(&self, logits: &Tensor, top_ids: &Tensor, top_w: &Tensor) -> Result<(Tensor, Tensor)> {
let e = logits.dim(D::Minus1)?;
let resident: HashSet<u32> = self.store.resident_ready(self.layer).map(|m| m.into_keys().collect()).unwrap_or_default();
let mask: Vec<f32> = (0..e as u32).map(|x| if resident.contains(&x) { 0.0 } else { -1e30 }).collect();
let mask = Tensor::from_vec(mask, (1, e), logits.device())?.to_dtype(logits.dtype())?;
if self.top_k > 2 {
let k = self.top_k;
let with_keep = |keep: usize| -> Result<Tensor> {
let kept = top_ids.narrow(D::Minus1, 0, keep)?.contiguous()?;
let cand = logits.broadcast_add(&mask)?.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, k)?.contiguous()?;
// Candidates already kept are pushed to the back, then the best k - keep of the rest are taken.
let taken = cand.reshape((k, 1))?.broadcast_eq(&kept.reshape((1, keep))?)?.to_dtype(DType::F32)?.max_keepdim(1)?.reshape((1, k))?;
let rank = (Tensor::arange(0f32, k as f32, logits.device())?.reshape((1, k))? + (taken * 1000.0)?)?;
let pick = rank.arg_sort_last_dim(true)?.narrow(D::Minus1, 0, k - keep)?.contiguous()?;
let ids = Tensor::cat(&[&kept, &cand.gather(&pick, D::Minus1)?], D::Minus1)?.contiguous()?;
Ok(ids)
};
let ids = match stand_in_w() {
Some(line) => {
let heavy = top_w.narrow(D::Minus1, 1, 1)?.to_dtype(DType::F32)?.ge(line)?.broadcast_as((1, k))?.contiguous()?;
heavy.where_cond(&with_keep(2)?, &with_keep(1)?)?
}
None => with_keep(stand_in_keep(k))?,
};
// The gap: a substitute too far below its real pick is not used; the real one is read.
let ids = match stand_in_gap() {
Some(gap) => {
let real_ids = top_ids.contiguous()?;
let real = logits.gather(&real_ids, D::Minus1)?;
let sub = logits.gather(&ids, D::Minus1)?;
let close = (real - sub)?.to_dtype(DType::F32)?.le(gap)?;
close.where_cond(&ids, &real_ids)?
}
None => ids,
};
let w = candle_nn::ops::softmax_last_dim(&logits.gather(&ids, D::Minus1)?)?;
return Ok((ids, w));
}
let ranked = logits.broadcast_add(&mask)?.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, 2)?.contiguous()?;
let first = top_ids.narrow(D::Minus1, 0, 1)?.contiguous()?;
let m0 = ranked.narrow(D::Minus1, 0, 1)?.contiguous()?;
let m1 = ranked.narrow(D::Minus1, 1, 1)?.contiguous()?;
let stand_in = m0.eq(&first)?.where_cond(&m1, &m0)?;
// Only a light second pick is swapped; a second pick that carries real weight is read for real.
let own = top_ids.narrow(D::Minus1, 1, 1)?.contiguous()?;
let light = top_w.narrow(D::Minus1, 1, 1)?.to_dtype(DType::F32)?.lt(stand_in_limit())?;
let second = light.where_cond(&stand_in, &own)?;
let ids = Tensor::cat(&[&first, &second], D::Minus1)?.contiguous()?;
let w = candle_nn::ops::softmax_last_dim(&logits.gather(&ids, D::Minus1)?)?;
Ok((ids, w))
}
/// The plain path: one quantised matmul per expert, experts read fresh from the file.
fn reference(&self, xs: &Tensor, ids: &[Vec<u32>], top_w: &Tensor) -> Result<Tensor> {
let device = xs.device();
let weights: Vec<Vec<f32>> = top_w.to_vec2()?;
let mut tokens_per_expert: BTreeMap<u32, (Vec<u32>, Vec<f32>)> = BTreeMap::new();
for (t, (row_ids, row_w)) in ids.iter().zip(weights.iter()).enumerate() {
for (e, w) in row_ids.iter().zip(row_w.iter()) {
let entry = tokens_per_expert.entry(*e).or_default();
entry.0.push(t as u32);
entry.1.push(*w);
}
}
let experts: Vec<u32> = tokens_per_expert.keys().copied().collect();
let work: Vec<(Vec<u32>, Vec<f32>)> = tokens_per_expert.into_values().collect();
// Each expert's result is kept and summed in expert order at the end, so the answer does
// not depend on the order the reads happened to land in.
let mut parts: Vec<Option<(Tensor, Tensor)>> = (0..experts.len()).map(|_| None).collect();
self.store.fresh_stream(self.layer, &experts, |j, [g, u, d]| {
let (tokens, w) = &work[j];
let (gate, up, down) = (QMatMul::from_arc(g)?, QMatMul::from_arc(u)?, QMatMul::from_arc(d)?);
let idx = Tensor::new(tokens.as_slice(), device)?;
let x = xs.index_select(&idx, 0)?;
let picked = || Tensor::new(vec![experts[j]; tokens.len()], device);
let hidden = self.act.hidden(&gate.forward(&x)?, &up.forward(&x)?, picked)?;
let y = self.act.finish(down.forward(&hidden)?, picked)?;
let w = Tensor::new(w.as_slice(), device)?.unsqueeze(1)?;
parts[j] = Some((idx, y.broadcast_mul(&w)?));
Ok(())
})?;
// One add for the whole layer: adding expert by expert would build a full copy of the
// output for every expert, a gigabyte of short-lived GPU memory on a long prompt.
let (idx, ys): (Vec<Tensor>, Vec<Tensor>) = parts.into_iter().flatten().unzip();
if idx.is_empty() {
return xs.zeros_like();
}
// The output is as wide as down's rows, which a padded input can exceed.
let ys = Tensor::cat(&ys, 0)?;
Tensor::zeros((xs.dim(0)?, ys.dim(1)?), ys.dtype(), xs.device())?.index_add(&Tensor::cat(&idx, 0)?, &ys, 0)
}
pub fn forward(&self, xs: &Tensor) -> Result<Tensor> {
let (b, s, h) = xs.dims3()?;
let prof = profiling() && b * s == 1 && self.store.s4.is_none();
if prof && self.layer == 0 && !self.store.drafting() {
self.store.device.synchronize()?;
let mut st = self.store.stats.lock().unwrap();
if let Some(t) = st.word_started {
if st.words >= 2 {
st.word_seconds += t.elapsed().as_secs_f64();
}
}
st.word_started = Some(Instant::now());
st.words += 1;
}
let started = Instant::now();
let dtype = xs.dtype();
let n = b * s;
if n == 1 && !self.store.drafting() && self.store.s4.is_none() {
self.store.reached(self.layer);
}
let xs = xs.reshape((n, h))?.to_dtype(DType::F32)?.contiguous()?;
// Resident mode: every expert is on the GPU; the GPU routes and its picks are the addresses.
if resident_on() && !self.store.drafting() {
let logits = if n == 1 && std::env::var("COACHWHIP_SKIP").map(|v| v.split(',').any(|x| x == "router")).unwrap_or(false) {
Tensor::zeros((1, self.store.slices[self.layer].len()), DType::F32, xs.device())?
} else if n == 1 && crate::fuse_kernel::router16_on() {
static R16: std::sync::Mutex<Option<HashMap<usize, Tensor>>> = std::sync::Mutex::new(None);
let w16 = {
let mut g = R16.lock().unwrap();
let m = g.get_or_insert_with(HashMap::new);
if !m.contains_key(&self.layer) {
m.insert(self.layer, self.gate.weight().to_dtype(DType::F16)?.contiguous()?);
}
m[&self.layer].clone()
};
crate::fuse_kernel::mv_f16(&w16, &xs)?
} else {
self.gate.forward(&xs)?
};
let fuse = crate::fuse_kernel::on();
let (top_ids, top_w) = if fuse { crate::fuse_kernel::route(&logits, self.top_k, self.norm_topk_prob)? } else { route(&logits, self.top_k, self.norm_topk_prob)? };
let ids = if fuse { top_ids.clone() } else { top_ids.to_dtype(DType::U32)?.contiguous()? };
let esk = |w: &str| n == 1 && std::env::var("COACHWHIP_SKIP").map(|v| v.split(',').any(|x| x == w)).unwrap_or(false);
let silu = matches!(self.act, ExpertAct::Silu);
if fuse && silu && gu_on() {
let gu = xs.apply_op2_no_bwd(&ids, &self.store.resident_op(self.layer, 0, self.top_k, false)?)?;
let gu_ok = gu.dim(D::Minus1)? == 2 * self.store.slices[self.layer][0][0].dims[0];
if gu_ok {
let hidden = crate::fuse_kernel::silu_mul_gu(&gu)?.reshape((n * self.top_k, ()))?;
let down = hidden.apply_op2_no_bwd(&ids, &self.store.resident_op(self.layer, 2, self.top_k, true)?)?;
let down = self.act.finish(down, || Ok(top_ids.clone()))?;
let out = crate::fuse_kernel::weighted_sum(&down, &top_w)?;
return out.reshape((b, s, ()))?.to_dtype(dtype);
}
}
let (gate, up) = if esk("egate") {
let up = xs.apply_op2_no_bwd(&ids, &self.store.resident_op(self.layer, 1, self.top_k, false)?)?;
(up.clone(), up)
} else if esk("eup") {
let gate = xs.apply_op2_no_bwd(&ids, &self.store.resident_op(self.layer, 0, self.top_k, false)?)?;
(gate.clone(), gate)
} else {
(xs.apply_op2_no_bwd(&ids, &self.store.resident_op(self.layer, 0, self.top_k, false)?)?,
xs.apply_op2_no_bwd(&ids, &self.store.resident_op(self.layer, 1, self.top_k, false)?)?)
};
let hidden = if fuse && silu { crate::fuse_kernel::silu_mul(&gate, &up)? } else { self.act.hidden(&gate, &up, || Ok(top_ids.clone()))? }.reshape((n * self.top_k, ()))?;
let down = if esk("edown") {
Tensor::zeros((n, self.top_k, xs.dim(1)?), DType::F32, xs.device())?
} else {
hidden.apply_op2_no_bwd(&ids, &self.store.resident_op(self.layer, 2, self.top_k, true)?)?
};
let down = self.act.finish(down, || Ok(top_ids.clone()))?;
let out = if fuse { crate::fuse_kernel::weighted_sum(&down, &top_w)? } else { down.broadcast_mul(&top_w.unsqueeze(2)?)?.sum(1)? };
return out.reshape((b, s, ()))?.to_dtype(dtype);
}
// S6 draft: the bank as it stands is the whole model for this word. No reads, no learning,
// no prefetch, no bookkeeping; the LRU order is left alone too.
if self.store.drafting() {
if n != 1 {
candle::bail!("draft: one word at a time")
}
// No round trip to the CPU: the bank's map goes up to the GPU as two small tensors, the
// GPU picks among the resident experts and looks their slots up itself. The foresight
// list (the unrestricted route) is kept on the GPU and read back once, after the pass.
let device = xs.device();
let logits = self.gate.forward(&xs)?;
let (mask, table) = self.store.peek_tables(self.layer, self.top_k, device)?;
if self.store.foreseeing() {
let want = (self.top_k + ExpertStore::foresight_extra()).min(logits.dim(D::Minus1)?);
let list = logits.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, want)?.flatten_all()?.contiguous()?;
self.store.keep_foresight(self.layer, list);
}
let (top_ids, top_w) = route(&logits.broadcast_add(&mask)?, self.top_k, self.norm_topk_prob)?;
let slots = table.gather(&top_ids.flatten_all()?, 0)?.reshape((n, self.top_k))?;
let gate = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 0, self.top_k, false))?;
let up = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 1, self.top_k, false))?;
let inter = gate.dim(2)?;
let hidden = self.act.hidden(&gate, &up, || Ok(top_ids.clone()))?.reshape((n * self.top_k, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 2, self.top_k, true))?;
let down = self.act.finish(down, || Ok(top_ids.clone()))?;
let out = down.broadcast_mul(&top_w.unsqueeze(2)?)?.sum(1)?;
return out.reshape((b, s, ()))?.to_dtype(dtype);
}
let logits = self.gate.forward(&xs)?;
let (top_ids, top_w) = route(&logits, self.top_k, self.norm_topk_prob)?;
if n == 1 && repair_on() {
self.store.last_x.lock().unwrap().insert(self.layer, xs.clone());
}
// S4, the parallel path: nothing here waits for the GPU. The picks stay on the GPU; a tiny
// kernel publishes them for the service thread, the maths is queued behind a wait on the
// shared event, and a second kernel looks the slots up once the service thread has placed
// the experts and signalled. The CPU moves straight on to the next layer.
if n == 1 && self.store.s4.is_some() {
let s4 = self.store.s4.as_ref().unwrap();
// Lab (`COACHWHIP_STANDIN=1`, 2 experts a word): stand-ins chosen on the GPU.
let (top_ids, top_w) = if self.top_k >= 2 && std::env::var("COACHWHIP_STANDIN").is_ok() { self.stand_in_gpu(&logits, &top_ids, &top_w)? } else { (top_ids, top_w) };
if self.layer == 0 {
self.store.ensure_all_banks()?;
}
let device = xs.device();
let (guess, n_guess) = match &self.next_gate {
Some(g) if self.prefetch_k > 0 => {
// With the register on, guess from the state it predicts, all on the GPU.
let x_in = if self.store.register_on() {
let mut x_hat = self.store.register_step_gpu(self.layer, &xs, &top_ids, &top_w)?;
if self.ahead >= 2 {
if let Some(g1) = &self.gate1 {
x_hat = self.store.register_chain(self.layer + 1, &x_hat, g1, self.top_k)?;
}
}
x_hat
} else {
xs.clone()
};
let scores = g.forward(&x_in)?;
let n_guess = 32.min(scores.dim(D::Minus1)?);
(scores.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, n_guess)?.flatten_all()?.contiguous()?, n_guess)
}
_ => (Tensor::zeros(1, DType::U32, device)?, 0),
};
// Metal's rule: a command buffer must be committed after the CPU writes the data it
// reads. The buffer holding the layer before's maths is committed by this layer's flush,
// so wait here until the service thread has placed and written that layer's experts.
// The GPU keeps its queue; only the CPU's run-ahead is capped at one layer.
let last = s4.seq.load(Ordering::Acquire);
let held = Instant::now();
while s4.event.value() < last {
if held.elapsed().as_secs() > 20 {
candle::bail!("parallel: layer {} waited too long for the layer before", self.layer)
}
std::hint::spin_loop();
}
let gpu = self.store.gpu_bank(self.layer)?;
gpu.reset_flag();
if std::env::var("COACHWHIP_S4_DEBUG").is_ok() {
eprintln!("s4: encode layer {}", self.layer);
}
let seq = s4.seq.fetch_add(1, Ordering::AcqRel) + 1;
let publish = crate::gpu_sync::Publish { bank: gpu.clone(), pipeline: s4.pipes.publish.clone(), n_guess, tag: (seq & 0xFFFF) as u32 };
let _published = top_ids.apply_op2_no_bwd(&guess, &publish)?;
crate::gpu_sync::flush()?;
s4.event.gpu_wait(seq)?;
let resolve = crate::gpu_sync::Resolve { bank: gpu.clone(), pipeline: s4.pipes.resolve.clone() };
let slots = top_ids.apply_op1_no_bwd(&resolve)?;
let gate = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 0, self.top_k, false))?;
let up = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 1, self.top_k, false))?;
let inter = gate.dim(2)?;
let hidden = self.act.hidden(&gate, &up, || Ok(top_ids.clone()))?.reshape((n * self.top_k, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 2, self.top_k, true))?;
let down = self.act.finish(down, || Ok(top_ids.clone()))?;
let out = down.broadcast_mul(&top_w.unsqueeze(2)?)?.sum(1)?;
if let Some(tx) = s4.tx.lock().unwrap().as_ref() {
let _ = tx.send(S4Job { layer: self.layer, seq, n_ids: self.top_k, n_guess, prefetch_k: self.prefetch_k, ahead: self.ahead });
}
return out.reshape((b, s, ()))?.to_dtype(dtype);
}
// S6 measurement (profile mode, one word at a time): what a bank-only draft would pick.
let draft_scores: Option<Vec<f32>> = if prof { Some(logits.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?) } else { None };
if n == 1 && self.store.dump.lock().unwrap().is_some() {
let x: Vec<f32> = xs.flatten_all()?.to_vec1()?;
let s: Vec<f32> = logits.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?;
self.store.dump_record(self.layer, &x, &s);
}
// The guess for a later layer is queued before the read-back below, so it costs no wait.
let register = n == 1 && self.store.register_on();
let early = match &self.next_gate {
Some(g) if n == 1 && self.prefetch_k > 0 && !register => Some(g.forward(&xs)?),
_ => None,
};
// Reading the picks back waits for the GPU to finish everything queued so far, so no
// kernel is still reading a bank slot when `ensure` and the prefetchers overwrite it.
// Keep this wait if this code ever changes.
let t_rb = Instant::now();
let ids: Vec<Vec<u32>> = top_ids.to_vec2()?;
// Lab (`COACHWHIP_STANDIN=1`): a word keeps its strongest pick (read from the SSD if it must
// be), and each weaker pick that is not in the bank is swapped for the resident expert the
// router scores highest for this word. The weights are the softmax over the final picks.
let (ids, top_ids, top_w) = if n == 1 && std::env::var("COACHWHIP_STANDIN").is_ok() {
let scores: Vec<f32> = logits.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?;
let resident: HashSet<u32> = self.store.resident(self.layer).into_iter().collect();
let w0: Vec<f32> = top_w.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?;
let mut row = ids[0].clone();
let keep = match (stand_in_w(), row.len() > 2) {
(Some(line), true) => if w0[1] >= line { 2 } else { 1 },
_ => stand_in_keep(row.len()),
};
for p in keep..row.len() {
if !resident.contains(&row[p]) && (row.len() > 2 || w0[p] < stand_in_limit()) {
let best = resident.iter().copied().filter(|e| !row.contains(e)).max_by(|a, b| scores[*a as usize].partial_cmp(&scores[*b as usize]).unwrap_or(std::cmp::Ordering::Equal));
if let Some(e) = best.filter(|&e| stand_in_gap().map_or(true, |g| scores[row[p] as usize] - scores[e as usize] <= g)) {
row[p] = e;
self.store.stats.lock().unwrap().stand_ins += 1;
}
}
}
let m = row.iter().map(|&e| scores[e as usize]).fold(f32::NEG_INFINITY, f32::max);
let ex: Vec<f32> = row.iter().map(|&e| (scores[e as usize] - m).exp()).collect();
let z: f32 = ex.iter().sum();
let k = row.len();
let new_ids = Tensor::from_vec(row.clone(), (1, k), xs.device())?;
let new_w = Tensor::from_vec(ex.iter().map(|v| v / z).collect::<Vec<f32>>(), (1, k), xs.device())?.to_dtype(top_w.dtype())?;
(vec![row], new_ids, new_w)
} else {
(ids, top_ids, top_w)
};
let t_rb = t_rb.elapsed().as_secs_f64();
// The GPU has drained: the misses the layer before used sharp can be squeezed now.
if use_sharp() {
self.store.squeeze_pending()?;
}
let t_bk = Instant::now();
if let Some(scores) = &draft_scores {
self.store.measure_draft(self.layer, scores, &ids[0], self.top_k);
}
self.store.learn(self.layer, &ids);
let mut needed: Vec<u32> = ids.iter().flatten().copied().collect();
needed.sort_unstable();
needed.dedup();
let books_a = t_bk.elapsed().as_secs_f64();
let loads_before = if prof { self.store.stats.lock().unwrap().load_seconds } else { 0.0 };
let t_pl = Instant::now();
// Split path: assign now, read the misses after the hits' maths has been queued.
let split = n == 1 && split_maths();
let mut pending: Option<(Vec<(u32, usize)>, Vec<(usize, usize, u64)>)> = None;
let slot_of = if split {
match self.store.ensure_split(self.layer, &needed)? {
Some((slot_of, misses, reads)) => {
pending = Some((misses, reads));
Some(slot_of)
}
None => None,
}
} else if n == 1 && ((self.store.sharp_n > 0 && sharp_weight().is_some()) || (use_sharp() && (admit_w().is_some() || reread_w().is_some()))) {
let w: Vec<f32> = top_w.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?;
let weights: HashMap<u32, f32> = ids[0].iter().copied().zip(w.into_iter()).collect();
self.store.ensure_weighted(self.layer, &needed, Some(&weights))?
} else {
self.store.ensure(self.layer, &needed)?
};
let t_pl = t_pl.elapsed().as_secs_f64();
let loads_in = if prof { self.store.stats.lock().unwrap().load_seconds - loads_before } else { 0.0 };
let t_bk2 = Instant::now();
// Our register: predict the state this layer hands on, chain to the layer before the
// target if need be, and run the target layer's router on the predicted state.
let early = match (&self.next_gate, register) {
(Some(g), true) if self.prefetch_k > 0 => {
let mut x_hat = self.store.register_step(self.layer, &xs, &ids[0], &top_w)?;
if self.ahead >= 2 {
if let Some(g1) = &self.gate1 {
x_hat = self.store.register_chain(self.layer + 1, &x_hat, g1, self.top_k)?;
}
}
Some(g.forward(&x_hat)?)
}
_ => early,
};
if let Some(logits) = early {
let scores: Vec<f32> = logits.flatten_all()?.to_dtype(DType::F32)?.to_vec1()?;
if let Ok(path) = std::env::var("COACHWHIP_GUESSLOG") {
guess_log(&path, self.layer, self.ahead, &ids[0], &top_w, &scores)?;
}
if let Some(p_min) = q_rule() {
// The probability rule: a candidate is read iff its router probability, calibrated
// on the route dumps, says it is more likely needed than its bytes cost. On the
// 122B that line sits at a router probability of about 1%.
let m = scores.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let ex: Vec<f32> = scores.iter().map(|&z| (z - m).exp()).collect();
let sum: f32 = ex.iter().sum();
let ranked = rank(&scores);
let cands: Vec<u32> = ranked.iter().copied().take_while(|&e| ex[e as usize] / sum >= p_min).take(q_cap()).collect();
if let (Some(line), true) = (sharp_weight(), self.store.sharp_n > 0) {
// By weight: the guess's share among the candidates stands in for the token's
// weight; above the line it is tagged for the sharp bank. Reader threads do the reads.
let tot: f32 = cands.iter().map(|&e| ex[e as usize]).sum::<f32>().max(1e-9);
let missing: Vec<u32> = {
let banks = self.store.banks.lock().unwrap();
let present = banks.get(&(self.layer + self.ahead));
cands.iter().copied().filter(|e| present.map_or(true, |b| !b.slots.slot_of.contains_key(e) && !b.sharp.as_ref().map_or(false, |sb| sb.slots.slot_of.contains_key(e)))).collect()
};
let tagged: Vec<u32> = missing.iter().map(|&e| if ex[e as usize] / tot >= line { e | SHARP_TAG } else { e }).collect();
for pair in tagged.chunks(2) {
self.store.request_prefetch(self.layer + self.ahead, pair.to_vec());
}
} else {
let k = cands.len().max(1);
self.store.prefetch_guess(self.layer + self.ahead, &cands, k);
}
} else {
self.store.prefetch_guess(self.layer + self.ahead, &rank(&scores), self.prefetch_k);
}
} else if n == 1 && self.prefetch_k > 0 {
for ahead in 1..=self.store.ahead {
if self.layer + ahead < self.store.slices.len() {
let predicted = self.store.predict_missing(self.layer, &needed, ahead, self.prefetch_k);
self.store.request_prefetch(self.layer + ahead, predicted);
}
}
}
let books_b = t_bk2.elapsed().as_secs_f64();
if prof {
self.store.device.synchronize()?;
let mut st = self.store.stats.lock().unwrap();
if st.words > 1 {
st.load_phase_seconds += started.elapsed().as_secs_f64();
st.t_readback += t_rb;
st.t_place += (t_pl - loads_in).max(0.0);
st.t_books += books_a + books_b;
}
}
let maths_started = Instant::now();
let device = xs.device();
let out = if let (Some(slot_of), Some((misses, reads))) = (&slot_of, pending.take()) {
// The resident experts first, queued and flushed so the GPU works while the CPU reads.
let miss_set: HashSet<u32> = misses.iter().map(|&(e, _)| e).collect();
let row = &ids[0];
let part = |positions: &[usize]| -> Result<Option<Tensor>> {
if positions.is_empty() {
return Ok(None);
}
let k = positions.len();
let slots: Vec<u32> = positions.iter().map(|&p| slot_of[&row[p]] as u32).collect();
let slots = Tensor::from_vec(slots, (1, k), device)?;
let idx = Tensor::new(positions.iter().map(|&p| p as u32).collect::<Vec<_>>(), device)?;
let w = top_w.index_select(&idx, 1)?; // (1, k)
let gate = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 0, k, false))?;
let up = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 1, k, false))?;
let inter = gate.dim(2)?;
let picked = || top_ids.index_select(&idx, 1);
let hidden = self.act.hidden(&gate, &up, picked)?.reshape((k, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 2, k, true))?;
let down = self.act.finish(down, picked)?;
Ok(Some(down.broadcast_mul(&w.unsqueeze(2)?)?.sum(1)?))
};
let hit_pos: Vec<usize> = (0..row.len()).filter(|&p| !miss_set.contains(&row[p])).collect();
let miss_pos: Vec<usize> = (0..row.len()).filter(|&p| miss_set.contains(&row[p])).collect();
let hits_out = part(&hit_pos)?;
if hits_out.is_some() && !misses.is_empty() {
crate::gpu_sync::flush()?;
}
self.store.read_misses(self.layer, &misses, &reads)?;
let miss_out = part(&miss_pos)?;
match (hits_out, miss_out) {
(Some(a), Some(b)) => (a + b)?,
(Some(a), None) | (None, Some(a)) => a,
(None, None) => xs.zeros_like()?,
}
} else if let (Some(slot_of), true) = (&slot_of, n == 1 && use_sharp() && self.store.requant.is_some()) {
// Use sharp, then squeeze: fresh misses compute from their stagings, the rest from the bank.
let row = &ids[0];
let fresh: HashMap<u32, [Arc<QMetalStorage>; 3]> = {
let f = self.store.fresh.lock().unwrap();
row.iter().filter_map(|e| f.get(&(self.layer, *e)).map(|p| (*e, p.clone()))).collect()
};
let mut out: Option<Tensor> = None;
let mut add = |t: Tensor| -> Result<()> {
out = Some(match out.take() {
Some(a) => (a + t)?,
None => t,
});
Ok(())
};
// the bank part
let bank_pos: Vec<usize> = (0..row.len()).filter(|&p| !fresh.contains_key(&row[p])).collect();
if !bank_pos.is_empty() {
let k = bank_pos.len();
let slots: Vec<u32> = bank_pos.iter().map(|&p| (slot_of[&row[p]] & !SHARP_FLAG) as u32).collect();
let slots = Tensor::from_vec(slots, (1, k), device)?;
let idx = Tensor::new(bank_pos.iter().map(|&p| p as u32).collect::<Vec<_>>(), device)?;
let w = top_w.index_select(&idx, 1)?;
let gate = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 0, k, false))?;
let up = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 1, k, false))?;
let inter = gate.dim(2)?;
let picked = || top_ids.index_select(&idx, 1);
let hidden = self.act.hidden(&gate, &up, picked)?.reshape((k, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 2, k, true))?; // (1, k, h)
let mut down = self.act.finish(down, picked)?;
if repair_on() {
let h = down.dim(2)?;
let rep = self.store.repair.lock().unwrap();
let rows: Vec<Tensor> = bank_pos
.iter()
.map(|&p| rep.get(&(self.layer, row[p])).cloned().map(|t| t.to_dtype(down.dtype())).transpose())
.collect::<Result<Vec<Option<Tensor>>>>()?
.into_iter()
.map(|t| t.map_or_else(|| Tensor::zeros(h, down.dtype(), device), Ok))
.collect::<Result<Vec<Tensor>>>()?;
let corr = Tensor::stack(&rows, 0)?.unsqueeze(0)?; // (1, k, h)
down = (down + corr)?;
}
add(down.broadcast_mul(&w.unsqueeze(2)?)?.sum(1)?)?;
}
// each fresh miss, one slot wide, from its sharp staging
for p in 0..row.len() {
let Some(parts) = fresh.get(&row[p]) else { continue };
let slots = Tensor::from_vec(vec![0u32], (1, 1), device)?;
let idx = Tensor::new(vec![p as u32], device)?;
let w = top_w.index_select(&idx, 1)?;
let gate = xs.apply_op2_no_bwd(&slots, &self.store.fresh_op(&parts[0], self.layer, 0, 1, false))?;
let up = xs.apply_op2_no_bwd(&slots, &self.store.fresh_op(&parts[1], self.layer, 1, 1, false))?;
let inter = gate.dim(2)?;
let picked = || top_ids.index_select(&idx, 1);
let hidden = self.act.hidden(&gate, &up, picked)?.reshape((1, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &self.store.fresh_op(&parts[2], self.layer, 2, 1, true))?;
let down = self.act.finish(down, picked)?;
add(down.broadcast_mul(&w.unsqueeze(2)?)?.sum(1)?)?;
}
match out {
Some(t) => t,
None => xs.zeros_like()?,
}
} else if let (Some(slot_of), true) = (&slot_of, n == 1 && self.store.sharp_n > 0) {
// Two kinds of slot: the sharp experts and the small ones each get their dispatches; the sums add.
let row = &ids[0];
let part = |positions: &[usize], sharp: bool| -> Result<Option<Tensor>> {
if positions.is_empty() {
return Ok(None);
}
let k = positions.len();
let slots: Vec<u32> = positions.iter().map(|&p| (slot_of[&row[p]] & !SHARP_FLAG) as u32).collect();
let slots = Tensor::from_vec(slots, (1, k), device)?;
let idx = Tensor::new(positions.iter().map(|&p| p as u32).collect::<Vec<_>>(), device)?;
let w = top_w.index_select(&idx, 1)?;
let op = |which: usize, nei0: usize, rpe: bool| if sharp { self.store.bank_op_sharp(self.layer, which, nei0, rpe) } else { self.store.bank_op(self.layer, which, nei0, rpe) };
let gate = xs.apply_op2_no_bwd(&slots, &op(0, k, false))?;
let up = xs.apply_op2_no_bwd(&slots, &op(1, k, false))?;
let inter = gate.dim(2)?;
let picked = || top_ids.index_select(&idx, 1);
let hidden = self.act.hidden(&gate, &up, picked)?.reshape((k, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &op(2, k, true))?;
let down = self.act.finish(down, picked)?;
Ok(Some(down.broadcast_mul(&w.unsqueeze(2)?)?.sum(1)?))
};
let sharp_pos: Vec<usize> = (0..row.len()).filter(|&p| slot_of[&row[p]] & SHARP_FLAG != 0).collect();
let small_pos: Vec<usize> = (0..row.len()).filter(|&p| slot_of[&row[p]] & SHARP_FLAG == 0).collect();
match (part(&sharp_pos, true)?, part(&small_pos, false)?) {
(Some(a), Some(b)) => (a + b)?,
(Some(a), None) | (None, Some(a)) => a,
(None, None) => xs.zeros_like()?,
}
} else if let Some(slot_of) = slot_of {
let slots: Vec<u32> = ids.iter().flatten().map(|e| slot_of[e] as u32).collect();
let slots = Tensor::from_vec(slots, (n, self.top_k), device)?;
let gate = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 0, self.top_k, false))?; // (n, k, inter)
let up = xs.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 1, self.top_k, false))?;
let inter = gate.dim(2)?;
let hidden = self.act.hidden(&gate, &up, || Ok(top_ids.clone()))?.reshape((n * self.top_k, ()))?;
let down = hidden.apply_op2_no_bwd(&slots, &self.store.bank_op(self.layer, 2, self.top_k, true))?; // (n, k, h)
let down = self.act.finish(down, || Ok(top_ids.clone()))?;
if n == 1 && shaped() && self.store.requant.is_some() {
self.store.note_shape(self.layer, &xs, &hidden)?;
}
down.broadcast_mul(&top_w.unsqueeze(2)?)?.sum(1)?
} else {
self.reference(&xs, &ids, &top_w)?
};
if n == 1 && toaster() {
self.store.pop_used(self.layer, &needed);
}
if prof {
self.store.device.synchronize()?;
let mut st = self.store.stats.lock().unwrap();
if st.words > 1 {
st.expert_seconds += started.elapsed().as_secs_f64();
st.maths_seconds += maths_started.elapsed().as_secs_f64();
}
}
out.reshape((b, s, ()))?.to_dtype(dtype)
}
}