The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
parallel path: stamp each published pick with its job, so a stale mix can never be placed
Merline606dfee5544
diff --git a/crates/engine/src/experts.rs b/crates/engine/src/experts.rsindex c479e6a..2e71d0a 100644--- a/crates/engine/src/experts.rs+++ b/crates/engine/src/experts.rs@@ -1638,10 +1638,7 @@ impl ExpertStore {}let mut picks: Option<Vec<u32>> = None;while picks.is_none() {- let flag = gpu.flag();- if flag != 0 {- picks = gpu.picks_checked(job.n_ids + job.n_guess, flag);- }+ picks = gpu.picks_tagged(job.n_ids + job.n_guess, (job.seq & 0xFFFF) as u32);if picks.is_none() && started.elapsed().as_secs() > 15 {eprintln!("coachwhip: parallel: layer {} never published its picks; releasing the GPU", job.layer);break;@@ -2243,10 +2240,10 @@ impl StreamedMoe {if std::env::var("COACHWHIP_S4_DEBUG").is_ok() {eprintln!("s4: encode layer {}", self.layer);}- let publish = crate::gpu_sync::Publish { bank: gpu.clone(), pipeline: s4.pipes.publish.clone(), n_guess };+ let seq = s4.seq.fetch_add(1, Ordering::AcqRel) + 1;+ let publish = crate::gpu_sync::Publish { bank: gpu.clone(), pipeline: s4.pipes.publish.clone(), n_guess, tag: (seq & 0xFFFF) as u32 };let _published = top_ids.apply_op2_no_bwd(&guess, &publish)?;crate::gpu_sync::flush()?;- let seq = s4.seq.fetch_add(1, Ordering::AcqRel) + 1;s4.event.gpu_wait(seq)?;let resolve = crate::gpu_sync::Resolve { bank: gpu.clone(), pipeline: s4.pipes.resolve.clone() };let slots = top_ids.apply_op1_no_bwd(&resolve)?;diff --git a/crates/engine/src/gpu_sync.rs b/crates/engine/src/gpu_sync.rsindex 8d21788..f8a9b16 100644--- a/crates/engine/src/gpu_sync.rs+++ b/crates/engine/src/gpu_sync.rs@@ -31,17 +31,17 @@ kernel void coachwhip_publish(device uint* out [[buffer(4)]],constant uint& n_ids [[buffer(5)]],constant uint& n_guess [[buffer(6)]],+ constant uint& tag [[buffer(7)]],uint tid [[thread_position_in_grid]],uint tgs [[threads_per_threadgroup]]){- for (uint i = tid; i < n_ids; i += tgs) picks[i] = ids[i];- for (uint i = tid; i < n_guess; i += tgs) picks[n_ids + i] = guess[i];+ // Each pick carries its job's stamp in the top half of the same word, so a reader can never+ // take a mix of this word's picks and the last word's left in the buffer.+ for (uint i = tid; i < n_ids; i += tgs) picks[i] = (tag << 16) | ids[i];+ for (uint i = tid; i < n_guess; i += tgs) picks[n_ids + i] = (tag << 16) | guess[i];threadgroup_barrier(mem_flags::mem_device);if (tid == 0) {- // The flag is 1 + the sum of the picks: the CPU reads picks until they add up to it.- uint sum = 0;- for (uint i = 0; i < n_ids + n_guess; ++i) sum += picks[i];- atomic_store_explicit(flag, 1u + sum, memory_order_relaxed);+ atomic_store_explicit(flag, 1u + tag, memory_order_relaxed);out[0] = 1u;}}@@ -132,11 +132,13 @@ impl GpuBank {(0..n).map(|i| atom(&self.picks, i).load(Ordering::SeqCst)).collect()}- /// The picks once they are complete: read until they add up to the flag's checksum.- pub fn picks_checked(&self, n: usize, flag: u32) -> Option<Vec<u32>> {+ /// The picks of job `tag`, once every one of them carries that stamp; None until then.+ pub fn picks_tagged(&self, n: usize, tag: u32) -> Option<Vec<u32>> {+ if self.flag() != 1 + tag {+ return None;+ }let picks = self.picks(n);- let sum = picks.iter().fold(0u32, |a, &b| a.wrapping_add(b));- if sum.wrapping_add(1) == flag { Some(picks) } else { None }+ if picks.iter().all(|&p| p >> 16 == tag) { Some(picks.iter().map(|&p| p & 0xFFFF).collect()) } else { None }}/// Picks the resolve kernel found no slot for so far.@@ -201,6 +203,8 @@ pub struct Publish {pub bank: Arc<GpuBank>,pub pipeline: Arc<ComputePipeline>,pub n_guess: usize,+ /// This job's stamp, the low 16 bits of its sequence number.+ pub tag: u32,}impl CustomOp2 for Publish {@@ -237,7 +241,8 @@ impl CustomOp2 for Publish {self.bank.flag.as_ref(),Output::with_offset(&dst, 0),n_ids_u,- n_guess_u+ n_guess_u,+ self.tag));encoder.dispatch_thread_groups(MTLSize { width: 1, height: 1, depth: 1 }, MTLSize { width: 64, height: 1, depth: 1 });