Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust

parallel path: stamp each published pick with its job, so a stale mix can never be placed

Merline606dfee5544
diff --git a/crates/engine/src/experts.rs b/crates/engine/src/experts.rs
index c479e6a..2e71d0a 100644
--- a/crates/engine/src/experts.rs
+++ b/crates/engine/src/experts.rs
@@ -1638,10 +1638,7 @@ impl ExpertStore {
}
let mut picks: Option<Vec<u32>> = None;
while picks.is_none() {
- let flag = gpu.flag();
- if flag != 0 {
- picks = gpu.picks_checked(job.n_ids + job.n_guess, flag);
- }
+ picks = gpu.picks_tagged(job.n_ids + job.n_guess, (job.seq & 0xFFFF) as u32);
if picks.is_none() && started.elapsed().as_secs() > 15 {
eprintln!("coachwhip: parallel: layer {} never published its picks; releasing the GPU", job.layer);
break;
@@ -2243,10 +2240,10 @@ impl StreamedMoe {
if std::env::var("COACHWHIP_S4_DEBUG").is_ok() {
eprintln!("s4: encode layer {}", self.layer);
}
- let publish = crate::gpu_sync::Publish { bank: gpu.clone(), pipeline: s4.pipes.publish.clone(), n_guess };
+ let seq = s4.seq.fetch_add(1, Ordering::AcqRel) + 1;
+ let publish = crate::gpu_sync::Publish { bank: gpu.clone(), pipeline: s4.pipes.publish.clone(), n_guess, tag: (seq & 0xFFFF) as u32 };
let _published = top_ids.apply_op2_no_bwd(&guess, &publish)?;
crate::gpu_sync::flush()?;
- let seq = s4.seq.fetch_add(1, Ordering::AcqRel) + 1;
s4.event.gpu_wait(seq)?;
let resolve = crate::gpu_sync::Resolve { bank: gpu.clone(), pipeline: s4.pipes.resolve.clone() };
let slots = top_ids.apply_op1_no_bwd(&resolve)?;
diff --git a/crates/engine/src/gpu_sync.rs b/crates/engine/src/gpu_sync.rs
index 8d21788..f8a9b16 100644
--- a/crates/engine/src/gpu_sync.rs
+++ b/crates/engine/src/gpu_sync.rs
@@ -31,17 +31,17 @@ kernel void coachwhip_publish(
device uint* out [[buffer(4)]],
constant uint& n_ids [[buffer(5)]],
constant uint& n_guess [[buffer(6)]],
+ constant uint& tag [[buffer(7)]],
uint tid [[thread_position_in_grid]],
uint tgs [[threads_per_threadgroup]])
{
- for (uint i = tid; i < n_ids; i += tgs) picks[i] = ids[i];
- for (uint i = tid; i < n_guess; i += tgs) picks[n_ids + i] = guess[i];
+ // Each pick carries its job's stamp in the top half of the same word, so a reader can never
+ // take a mix of this word's picks and the last word's left in the buffer.
+ for (uint i = tid; i < n_ids; i += tgs) picks[i] = (tag << 16) | ids[i];
+ for (uint i = tid; i < n_guess; i += tgs) picks[n_ids + i] = (tag << 16) | guess[i];
threadgroup_barrier(mem_flags::mem_device);
if (tid == 0) {
- // The flag is 1 + the sum of the picks: the CPU reads picks until they add up to it.
- uint sum = 0;
- for (uint i = 0; i < n_ids + n_guess; ++i) sum += picks[i];
- atomic_store_explicit(flag, 1u + sum, memory_order_relaxed);
+ atomic_store_explicit(flag, 1u + tag, memory_order_relaxed);
out[0] = 1u;
}
}
@@ -132,11 +132,13 @@ impl GpuBank {
(0..n).map(|i| atom(&self.picks, i).load(Ordering::SeqCst)).collect()
}
- /// The picks once they are complete: read until they add up to the flag's checksum.
- pub fn picks_checked(&self, n: usize, flag: u32) -> Option<Vec<u32>> {
+ /// The picks of job `tag`, once every one of them carries that stamp; None until then.
+ pub fn picks_tagged(&self, n: usize, tag: u32) -> Option<Vec<u32>> {
+ if self.flag() != 1 + tag {
+ return None;
+ }
let picks = self.picks(n);
- let sum = picks.iter().fold(0u32, |a, &b| a.wrapping_add(b));
- if sum.wrapping_add(1) == flag { Some(picks) } else { None }
+ if picks.iter().all(|&p| p >> 16 == tag) { Some(picks.iter().map(|&p| p & 0xFFFF).collect()) } else { None }
}
/// Picks the resolve kernel found no slot for so far.
@@ -201,6 +203,8 @@ pub struct Publish {
pub bank: Arc<GpuBank>,
pub pipeline: Arc<ComputePipeline>,
pub n_guess: usize,
+ /// This job's stamp, the low 16 bits of its sequence number.
+ pub tag: u32,
}
impl CustomOp2 for Publish {
@@ -237,7 +241,8 @@ impl CustomOp2 for Publish {
self.bank.flag.as_ref(),
Output::with_offset(&dst, 0),
n_ids_u,
- n_guess_u
+ n_guess_u,
+ self.tag
)
);
encoder.dispatch_thread_groups(MTLSize { width: 1, height: 1, depth: 1 }, MTLSize { width: 64, height: 1, depth: 1 });