Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust

Tests: router, bank, prefetch guess and API door; their decisions become small pure functions

1picassoaifcb1cf6cee9f
diff --git a/crates/engine/src/experts.rs b/crates/engine/src/experts.rs
index 58cd207..7a58a76 100644
--- a/crates/engine/src/experts.rs
+++ b/crates/engine/src/experts.rs
@@ -46,17 +46,117 @@ struct BankMat {
struct Bank {
mats: [BankMat; 3],
- holder: Vec<Option<u32>>,
- slot_of: HashMap<u32, usize>,
- lru: VecDeque<usize>,
+ slots: Slots,
/// False while the prefetch thread is still reading an expert into the slot.
ready: Vec<Arc<AtomicBool>>,
- /// Experts the prefetch thread brought in that no token has asked for yet.
- prefetched: HashSet<u32>,
+}
+
+/// A bank's bookkeeping, apart from the GPU memory it describes: which expert sits in which
+/// slot, and the order slots were last used in. Pure, so it can be tested without a GPU.
+#[derive(Clone, Debug)]
+pub struct Slots {
+ pub holder: Vec<Option<u32>>,
+ pub slot_of: HashMap<u32, usize>,
+ /// Slots from least to most recently used.
+ pub lru: VecDeque<usize>,
+ /// Experts a prefetch brought in that no token has asked for yet.
+ pub prefetched: HashSet<u32>,
+}
+
+/// What one forward needs from a layer's bank: the experts already there and the ones to read in.
+#[derive(Debug, Default, PartialEq)]
+pub struct Plan {
+ pub hits: Vec<(u32, usize)>,
+ pub misses: Vec<(u32, usize)>,
+ /// Hits that a prefetch had brought in.
+ pub prefetch_hits: u64,
+}
+
+impl Slots {
+ pub fn new(n: usize) -> Self {
+ Self { holder: vec![None; n], slot_of: HashMap::new(), lru: (0..n).collect(), prefetched: HashSet::new() }
+ }
+
+ /// The least recently used slot that is not being filled (`ready`) and does not hold one of `keep`.
+ fn victim(&self, ready: &impl Fn(usize) -> bool, keep: &[u32]) -> Option<usize> {
+ self.lru.iter().position(|&slot| ready(slot) && self.holder[slot].map_or(true, |h| !keep.contains(&h)))
+ }
+
+ fn put(&mut self, pos: usize, e: u32) -> usize {
+ let slot = self.lru.remove(pos).unwrap();
+ if let Some(old) = self.holder[slot] {
+ self.slot_of.remove(&old);
+ self.prefetched.remove(&old);
+ }
+ self.holder[slot] = Some(e);
+ self.slot_of.insert(e, slot);
+ self.lru.push_back(slot);
+ slot
+ }
+
+ /// Give every expert in `needed` a slot: hits move to most recently used, misses take the least
+ /// recently used slot that is ready and holds nothing this forward needs. None, and nothing
+ /// changed, when too few such slots are free (the caller then reads the experts fresh).
+ pub fn assign(&mut self, needed: &[u32], ready: impl Fn(usize) -> bool) -> Option<Plan> {
+ let missing = needed.iter().filter(|e| !self.slot_of.contains_key(e)).count();
+ let free = self.lru.iter().filter(|&&slot| ready(slot) && self.holder[slot].map_or(true, |h| !needed.contains(&h))).count();
+ if free < missing {
+ return None;
+ }
+ let mut plan = Plan::default();
+ for &e in needed {
+ if let Some(&slot) = self.slot_of.get(&e) {
+ self.lru.retain(|&x| x != slot);
+ self.lru.push_back(slot);
+ if self.prefetched.remove(&e) {
+ plan.prefetch_hits += 1;
+ }
+ plan.hits.push((e, slot));
+ continue;
+ }
+ let pos = self.victim(&ready, needed).expect("a bank always has a slot that this forward does not need");
+ plan.misses.push((e, self.put(pos, e)));
+ }
+ Some(plan)
+ }
+
+ /// Reserve slots for the predicted experts not already held, best first, as far as free slots
+ /// go; never evicts another predicted expert or a slot still being filled.
+ pub fn reserve(&mut self, predicted: &[u32], ready: impl Fn(usize) -> bool) -> Vec<(u32, usize)> {
+ let mut out = Vec::new();
+ for &e in predicted {
+ if self.slot_of.contains_key(&e) {
+ continue;
+ }
+ let Some(pos) = self.victim(&ready, predicted) else { continue };
+ let slot = self.put(pos, e);
+ self.prefetched.insert(e);
+ out.push((e, slot));
+ }
+ out
+ }
}
pub type Key = (usize, u32);
+/// Scores for layer `layer + 1`'s experts given `layer`'s picks (expert, weight), from the route
+/// table: each pick hands its weight on in proportion to where it has led before. Best first;
+/// equal scores in expert order, so the same table always gives the same guess.
+pub fn next_scores(routes: &Counts, layer: usize, now: &[(u32, f32)]) -> Vec<(u32, f32)> {
+ let mut score: HashMap<u32, f32> = HashMap::new();
+ for &(a, w) in now {
+ if let Some(row) = routes.get(&(layer, a)) {
+ let total = row.values().sum::<u64>().max(1) as f32;
+ for (&b, &n) in row {
+ *score.entry(b).or_insert(0.0) += w * n as f32 / total;
+ }
+ }
+ }
+ let mut v: Vec<(u32, f32)> = score.into_iter().collect();
+ v.sort_by(|x, y| y.1.partial_cmp(&x.1).unwrap().then(x.0.cmp(&y.0)));
+ v
+}
+
/// How the store behaves. `Default` is what the Mac mini M4 with 16 GB was tuned on.
#[derive(Clone, Debug)]
pub struct Settings {
@@ -268,21 +368,8 @@ impl ExpertStore {
Ok(store)
}
- /// Scores for the next layer's experts given this layer's picks, from the route table.
fn next_scores(&self, layer: usize, now: &[(u32, f32)]) -> Vec<(u32, f32)> {
- let mut score: HashMap<u32, f32> = HashMap::new();
- let routes = self.routes.lock().unwrap();
- for &(a, w) in now {
- if let Some(row) = routes.get(&(layer, a)) {
- let total = row.values().sum::<u64>().max(1) as f32;
- for (&b, &n) in row {
- *score.entry(b).or_insert(0.0) += w * n as f32 / total;
- }
- }
- }
- let mut v: Vec<(u32, f32)> = score.into_iter().collect();
- v.sort_by(|x, y| y.1.partial_cmp(&x.1).unwrap());
- v
+ next_scores(&self.routes.lock().unwrap(), layer, now)
}
/// The `k` likeliest experts of `layer` that are not in its bank yet, one layer ahead of `now`
@@ -298,7 +385,7 @@ impl ExpertStore {
scores
.into_iter()
.map(|(e, _)| e)
- .filter(|e| present.map_or(true, |b| !b.slot_of.contains_key(e)))
+ .filter(|e| present.map_or(true, |b| !b.slots.slot_of.contains_key(e)))
.take(k)
.collect()
}
@@ -320,29 +407,11 @@ impl ExpertStore {
{
let mut banks = self.banks.lock().unwrap();
let Some(bank) = banks.get_mut(&layer) else { return Ok(()) };
- for &e in predicted {
- if bank.slot_of.contains_key(&e) {
- continue;
- }
- let Some(pos) = bank
- .lru
- .iter()
- .position(|&slot| bank.ready[slot].load(Ordering::Acquire) && bank.holder[slot].map_or(true, |h| !predicted.contains(&h)))
- else {
- continue;
- };
- let slot = bank.lru.remove(pos).unwrap();
- if let Some(old) = bank.holder[slot] {
- bank.slot_of.remove(&old);
- bank.prefetched.remove(&old);
- }
- bank.ready[slot].store(false, Ordering::Release);
- bank.holder[slot] = Some(e);
- bank.slot_of.insert(e, slot);
- bank.prefetched.insert(e);
- bank.lru.push_back(slot);
+ let ready = &bank.ready;
+ for (e, slot) in bank.slots.reserve(predicted, |s| ready[s].load(Ordering::Acquire)) {
+ ready[slot].store(false, Ordering::Release);
let ptrs = [0, 1, 2].map(|i| (bank.mats[i].base + slot * bank.mats[i].bytes, bank.mats[i].bytes));
- jobs.push((e, ptrs, bank.ready[slot].clone()));
+ jobs.push((e, ptrs, ready[slot].clone()));
}
}
for (e, ptrs, ready) in jobs {
@@ -383,11 +452,8 @@ impl ExpertStore {
self.device.synchronize()?;
Ok(Bank {
mats,
- holder: vec![None; self.per_layer],
- slot_of: HashMap::new(),
- lru: (0..self.per_layer).collect(),
+ slots: Slots::new(self.per_layer),
ready: (0..self.per_layer).map(|_| Arc::new(AtomicBool::new(true))).collect(),
- prefetched: HashSet::new(),
})
}
@@ -448,42 +514,22 @@ impl ExpertStore {
let bank = banks.get_mut(&layer).unwrap();
// A slot still being filled by a prefetch cannot be taken; if too few are free for this
// layer's misses, read them fresh instead, before the bank is touched.
- let missing = needed.iter().filter(|e| !bank.slot_of.contains_key(e)).count();
- let free = bank.lru.iter().filter(|&&slot| bank.ready[slot].load(Ordering::Acquire) && bank.holder[slot].map_or(true, |h| !needed.contains(&h))).count();
- if free < missing {
+ let ready = &bank.ready;
+ let Some(plan) = bank.slots.assign(needed, |s| ready[s].load(Ordering::Acquire)) else {
return Ok(None);
- }
- let mut misses: Vec<(u32, usize)> = Vec::new();
- for &e in needed {
- if let Some(&slot) = bank.slot_of.get(&e) {
- // A prefetch may still be reading this one in; wait for it rather than read it twice.
- while !bank.ready[slot].load(Ordering::Acquire) {
- std::thread::yield_now();
- }
- bank.lru.retain(|&x| x != slot);
- bank.lru.push_back(slot);
- let mut st = self.stats.lock().unwrap();
- st.hits += 1;
- if bank.prefetched.remove(&e) {
- st.prefetch_hits += 1;
- }
- continue;
- }
- let pos = bank
- .lru
- .iter()
- .position(|&slot| bank.ready[slot].load(Ordering::Acquire) && bank.holder[slot].map_or(true, |h| !needed.contains(&h)))
- .expect("a bank always has a slot that this forward does not need");
- let slot = bank.lru.remove(pos).unwrap();
- if let Some(old) = bank.holder[slot] {
- bank.slot_of.remove(&old);
- bank.prefetched.remove(&old);
+ };
+ // A prefetch may still be reading a hit in; wait for it rather than read it twice.
+ for &(_, slot) in &plan.hits {
+ while !bank.ready[slot].load(Ordering::Acquire) {
+ std::thread::yield_now();
}
- bank.holder[slot] = Some(e);
- bank.slot_of.insert(e, slot);
- bank.lru.push_back(slot);
- misses.push((e, slot));
}
+ {
+ let mut st = self.stats.lock().unwrap();
+ st.hits += plan.hits.len() as u64;
+ st.prefetch_hits += plan.prefetch_hits;
+ }
+ let misses = plan.misses;
if !misses.is_empty() {
// All of this layer's misses go to the SSD at once: it serves several requests in
// parallel far faster than one after another. Each matrix of each expert is one read.
@@ -509,7 +555,7 @@ impl ExpertStore {
st.bytes_loaded += reads.iter().map(|r| r.1 as u64).sum::<u64>();
st.load_seconds += started.elapsed().as_secs_f64();
}
- Ok(Some(bank.slot_of.clone()))
+ Ok(Some(bank.slots.slot_of.clone()))
}
/// One `mul_mv_id` dispatch over a bank matrix.
@@ -686,6 +732,18 @@ impl CustomOp2 for MoeMatvec {
}
}
+/// The router's choice for each token: the `top_k` likeliest experts from the router's scores,
+/// best first, and their weights, rescaled to sum to 1 when `norm`.
+pub fn route(logits: &Tensor, top_k: usize, norm: bool) -> Result<(Tensor, Tensor)> {
+ let probs = candle_nn::ops::softmax_last_dim(logits)?;
+ let top_ids = probs.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, top_k)?.contiguous()?;
+ let mut top_w = probs.gather(&top_ids, D::Minus1)?;
+ if norm {
+ top_w = top_w.broadcast_div(&top_w.sum_keepdim(D::Minus1)?)?;
+ }
+ Ok((top_ids, top_w))
+}
+
/// A MoE layer whose experts live in the ExpertStore instead of on the device.
pub struct StreamedMoe {
pub layer: usize,
@@ -753,12 +811,7 @@ impl StreamedMoe {
let n = b * s;
let xs = xs.reshape((n, h))?.to_dtype(DType::F32)?.contiguous()?;
- let probs = candle_nn::ops::softmax_last_dim(&self.gate.forward(&xs)?)?;
- let top_ids = probs.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, self.top_k)?.contiguous()?;
- let mut top_w = probs.gather(&top_ids, D::Minus1)?;
- if self.norm_topk_prob {
- top_w = top_w.broadcast_div(&top_w.sum_keepdim(D::Minus1)?)?;
- }
+ let (top_ids, top_w) = route(&self.gate.forward(&xs)?, self.top_k, self.norm_topk_prob)?;
// Reading the picks back waits for the GPU to finish everything queued so far, so no
// kernel is still reading a bank slot when `ensure` and the prefetchers overwrite it.
// Keep this wait if this code ever changes.
diff --git a/crates/engine/tests/bank.rs b/crates/engine/tests/bank.rs
new file mode 100644
index 0000000..33d7ec8
--- /dev/null
+++ b/crates/engine/tests/bank.rs
@@ -0,0 +1,154 @@
+//! The expert bank's bookkeeping: which expert sits in which slot, and who gets evicted.
+//! No GPU, no model file: `Slots` is the bank without its memory.
+
+use coachwhip_engine::experts::{Plan, Slots};
+
+fn all_ready(_: usize) -> bool {
+ true
+}
+
+/// The bank's rules that must hold after any sequence of forwards and prefetches.
+fn check_consistent(s: &Slots, n: usize) {
+ let mut seen: Vec<usize> = s.lru.iter().copied().collect();
+ seen.sort_unstable();
+ assert_eq!(seen, (0..n).collect::<Vec<_>>(), "the use order lists every slot exactly once");
+ for (&e, &slot) in &s.slot_of {
+ assert_eq!(s.holder[slot], Some(e), "slot_of and holder agree for expert {e}");
+ }
+ let held = s.holder.iter().filter(|h| h.is_some()).count();
+ assert_eq!(held, s.slot_of.len(), "every held expert is findable");
+ for e in &s.prefetched {
+ assert!(s.slot_of.contains_key(e), "a prefetched expert is still in the bank");
+ }
+}
+
+#[test]
+fn a_fresh_bank_misses_everything_and_fills_slots_in_order() {
+ let mut s = Slots::new(4);
+ let plan = s.assign(&[7, 3], all_ready).unwrap();
+ assert_eq!(plan, Plan { hits: vec![], misses: vec![(7, 0), (3, 1)], prefetch_hits: 0 });
+ check_consistent(&s, 4);
+}
+
+#[test]
+fn a_second_ask_is_a_hit_in_the_same_slot() {
+ let mut s = Slots::new(4);
+ s.assign(&[7, 3], all_ready).unwrap();
+ let plan = s.assign(&[3], all_ready).unwrap();
+ assert_eq!(plan.hits, vec![(3, 1)]);
+ assert!(plan.misses.is_empty());
+}
+
+#[test]
+fn the_least_recently_used_expert_is_evicted() {
+ let mut s = Slots::new(2);
+ s.assign(&[1], all_ready).unwrap();
+ s.assign(&[2], all_ready).unwrap();
+ s.assign(&[1], all_ready).unwrap(); // 1 is now the most recently used
+ let plan = s.assign(&[3], all_ready).unwrap();
+ assert_eq!(plan.misses, vec![(3, 1)], "3 takes 2's slot");
+ assert!(!s.slot_of.contains_key(&2) && s.slot_of.contains_key(&1));
+}
+
+#[test]
+fn never_evicts_an_expert_the_same_word_needs() {
+ let mut s = Slots::new(3);
+ s.assign(&[1, 2, 3], all_ready).unwrap(); // 1 is the least recently used
+ let plan = s.assign(&[4, 1, 2], all_ready).unwrap();
+ assert_eq!(plan.misses, vec![(4, 2)], "4 must take 3's slot, not 1's or 2's");
+ assert_eq!(plan.hits.len(), 2);
+ check_consistent(&s, 3);
+}
+
+#[test]
+fn never_hands_out_a_slot_still_being_filled() {
+ let mut s = Slots::new(3);
+ let plan = s.assign(&[9], |slot| slot != 0).unwrap();
+ assert_eq!(plan.misses, vec![(9, 1)], "slot 0 is being filled, so 9 goes to slot 1");
+}
+
+#[test]
+fn a_crowded_bank_says_none_and_changes_nothing() {
+ let mut s = Slots::new(2);
+ s.assign(&[1, 2], all_ready).unwrap();
+ let before = s.clone();
+ assert!(s.assign(&[3, 4], |_| false).is_none(), "no slot is ready");
+ assert!(s.assign(&[1, 3, 4], all_ready).is_none(), "three experts cannot fit two slots");
+ assert_eq!(format!("{:?}", s.holder), format!("{:?}", before.holder));
+ assert_eq!(s.lru, before.lru);
+}
+
+#[test]
+fn reserve_skips_what_is_held_and_never_evicts_another_guess() {
+ let mut s = Slots::new(3);
+ s.assign(&[1, 2], all_ready).unwrap();
+ let got = s.reserve(&[2, 5, 6], all_ready);
+ assert_eq!(got.len(), 2, "2 is already held; 5 and 6 get slots");
+ assert!(got.iter().all(|&(e, _)| e == 5 || e == 6));
+ assert!(s.slot_of.contains_key(&2), "a guessed expert already in the bank is kept");
+ assert!(s.prefetched.contains(&5) && s.prefetched.contains(&6));
+ check_consistent(&s, 3);
+}
+
+#[test]
+fn reserve_stops_when_no_slot_is_free() {
+ let mut s = Slots::new(2);
+ let got = s.reserve(&[1, 2, 3], all_ready);
+ assert_eq!(got.len(), 2, "two slots, three guesses: the best two win");
+ assert_eq!(got.iter().map(|g| g.0).collect::<Vec<_>>(), vec![1, 2]);
+}
+
+#[test]
+fn a_used_guess_counts_as_a_prefetch_hit_once() {
+ let mut s = Slots::new(4);
+ s.reserve(&[8], all_ready);
+ let plan = s.assign(&[8], all_ready).unwrap();
+ assert_eq!(plan.prefetch_hits, 1);
+ let again = s.assign(&[8], all_ready).unwrap();
+ assert_eq!(again.prefetch_hits, 0, "the second use is an ordinary hit");
+}
+
+#[test]
+fn an_unused_guess_that_is_evicted_is_forgotten() {
+ let mut s = Slots::new(1);
+ s.reserve(&[8], all_ready);
+ s.assign(&[9], all_ready).unwrap();
+ assert!(!s.prefetched.contains(&8));
+ check_consistent(&s, 1);
+}
+
+/// A long random run, checked against a plain least-recently-used model after every step.
+#[test]
+fn matches_a_reference_lru_over_a_long_random_run() {
+ let n = 8;
+ let mut s = Slots::new(n);
+ let mut reference: Vec<u32> = Vec::new(); // least recently used first
+ let mut x: u64 = 42;
+ let mut next = |m: u64| {
+ x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
+ (x >> 33) % m
+ };
+ for _ in 0..5000 {
+ let k = 1 + next(4) as usize;
+ let mut needed: Vec<u32> = Vec::new();
+ while needed.len() < k {
+ let e = next(20) as u32;
+ if !needed.contains(&e) {
+ needed.push(e);
+ }
+ }
+ let plan = s.assign(&needed, all_ready).unwrap();
+ for &e in &needed {
+ let was_held = reference.contains(&e);
+ assert_eq!(plan.hits.iter().any(|h| h.0 == e), was_held, "hit or miss for {e}");
+ if was_held {
+ reference.retain(|&r| r != e);
+ } else if reference.len() == n {
+ let victim = *reference.iter().find(|r| !needed.contains(r)).unwrap();
+ reference.retain(|&r| r != victim);
+ }
+ reference.push(e);
+ }
+ check_consistent(&s, n);
+ }
+}
diff --git a/crates/engine/tests/prefetch.rs b/crates/engine/tests/prefetch.rs
new file mode 100644
index 0000000..7443aaa
--- /dev/null
+++ b/crates/engine/tests/prefetch.rs
@@ -0,0 +1,57 @@
+//! The hot path's guess: scoring the next layer's experts from the route table.
+//! No GPU, no model file.
+
+use coachwhip_engine::experts::{count_step, next_scores, Counts};
+
+fn table() -> Counts {
+ let mut c = Counts::new();
+ // Layer 3 -> 4: expert 1 led to 10 three times and to 11 once; expert 2 led to 11 twice.
+ count_step(&mut c, 3, &[vec![1]], &[vec![10]]);
+ count_step(&mut c, 3, &[vec![1]], &[vec![10]]);
+ count_step(&mut c, 3, &[vec![1]], &[vec![10]]);
+ count_step(&mut c, 3, &[vec![1]], &[vec![11]]);
+ count_step(&mut c, 3, &[vec![2]], &[vec![11]]);
+ count_step(&mut c, 3, &[vec![2]], &[vec![11]]);
+ c
+}
+
+#[test]
+fn a_pick_hands_its_weight_on_in_proportion_to_where_it_led() {
+ let s = next_scores(&table(), 3, &[(1, 1.0)]);
+ assert_eq!(s.len(), 2);
+ assert_eq!(s[0].0, 10);
+ assert!((s[0].1 - 0.75).abs() < 1e-6 && (s[1].1 - 0.25).abs() < 1e-6);
+}
+
+#[test]
+fn several_picks_add_up() {
+ let s = next_scores(&table(), 3, &[(1, 1.0), (2, 1.0)]);
+ // 10: 0.75; 11: 0.25 + 1.0
+ assert_eq!(s[0].0, 11);
+ assert!((s[0].1 - 1.25).abs() < 1e-6);
+ assert!((s[1].1 - 0.75).abs() < 1e-6);
+}
+
+#[test]
+fn pick_weights_scale_their_share() {
+ let s = next_scores(&table(), 3, &[(1, 0.5)]);
+ assert!((s[0].1 - 0.375).abs() < 1e-6);
+}
+
+#[test]
+fn unknown_experts_and_other_layers_add_nothing() {
+ assert!(next_scores(&table(), 3, &[(99, 1.0)]).is_empty());
+ assert!(next_scores(&table(), 4, &[(1, 1.0)]).is_empty());
+}
+
+#[test]
+fn equal_scores_come_back_in_expert_order_every_time() {
+ let mut c = Counts::new();
+ for b in [30u32, 7, 19, 2] {
+ count_step(&mut c, 0, &[vec![1]], &[vec![b]]);
+ }
+ for _ in 0..20 {
+ let ids: Vec<u32> = next_scores(&c, 0, &[(1, 1.0)]).into_iter().map(|x| x.0).collect();
+ assert_eq!(ids, vec![2, 7, 19, 30]);
+ }
+}
diff --git a/crates/engine/tests/router.rs b/crates/engine/tests/router.rs
new file mode 100644
index 0000000..942e8d3
--- /dev/null
+++ b/crates/engine/tests/router.rs
@@ -0,0 +1,60 @@
+//! The router's choice: which experts each word goes to, and with what weight.
+//! No GPU, no model file. Run with `cargo test --release` (debug builds of candle are slow).
+
+use candle::{Device, Tensor};
+use coachwhip_engine::experts::route;
+
+fn logits(rows: &[&[f32]]) -> Tensor {
+ let n = rows[0].len();
+ let flat: Vec<f32> = rows.iter().flat_map(|r| r.iter().copied()).collect();
+ Tensor::from_vec(flat, (rows.len(), n), &Device::Cpu).unwrap()
+}
+
+fn softmax(row: &[f32]) -> Vec<f32> {
+ let m = row.iter().cloned().fold(f32::MIN, f32::max);
+ let e: Vec<f32> = row.iter().map(|x| (x - m).exp()).collect();
+ let s: f32 = e.iter().sum();
+ e.iter().map(|x| x / s).collect()
+}
+
+#[test]
+fn picks_the_top_k_best_first() {
+ let (ids, _) = route(&logits(&[&[0.1, 3.0, 2.0, -1.0, 1.0]]), 3, true).unwrap();
+ assert_eq!(ids.to_vec2::<u32>().unwrap(), vec![vec![1, 2, 4]]);
+}
+
+#[test]
+fn each_word_is_routed_on_its_own() {
+ let (ids, _) = route(&logits(&[&[5.0, 0.0, 1.0, 2.0], &[0.0, 1.0, 9.0, 3.0]]), 2, true).unwrap();
+ assert_eq!(ids.to_vec2::<u32>().unwrap(), vec![vec![0, 3], vec![2, 3]]);
+}
+
+#[test]
+fn renormalised_weights_sum_to_one_and_keep_their_ratios() {
+ let row = [0.5f32, 2.0, 1.0, -0.5, 1.5];
+ let (_, w) = route(&logits(&[&row]), 3, true).unwrap();
+ let w = &w.to_vec2::<f32>().unwrap()[0];
+ assert!((w.iter().sum::<f32>() - 1.0).abs() < 1e-5);
+ let p = softmax(&row);
+ let kept = p[1] + p[4] + p[2];
+ for (got, i) in w.iter().zip([1usize, 4, 2]) {
+ assert!((got - p[i] / kept).abs() < 1e-5, "weight of expert {i}");
+ }
+}
+
+#[test]
+fn without_renormalising_the_weights_are_the_router_probabilities() {
+ let row = [1.0f32, 0.0, 3.0, 2.0];
+ let (_, w) = route(&logits(&[&row]), 2, false).unwrap();
+ let w = &w.to_vec2::<f32>().unwrap()[0];
+ let p = softmax(&row);
+ assert!((w[0] - p[2]).abs() < 1e-5 && (w[1] - p[3]).abs() < 1e-5);
+ assert!(w.iter().sum::<f32>() < 1.0);
+}
+
+#[test]
+fn weights_follow_the_order_of_the_picks() {
+ let (_, w) = route(&logits(&[&[0.0, 4.0, 1.0, 2.0, 3.0]]), 4, true).unwrap();
+ let w = &w.to_vec2::<f32>().unwrap()[0];
+ assert!(w.windows(2).all(|p| p[0] >= p[1]));
+}
diff --git a/crates/server/src/chat.rs b/crates/server/src/chat.rs
index b0b4e69..1522e4c 100644
--- a/crates/server/src/chat.rs
+++ b/crates/server/src/chat.rs
@@ -29,7 +29,7 @@ const MAX_BODY: usize = 1 << 20;
const PROMPT_CHUNK: usize = 512;
/// When a chat would grow past this many words, it is summarised first to make room.
-const CONTEXT_LIMIT: usize = 16384;
+pub const CONTEXT_LIMIT: usize = 16384;
/// Longest summary written when making room.
const SUMMARY_TOKENS: usize = 500;
@@ -317,7 +317,7 @@ fn summarise(model: &mut Model, tokenizer: &Tokenizer, device: &Device, session:
}
/// The text of an OpenAI message: a plain string, or a list of parts of which the text ones count.
-fn message_text(content: &serde_json::Value) -> String {
+pub fn message_text(content: &serde_json::Value) -> String {
match content {
serde_json::Value::String(t) => t.clone(),
serde_json::Value::Array(parts) => parts.iter().filter_map(|p| p["text"].as_str()).collect::<Vec<_>>().join("\n"),
@@ -359,9 +359,7 @@ fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, to
let stream = q["stream"].as_bool().unwrap_or(false);
let asked = q["max_completion_tokens"].as_u64().or(q["max_tokens"].as_u64());
let prompt = tokenizer.encode(chat_prompt(messages), true).map_err(anyhow::Error::msg)?.get_ids().to_vec();
- // With no limit asked for, the answer gets whatever room the conversation leaves.
- let room = CONTEXT_LIMIT.saturating_sub(prompt.len());
- let max_new = asked.map_or(max_new.min(room), |m| (m as usize).min(max_new));
+ let max_new = answer_budget(asked, max_new, prompt.len());
let id = format!("chatcmpl-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()));
let created = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs());
let prompt_len = prompt.len();
@@ -429,6 +427,21 @@ fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, to
}
}
+/// Only this machine's own page, or a program on this machine, may talk to the model: the Host
+/// must name this port on loopback, and an Origin, when a browser sends one, must too. Another
+/// site open in the browser could otherwise post prompts here. Headers arrive lower-cased.
+pub fn own_page(host: &str, origin: &str, port: u16) -> bool {
+ let local = |h: &str| h == format!("127.0.0.1:{port}") || h == format!("localhost:{port}");
+ local(host) && (origin.is_empty() || origin.strip_prefix("http://").is_some_and(local))
+}
+
+/// How many tokens an API answer may write: what the client asked for, capped by Coachwhip's own
+/// limit; with no limit asked for, whatever room the 16K conversation leaves.
+pub fn answer_budget(asked: Option<u64>, max_new: usize, prompt_len: usize) -> usize {
+ let room = CONTEXT_LIMIT.saturating_sub(prompt_len);
+ asked.map_or(max_new.min(room), |m| (m as usize).min(max_new))
+}
+
pub fn html_escape(t: &str) -> String {
t.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;")
}
@@ -468,11 +481,7 @@ fn handle(mut s: TcpStream, model: &mut Model, model_name: &str, tokenizer: &Tok
origin = v.trim().to_string();
}
}
- // Only this machine's own page may talk to the model: another site open in the browser
- // could otherwise post prompts here.
- let port = s.local_addr()?.port();
- let local = |h: &str| h == format!("127.0.0.1:{port}") || h == format!("localhost:{port}");
- let own_page = local(&host) && (origin.is_empty() || origin.strip_prefix("http://").is_some_and(local));
+ let own_page = own_page(&host, &origin, s.local_addr()?.port());
let path = path.split('?').next().unwrap_or("/").to_string();
let api = path.starts_with("/v1/");
if (path == "/ask" || api) && (!own_page || len > MAX_BODY) {
diff --git a/crates/server/tests/api.rs b/crates/server/tests/api.rs
new file mode 100644
index 0000000..f3152f3
--- /dev/null
+++ b/crates/server/tests/api.rs
@@ -0,0 +1,70 @@
+//! The OpenAI door: who may call it, how long an answer may be, and the chat format the model sees.
+//! No GPU, no model file.
+
+use coachwhip_server::chat::{answer_budget, chat_prompt, message_text, own_page, CONTEXT_LIMIT};
+use serde_json::json;
+
+const PORT: u16 = 8090;
+
+#[test]
+fn this_machine_may_call() {
+ assert!(own_page("127.0.0.1:8090", "", PORT));
+ assert!(own_page("localhost:8090", "", PORT));
+ assert!(own_page("127.0.0.1:8090", "http://127.0.0.1:8090", PORT));
+ assert!(own_page("localhost:8090", "http://localhost:8090", PORT));
+}
+
+#[test]
+fn another_site_in_the_browser_may_not() {
+ assert!(!own_page("127.0.0.1:8090", "http://evil.example", PORT));
+ assert!(!own_page("127.0.0.1:8090", "https://127.0.0.1:8090", PORT), "only the page's own http origin");
+ assert!(!own_page("127.0.0.1:8090", "null", PORT));
+ assert!(!own_page("127.0.0.1:8090", "http://127.0.0.1:9999", PORT));
+}
+
+#[test]
+fn a_rebound_or_wrong_host_may_not() {
+ assert!(!own_page("evil.example:8090", "", PORT));
+ assert!(!own_page("127.0.0.1:9999", "", PORT));
+ assert!(!own_page("127.0.0.1:8090.evil.example", "", PORT));
+ assert!(!own_page("", "", PORT));
+}
+
+#[test]
+fn no_limit_asked_gets_the_room_left() {
+ assert_eq!(answer_budget(None, 4000, 100), 4000);
+ assert_eq!(answer_budget(None, 4000, CONTEXT_LIMIT - 300), 300);
+ assert_eq!(answer_budget(None, 4000, CONTEXT_LIMIT + 5), 0);
+}
+
+#[test]
+fn an_asked_limit_is_kept_but_never_above_coachwhips() {
+ assert_eq!(answer_budget(Some(200), 4000, 100), 200);
+ assert_eq!(answer_budget(Some(10_000), 4000, 100), 4000);
+}
+
+#[test]
+fn the_context_limit_is_16k() {
+ assert_eq!(CONTEXT_LIMIT, 16384);
+}
+
+#[test]
+fn messages_become_the_models_chat_format() {
+ let p = chat_prompt(&[json!({"role": "system", "content": "Be brief."}), json!({"role": "user", "content": "Hi"})]);
+ assert_eq!(p, "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n");
+}
+
+#[test]
+fn unknown_roles_are_mapped_safely() {
+ let p = chat_prompt(&[json!({"role": "developer", "content": "a"}), json!({"role": "tool", "content": "b"}), json!({"content": "c"})]);
+ assert!(p.starts_with("<|im_start|>system\na<|im_end|>\n<|im_start|>user\nb<|im_end|>\n<|im_start|>user\nc<|im_end|>\n"));
+ assert!(p.ends_with("<|im_start|>assistant\n"));
+}
+
+#[test]
+fn content_parts_keep_only_their_text() {
+ assert_eq!(message_text(&json!("plain")), "plain");
+ assert_eq!(message_text(&json!([{"type": "text", "text": "one"}, {"type": "image_url", "image_url": {"url": "x"}}, {"type": "text", "text": "two"}])), "one\ntwo");
+ assert_eq!(message_text(&json!(null)), "");
+ assert_eq!(message_text(&json!(42)), "");
+}