The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
Tests: router, bank, prefetch guess and API door; their decisions become small pure functions
1picassoaifcb1cf6cee9f
diff --git a/crates/engine/src/experts.rs b/crates/engine/src/experts.rsindex 58cd207..7a58a76 100644--- a/crates/engine/src/experts.rs+++ b/crates/engine/src/experts.rs@@ -46,17 +46,117 @@ struct BankMat {struct Bank {mats: [BankMat; 3],- holder: Vec<Option<u32>>,- slot_of: HashMap<u32, usize>,- lru: VecDeque<usize>,+ slots: Slots,/// False while the prefetch thread is still reading an expert into the slot.ready: Vec<Arc<AtomicBool>>,- /// Experts the prefetch thread brought in that no token has asked for yet.- prefetched: HashSet<u32>,+}++/// A bank's bookkeeping, apart from the GPU memory it describes: which expert sits in which+/// slot, and the order slots were last used in. Pure, so it can be tested without a GPU.+#[derive(Clone, Debug)]+pub struct Slots {+ pub holder: Vec<Option<u32>>,+ pub slot_of: HashMap<u32, usize>,+ /// Slots from least to most recently used.+ pub lru: VecDeque<usize>,+ /// Experts a prefetch brought in that no token has asked for yet.+ pub prefetched: HashSet<u32>,+}++/// What one forward needs from a layer's bank: the experts already there and the ones to read in.+#[derive(Debug, Default, PartialEq)]+pub struct Plan {+ pub hits: Vec<(u32, usize)>,+ pub misses: Vec<(u32, usize)>,+ /// Hits that a prefetch had brought in.+ pub prefetch_hits: u64,+}++impl Slots {+ pub fn new(n: usize) -> Self {+ Self { holder: vec![None; n], slot_of: HashMap::new(), lru: (0..n).collect(), prefetched: HashSet::new() }+ }++ /// The least recently used slot that is not being filled (`ready`) and does not hold one of `keep`.+ fn victim(&self, ready: &impl Fn(usize) -> bool, keep: &[u32]) -> Option<usize> {+ self.lru.iter().position(|&slot| ready(slot) && self.holder[slot].map_or(true, |h| !keep.contains(&h)))+ }++ fn put(&mut self, pos: usize, e: u32) -> usize {+ let slot = self.lru.remove(pos).unwrap();+ if let Some(old) = self.holder[slot] {+ self.slot_of.remove(&old);+ self.prefetched.remove(&old);+ }+ self.holder[slot] = Some(e);+ self.slot_of.insert(e, slot);+ self.lru.push_back(slot);+ slot+ }++ /// Give every expert in `needed` a slot: hits move to most recently used, misses take the least+ /// recently used slot that is ready and holds nothing this forward needs. None, and nothing+ /// changed, when too few such slots are free (the caller then reads the experts fresh).+ pub fn assign(&mut self, needed: &[u32], ready: impl Fn(usize) -> bool) -> Option<Plan> {+ let missing = needed.iter().filter(|e| !self.slot_of.contains_key(e)).count();+ let free = self.lru.iter().filter(|&&slot| ready(slot) && self.holder[slot].map_or(true, |h| !needed.contains(&h))).count();+ if free < missing {+ return None;+ }+ let mut plan = Plan::default();+ for &e in needed {+ if let Some(&slot) = self.slot_of.get(&e) {+ self.lru.retain(|&x| x != slot);+ self.lru.push_back(slot);+ if self.prefetched.remove(&e) {+ plan.prefetch_hits += 1;+ }+ plan.hits.push((e, slot));+ continue;+ }+ let pos = self.victim(&ready, needed).expect("a bank always has a slot that this forward does not need");+ plan.misses.push((e, self.put(pos, e)));+ }+ Some(plan)+ }++ /// Reserve slots for the predicted experts not already held, best first, as far as free slots+ /// go; never evicts another predicted expert or a slot still being filled.+ pub fn reserve(&mut self, predicted: &[u32], ready: impl Fn(usize) -> bool) -> Vec<(u32, usize)> {+ let mut out = Vec::new();+ for &e in predicted {+ if self.slot_of.contains_key(&e) {+ continue;+ }+ let Some(pos) = self.victim(&ready, predicted) else { continue };+ let slot = self.put(pos, e);+ self.prefetched.insert(e);+ out.push((e, slot));+ }+ out+ }}pub type Key = (usize, u32);+/// Scores for layer `layer + 1`'s experts given `layer`'s picks (expert, weight), from the route+/// table: each pick hands its weight on in proportion to where it has led before. Best first;+/// equal scores in expert order, so the same table always gives the same guess.+pub fn next_scores(routes: &Counts, layer: usize, now: &[(u32, f32)]) -> Vec<(u32, f32)> {+ let mut score: HashMap<u32, f32> = HashMap::new();+ for &(a, w) in now {+ if let Some(row) = routes.get(&(layer, a)) {+ let total = row.values().sum::<u64>().max(1) as f32;+ for (&b, &n) in row {+ *score.entry(b).or_insert(0.0) += w * n as f32 / total;+ }+ }+ }+ let mut v: Vec<(u32, f32)> = score.into_iter().collect();+ v.sort_by(|x, y| y.1.partial_cmp(&x.1).unwrap().then(x.0.cmp(&y.0)));+ v+}+/// How the store behaves. `Default` is what the Mac mini M4 with 16 GB was tuned on.#[derive(Clone, Debug)]pub struct Settings {@@ -268,21 +368,8 @@ impl ExpertStore {Ok(store)}- /// Scores for the next layer's experts given this layer's picks, from the route table.fn next_scores(&self, layer: usize, now: &[(u32, f32)]) -> Vec<(u32, f32)> {- let mut score: HashMap<u32, f32> = HashMap::new();- let routes = self.routes.lock().unwrap();- for &(a, w) in now {- if let Some(row) = routes.get(&(layer, a)) {- let total = row.values().sum::<u64>().max(1) as f32;- for (&b, &n) in row {- *score.entry(b).or_insert(0.0) += w * n as f32 / total;- }- }- }- let mut v: Vec<(u32, f32)> = score.into_iter().collect();- v.sort_by(|x, y| y.1.partial_cmp(&x.1).unwrap());- v+ next_scores(&self.routes.lock().unwrap(), layer, now)}/// The `k` likeliest experts of `layer` that are not in its bank yet, one layer ahead of `now`@@ -298,7 +385,7 @@ impl ExpertStore {scores.into_iter().map(|(e, _)| e)- .filter(|e| present.map_or(true, |b| !b.slot_of.contains_key(e)))+ .filter(|e| present.map_or(true, |b| !b.slots.slot_of.contains_key(e))).take(k).collect()}@@ -320,29 +407,11 @@ impl ExpertStore {{let mut banks = self.banks.lock().unwrap();let Some(bank) = banks.get_mut(&layer) else { return Ok(()) };- for &e in predicted {- if bank.slot_of.contains_key(&e) {- continue;- }- let Some(pos) = bank- .lru- .iter()- .position(|&slot| bank.ready[slot].load(Ordering::Acquire) && bank.holder[slot].map_or(true, |h| !predicted.contains(&h)))- else {- continue;- };- let slot = bank.lru.remove(pos).unwrap();- if let Some(old) = bank.holder[slot] {- bank.slot_of.remove(&old);- bank.prefetched.remove(&old);- }- bank.ready[slot].store(false, Ordering::Release);- bank.holder[slot] = Some(e);- bank.slot_of.insert(e, slot);- bank.prefetched.insert(e);- bank.lru.push_back(slot);+ let ready = &bank.ready;+ for (e, slot) in bank.slots.reserve(predicted, |s| ready[s].load(Ordering::Acquire)) {+ ready[slot].store(false, Ordering::Release);let ptrs = [0, 1, 2].map(|i| (bank.mats[i].base + slot * bank.mats[i].bytes, bank.mats[i].bytes));- jobs.push((e, ptrs, bank.ready[slot].clone()));+ jobs.push((e, ptrs, ready[slot].clone()));}}for (e, ptrs, ready) in jobs {@@ -383,11 +452,8 @@ impl ExpertStore {self.device.synchronize()?;Ok(Bank {mats,- holder: vec![None; self.per_layer],- slot_of: HashMap::new(),- lru: (0..self.per_layer).collect(),+ slots: Slots::new(self.per_layer),ready: (0..self.per_layer).map(|_| Arc::new(AtomicBool::new(true))).collect(),- prefetched: HashSet::new(),})}@@ -448,42 +514,22 @@ impl ExpertStore {let bank = banks.get_mut(&layer).unwrap();// A slot still being filled by a prefetch cannot be taken; if too few are free for this// layer's misses, read them fresh instead, before the bank is touched.- let missing = needed.iter().filter(|e| !bank.slot_of.contains_key(e)).count();- let free = bank.lru.iter().filter(|&&slot| bank.ready[slot].load(Ordering::Acquire) && bank.holder[slot].map_or(true, |h| !needed.contains(&h))).count();- if free < missing {+ let ready = &bank.ready;+ let Some(plan) = bank.slots.assign(needed, |s| ready[s].load(Ordering::Acquire)) else {return Ok(None);- }- let mut misses: Vec<(u32, usize)> = Vec::new();- for &e in needed {- if let Some(&slot) = bank.slot_of.get(&e) {- // A prefetch may still be reading this one in; wait for it rather than read it twice.- while !bank.ready[slot].load(Ordering::Acquire) {- std::thread::yield_now();- }- bank.lru.retain(|&x| x != slot);- bank.lru.push_back(slot);- let mut st = self.stats.lock().unwrap();- st.hits += 1;- if bank.prefetched.remove(&e) {- st.prefetch_hits += 1;- }- continue;- }- let pos = bank- .lru- .iter()- .position(|&slot| bank.ready[slot].load(Ordering::Acquire) && bank.holder[slot].map_or(true, |h| !needed.contains(&h)))- .expect("a bank always has a slot that this forward does not need");- let slot = bank.lru.remove(pos).unwrap();- if let Some(old) = bank.holder[slot] {- bank.slot_of.remove(&old);- bank.prefetched.remove(&old);+ };+ // A prefetch may still be reading a hit in; wait for it rather than read it twice.+ for &(_, slot) in &plan.hits {+ while !bank.ready[slot].load(Ordering::Acquire) {+ std::thread::yield_now();}- bank.holder[slot] = Some(e);- bank.slot_of.insert(e, slot);- bank.lru.push_back(slot);- misses.push((e, slot));}+ {+ let mut st = self.stats.lock().unwrap();+ st.hits += plan.hits.len() as u64;+ st.prefetch_hits += plan.prefetch_hits;+ }+ let misses = plan.misses;if !misses.is_empty() {// All of this layer's misses go to the SSD at once: it serves several requests in// parallel far faster than one after another. Each matrix of each expert is one read.@@ -509,7 +555,7 @@ impl ExpertStore {st.bytes_loaded += reads.iter().map(|r| r.1 as u64).sum::<u64>();st.load_seconds += started.elapsed().as_secs_f64();}- Ok(Some(bank.slot_of.clone()))+ Ok(Some(bank.slots.slot_of.clone()))}/// One `mul_mv_id` dispatch over a bank matrix.@@ -686,6 +732,18 @@ impl CustomOp2 for MoeMatvec {}}+/// The router's choice for each token: the `top_k` likeliest experts from the router's scores,+/// best first, and their weights, rescaled to sum to 1 when `norm`.+pub fn route(logits: &Tensor, top_k: usize, norm: bool) -> Result<(Tensor, Tensor)> {+ let probs = candle_nn::ops::softmax_last_dim(logits)?;+ let top_ids = probs.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, top_k)?.contiguous()?;+ let mut top_w = probs.gather(&top_ids, D::Minus1)?;+ if norm {+ top_w = top_w.broadcast_div(&top_w.sum_keepdim(D::Minus1)?)?;+ }+ Ok((top_ids, top_w))+}+/// A MoE layer whose experts live in the ExpertStore instead of on the device.pub struct StreamedMoe {pub layer: usize,@@ -753,12 +811,7 @@ impl StreamedMoe {let n = b * s;let xs = xs.reshape((n, h))?.to_dtype(DType::F32)?.contiguous()?;- let probs = candle_nn::ops::softmax_last_dim(&self.gate.forward(&xs)?)?;- let top_ids = probs.arg_sort_last_dim(false)?.narrow(D::Minus1, 0, self.top_k)?.contiguous()?;- let mut top_w = probs.gather(&top_ids, D::Minus1)?;- if self.norm_topk_prob {- top_w = top_w.broadcast_div(&top_w.sum_keepdim(D::Minus1)?)?;- }+ let (top_ids, top_w) = route(&self.gate.forward(&xs)?, self.top_k, self.norm_topk_prob)?;// Reading the picks back waits for the GPU to finish everything queued so far, so no// kernel is still reading a bank slot when `ensure` and the prefetchers overwrite it.// Keep this wait if this code ever changes.diff --git a/crates/engine/tests/bank.rs b/crates/engine/tests/bank.rsnew file mode 100644index 0000000..33d7ec8--- /dev/null+++ b/crates/engine/tests/bank.rs@@ -0,0 +1,154 @@+//! The expert bank's bookkeeping: which expert sits in which slot, and who gets evicted.+//! No GPU, no model file: `Slots` is the bank without its memory.++use coachwhip_engine::experts::{Plan, Slots};++fn all_ready(_: usize) -> bool {+ true+}++/// The bank's rules that must hold after any sequence of forwards and prefetches.+fn check_consistent(s: &Slots, n: usize) {+ let mut seen: Vec<usize> = s.lru.iter().copied().collect();+ seen.sort_unstable();+ assert_eq!(seen, (0..n).collect::<Vec<_>>(), "the use order lists every slot exactly once");+ for (&e, &slot) in &s.slot_of {+ assert_eq!(s.holder[slot], Some(e), "slot_of and holder agree for expert {e}");+ }+ let held = s.holder.iter().filter(|h| h.is_some()).count();+ assert_eq!(held, s.slot_of.len(), "every held expert is findable");+ for e in &s.prefetched {+ assert!(s.slot_of.contains_key(e), "a prefetched expert is still in the bank");+ }+}++#[test]+fn a_fresh_bank_misses_everything_and_fills_slots_in_order() {+ let mut s = Slots::new(4);+ let plan = s.assign(&[7, 3], all_ready).unwrap();+ assert_eq!(plan, Plan { hits: vec![], misses: vec![(7, 0), (3, 1)], prefetch_hits: 0 });+ check_consistent(&s, 4);+}++#[test]+fn a_second_ask_is_a_hit_in_the_same_slot() {+ let mut s = Slots::new(4);+ s.assign(&[7, 3], all_ready).unwrap();+ let plan = s.assign(&[3], all_ready).unwrap();+ assert_eq!(plan.hits, vec![(3, 1)]);+ assert!(plan.misses.is_empty());+}++#[test]+fn the_least_recently_used_expert_is_evicted() {+ let mut s = Slots::new(2);+ s.assign(&[1], all_ready).unwrap();+ s.assign(&[2], all_ready).unwrap();+ s.assign(&[1], all_ready).unwrap(); // 1 is now the most recently used+ let plan = s.assign(&[3], all_ready).unwrap();+ assert_eq!(plan.misses, vec![(3, 1)], "3 takes 2's slot");+ assert!(!s.slot_of.contains_key(&2) && s.slot_of.contains_key(&1));+}++#[test]+fn never_evicts_an_expert_the_same_word_needs() {+ let mut s = Slots::new(3);+ s.assign(&[1, 2, 3], all_ready).unwrap(); // 1 is the least recently used+ let plan = s.assign(&[4, 1, 2], all_ready).unwrap();+ assert_eq!(plan.misses, vec![(4, 2)], "4 must take 3's slot, not 1's or 2's");+ assert_eq!(plan.hits.len(), 2);+ check_consistent(&s, 3);+}++#[test]+fn never_hands_out_a_slot_still_being_filled() {+ let mut s = Slots::new(3);+ let plan = s.assign(&[9], |slot| slot != 0).unwrap();+ assert_eq!(plan.misses, vec![(9, 1)], "slot 0 is being filled, so 9 goes to slot 1");+}++#[test]+fn a_crowded_bank_says_none_and_changes_nothing() {+ let mut s = Slots::new(2);+ s.assign(&[1, 2], all_ready).unwrap();+ let before = s.clone();+ assert!(s.assign(&[3, 4], |_| false).is_none(), "no slot is ready");+ assert!(s.assign(&[1, 3, 4], all_ready).is_none(), "three experts cannot fit two slots");+ assert_eq!(format!("{:?}", s.holder), format!("{:?}", before.holder));+ assert_eq!(s.lru, before.lru);+}++#[test]+fn reserve_skips_what_is_held_and_never_evicts_another_guess() {+ let mut s = Slots::new(3);+ s.assign(&[1, 2], all_ready).unwrap();+ let got = s.reserve(&[2, 5, 6], all_ready);+ assert_eq!(got.len(), 2, "2 is already held; 5 and 6 get slots");+ assert!(got.iter().all(|&(e, _)| e == 5 || e == 6));+ assert!(s.slot_of.contains_key(&2), "a guessed expert already in the bank is kept");+ assert!(s.prefetched.contains(&5) && s.prefetched.contains(&6));+ check_consistent(&s, 3);+}++#[test]+fn reserve_stops_when_no_slot_is_free() {+ let mut s = Slots::new(2);+ let got = s.reserve(&[1, 2, 3], all_ready);+ assert_eq!(got.len(), 2, "two slots, three guesses: the best two win");+ assert_eq!(got.iter().map(|g| g.0).collect::<Vec<_>>(), vec![1, 2]);+}++#[test]+fn a_used_guess_counts_as_a_prefetch_hit_once() {+ let mut s = Slots::new(4);+ s.reserve(&[8], all_ready);+ let plan = s.assign(&[8], all_ready).unwrap();+ assert_eq!(plan.prefetch_hits, 1);+ let again = s.assign(&[8], all_ready).unwrap();+ assert_eq!(again.prefetch_hits, 0, "the second use is an ordinary hit");+}++#[test]+fn an_unused_guess_that_is_evicted_is_forgotten() {+ let mut s = Slots::new(1);+ s.reserve(&[8], all_ready);+ s.assign(&[9], all_ready).unwrap();+ assert!(!s.prefetched.contains(&8));+ check_consistent(&s, 1);+}++/// A long random run, checked against a plain least-recently-used model after every step.+#[test]+fn matches_a_reference_lru_over_a_long_random_run() {+ let n = 8;+ let mut s = Slots::new(n);+ let mut reference: Vec<u32> = Vec::new(); // least recently used first+ let mut x: u64 = 42;+ let mut next = |m: u64| {+ x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);+ (x >> 33) % m+ };+ for _ in 0..5000 {+ let k = 1 + next(4) as usize;+ let mut needed: Vec<u32> = Vec::new();+ while needed.len() < k {+ let e = next(20) as u32;+ if !needed.contains(&e) {+ needed.push(e);+ }+ }+ let plan = s.assign(&needed, all_ready).unwrap();+ for &e in &needed {+ let was_held = reference.contains(&e);+ assert_eq!(plan.hits.iter().any(|h| h.0 == e), was_held, "hit or miss for {e}");+ if was_held {+ reference.retain(|&r| r != e);+ } else if reference.len() == n {+ let victim = *reference.iter().find(|r| !needed.contains(r)).unwrap();+ reference.retain(|&r| r != victim);+ }+ reference.push(e);+ }+ check_consistent(&s, n);+ }+}diff --git a/crates/engine/tests/prefetch.rs b/crates/engine/tests/prefetch.rsnew file mode 100644index 0000000..7443aaa--- /dev/null+++ b/crates/engine/tests/prefetch.rs@@ -0,0 +1,57 @@+//! The hot path's guess: scoring the next layer's experts from the route table.+//! No GPU, no model file.++use coachwhip_engine::experts::{count_step, next_scores, Counts};++fn table() -> Counts {+ let mut c = Counts::new();+ // Layer 3 -> 4: expert 1 led to 10 three times and to 11 once; expert 2 led to 11 twice.+ count_step(&mut c, 3, &[vec![1]], &[vec![10]]);+ count_step(&mut c, 3, &[vec![1]], &[vec![10]]);+ count_step(&mut c, 3, &[vec![1]], &[vec![10]]);+ count_step(&mut c, 3, &[vec![1]], &[vec![11]]);+ count_step(&mut c, 3, &[vec![2]], &[vec![11]]);+ count_step(&mut c, 3, &[vec![2]], &[vec![11]]);+ c+}++#[test]+fn a_pick_hands_its_weight_on_in_proportion_to_where_it_led() {+ let s = next_scores(&table(), 3, &[(1, 1.0)]);+ assert_eq!(s.len(), 2);+ assert_eq!(s[0].0, 10);+ assert!((s[0].1 - 0.75).abs() < 1e-6 && (s[1].1 - 0.25).abs() < 1e-6);+}++#[test]+fn several_picks_add_up() {+ let s = next_scores(&table(), 3, &[(1, 1.0), (2, 1.0)]);+ // 10: 0.75; 11: 0.25 + 1.0+ assert_eq!(s[0].0, 11);+ assert!((s[0].1 - 1.25).abs() < 1e-6);+ assert!((s[1].1 - 0.75).abs() < 1e-6);+}++#[test]+fn pick_weights_scale_their_share() {+ let s = next_scores(&table(), 3, &[(1, 0.5)]);+ assert!((s[0].1 - 0.375).abs() < 1e-6);+}++#[test]+fn unknown_experts_and_other_layers_add_nothing() {+ assert!(next_scores(&table(), 3, &[(99, 1.0)]).is_empty());+ assert!(next_scores(&table(), 4, &[(1, 1.0)]).is_empty());+}++#[test]+fn equal_scores_come_back_in_expert_order_every_time() {+ let mut c = Counts::new();+ for b in [30u32, 7, 19, 2] {+ count_step(&mut c, 0, &[vec![1]], &[vec![b]]);+ }+ for _ in 0..20 {+ let ids: Vec<u32> = next_scores(&c, 0, &[(1, 1.0)]).into_iter().map(|x| x.0).collect();+ assert_eq!(ids, vec![2, 7, 19, 30]);+ }+}diff --git a/crates/engine/tests/router.rs b/crates/engine/tests/router.rsnew file mode 100644index 0000000..942e8d3--- /dev/null+++ b/crates/engine/tests/router.rs@@ -0,0 +1,60 @@+//! The router's choice: which experts each word goes to, and with what weight.+//! No GPU, no model file. Run with `cargo test --release` (debug builds of candle are slow).++use candle::{Device, Tensor};+use coachwhip_engine::experts::route;++fn logits(rows: &[&[f32]]) -> Tensor {+ let n = rows[0].len();+ let flat: Vec<f32> = rows.iter().flat_map(|r| r.iter().copied()).collect();+ Tensor::from_vec(flat, (rows.len(), n), &Device::Cpu).unwrap()+}++fn softmax(row: &[f32]) -> Vec<f32> {+ let m = row.iter().cloned().fold(f32::MIN, f32::max);+ let e: Vec<f32> = row.iter().map(|x| (x - m).exp()).collect();+ let s: f32 = e.iter().sum();+ e.iter().map(|x| x / s).collect()+}++#[test]+fn picks_the_top_k_best_first() {+ let (ids, _) = route(&logits(&[&[0.1, 3.0, 2.0, -1.0, 1.0]]), 3, true).unwrap();+ assert_eq!(ids.to_vec2::<u32>().unwrap(), vec![vec![1, 2, 4]]);+}++#[test]+fn each_word_is_routed_on_its_own() {+ let (ids, _) = route(&logits(&[&[5.0, 0.0, 1.0, 2.0], &[0.0, 1.0, 9.0, 3.0]]), 2, true).unwrap();+ assert_eq!(ids.to_vec2::<u32>().unwrap(), vec![vec![0, 3], vec![2, 3]]);+}++#[test]+fn renormalised_weights_sum_to_one_and_keep_their_ratios() {+ let row = [0.5f32, 2.0, 1.0, -0.5, 1.5];+ let (_, w) = route(&logits(&[&row]), 3, true).unwrap();+ let w = &w.to_vec2::<f32>().unwrap()[0];+ assert!((w.iter().sum::<f32>() - 1.0).abs() < 1e-5);+ let p = softmax(&row);+ let kept = p[1] + p[4] + p[2];+ for (got, i) in w.iter().zip([1usize, 4, 2]) {+ assert!((got - p[i] / kept).abs() < 1e-5, "weight of expert {i}");+ }+}++#[test]+fn without_renormalising_the_weights_are_the_router_probabilities() {+ let row = [1.0f32, 0.0, 3.0, 2.0];+ let (_, w) = route(&logits(&[&row]), 2, false).unwrap();+ let w = &w.to_vec2::<f32>().unwrap()[0];+ let p = softmax(&row);+ assert!((w[0] - p[2]).abs() < 1e-5 && (w[1] - p[3]).abs() < 1e-5);+ assert!(w.iter().sum::<f32>() < 1.0);+}++#[test]+fn weights_follow_the_order_of_the_picks() {+ let (_, w) = route(&logits(&[&[0.0, 4.0, 1.0, 2.0, 3.0]]), 4, true).unwrap();+ let w = &w.to_vec2::<f32>().unwrap()[0];+ assert!(w.windows(2).all(|p| p[0] >= p[1]));+}diff --git a/crates/server/src/chat.rs b/crates/server/src/chat.rsindex b0b4e69..1522e4c 100644--- a/crates/server/src/chat.rs+++ b/crates/server/src/chat.rs@@ -29,7 +29,7 @@ const MAX_BODY: usize = 1 << 20;const PROMPT_CHUNK: usize = 512;/// When a chat would grow past this many words, it is summarised first to make room.-const CONTEXT_LIMIT: usize = 16384;+pub const CONTEXT_LIMIT: usize = 16384;/// Longest summary written when making room.const SUMMARY_TOKENS: usize = 500;@@ -317,7 +317,7 @@ fn summarise(model: &mut Model, tokenizer: &Tokenizer, device: &Device, session:}/// The text of an OpenAI message: a plain string, or a list of parts of which the text ones count.-fn message_text(content: &serde_json::Value) -> String {+pub fn message_text(content: &serde_json::Value) -> String {match content {serde_json::Value::String(t) => t.clone(),serde_json::Value::Array(parts) => parts.iter().filter_map(|p| p["text"].as_str()).collect::<Vec<_>>().join("\n"),@@ -359,9 +359,7 @@ fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, tolet stream = q["stream"].as_bool().unwrap_or(false);let asked = q["max_completion_tokens"].as_u64().or(q["max_tokens"].as_u64());let prompt = tokenizer.encode(chat_prompt(messages), true).map_err(anyhow::Error::msg)?.get_ids().to_vec();- // With no limit asked for, the answer gets whatever room the conversation leaves.- let room = CONTEXT_LIMIT.saturating_sub(prompt.len());- let max_new = asked.map_or(max_new.min(room), |m| (m as usize).min(max_new));+ let max_new = answer_budget(asked, max_new, prompt.len());let id = format!("chatcmpl-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()));let created = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs());let prompt_len = prompt.len();@@ -429,6 +427,21 @@ fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, to}}+/// Only this machine's own page, or a program on this machine, may talk to the model: the Host+/// must name this port on loopback, and an Origin, when a browser sends one, must too. Another+/// site open in the browser could otherwise post prompts here. Headers arrive lower-cased.+pub fn own_page(host: &str, origin: &str, port: u16) -> bool {+ let local = |h: &str| h == format!("127.0.0.1:{port}") || h == format!("localhost:{port}");+ local(host) && (origin.is_empty() || origin.strip_prefix("http://").is_some_and(local))+}++/// How many tokens an API answer may write: what the client asked for, capped by Coachwhip's own+/// limit; with no limit asked for, whatever room the 16K conversation leaves.+pub fn answer_budget(asked: Option<u64>, max_new: usize, prompt_len: usize) -> usize {+ let room = CONTEXT_LIMIT.saturating_sub(prompt_len);+ asked.map_or(max_new.min(room), |m| (m as usize).min(max_new))+}+pub fn html_escape(t: &str) -> String {t.replace('&', "&").replace('<', "<").replace('>', ">")}@@ -468,11 +481,7 @@ fn handle(mut s: TcpStream, model: &mut Model, model_name: &str, tokenizer: &Tokorigin = v.trim().to_string();}}- // Only this machine's own page may talk to the model: another site open in the browser- // could otherwise post prompts here.- let port = s.local_addr()?.port();- let local = |h: &str| h == format!("127.0.0.1:{port}") || h == format!("localhost:{port}");- let own_page = local(&host) && (origin.is_empty() || origin.strip_prefix("http://").is_some_and(local));+ let own_page = own_page(&host, &origin, s.local_addr()?.port());let path = path.split('?').next().unwrap_or("/").to_string();let api = path.starts_with("/v1/");if (path == "/ask" || api) && (!own_page || len > MAX_BODY) {diff --git a/crates/server/tests/api.rs b/crates/server/tests/api.rsnew file mode 100644index 0000000..f3152f3--- /dev/null+++ b/crates/server/tests/api.rs@@ -0,0 +1,70 @@+//! The OpenAI door: who may call it, how long an answer may be, and the chat format the model sees.+//! No GPU, no model file.++use coachwhip_server::chat::{answer_budget, chat_prompt, message_text, own_page, CONTEXT_LIMIT};+use serde_json::json;++const PORT: u16 = 8090;++#[test]+fn this_machine_may_call() {+ assert!(own_page("127.0.0.1:8090", "", PORT));+ assert!(own_page("localhost:8090", "", PORT));+ assert!(own_page("127.0.0.1:8090", "http://127.0.0.1:8090", PORT));+ assert!(own_page("localhost:8090", "http://localhost:8090", PORT));+}++#[test]+fn another_site_in_the_browser_may_not() {+ assert!(!own_page("127.0.0.1:8090", "http://evil.example", PORT));+ assert!(!own_page("127.0.0.1:8090", "https://127.0.0.1:8090", PORT), "only the page's own http origin");+ assert!(!own_page("127.0.0.1:8090", "null", PORT));+ assert!(!own_page("127.0.0.1:8090", "http://127.0.0.1:9999", PORT));+}++#[test]+fn a_rebound_or_wrong_host_may_not() {+ assert!(!own_page("evil.example:8090", "", PORT));+ assert!(!own_page("127.0.0.1:9999", "", PORT));+ assert!(!own_page("127.0.0.1:8090.evil.example", "", PORT));+ assert!(!own_page("", "", PORT));+}++#[test]+fn no_limit_asked_gets_the_room_left() {+ assert_eq!(answer_budget(None, 4000, 100), 4000);+ assert_eq!(answer_budget(None, 4000, CONTEXT_LIMIT - 300), 300);+ assert_eq!(answer_budget(None, 4000, CONTEXT_LIMIT + 5), 0);+}++#[test]+fn an_asked_limit_is_kept_but_never_above_coachwhips() {+ assert_eq!(answer_budget(Some(200), 4000, 100), 200);+ assert_eq!(answer_budget(Some(10_000), 4000, 100), 4000);+}++#[test]+fn the_context_limit_is_16k() {+ assert_eq!(CONTEXT_LIMIT, 16384);+}++#[test]+fn messages_become_the_models_chat_format() {+ let p = chat_prompt(&[json!({"role": "system", "content": "Be brief."}), json!({"role": "user", "content": "Hi"})]);+ assert_eq!(p, "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n");+}++#[test]+fn unknown_roles_are_mapped_safely() {+ let p = chat_prompt(&[json!({"role": "developer", "content": "a"}), json!({"role": "tool", "content": "b"}), json!({"content": "c"})]);+ assert!(p.starts_with("<|im_start|>system\na<|im_end|>\n<|im_start|>user\nb<|im_end|>\n<|im_start|>user\nc<|im_end|>\n"));+ assert!(p.ends_with("<|im_start|>assistant\n"));+}++#[test]+fn content_parts_keep_only_their_text() {+ assert_eq!(message_text(&json!("plain")), "plain");+ assert_eq!(message_text(&json!([{"type": "text", "text": "one"}, {"type": "image_url", "image_url": {"url": "x"}}, {"type": "text", "text": "two"}])), "one\ntwo");+ assert_eq!(message_text(&json!(null)), "");+ assert_eq!(message_text(&json!(42)), "");+}