The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
Workspace: split into engine, server and cli crates; tests move beside their crate
1picassoai72b97c8a4eb4
diff --git a/Cargo.toml b/Cargo.tomlindex 9345ee9..455d2a9 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,12 +1,14 @@-[package]-name = "coachwhip"-version = "0.2.0"+[workspace]+members = ["crates/engine", "crates/server", "crates/cli"]+resolver = "2"++[workspace.package]+version = "0.2.1"edition = "2021"-description = "Run mixture-of-experts models bigger than your memory on a Mac: an 80B model on 16 GB. The experts stream from the SSD into a small GPU bank."license = "Apache-2.0"publish = false-[dependencies]+[workspace.dependencies]candle = { package = "candle-core", git = "https://github.com/huggingface/candle", rev = "5ba5d5b468b5b1df40e82dd3d556987bedeea041", features = ["metal"] }candle-nn = { git = "https://github.com/huggingface/candle", rev = "5ba5d5b468b5b1df40e82dd3d556987bedeea041", features = ["metal"] }candle-transformers = { git = "https://github.com/huggingface/candle", rev = "5ba5d5b468b5b1df40e82dd3d556987bedeea041", features = ["metal"] }@@ -17,6 +19,8 @@ clap = { version = "4.2.4", features = ["derive"] }anyhow = "1"serde_json = "1.0.99"rayon = "1"+coachwhip-engine = { path = "crates/engine" }+coachwhip-server = { path = "crates/server" }[profile.release]debug = falsediff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.tomlnew file mode 100644index 0000000..03eed86--- /dev/null+++ b/crates/cli/Cargo.toml@@ -0,0 +1,20 @@+[package]+name = "coachwhip"+description = "Run mixture-of-experts models bigger than your memory on a Mac: an 80B model on 16 GB. The experts stream from the SSD into a small GPU bank."+version.workspace = true+edition.workspace = true+license.workspace = true+publish.workspace = true++[[bin]]+name = "coachwhip"+path = "src/main.rs"++[dependencies]+coachwhip-engine.workspace = true+coachwhip-server.workspace = true+candle.workspace = true+candle-transformers.workspace = true+tokenizers.workspace = true+clap.workspace = true+anyhow.workspace = truediff --git a/src/main.rs b/crates/cli/src/main.rssimilarity index 98%rename from src/main.rsrename to crates/cli/src/main.rsindex 475e720..80d7b91 100644--- a/src/main.rs+++ b/crates/cli/src/main.rs@@ -2,13 +2,8 @@//! model runs on 16 GB: the experts stream from the SSD into a small bank on the GPU, and the//! ones the next layer will probably need are fetched while the current layer computes.-mod chat;-mod experts;-mod model;-mod model_next;-mod mv_id;-#[cfg(test)]-mod tests;+use coachwhip_engine::{experts, model};+use coachwhip_server::chat;use anyhow::Result;use candle::Device;diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.tomlnew file mode 100644index 0000000..c1e9276--- /dev/null+++ b/crates/engine/Cargo.toml@@ -0,0 +1,15 @@+[package]+name = "coachwhip-engine"+description = "The Coachwhip engine: MoE model families, the expert bank on the GPU, and streaming experts from the SSD."+version.workspace = true+edition.workspace = true+license.workspace = true+publish.workspace = true++[dependencies]+candle.workspace = true+candle-nn.workspace = true+candle-transformers.workspace = true+candle-metal-kernels.workspace = true+objc2-metal.workspace = true+rayon.workspace = truediff --git a/src/experts.rs b/crates/engine/src/experts.rssimilarity index 99%rename from src/experts.rsrename to crates/engine/src/experts.rsindex c3aed55..58cd207 100644--- a/src/experts.rs+++ b/crates/engine/src/experts.rs@@ -55,7 +55,7 @@ struct Bank {prefetched: HashSet<u32>,}-type Key = (usize, u32);+pub type Key = (usize, u32);/// How the store behaves. `Default` is what the Mac mini M4 with 16 GB was tuned on.#[derive(Clone, Debug)]@@ -83,11 +83,11 @@ impl Default for Settings {}/// Route table: for each (layer, expert), how often each expert of the next layer followed it.-pub(crate) type Counts = HashMap<Key, HashMap<u32, u64>>;+pub type Counts = HashMap<Key, HashMap<u32, u64>>;/// Adds one layer-to-layer step to the table: every expert picked now, for every expert picked/// in the layer before, token by token.-pub(crate) fn count_step(counts: &mut Counts, layer: usize, before: &[Vec<u32>], now: &[Vec<u32>]) {+pub fn count_step(counts: &mut Counts, layer: usize, before: &[Vec<u32>], now: &[Vec<u32>]) {for (a_row, b_row) in before.iter().zip(now.iter()) {// Nothing followed (a broken log line): leave no empty row behind to look like knowledge.if b_row.is_empty() {@@ -104,7 +104,7 @@ pub(crate) fn count_step(counts: &mut Counts, layer: usize, before: &[Vec<u32>],/// Reads route logs from folders. A log is a "T ..." line followed by one line per layer:/// comma-separated tokens, each the space-separated experts the router picked for it.-pub(crate) fn load_routes(dirs: &[&Path], layers: usize) -> Counts {+pub fn load_routes(dirs: &[&Path], layers: usize) -> Counts {let mut counts = Counts::new();for dir in dirs {let Ok(entries) = std::fs::read_dir(dir) else { continue };diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rsnew file mode 100644index 0000000..f36dfc0--- /dev/null+++ b/crates/engine/src/lib.rs@@ -0,0 +1,7 @@+//! The Coachwhip engine: runs mixture-of-experts models bigger than the memory on Apple silicon.+//! The experts stream from the SSD into a small bank on the GPU; the model families sit on top.++pub mod experts;+pub mod model;+pub mod model_next;+mod mv_id;diff --git a/src/model.rs b/crates/engine/src/model.rssimilarity index 100%rename from src/model.rsrename to crates/engine/src/model.rsdiff --git a/src/model_next.rs b/crates/engine/src/model_next.rssimilarity index 99%rename from src/model_next.rsrename to crates/engine/src/model_next.rsindex f79242a..3eaff58 100644--- a/src/model_next.rs+++ b/crates/engine/src/model_next.rs@@ -27,7 +27,7 @@ fn softplus(x: &Tensor) -> Result<Tensor> {}/// Words per block when a running-memory layer reads a prompt.-pub(crate) const DELTA_CHUNK: usize = 64;+pub const DELTA_CHUNK: usize = 64;/// The delta rule over a whole prompt, a block of words at a time (Gated DeltaNet, Yang, Kautz and/// Hatamizadeh, ICLR 2025, section 3). Gives the same answer as stepping word by word, but each@@ -36,7 +36,7 @@ pub(crate) const DELTA_CHUNK: usize = 64;/// system, solved once per block on the CPU. Decays are kept as logarithms so long blocks never/// divide by a vanishing number./// q, k: (l, n_v, dk); v: (l, n_v, dv); beta, log_decay: (l, n_v); s: (n_v, dk, dv).-pub(crate) fn delta_chunked(q: &Tensor, k: &Tensor, v: &Tensor, beta: &Tensor, log_decay: &Tensor, mut s: Tensor) -> Result<(Tensor, Tensor)> {+pub fn delta_chunked(q: &Tensor, k: &Tensor, v: &Tensor, beta: &Tensor, log_decay: &Tensor, mut s: Tensor) -> Result<(Tensor, Tensor)> {let (l, n_v, dk) = q.dims3()?;let dv = v.dim(2)?;let c = DELTA_CHUNK;diff --git a/src/mv_id.rs b/crates/engine/src/mv_id.rssimilarity index 100%rename from src/mv_id.rsrename to crates/engine/src/mv_id.rsdiff --git a/crates/engine/tests/linear_attention.rs b/crates/engine/tests/linear_attention.rsnew file mode 100644index 0000000..6319731--- /dev/null+++ b/crates/engine/tests/linear_attention.rs@@ -0,0 +1,115 @@+//! The 80B's linear attention: the block-at-a-time delta rule against the word-by-word one.+//! No GPU, no model file. Run with `cargo test --release` (debug builds of candle are slow).++// ---------------------------------------------------------------- model_next.rs delta rule (CPU)++use candle::{Device, Tensor};++/// A tiny deterministic generator so the test needs no `rand` crate.+struct Lcg(u64);+impl Lcg {+ fn next(&mut self) -> f32 {+ self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);+ ((self.0 >> 33) as f32 / (1u64 << 31) as f32) * 2.0 - 1.0+ }+ fn tensor(&mut self, shape: &[usize], scale: f32) -> Tensor {+ let n: usize = shape.iter().product();+ let v: Vec<f32> = (0..n).map(|_| self.next() * scale).collect();+ Tensor::from_vec(v, shape, &Device::Cpu).unwrap()+ }+}++/// The word-by-word delta rule exactly as model_next.rs steps it for one word (its `l == 1` path).+fn delta_word_by_word(q: &Tensor, k: &Tensor, v: &Tensor, beta: &Tensor, log_decay: &Tensor, mut s: Tensor) -> (Tensor, Tensor) {+ let (l, n_v, _dk) = q.dims3().unwrap();+ let decay = log_decay.exp().unwrap();+ let mut outs = Vec::with_capacity(l);+ for t in 0..l {+ let kt = k.get(t).unwrap().unsqueeze(1).unwrap().contiguous().unwrap();+ let qt = q.get(t).unwrap().unsqueeze(1).unwrap().contiguous().unwrap();+ let vt = v.get(t).unwrap().unsqueeze(1).unwrap();+ let bt = beta.get(t).unwrap().reshape((n_v, 1, 1)).unwrap();+ let gt = decay.get(t).unwrap().reshape((n_v, 1, 1)).unwrap();+ s = s.broadcast_mul(>).unwrap();+ let sk = kt.matmul(&s).unwrap();+ let d = (vt - sk).unwrap().broadcast_mul(&bt).unwrap();+ s = (s + kt.transpose(1, 2).unwrap().contiguous().unwrap().matmul(&d).unwrap()).unwrap();+ outs.push(qt.matmul(&s).unwrap().squeeze(1).unwrap());+ }+ (Tensor::stack(&outs, 0).unwrap(), s)+}++fn max_abs_diff(a: &Tensor, b: &Tensor) -> f32 {+ (a - b).unwrap().abs().unwrap().flatten_all().unwrap().max(0).unwrap().to_scalar::<f32>().unwrap()+}++fn inputs(seed: u64, l: usize, n_v: usize, dk: usize, dv: usize) -> (Tensor, Tensor, Tensor, Tensor, Tensor) {+ let mut g = Lcg(seed);+ let unit = |t: Tensor| -> Tensor {+ let n = t.sqr().unwrap().sum_keepdim(2).unwrap().sqrt().unwrap();+ t.broadcast_div(&n).unwrap()+ };+ let q = unit(g.tensor(&[l, n_v, dk], 1.0));+ let k = unit(g.tensor(&[l, n_v, dk], 1.0));+ let v = g.tensor(&[l, n_v, dv], 1.0);+ // beta in (0, 1); log decay in about (-0.6, 0): what sigmoid and softplus * A produce.+ let beta = candle_nn::ops::sigmoid(&g.tensor(&[l, n_v], 2.0)).unwrap();+ let log_decay = (g.tensor(&[l, n_v], 0.3).abs().unwrap() * -1.0).unwrap();+ (q, k, v, beta, log_decay)+}++fn check_chunked_against_word_by_word(l: usize, seed: u64) {+ let (n_v, dk, dv) = (3, 8, 6);+ let (q, k, v, beta, log_decay) = inputs(seed, l, n_v, dk, dv);+ let s0 = Tensor::zeros((n_v, dk, dv), candle::DType::F32, &Device::Cpu).unwrap();+ let (o_ref, s_ref) = delta_word_by_word(&q, &k, &v, &beta, &log_decay, s0.clone());+ let (o, s) = coachwhip_engine::model_next::delta_chunked(&q, &k, &v, &beta, &log_decay, s0).unwrap();+ assert_eq!(o.dims(), &[l, n_v, dv], "output shape for l = {l}");+ assert_eq!(s.dims(), &[n_v, dk, dv]);+ let od = max_abs_diff(&o, &o_ref);+ let sd = max_abs_diff(&s, &s_ref);+ assert!(od < 1e-4, "l = {l}: output differs from word-by-word by {od}");+ assert!(sd < 1e-4, "l = {l}: state differs from word-by-word by {sd}");+}++#[test]+fn delta_chunked_matches_word_by_word_when_l_is_a_whole_number_of_blocks() {+ let c = coachwhip_engine::model_next::DELTA_CHUNK;+ check_chunked_against_word_by_word(c, 1);+ check_chunked_against_word_by_word(2 * c, 2);+}++#[test]+fn delta_chunked_matches_word_by_word_with_padding() {+ let c = coachwhip_engine::model_next::DELTA_CHUNK;+ for l in [2, 5, c - 1, c + 1, 2 * c + 17] {+ check_chunked_against_word_by_word(l, 10 + l as u64);+ }+}++#[test]+fn delta_chunked_carries_state_across_calls() {+ // One call over 70 words must equal two calls, 70 = 45 + 25, with the state handed on.+ let (l, n_v, dk, dv) = (70, 3, 8, 6);+ let (q, k, v, beta, log_decay) = inputs(99, l, n_v, dk, dv);+ let s0 = Tensor::zeros((n_v, dk, dv), candle::DType::F32, &Device::Cpu).unwrap();+ let (o_all, s_all) = coachwhip_engine::model_next::delta_chunked(&q, &k, &v, &beta, &log_decay, s0.clone()).unwrap();+ let cut = 45;+ let part = |t: &Tensor, from: usize, len: usize| t.narrow(0, from, len).unwrap().contiguous().unwrap();+ let (o1, s1) = coachwhip_engine::model_next::delta_chunked(&part(&q, 0, cut), &part(&k, 0, cut), &part(&v, 0, cut), &part(&beta, 0, cut), &part(&log_decay, 0, cut), s0).unwrap();+ let (o2, s2) = coachwhip_engine::model_next::delta_chunked(&part(&q, cut, l - cut), &part(&k, cut, l - cut), &part(&v, cut, l - cut), &part(&beta, cut, l - cut), &part(&log_decay, cut, l - cut), s1).unwrap();+ let o_split = Tensor::cat(&[&o1, &o2], 0).unwrap();+ assert!(max_abs_diff(&o_all, &o_split) < 1e-4);+ assert!(max_abs_diff(&s_all, &s2) < 1e-4);+}++#[test]+fn delta_chunked_with_a_nonzero_starting_state() {+ let (l, n_v, dk, dv) = (37, 2, 8, 6);+ let (q, k, v, beta, log_decay) = inputs(7, l, n_v, dk, dv);+ let s0 = Lcg(123).tensor(&[n_v, dk, dv], 0.5);+ let (o_ref, s_ref) = delta_word_by_word(&q, &k, &v, &beta, &log_decay, s0.clone());+ let (o, s) = coachwhip_engine::model_next::delta_chunked(&q, &k, &v, &beta, &log_decay, s0).unwrap();+ assert!(max_abs_diff(&o, &o_ref) < 1e-4);+ assert!(max_abs_diff(&s, &s_ref) < 1e-4);+}diff --git a/crates/engine/tests/routes.rs b/crates/engine/tests/routes.rsnew file mode 100644index 0000000..d30a4fb--- /dev/null+++ b/crates/engine/tests/routes.rs@@ -0,0 +1,56 @@+//! The route table the hot path learns from: counting steps and reading the logs back.+//! No GPU, no model file. Run with `cargo test --release` (debug builds of candle are slow).++// ---------------------------------------------------------------- experts.rs route table++#[test]+fn count_step_counts_every_pair_token_by_token() {+ let mut counts = coachwhip_engine::experts::Counts::new();+ let before = vec![vec![1, 2], vec![1]];+ let now = vec![vec![7], vec![7, 9]];+ coachwhip_engine::experts::count_step(&mut counts, 4, &before, &now);+ assert_eq!(counts[&(4, 1)][&7], 2);+ assert_eq!(counts[&(4, 1)][&9], 1);+ assert_eq!(counts[&(4, 2)][&7], 1);+ assert!(counts.get(&(4, 2)).map_or(true, |r| !r.contains_key(&9)));+ assert!(!counts.contains_key(&(5, 1)));+}++#[test]+fn count_step_ignores_a_token_count_mismatch_beyond_the_shorter_side() {+ let mut counts = coachwhip_engine::experts::Counts::new();+ coachwhip_engine::experts::count_step(&mut counts, 0, &[vec![1]], &[vec![2], vec![3]]);+ assert_eq!(counts[&(0, 1)][&2], 1);+ assert_eq!(counts.len(), 1);+}++#[test]+fn load_routes_reads_the_log_shape_and_skips_junk() {+ let dir = std::env::temp_dir().join(format!("coachwhip-routes-{}", std::process::id()));+ std::fs::create_dir_all(&dir).unwrap();+ // layers = 3: a "T" line then 3 layer lines; a stray header line and a short block are skipped.+ let log = "T -\n1 2,3\n4,5 6\n7,7\nT 1 2 3\n9\n9\nnot-a-number\nT -\n1\n2\n";+ std::fs::write(dir.join("a.log"), log).unwrap();+ std::fs::write(dir.join("ignored.txt"), "T -\n1\n2\n3\n").unwrap();+ let counts = coachwhip_engine::experts::load_routes(&[dir.as_path()], 3);+ std::fs::remove_dir_all(&dir).unwrap();+ // block 1, layer 0 -> 1: token0 (1,2)->(4), token1 (3)->(5,6)+ assert_eq!(counts[&(0, 1)][&4], 1);+ assert_eq!(counts[&(0, 2)][&4], 1);+ assert_eq!(counts[&(0, 3)][&5], 1);+ assert_eq!(counts[&(0, 3)][&6], 1);+ // block 1, layer 1 -> 2: (4)->(7), (5,6)->(7)+ assert_eq!(counts[&(1, 4)][&7], 1);+ assert_eq!(counts[&(1, 5)][&7], 1);+ // block 2: "not-a-number" parses to no experts, so (9)->(9) counted once and nothing from layer 2+ assert_eq!(counts[&(0, 9)][&9], 1);+ assert!(!counts.contains_key(&(1, 9)));+ // the last block is too short and must not be read+ assert!(!counts.contains_key(&(0, 1)) || counts[&(0, 1)].get(&2).is_none());+}++#[test]+fn load_routes_with_a_missing_folder_is_empty_not_an_error() {+ let counts = coachwhip_engine::experts::load_routes(&[std::path::Path::new("/nonexistent/coachwhip")], 4);+ assert!(counts.is_empty());+}diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.tomlnew file mode 100644index 0000000..8657bf1--- /dev/null+++ b/crates/server/Cargo.toml@@ -0,0 +1,15 @@+[package]+name = "coachwhip-server"+description = "Coachwhip's doors: the chat page and the OpenAI-compatible API, on loopback."+version.workspace = true+edition.workspace = true+license.workspace = true+publish.workspace = true++[dependencies]+coachwhip-engine.workspace = true+candle.workspace = true+candle-transformers.workspace = true+tokenizers.workspace = true+anyhow.workspace = true+serde_json.workspace = truediff --git a/src/chat.rs b/crates/server/src/chat.rssimilarity index 98%rename from src/chat.rsrename to crates/server/src/chat.rsindex 896863b..b0b4e69 100644--- a/src/chat.rs+++ b/crates/server/src/chat.rs@@ -19,7 +19,7 @@ use std::rc::Rc;use candle::{Device, Tensor};use candle_transformers::generation::{LogitsProcessor, Sampling};-use crate::model::{Model, Progress, Step};+use coachwhip_engine::model::{Model, Progress, Step};use tokenizers::Tokenizer;/// Largest request body accepted: a prompt of a few hundred pages.@@ -37,13 +37,13 @@ const SUMMARY_TOKENS: usize = 500;const SUMMARY_ASK: &str = "Summarise our conversation so far in at most 300 words, for someone who will continue it: what was asked, what was decided, and every name, file, function or fact the next answer will need. Plain text, no preamble.";/// Separates the streamed text from the closing stats line.-pub(crate) const STATS_MARK: &str = "\u{1}STATS:";+pub const STATS_MARK: &str = "\u{1}STATS:";/// A progress line sent while the prompt is read, before any text: `\u{1}PROG:<json>\n`.-pub(crate) const PROG_MARK: &str = "\u{1}PROG:";+pub const PROG_MARK: &str = "\u{1}PROG:";/// Ends the stream when the answer could not be finished; the page shows what follows it.-pub(crate) const ERROR_MARK: &str = "\u{1}ERROR:";+pub const ERROR_MARK: &str = "\u{1}ERROR:";/// After the GPU has run out of memory once, Metal does not recover in this process.fn gpu_out_of_memory(e: &anyhow::Error) -> bool {@@ -326,7 +326,7 @@ fn message_text(content: &serde_json::Value) -> String {}/// An OpenAI conversation in the model's chat format, ready for the assistant's turn.-pub(crate) fn chat_prompt(messages: &[serde_json::Value]) -> String {+pub fn chat_prompt(messages: &[serde_json::Value]) -> String {let mut text = String::new();for m in messages {let role = match m["role"].as_str().unwrap_or("user") {@@ -429,11 +429,11 @@ fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, to}}-pub(crate) fn html_escape(t: &str) -> String {+pub fn html_escape(t: &str) -> String {t.replace('&', "&").replace('<', "<").replace('>', ">")}-pub(crate) fn chunk(s: &mut TcpStream, bytes: &[u8]) -> Result<()> {+pub fn chunk(s: &mut TcpStream, bytes: &[u8]) -> Result<()> {if bytes.is_empty() {return Ok(());}@@ -560,9 +560,9 @@ fn handle(mut s: TcpStream, model: &mut Model, model_name: &str, tokenizer: &Tok}/// The page's thinking switch, left out for a model that has no thinking mode.-pub(crate) const THINK_BOX: &str = r#"<label><input type="checkbox" id="think"> let it think first (slower)</label>"#;+pub const THINK_BOX: &str = r#"<label><input type="checkbox" id="think"> let it think first (slower)</label>"#;-pub(crate) const PAGE: &str = r##"<!doctype html>+pub const PAGE: &str = r##"<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Coachwhip</title><style>:root{--bg:#0d1117;--panel:#161b22;--line:#30363d;--text:#e6edf3;--dim:#8b949e;--on:#3fb950;--off:#6e7681}*{box-sizing:border-box} body{margin:0;background:var(--bg);color:var(--text);diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rsnew file mode 100644index 0000000..3a5f108--- /dev/null+++ b/crates/server/src/lib.rs@@ -0,0 +1,3 @@+//! Coachwhip's doors: the chat page and the OpenAI-compatible API, served on loopback only.++pub mod chat;diff --git a/crates/server/tests/chat_page.rs b/crates/server/tests/chat_page.rsnew file mode 100644index 0000000..41b0666--- /dev/null+++ b/crates/server/tests/chat_page.rs@@ -0,0 +1,131 @@+//! The chat page and the HTTP framing: the pieces the browser depends on.+//! No GPU, no model file. Run with `cargo test --release` (debug builds of candle are slow).++use std::io::Read;+use std::net::{TcpListener, TcpStream};++// ---------------------------------------------------------------- chat.rs helpers++#[test]+fn html_escape_escapes_the_three_that_matter() {+ assert_eq!(coachwhip_server::chat::html_escape("a&b<c>d"), "a&b<c>d");+ assert_eq!(coachwhip_server::chat::html_escape("plain"), "plain");+ assert_eq!(coachwhip_server::chat::html_escape(""), "");+ // Escaping is applied once: an already-escaped amp is escaped again, as it must be.+ assert_eq!(coachwhip_server::chat::html_escape("&"), "&amp;");+}++#[test]+fn markers_are_the_control_one_prefix_the_page_strips() {+ // The page's JS: MARK = '\u0001STATS:' and PROG = /\u0001PROG:(\{[^\n]*\})\n/+ assert_eq!(coachwhip_server::chat::STATS_MARK, "\u{1}STATS:");+ assert_eq!(coachwhip_server::chat::PROG_MARK, "\u{1}PROG:");+ assert!(coachwhip_server::chat::PAGE.contains("const MARK = '\\u0001STATS:'"));+ assert!(coachwhip_server::chat::PAGE.contains("const PROG = /\\u0001PROG:(\\{[^\\n]*\\})\\n/g"));+}++#[test]+fn progress_lines_match_the_page_regex() {+ // Same format strings as chat.rs builds in `handle`.+ let layer = format!("{}{{\"layer\":{},\"of\":{}}}\n", coachwhip_server::chat::PROG_MARK, 3, 48);+ let summary = format!("{}{{\"summary\":{}}}\n", coachwhip_server::chat::PROG_MARK, 16);+ for line in [&layer, &summary] {+ assert!(line.starts_with("\u{1}PROG:{"));+ assert!(line.ends_with("}\n"));+ let json = &line["\u{1}PROG:".len()..line.len() - 1];+ let v: serde_json::Value = serde_json::from_str(json).expect("progress line is JSON");+ assert!(v.is_object());+ assert!(!json.contains('\n'));+ }+}++#[test]+fn think_box_is_a_verbatim_substring_of_the_page() {+ // `PAGE.replace(THINK_BOX, "")` is a silent no-op if the two ever drift apart.+ assert!(coachwhip_server::chat::PAGE.contains(coachwhip_server::chat::THINK_BOX), "THINK_BOX no longer matches the page");+ assert_eq!(coachwhip_server::chat::PAGE.matches(coachwhip_server::chat::THINK_BOX).count(), 1);+ let without = coachwhip_server::chat::PAGE.replace(coachwhip_server::chat::THINK_BOX, "");+ assert!(!without.contains("id=\"think\""));+ // The JS still guards the missing element.+ assert!(without.contains("$('think')&&$('think').checked"));+}++#[test]+fn page_has_the_model_name_placeholder_once() {+ assert_eq!(coachwhip_server::chat::PAGE.matches("MODEL_NAME").count(), 1);+}++/// Sends `bytes` through `chunk` over a loopback socket and returns what the other end saw.+fn framed(bytes: &[u8]) -> Vec<u8> {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = listener.local_addr().unwrap();+ let mut client = TcpStream::connect(addr).unwrap();+ let (mut server, _) = listener.accept().unwrap();+ coachwhip_server::chat::chunk(&mut server, bytes).unwrap();+ drop(server);+ let mut got = Vec::new();+ client.read_to_end(&mut got).unwrap();+ got+}++#[test]+fn chunk_frames_as_hex_length_crlf_body_crlf() {+ assert_eq!(framed(b"hello"), b"5\r\nhello\r\n");+ assert_eq!(framed(&[b'x'; 255]), [b"ff\r\n".as_slice(), &[b'x'; 255], b"\r\n"].concat());+ assert_eq!(framed("é".as_bytes()), b"2\r\n\xc3\xa9\r\n");+}++#[test]+fn chunk_sends_nothing_for_empty_bytes() {+ // An empty chunk ("0\r\n\r\n") would end the HTTP response early.+ assert_eq!(framed(b""), b"");+}++#[test]+fn chunk_to_a_closed_socket_is_an_error_not_a_panic() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let addr = listener.local_addr().unwrap();+ let client = TcpStream::connect(addr).unwrap();+ let (mut server, _) = listener.accept().unwrap();+ drop(client);+ // The first write may still succeed into the kernel buffer; keep writing until it fails.+ let mut failed = false;+ for _ in 0..64 {+ if coachwhip_server::chat::chunk(&mut server, &[b'x'; 65536]).is_err() {+ failed = true;+ break;+ }+ }+ assert!(failed, "writing to a closed socket never reported an error");+}++// ---------------------------------------------------------------- loopback HTTP, no model++#[test]+fn chunk_output_is_valid_for_a_chunked_http_client() {+ // What the page receives per chunk: hex length, CRLF, body, CRLF. A zero-length chunk is never+ // produced by `chunk`; `handle` writes the terminator itself.+ let mut server_side: Vec<u8> = Vec::new();+ for piece in ["Here", "'s", " a", "\u{1}STATS:{\"written\":2}"] {+ server_side.extend(framed(piece.as_bytes()));+ }+ server_side.extend(b"0\r\n\r\n");+ // Decode by hand.+ let mut i = 0;+ let mut body = Vec::new();+ loop {+ let nl = server_side[i..].windows(2).position(|w| w == b"\r\n").unwrap() + i;+ let n = usize::from_str_radix(std::str::from_utf8(&server_side[i..nl]).unwrap(), 16).unwrap();+ if n == 0 {+ break;+ }+ body.extend_from_slice(&server_side[nl + 2..nl + 2 + n]);+ assert_eq!(&server_side[nl + 2 + n..nl + 4 + n], b"\r\n");+ i = nl + 4 + n;+ }+ let text = String::from_utf8(body).unwrap();+ let k = text.find(coachwhip_server::chat::STATS_MARK).unwrap();+ assert_eq!(&text[..k], "Here's a");+ let stats: serde_json::Value = serde_json::from_str(&text[k + coachwhip_server::chat::STATS_MARK.len()..]).unwrap();+ assert_eq!(stats["written"], 2);+}diff --git a/src/tests.rs b/src/tests.rsdeleted file mode 100644index 02d6508..0000000--- a/src/tests.rs+++ /dev/null@@ -1,301 +0,0 @@-//! Unit tests. No GPU, no model file: only the pure helpers and the CPU maths.-//! Run with `cargo test --release` (debug builds of candle are slow).--use std::io::{Read, Write};-use std::net::{TcpListener, TcpStream};--// ---------------------------------------------------------------- chat.rs helpers--#[test]-fn html_escape_escapes_the_three_that_matter() {- assert_eq!(crate::chat::html_escape("a&b<c>d"), "a&b<c>d");- assert_eq!(crate::chat::html_escape("plain"), "plain");- assert_eq!(crate::chat::html_escape(""), "");- // Escaping is applied once: an already-escaped amp is escaped again, as it must be.- assert_eq!(crate::chat::html_escape("&"), "&amp;");-}--#[test]-fn markers_are_the_control_one_prefix_the_page_strips() {- // The page's JS: MARK = '\u0001STATS:' and PROG = /\u0001PROG:(\{[^\n]*\})\n/- assert_eq!(crate::chat::STATS_MARK, "\u{1}STATS:");- assert_eq!(crate::chat::PROG_MARK, "\u{1}PROG:");- assert!(crate::chat::PAGE.contains("const MARK = '\\u0001STATS:'"));- assert!(crate::chat::PAGE.contains("const PROG = /\\u0001PROG:(\\{[^\\n]*\\})\\n/g"));-}--#[test]-fn progress_lines_match_the_page_regex() {- // Same format strings as chat.rs builds in `handle`.- let layer = format!("{}{{\"layer\":{},\"of\":{}}}\n", crate::chat::PROG_MARK, 3, 48);- let summary = format!("{}{{\"summary\":{}}}\n", crate::chat::PROG_MARK, 16);- for line in [&layer, &summary] {- assert!(line.starts_with("\u{1}PROG:{"));- assert!(line.ends_with("}\n"));- let json = &line["\u{1}PROG:".len()..line.len() - 1];- let v: serde_json::Value = serde_json::from_str(json).expect("progress line is JSON");- assert!(v.is_object());- assert!(!json.contains('\n'));- }-}--#[test]-fn think_box_is_a_verbatim_substring_of_the_page() {- // `PAGE.replace(THINK_BOX, "")` is a silent no-op if the two ever drift apart.- assert!(crate::chat::PAGE.contains(crate::chat::THINK_BOX), "THINK_BOX no longer matches the page");- assert_eq!(crate::chat::PAGE.matches(crate::chat::THINK_BOX).count(), 1);- let without = crate::chat::PAGE.replace(crate::chat::THINK_BOX, "");- assert!(!without.contains("id=\"think\""));- // The JS still guards the missing element.- assert!(without.contains("$('think')&&$('think').checked"));-}--#[test]-fn page_has_the_model_name_placeholder_once() {- assert_eq!(crate::chat::PAGE.matches("MODEL_NAME").count(), 1);-}--/// Sends `bytes` through `chunk` over a loopback socket and returns what the other end saw.-fn framed(bytes: &[u8]) -> Vec<u8> {- let listener = TcpListener::bind("127.0.0.1:0").unwrap();- let addr = listener.local_addr().unwrap();- let mut client = TcpStream::connect(addr).unwrap();- let (mut server, _) = listener.accept().unwrap();- crate::chat::chunk(&mut server, bytes).unwrap();- drop(server);- let mut got = Vec::new();- client.read_to_end(&mut got).unwrap();- got-}--#[test]-fn chunk_frames_as_hex_length_crlf_body_crlf() {- assert_eq!(framed(b"hello"), b"5\r\nhello\r\n");- assert_eq!(framed(&[b'x'; 255]), [b"ff\r\n".as_slice(), &[b'x'; 255], b"\r\n"].concat());- assert_eq!(framed("é".as_bytes()), b"2\r\n\xc3\xa9\r\n");-}--#[test]-fn chunk_sends_nothing_for_empty_bytes() {- // An empty chunk ("0\r\n\r\n") would end the HTTP response early.- assert_eq!(framed(b""), b"");-}--#[test]-fn chunk_to_a_closed_socket_is_an_error_not_a_panic() {- let listener = TcpListener::bind("127.0.0.1:0").unwrap();- let addr = listener.local_addr().unwrap();- let client = TcpStream::connect(addr).unwrap();- let (mut server, _) = listener.accept().unwrap();- drop(client);- // The first write may still succeed into the kernel buffer; keep writing until it fails.- let mut failed = false;- for _ in 0..64 {- if crate::chat::chunk(&mut server, &[b'x'; 65536]).is_err() {- failed = true;- break;- }- }- assert!(failed, "writing to a closed socket never reported an error");-}--// ---------------------------------------------------------------- experts.rs route table--#[test]-fn count_step_counts_every_pair_token_by_token() {- let mut counts = crate::experts::Counts::new();- let before = vec![vec![1, 2], vec![1]];- let now = vec![vec![7], vec![7, 9]];- crate::experts::count_step(&mut counts, 4, &before, &now);- assert_eq!(counts[&(4, 1)][&7], 2);- assert_eq!(counts[&(4, 1)][&9], 1);- assert_eq!(counts[&(4, 2)][&7], 1);- assert!(counts.get(&(4, 2)).map_or(true, |r| !r.contains_key(&9)));- assert!(!counts.contains_key(&(5, 1)));-}--#[test]-fn count_step_ignores_a_token_count_mismatch_beyond_the_shorter_side() {- let mut counts = crate::experts::Counts::new();- crate::experts::count_step(&mut counts, 0, &[vec![1]], &[vec![2], vec![3]]);- assert_eq!(counts[&(0, 1)][&2], 1);- assert_eq!(counts.len(), 1);-}--#[test]-fn load_routes_reads_the_log_shape_and_skips_junk() {- let dir = std::env::temp_dir().join(format!("coachwhip-routes-{}", std::process::id()));- std::fs::create_dir_all(&dir).unwrap();- // layers = 3: a "T" line then 3 layer lines; a stray header line and a short block are skipped.- let log = "T -\n1 2,3\n4,5 6\n7,7\nT 1 2 3\n9\n9\nnot-a-number\nT -\n1\n2\n";- std::fs::write(dir.join("a.log"), log).unwrap();- std::fs::write(dir.join("ignored.txt"), "T -\n1\n2\n3\n").unwrap();- let counts = crate::experts::load_routes(&[dir.as_path()], 3);- std::fs::remove_dir_all(&dir).unwrap();- // block 1, layer 0 -> 1: token0 (1,2)->(4), token1 (3)->(5,6)- assert_eq!(counts[&(0, 1)][&4], 1);- assert_eq!(counts[&(0, 2)][&4], 1);- assert_eq!(counts[&(0, 3)][&5], 1);- assert_eq!(counts[&(0, 3)][&6], 1);- // block 1, layer 1 -> 2: (4)->(7), (5,6)->(7)- assert_eq!(counts[&(1, 4)][&7], 1);- assert_eq!(counts[&(1, 5)][&7], 1);- // block 2: "not-a-number" parses to no experts, so (9)->(9) counted once and nothing from layer 2- assert_eq!(counts[&(0, 9)][&9], 1);- assert!(!counts.contains_key(&(1, 9)));- // the last block is too short and must not be read- assert!(!counts.contains_key(&(0, 1)) || counts[&(0, 1)].get(&2).is_none());-}--#[test]-fn load_routes_with_a_missing_folder_is_empty_not_an_error() {- let counts = crate::experts::load_routes(&[std::path::Path::new("/nonexistent/coachwhip")], 4);- assert!(counts.is_empty());-}--// ---------------------------------------------------------------- model_next.rs delta rule (CPU)--use candle::{Device, Tensor};--/// A tiny deterministic generator so the test needs no `rand` crate.-struct Lcg(u64);-impl Lcg {- fn next(&mut self) -> f32 {- self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);- ((self.0 >> 33) as f32 / (1u64 << 31) as f32) * 2.0 - 1.0- }- fn tensor(&mut self, shape: &[usize], scale: f32) -> Tensor {- let n: usize = shape.iter().product();- let v: Vec<f32> = (0..n).map(|_| self.next() * scale).collect();- Tensor::from_vec(v, shape, &Device::Cpu).unwrap()- }-}--/// The word-by-word delta rule exactly as model_next.rs steps it for one word (its `l == 1` path).-fn delta_word_by_word(q: &Tensor, k: &Tensor, v: &Tensor, beta: &Tensor, log_decay: &Tensor, mut s: Tensor) -> (Tensor, Tensor) {- let (l, n_v, _dk) = q.dims3().unwrap();- let decay = log_decay.exp().unwrap();- let mut outs = Vec::with_capacity(l);- for t in 0..l {- let kt = k.get(t).unwrap().unsqueeze(1).unwrap().contiguous().unwrap();- let qt = q.get(t).unwrap().unsqueeze(1).unwrap().contiguous().unwrap();- let vt = v.get(t).unwrap().unsqueeze(1).unwrap();- let bt = beta.get(t).unwrap().reshape((n_v, 1, 1)).unwrap();- let gt = decay.get(t).unwrap().reshape((n_v, 1, 1)).unwrap();- s = s.broadcast_mul(>).unwrap();- let sk = kt.matmul(&s).unwrap();- let d = (vt - sk).unwrap().broadcast_mul(&bt).unwrap();- s = (s + kt.transpose(1, 2).unwrap().contiguous().unwrap().matmul(&d).unwrap()).unwrap();- outs.push(qt.matmul(&s).unwrap().squeeze(1).unwrap());- }- (Tensor::stack(&outs, 0).unwrap(), s)-}--fn max_abs_diff(a: &Tensor, b: &Tensor) -> f32 {- (a - b).unwrap().abs().unwrap().flatten_all().unwrap().max(0).unwrap().to_scalar::<f32>().unwrap()-}--fn inputs(seed: u64, l: usize, n_v: usize, dk: usize, dv: usize) -> (Tensor, Tensor, Tensor, Tensor, Tensor) {- let mut g = Lcg(seed);- let unit = |t: Tensor| -> Tensor {- let n = t.sqr().unwrap().sum_keepdim(2).unwrap().sqrt().unwrap();- t.broadcast_div(&n).unwrap()- };- let q = unit(g.tensor(&[l, n_v, dk], 1.0));- let k = unit(g.tensor(&[l, n_v, dk], 1.0));- let v = g.tensor(&[l, n_v, dv], 1.0);- // beta in (0, 1); log decay in about (-0.6, 0): what sigmoid and softplus * A produce.- let beta = candle_nn::ops::sigmoid(&g.tensor(&[l, n_v], 2.0)).unwrap();- let log_decay = (g.tensor(&[l, n_v], 0.3).abs().unwrap() * -1.0).unwrap();- (q, k, v, beta, log_decay)-}--fn check_chunked_against_word_by_word(l: usize, seed: u64) {- let (n_v, dk, dv) = (3, 8, 6);- let (q, k, v, beta, log_decay) = inputs(seed, l, n_v, dk, dv);- let s0 = Tensor::zeros((n_v, dk, dv), candle::DType::F32, &Device::Cpu).unwrap();- let (o_ref, s_ref) = delta_word_by_word(&q, &k, &v, &beta, &log_decay, s0.clone());- let (o, s) = crate::model_next::delta_chunked(&q, &k, &v, &beta, &log_decay, s0).unwrap();- assert_eq!(o.dims(), &[l, n_v, dv], "output shape for l = {l}");- assert_eq!(s.dims(), &[n_v, dk, dv]);- let od = max_abs_diff(&o, &o_ref);- let sd = max_abs_diff(&s, &s_ref);- assert!(od < 1e-4, "l = {l}: output differs from word-by-word by {od}");- assert!(sd < 1e-4, "l = {l}: state differs from word-by-word by {sd}");-}--#[test]-fn delta_chunked_matches_word_by_word_when_l_is_a_whole_number_of_blocks() {- let c = crate::model_next::DELTA_CHUNK;- check_chunked_against_word_by_word(c, 1);- check_chunked_against_word_by_word(2 * c, 2);-}--#[test]-fn delta_chunked_matches_word_by_word_with_padding() {- let c = crate::model_next::DELTA_CHUNK;- for l in [2, 5, c - 1, c + 1, 2 * c + 17] {- check_chunked_against_word_by_word(l, 10 + l as u64);- }-}--#[test]-fn delta_chunked_carries_state_across_calls() {- // One call over 70 words must equal two calls, 70 = 45 + 25, with the state handed on.- let (l, n_v, dk, dv) = (70, 3, 8, 6);- let (q, k, v, beta, log_decay) = inputs(99, l, n_v, dk, dv);- let s0 = Tensor::zeros((n_v, dk, dv), candle::DType::F32, &Device::Cpu).unwrap();- let (o_all, s_all) = crate::model_next::delta_chunked(&q, &k, &v, &beta, &log_decay, s0.clone()).unwrap();- let cut = 45;- let part = |t: &Tensor, from: usize, len: usize| t.narrow(0, from, len).unwrap().contiguous().unwrap();- let (o1, s1) = crate::model_next::delta_chunked(&part(&q, 0, cut), &part(&k, 0, cut), &part(&v, 0, cut), &part(&beta, 0, cut), &part(&log_decay, 0, cut), s0).unwrap();- let (o2, s2) = crate::model_next::delta_chunked(&part(&q, cut, l - cut), &part(&k, cut, l - cut), &part(&v, cut, l - cut), &part(&beta, cut, l - cut), &part(&log_decay, cut, l - cut), s1).unwrap();- let o_split = Tensor::cat(&[&o1, &o2], 0).unwrap();- assert!(max_abs_diff(&o_all, &o_split) < 1e-4);- assert!(max_abs_diff(&s_all, &s2) < 1e-4);-}--#[test]-fn delta_chunked_with_a_nonzero_starting_state() {- let (l, n_v, dk, dv) = (37, 2, 8, 6);- let (q, k, v, beta, log_decay) = inputs(7, l, n_v, dk, dv);- let s0 = Lcg(123).tensor(&[n_v, dk, dv], 0.5);- let (o_ref, s_ref) = delta_word_by_word(&q, &k, &v, &beta, &log_decay, s0.clone());- let (o, s) = crate::model_next::delta_chunked(&q, &k, &v, &beta, &log_decay, s0).unwrap();- assert!(max_abs_diff(&o, &o_ref) < 1e-4);- assert!(max_abs_diff(&s, &s_ref) < 1e-4);-}--// ---------------------------------------------------------------- loopback HTTP, no model--#[test]-fn chunk_output_is_valid_for_a_chunked_http_client() {- // What the page receives per chunk: hex length, CRLF, body, CRLF. A zero-length chunk is never- // produced by `chunk`; `handle` writes the terminator itself.- let mut server_side: Vec<u8> = Vec::new();- for piece in ["Here", "'s", " a", "\u{1}STATS:{\"written\":2}"] {- server_side.extend(framed(piece.as_bytes()));- }- server_side.extend(b"0\r\n\r\n");- // Decode by hand.- let mut i = 0;- let mut body = Vec::new();- loop {- let nl = server_side[i..].windows(2).position(|w| w == b"\r\n").unwrap() + i;- let n = usize::from_str_radix(std::str::from_utf8(&server_side[i..nl]).unwrap(), 16).unwrap();- if n == 0 {- break;- }- body.extend_from_slice(&server_side[nl + 2..nl + 2 + n]);- assert_eq!(&server_side[nl + 2 + n..nl + 4 + n], b"\r\n");- i = nl + 4 + n;- }- let text = String::from_utf8(body).unwrap();- let k = text.find(crate::chat::STATS_MARK).unwrap();- assert_eq!(&text[..k], "Here's a");- let stats: serde_json::Value = serde_json::from_str(&text[k + crate::chat::STATS_MARK.len()..]).unwrap();- assert_eq!(stats["written"], 2);-}--#[allow(dead_code)]-fn _unused(_: &mut dyn Write) {}