The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
//! The chat: the model loads once and stays hot; a page on loopback sends each prompt in.
//!
//! One thread, no framework - `std::net::TcpListener` and a string of HTML. Nothing leaves the
//! machine: the page is served on 127.0.0.1 only. The answer streams: each word goes out as an
//! HTTP chunk the moment it is written.
//!
//! A chat is one long conversation the model has already read: a follow-up reads only its own new
//! words, then the model carries on from where it stopped.
//!
//! The same server also speaks the OpenAI chat API (`/v1/chat/completions`, `/v1/models`), so
//! coding tools on this machine can use the model. Those clients send the whole conversation every
//! time; when it begins with what the model has already read, only the new part is read.
use anyhow::Result;
use std::cell::RefCell;
use std::io::{BufRead, BufReader, Read, Write};
use std::net::{TcpListener, TcpStream};
use std::rc::Rc;
use candle::{Device, Tensor};
use candle_transformers::generation::{LogitsProcessor, Sampling};
use coachwhip_engine::model::{Model, Progress, Step};
use tokenizers::Tokenizer;
/// Largest request body accepted: a prompt of a few hundred pages.
const MAX_BODY: usize = 1 << 20;
/// Longest piece of a prompt read in one forward pass.
const PROMPT_CHUNK_DEFAULT: usize = 512;
fn fast_chat() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var("COACHWHIP_FAST_CHAT").map(|v| v == "1").unwrap_or(false))
}
/// Lab (`COACHWHIP_PROMPT_CHUNK=n`): smaller pieces when the experts are all resident and memory is tight.
fn prompt_chunk() -> usize {
static N: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*N.get_or_init(|| std::env::var("COACHWHIP_PROMPT_CHUNK").ok().and_then(|v| v.parse().ok()).filter(|&n| n > 0).unwrap_or(PROMPT_CHUNK_DEFAULT))
}
/// When a chat would grow past this many words, it is summarised first to make room.
pub const CONTEXT_LIMIT_MAX: usize = 16384;
/// The most a chat can ever hold (the reserved space may set less).
pub const CONTEXT_LIMIT: usize = CONTEXT_LIMIT_MAX;
/// The chat limit: the reserved conversation space when the wrangler set one, else 16384.
pub fn context_limit() -> usize {
std::env::var("COACHWHIP_CONV_TOKENS").ok().and_then(|v| v.parse().ok()).filter(|&n: &usize| n > 0).map_or(CONTEXT_LIMIT_MAX, |n: usize| n.min(CONTEXT_LIMIT_MAX))
}
/// Longest summary written when making room.
const SUMMARY_TOKENS: usize = 500;
const SUMMARY_ASK: &str = "Summarise our conversation so far in at most 300 words, for someone who will continue it: what was asked, what was decided, and every name, file, function or fact the next answer will need. Plain text, no preamble.";
/// The start of every gpt-oss chat: the model's own default system message from its chat
/// template, at its default reasoning level.
const HARMONY_SYSTEM: &str = "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\nKnowledge cutoff: 2024-06\n\nReasoning: medium\n\n# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|>";
/// The chat format the model was trained on: ChatML (the Qwen families) or harmony (gpt-oss).
struct Fmt {
harmony: bool,
}
impl Fmt {
fn of(model: &Model) -> Self {
Self { harmony: model.harmony() }
}
/// The words that end the assistant's turn. A stop word is never part of the answer.
fn stops(&self, tokenizer: &Tokenizer) -> Vec<u32> {
// Harmony: <|end|> only closes a message (the thinking ends with it); <|return|> ends the answer.
let names: &[&str] = if self.harmony { &["<|return|>", "<|call|>"] } else { &["<|im_end|>"] };
let vocab = tokenizer.get_vocab(true);
names.iter().filter_map(|n| vocab.get(*n).copied()).collect()
}
/// What a new chat opens with, before its first turn.
fn start(&self) -> &'static str {
if self.harmony { HARMONY_SYSTEM } else { "" }
}
/// What closes the last answer before the next turn (the stop word itself was never read).
fn close(&self) -> &'static str {
if self.harmony { "<|end|>" } else { "<|im_end|>\n" }
}
fn user(&self, text: &str) -> String {
if self.harmony { format!("<|start|>user<|message|>{text}<|end|>") } else { format!("<|im_start|>user\n{text}<|im_end|>\n") }
}
/// Opens the assistant's turn. gpt-oss with thinking on picks its own channel (analysis first);
/// with thinking off it goes straight to its final channel.
fn assistant(&self, model: &Model, think: bool) -> String {
match (self.harmony, think) {
(true, true) => "<|start|>assistant".to_string(),
(true, false) => "<|start|>assistant<|channel|>final<|message|>".to_string(),
_ => format!("<|im_start|>assistant\n{}", model.assistant_open(think)),
}
}
}
/// What the page shows of a gpt-oss answer that thinks: its analysis channel as a think block,
/// then its final channel. Format markers never show; one still arriving holds the text back.
fn harmony_view(raw: &str) -> String {
let t = raw
.replace("<|channel|>analysis<|message|>", "<think>\n")
.replace("<|end|><|start|>assistant<|channel|>final<|message|>", "\n</think>\n\n")
.replace("<|channel|>final<|message|>", "");
match t.find("<|") {
Some(i) => t[..i].to_string(),
None => t,
}
}
/// Separates the streamed text from the closing stats line.
pub const STATS_MARK: &str = "\u{1}STATS:";
/// A progress line sent while the prompt is read, before any text: `\u{1}PROG:<json>\n`.
pub const PROG_MARK: &str = "\u{1}PROG:";
/// Ends the stream when the answer could not be finished; the page shows what follows it.
pub const ERROR_MARK: &str = "\u{1}ERROR:";
/// After the GPU has run out of memory once, Metal does not recover in this process.
fn gpu_out_of_memory(e: &anyhow::Error) -> bool {
let m = format!("{e:#}");
m.contains("OutOfMemory") || m.contains("Insufficient Memory")
}
pub fn serve(port: u16, model: &mut Model, model_name: &str, tokenizer: Tokenizer, device: &Device, max_new: usize, sampling: Sampling) -> Result<()> {
let listener = TcpListener::bind(("127.0.0.1", port))?;
println!("\n coachwhip chat on http://127.0.0.1:{port} (model hot, loopback only)\n");
let mut session = Session::default();
for stream in listener.incoming() {
let Ok(stream) = stream else { continue };
if let Err(e) = handle(stream, model, model_name, &tokenizer, device, max_new, &sampling, &mut session) {
eprintln!(" request failed: {e}");
}
}
Ok(())
}
/// Where the model is in the current chat.
#[derive(Default)]
pub struct Session {
/// Words the model has read so far, its own answers included.
pos: usize,
/// The last word of an answer cut off at the length limit: written, but not yet read back.
pending: Option<u32>,
/// Every word the model has read so far, in order: `pos` of them.
fed: Vec<u32>,
/// Who wrote the conversation the model holds: the page continues only its own.
by_api: bool,
}
pub struct Answer {
/// New words read for this question; the earlier chat was not read again.
pub prompt_tokens: usize,
/// Words in the chat after this answer.
pub context: usize,
pub fresh: bool,
/// The chat was summarised to make room before this answer.
pub compacted: bool,
pub written: usize,
pub read_tps: f64,
pub write_tps: f64,
/// The model ended its answer itself, rather than hitting the length limit.
pub finished: bool,
}
/// Runs the model and hands every new piece of text to `emit` as soon as it exists.
#[allow(clippy::too_many_arguments)]
/// Foresight (lab): COACHWHIP_FORESIGHT=N drafts N words from the bank before each real word, only
/// to start the reads those words will need; the text is unchanged.
fn foresight() -> usize {
std::env::var("COACHWHIP_FORESIGHT").ok().and_then(|v| v.parse().ok()).unwrap_or(0)
}
pub fn generate(
model: &mut Model,
tokenizer: &Tokenizer,
device: &Device,
session: &mut Session,
new_chat: bool,
prompt: &str,
think: bool,
max_new: usize,
sampling: &Sampling,
progress: Option<Progress>,
emit: &mut dyn FnMut(&str) -> Result<()>,
) -> Result<Answer> {
let think = think && model.thinks();
let fmt = Fmt::of(model);
// The template's way of switching thinking on or off is in what opens the assistant's turn.
let open = fmt.assistant(model, think);
let encode = |text: String| -> Result<Vec<u32>> { Ok(tokenizer.encode(text, true).map_err(anyhow::Error::msg)?.get_ids().to_vec()) };
let stops = fmt.stops(tokenizer);
let tell = progress.map(|p| Rc::new(RefCell::new(p)));
// A follow-up closes the last answer and adds the new question; nothing earlier is read again.
let turn = format!("{}{open}", fmt.user(prompt));
let mut tokens = Vec::new();
let mut fresh = new_chat || session.pos == 0 || session.by_api;
let mut summary = None;
if !fresh {
tokens.extend(session.pending);
tokens.extend(encode(format!("{}{turn}", fmt.close()))?);
// Too long to go on: the model summarises the chat, and the summary opens a new one.
if session.pos + tokens.len() + max_new > context_limit() {
summary = Some(summarise(model, tokenizer, device, session, &stops, &tell)?);
fresh = true;
}
}
if fresh {
model.clear_kv_cache();
*session = Session::default();
tokens = encode(match &summary {
Some(s) => format!("{}{}{open}", fmt.start(), fmt.user(&format!("Summary of our conversation so far, for context:\n{s}\n\nNow: {prompt}"))),
None => format!("{}{turn}", fmt.start()),
})?;
}
let a = answer(model, tokenizer, device, session, tokens, fresh, summary.is_some(), think, max_new, sampling, &tell, emit);
session.by_api = false;
a
}
/// Request-level extras the chain sampler reads (the door answers one request at a time).
static SAMPLER_EXTRA: std::sync::Mutex<(Option<usize>, Option<f64>)> = std::sync::Mutex::new((None, None));
static REPEAT_REQ: std::sync::Mutex<Option<f64>> = std::sync::Mutex::new(None);
fn default_repeat_penalty() -> f64 { std::env::var("COACHWHIP_REPEAT_PENALTY").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0) }
fn repeat_last_n() -> usize { std::env::var("COACHWHIP_REPEAT_LAST_N").ok().and_then(|v| v.parse().ok()).unwrap_or(64) }
fn default_top_k() -> usize { std::env::var("COACHWHIP_TOP_K").ok().and_then(|v| v.parse().ok()).unwrap_or(20) }
fn default_min_p() -> f64 { std::env::var("COACHWHIP_MIN_P").ok().and_then(|v| v.parse().ok()).unwrap_or(0.0) }
/// The word pick: argmax at temperature 0, else top-k -> top-p -> min-p -> temperature -> draw.
struct Picker { s: Sampling, k: usize, min_p: f64, rng: u64, penalty: f64, hist: std::collections::VecDeque<u32> }
impl Picker {
fn new(s: &Sampling) -> Self {
let (k, mp) = *SAMPLER_EXTRA.lock().unwrap();
let seed = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0x9E3779B97F4A7C15, |d| d.as_nanos() as u64) | 1;
let penalty = REPEAT_REQ.lock().unwrap().unwrap_or_else(default_repeat_penalty);
Picker { s: s.clone(), k: k.unwrap_or_else(default_top_k), min_p: mp.unwrap_or_else(default_min_p), rng: seed, penalty, hist: std::collections::VecDeque::new() }
}
/// The words the penalty looks back over: the end of the prompt, then every word picked.
fn remember(&mut self, words: &[u32]) {
for &w in words { self.hist.push_back(w); }
while self.hist.len() > repeat_last_n() { self.hist.pop_front(); }
}
fn penalised(&self) -> bool { self.penalty != 1.0 }
fn uniform(&mut self) -> f64 {
let mut x = self.rng;
x ^= x >> 12; x ^= x << 25; x ^= x >> 27;
self.rng = x;
(x.wrapping_mul(0x2545F4914F6CDD1D) >> 11) as f64 / (1u64 << 53) as f64
}
fn sample(&mut self, logits: &Tensor) -> candle::Result<u32> {
// Fast path: no penalty, a temperature, on the GPU: only the k best come back.
let t_now = match &self.s { Sampling::ArgMax => 0.0, Sampling::All { temperature } | Sampling::TopK { temperature, .. } | Sampling::TopP { temperature, .. } | Sampling::TopKThenTopP { temperature, .. } => *temperature, #[allow(unreachable_patterns)] _ => 0.0 };
let k_now = match &self.s { Sampling::TopK { k, .. } | Sampling::TopKThenTopP { k, .. } => *k, _ => self.k };
if !self.penalised() && t_now > 0.0 && k_now >= 1 && k_now <= 64 && logits.device().is_metal() && std::env::var("COACHWHIP_GPU_TOPK").map(|v| v != "0").unwrap_or(true) {
let (vals, ids) = coachwhip_engine::fuse_kernel::topk(logits, k_now)?;
let p_now = match &self.s { Sampling::TopP { p, .. } | Sampling::TopKThenTopP { p, .. } => *p, _ => 1.0 };
let pick = self.draw(&vals, &ids, p_now, t_now);
self.remember(&[pick]);
return Ok(pick);
}
let mut v: Vec<f32> = logits.flatten_all()?.to_dtype(candle::DType::F32)?.to_vec1()?;
if self.penalised() {
let mut seen = std::collections::HashSet::new();
for &w in &self.hist {
if seen.insert(w) {
if let Some(x) = v.get_mut(w as usize) {
*x = if *x > 0.0 { *x / self.penalty as f32 } else { *x * self.penalty as f32 };
}
}
}
}
let pick = self.pick(&v)?;
self.remember(&[pick]);
Ok(pick)
}
/// Top-p, min-p, temperature and the draw over k candidates already in best-first order.
fn draw(&mut self, vals: &[f32], ids: &[u32], p: f64, t: f64) -> u32 {
let m = vals[0] as f64;
let mut pr: Vec<f64> = vals.iter().map(|&x| (x as f64 - m).exp()).collect();
let z: f64 = pr.iter().sum();
for x in pr.iter_mut() { *x /= z; }
let mut cum = 0.0; let mut keep = pr.len();
for (j, &x) in pr.iter().enumerate() { cum += x; if cum >= p { keep = j + 1; break; } }
let cut = self.min_p * pr[0];
let keep = pr[..keep].iter().take_while(|&&x| x >= cut).count().max(1);
let w: Vec<f64> = vals[..keep].iter().map(|&x| ((x as f64 - m) / t).exp()).collect();
let total: f64 = w.iter().sum();
let mut r = self.uniform() * total;
for (j, &x) in w.iter().enumerate() { r -= x; if r <= 0.0 { return ids[j]; } }
ids[keep - 1]
}
fn pick(&mut self, v: &[f32]) -> candle::Result<u32> {
let v = v.to_vec();
let argmax = |v: &[f32]| -> u32 { let mut b = 0usize; for (i, &x) in v.iter().enumerate() { if x > v[b] { b = i; } } b as u32 };
let (k, p, t) = match &self.s {
Sampling::ArgMax => return Ok(argmax(&v)),
Sampling::All { temperature } => (self.k, 1.0, *temperature),
Sampling::TopK { k, temperature } => (*k, 1.0, *temperature),
Sampling::TopP { p, temperature } => (self.k, *p, *temperature),
Sampling::TopKThenTopP { k, p, temperature } => (*k, *p, *temperature),
#[allow(unreachable_patterns)]
_ => (self.k, 1.0, 1.0),
};
if t <= 0.0 { return Ok(argmax(&v)); }
let k = k.max(1).min(v.len());
let mut idx: Vec<usize> = (0..v.len()).collect();
let desc = |a: &usize, b: &usize| v[*b].partial_cmp(&v[*a]).unwrap_or(std::cmp::Ordering::Equal);
if k < idx.len() { idx.select_nth_unstable_by(k - 1, desc); idx.truncate(k); }
idx.sort_by(desc);
let m = v[idx[0]] as f64;
let mut pr: Vec<f64> = idx.iter().map(|&i| (v[i] as f64 - m).exp()).collect();
let z: f64 = pr.iter().sum();
for x in pr.iter_mut() { *x /= z; }
// top-p
let mut cum = 0.0; let mut keep = pr.len();
for (j, &x) in pr.iter().enumerate() { cum += x; if cum >= p { keep = j + 1; break; } }
// min-p
let cut = self.min_p * pr[0];
let keep = pr[..keep].iter().take_while(|&&x| x >= cut).count().max(1);
idx.truncate(keep);
// temperature, then one draw
let w: Vec<f64> = idx.iter().map(|&i| ((v[i] as f64 - m) / t).exp()).collect();
let total: f64 = w.iter().sum();
let mut r = self.uniform() * total;
for (j, &x) in w.iter().enumerate() { r -= x; if r <= 0.0 { return Ok(idx[j] as u32); } }
Ok(idx[idx.len() - 1] as u32)
}
}
/// Compaction reuse (`COACHWHIP_SNAPSHOTS=1`): conversation states saved on the SSD, keyed by the exact
/// words they cover, kept to a disk budget, the least used first out.
mod snaps {
use std::io::{Read, Write};
use std::path::PathBuf;
pub struct Entry { pub tokens: Vec<u32>, pub file: PathBuf, pub bytes: u64, pub hits: u64, pub used: u64 }
pub struct Store { pub dir: PathBuf, pub budget: u64, pub list: Vec<Entry>, pub clock: u64 }
pub fn on() -> bool { std::env::var("COACHWHIP_SNAPSHOTS").map(|v| v == "1").unwrap_or(false) }
impl Store {
pub fn open() -> Store {
let dir = std::env::var("COACHWHIP_SNAP_DIR").map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(std::env::var("HOME").unwrap_or_else(|_| ".".into())).join(".coachwhip").join("snapshots"));
let _ = std::fs::create_dir_all(&dir);
let gb: f64 = std::env::var("COACHWHIP_SNAP_GB").ok().and_then(|v| v.parse().ok()).unwrap_or(2.0);
let mut list = Vec::new();
if let Ok(rd) = std::fs::read_dir(&dir) {
for e in rd.flatten() {
let p = e.path();
if p.extension().map(|x| x == "tok").unwrap_or(false) {
if let Ok(b) = std::fs::read(&p) {
let tokens: Vec<u32> = b.chunks_exact(4).map(|c| u32::from_le_bytes([c[0], c[1], c[2], c[3]])).collect();
let file = p.with_extension("state");
let bytes = std::fs::metadata(&file).map(|m| m.len()).unwrap_or(0);
if bytes > 0 { list.push(Entry { tokens, file, bytes, hits: 0, used: 0 }); }
}
}
}
}
Store { dir, budget: (gb * 1e9) as u64, list, clock: 0 }
}
/// The longest saved conversation the prompt begins with (at least `min` words, shorter than the prompt).
pub fn find(&mut self, prompt: &[u32], min: usize) -> Option<usize> {
let mut best: Option<usize> = None;
for (i, e) in self.list.iter().enumerate() {
let n = e.tokens.len();
if n >= min && n < prompt.len() && prompt[..n] == e.tokens[..] && best.map_or(true, |b| self.list[b].tokens.len() < n) {
best = Some(i);
}
}
if let Some(i) = best { self.clock += 1; self.list[i].hits += 1; self.list[i].used = self.clock; }
best
}
pub fn read(&self, i: usize) -> std::io::Result<Vec<(Vec<usize>, Vec<f32>)>> {
let mut b = Vec::new();
std::fs::File::open(&self.list[i].file)?.read_to_end(&mut b)?;
let mut at = 0usize;
let mut u64_at = |b: &[u8], at: &mut usize| -> u64 { let v = u64::from_le_bytes(b[*at..*at + 8].try_into().unwrap()); *at += 8; v };
let parts = u64_at(&b, &mut at) as usize;
let mut out = Vec::with_capacity(parts);
for _ in 0..parts {
let rank = u64_at(&b, &mut at) as usize;
let dims: Vec<usize> = (0..rank).map(|_| u64_at(&b, &mut at) as usize).collect();
let n: usize = if rank == 0 { 0 } else { dims.iter().product() };
let vals: Vec<f32> = b[at..at + 4 * n].chunks_exact(4).map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])).collect();
at += 4 * n;
out.push((dims, vals));
}
Ok(out)
}
pub fn save(&mut self, tokens: &[u32], parts: &[(Vec<usize>, Vec<f32>)]) -> std::io::Result<()> {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
tokens.hash(&mut h);
let name = format!("{:016x}", h.finish());
let file = self.dir.join(format!("{name}.state"));
let mut w = std::io::BufWriter::new(std::fs::File::create(&file)?);
w.write_all(&(parts.len() as u64).to_le_bytes())?;
for (dims, vals) in parts {
w.write_all(&(dims.len() as u64).to_le_bytes())?;
for d in dims { w.write_all(&(*d as u64).to_le_bytes())?; }
for v in vals { w.write_all(&v.to_le_bytes())?; }
}
w.flush()?;
let tok: Vec<u8> = tokens.iter().flat_map(|t| t.to_le_bytes()).collect();
std::fs::write(self.dir.join(format!("{name}.tok")), tok)?;
let bytes = std::fs::metadata(&file)?.len();
self.clock += 1;
self.list.retain(|e| e.tokens != tokens);
self.list.push(Entry { tokens: tokens.to_vec(), file, bytes, hits: 0, used: self.clock });
// over budget: the least used go first (fewest hits, then oldest)
while self.list.iter().map(|e| e.bytes).sum::<u64>() > self.budget && self.list.len() > 1 {
let (i, _) = self.list.iter().enumerate().min_by_key(|(_, e)| (e.hits, e.used)).unwrap();
let e = self.list.remove(i);
let _ = std::fs::remove_file(&e.file);
let _ = std::fs::remove_file(e.file.with_extension("tok"));
}
Ok(())
}
}
pub fn store() -> &'static std::sync::Mutex<Store> {
static S: std::sync::OnceLock<std::sync::Mutex<Store>> = std::sync::OnceLock::new();
S.get_or_init(|| std::sync::Mutex::new(Store::open()))
}
}
/// Answers a whole conversation sent at once, as the OpenAI API does. When it begins with what the
/// model has already read, only the rest is read; otherwise the model starts afresh.
#[allow(clippy::too_many_arguments)]
pub fn generate_full(
model: &mut Model,
tokenizer: &Tokenizer,
device: &Device,
session: &mut Session,
prompt: Vec<u32>,
max_new: usize,
sampling: &Sampling,
emit: &mut dyn FnMut(&str) -> Result<()>,
) -> Result<Answer> {
// No refusal: past the reserved space the conversation rolls (see the engine's KvBuf).
let known = session.fed.len();
let mut fresh = known == 0 || prompt.len() <= known || !prompt.starts_with(&session.fed);
if fresh {
model.clear_kv_cache();
*session = Session::default();
// A conversation that begins with words already saved starts from that snapshot.
if snaps::on() {
let mut st = snaps::store().lock().unwrap();
if let Some(i) = st.find(&prompt, 256) {
let n = st.list[i].tokens.len();
match st.read(i).map_err(anyhow::Error::from).and_then(|parts| model.load_state(&parts).map_err(anyhow::Error::from)) {
Ok(true) => {
session.fed = prompt[..n].to_vec();
session.pos = n;
fresh = false;
eprintln!(" snapshot: restored {n} words, reading only the {} new", prompt.len() - n);
}
Ok(false) => {}
Err(e) => { eprintln!(" snapshot: could not restore ({e}); reading afresh"); model.clear_kv_cache(); *session = Session::default(); }
}
}
}
} else {
// The word an earlier answer was cut off on is part of what the client sent back.
session.pending = None;
}
let tokens = prompt[session.fed.len()..].to_vec();
let think = model.harmony();
let a = answer(model, tokenizer, device, session, tokens, fresh, false, think, max_new, sampling, &None, emit);
session.by_api = true;
// Kept for the next conversation that begins the same way.
if snaps::on() && a.is_ok() && session.fed.len() >= 256 {
if let Ok(Some(parts)) = model.save_state() {
if let Err(e) = snaps::store().lock().unwrap().save(&session.fed, &parts) {
eprintln!(" snapshot: could not save ({e})");
}
}
}
a
}
/// Reads `tokens` after what the session has read, then writes the answer, streaming it to `emit`.
#[allow(clippy::too_many_arguments)]
fn answer(
model: &mut Model,
tokenizer: &Tokenizer,
device: &Device,
session: &mut Session,
tokens: Vec<u32>,
fresh: bool,
compacted: bool,
think: bool,
max_new: usize,
sampling: &Sampling,
tell: &Option<Rc<RefCell<Progress>>>,
emit: &mut dyn FnMut(&str) -> Result<()>,
) -> Result<Answer> {
// Lab (`COACHWHIP_QOS=1`): this thread at user-interactive priority, so macOS wakes it at once
// when the GPU finishes a word.
if std::env::var("COACHWHIP_QOS").map(|v| v == "1").unwrap_or(false) {
extern "C" { fn pthread_set_qos_class_self_np(qos: u32, rel: i32) -> i32; }
unsafe { pthread_set_qos_class_self_np(0x21, 0); }
}
let fmt = Fmt::of(model);
let stops = fmt.stops(tokenizer);
// gpt-oss thinking: the channels are read from the format markers, so keep them when decoding.
let channels = fmt.harmony && think;
let mut sampler = Picker::new(sampling);
sampler.remember(&tokens[tokens.len().saturating_sub(repeat_last_n())..]);
let t0 = std::time::Instant::now();
let logits = read_prompt(model, device, session.pos, &tokens, tell)?;
session.fed.extend(&tokens);
let base = session.pos + tokens.len();
let mut next = sampler.sample(&logits)?;
let read = t0.elapsed().as_secs_f64();
// The answer so far; an end-of-turn word is never part of it.
let mut out = if stops.contains(&next) { Vec::new() } else { vec![next] };
let mut sent = 0usize;
let mut stream_text = |out: &[u32], sent: &mut usize| -> Result<()> {
let full = if channels {
harmony_view(&tokenizer.decode(out, false).map_err(anyhow::Error::msg)?)
} else {
tokenizer.decode(out, true).map_err(anyhow::Error::msg)?
};
// A token can be half of a multi-byte character; wait until the text is whole.
if full.ends_with('\u{FFFD}') {
return Ok(());
}
// With thinking off the model still opens and closes an empty think block; skip it.
let start = if !think && full.starts_with("<think>") {
match full.find("</think>") {
Some(i) => {
let after = &full[i + "</think>".len()..];
i + "</think>".len() + (after.len() - after.trim_start_matches('\n').len())
}
None => return Ok(()),
}
} else {
0
};
let from = start.max(*sent);
// A later decode can re-space earlier text; wait for a clean place to carry on from.
if !full.is_char_boundary(from) {
return Ok(());
}
if full.len() > from {
emit(&full[from..])?;
*sent = full.len();
}
Ok(())
};
stream_text(&out, &mut sent)?;
let t1 = std::time::Instant::now();
// S6 (lab): COACHWHIP_SPECULATE=K drafts K words from the bank and checks them in one pass;
// COACHWHIP_SPECULATE_MEASURE=K only measures, writing the answer the plain way.
let speculate: usize = std::env::var("COACHWHIP_SPECULATE").ok().and_then(|v| v.parse().ok()).unwrap_or(0);
let measure: usize = std::env::var("COACHWHIP_SPECULATE_MEASURE").ok().and_then(|v| v.parse().ok()).unwrap_or(0);
let greedy = matches!(sampling, Sampling::ArgMax);
while out.len() < max_new && !stops.contains(&next) {
if speculate > 0 && greedy && max_new - out.len() > speculate {
if let Some(seq) = model.speculate(next, base + out.len() - 1, speculate)? {
for &w in &seq {
next = w;
if stops.contains(&w) {
break;
}
out.push(w);
stream_text(&out, &mut sent)?;
}
continue;
}
}
if measure > 0 && greedy && max_new - out.len() >= measure {
if let Some((plain, _accepted)) = model.speculate_measure(next, base + out.len() - 1, measure)? {
for &w in &plain {
next = w;
if stops.contains(&w) {
break;
}
out.push(w);
stream_text(&out, &mut sent)?;
}
continue;
}
}
if foresight() > 0 {
model.foresee(next, base + out.len() - 1, foresight())?;
}
// Lab (`COACHWHIP_FAST_CHAT=1`): the word goes in from the CPU (no GPU round trip before the
// pass) and, when greedy, the GPU picks the next one and only 4 bytes come back.
let fast = fast_chat() && device.is_metal();
let ta = std::time::Instant::now();
let logits = model.forward(&Tensor::new(&[next], if fast { &Device::Cpu } else { device })?.unsqueeze(0)?, base + out.len() - 1)?.squeeze(0)?;
let tb = std::time::Instant::now();
next = if fast && greedy && !sampler.penalised() { coachwhip_engine::fuse_kernel::argmax(&logits)?.flatten_all()?.to_vec1::<u32>()?[0] } else { sampler.sample(&logits)? };
let tc = std::time::Instant::now();
if !stops.contains(&next) {
out.push(next);
stream_text(&out, &mut sent)?;
}
if std::env::var("COACHWHIP_CHAT_TIME").is_ok() {
static T: std::sync::Mutex<[f64; 4]> = std::sync::Mutex::new([0.0; 4]);
let mut g = T.lock().unwrap();
g[0] += (tb - ta).as_secs_f64(); g[1] += (tc - tb).as_secs_f64(); g[2] += tc.elapsed().as_secs_f64(); g[3] += 1.0;
if g[3] as u64 % 100 == 0 {
eprintln!("coachwhip chat time per word over {}: queue {:.2} ms, pick (GPU wait) {:.2} ms, decode+send {:.2} ms", g[3], 1000.0 * g[0] / g[3], 1000.0 * g[1] / g[3], 1000.0 * g[2] / g[3]);
}
}
}
let write = t1.elapsed().as_secs_f64();
let written = out.len();
// Every word of the answer was read back, except the last one when the length limit cut it off.
let finished = stops.contains(&next);
let read_back = if finished { out.len() } else { out.len() - 1 };
session.pos = base + read_back;
session.pending = if finished { None } else { out.last().copied() };
session.fed.extend(&out[..read_back]);
Ok(Answer {
prompt_tokens: tokens.len(),
context: session.pos,
fresh,
compacted,
finished,
written,
read_tps: tokens.len() as f64 / read.max(1e-9),
// The first word comes out of reading the prompt; the rate counts the ones written after it.
write_tps: written.saturating_sub(1) as f64 / write.max(1e-9),
})
}
/// Reads a prompt that continues the chat at `pos`, and returns the logits for the next word.
/// The chunker: a long prompt is read in pieces, so the memory a piece needs stays bounded.
/// While it reads, every finished layer is reported, counted across all the pieces, so the page
/// is never silent.
fn read_prompt(model: &mut Model, device: &Device, pos: usize, tokens: &[u32], tell: &Option<Rc<RefCell<Progress>>>) -> Result<Tensor> {
// The pieces shrink as the chat grows, so the attention scores of a piece (piece x chat) never
// need more memory than a full piece at 4K of chat.
let space: Option<usize> = std::env::var("COACHWHIP_CONV_TOKENS").ok().and_then(|v| v.parse().ok()).filter(|&n: &usize| n > 0);
let piece_at = |at: usize| -> usize {
let full = prompt_chunk();
let at = space.map_or(at, |s| at.min(s));
if at <= 4096 { full } else { (full * 4096 / at).max(32) }
};
let mut bounds = Vec::new();
let mut at = 0usize;
while at < tokens.len() {
let n = piece_at(pos + at + piece_at(pos + at)).min(tokens.len() - at);
bounds.push((at, n));
at += n;
}
let pieces = bounds.len();
let read = (|| -> Result<Tensor> {
let mut logits = None;
for (i, &(start, n)) in bounds.iter().enumerate() {
let piece = &tokens[start..start + n];
model.set_progress(tell.clone().map(|t| -> Progress {
Box::new(move |step| {
if let Step::Layer(layer, of) = step {
(t.borrow_mut())(Step::Layer(i * of + layer, pieces * of))
}
})
}));
logits = Some(model.forward(&Tensor::new(piece, device)?.unsqueeze(0)?, pos + start)?.squeeze(0)?);
}
logits.ok_or_else(|| anyhow::anyhow!("empty prompt"))
})();
// Taken back on every path: the hook holds the page's connection open.
model.set_progress(None);
read
}
/// Asks the model, in the chat as it stands, for a summary of the chat; the chat is then thrown
/// away and the summary opens the next one. Greedy, so it is the same every time.
fn summarise(model: &mut Model, tokenizer: &Tokenizer, device: &Device, session: &Session, stops: &[u32], tell: &Option<Rc<RefCell<Progress>>>) -> Result<String> {
let mut tokens: Vec<u32> = session.pending.into_iter().collect();
let fmt = Fmt::of(model);
let ask = format!("{}{}{}", fmt.close(), fmt.user(SUMMARY_ASK), fmt.assistant(model, false));
tokens.extend(tokenizer.encode(ask, true).map_err(anyhow::Error::msg)?.get_ids().to_vec());
let mut sampler = LogitsProcessor::from_sampling(42, Sampling::ArgMax);
let logits = read_prompt(model, device, session.pos, &tokens, tell)?;
let mut next = sampler.sample(&logits)?;
let base = session.pos + tokens.len();
let mut out = Vec::new();
while !stops.contains(&next) && out.len() < SUMMARY_TOKENS {
out.push(next);
if let Some(t) = tell {
if out.len() % 8 == 0 {
(t.borrow_mut())(Step::Summary(out.len()));
}
}
if foresight() > 0 {
model.foresee(next, base + out.len() - 1, foresight())?;
}
let logits = model.forward(&Tensor::new(&[next], device)?.unsqueeze(0)?, base + out.len() - 1)?.squeeze(0)?;
next = sampler.sample(&logits)?;
}
let text = tokenizer.decode(&out, true).map_err(anyhow::Error::msg)?;
let text = match text.find("</think>") {
Some(i) if text.starts_with("<think>") => text[i + "</think>".len()..].to_string(),
_ => text,
};
let text = text.trim().to_string();
println!(" made room: {} tokens of chat summarised into {} tokens", session.pos, out.len());
Ok(text)
}
/// The text of an OpenAI message: a plain string, or a list of parts of which the text ones count.
pub fn message_text(content: &serde_json::Value) -> String {
match content {
serde_json::Value::String(t) => t.clone(),
serde_json::Value::Array(parts) => parts.iter().filter_map(|p| p["text"].as_str()).collect::<Vec<_>>().join("\n"),
_ => String::new(),
}
}
/// An OpenAI conversation in harmony (gpt-oss), ready for the assistant's turn. The client's own
/// system message becomes harmony's developer message; the model thinks before its final answer.
fn harmony_prompt(messages: &[serde_json::Value]) -> String {
let mut text = HARMONY_SYSTEM.to_string();
for m in messages {
let content = message_text(&m["content"]);
text.push_str(&match m["role"].as_str().unwrap_or("user") {
"system" | "developer" => format!("<|start|>developer<|message|># Instructions\n\n{content}<|end|>"),
"assistant" => format!("<|start|>assistant<|channel|>final<|message|>{content}<|end|>"),
_ => format!("<|start|>user<|message|>{content}<|end|>"),
});
}
text.push_str("<|start|>assistant");
text
}
/// An OpenAI conversation in the model's chat format, ready for the assistant's turn; `open` is
/// what the model's template puts at the start of that turn (see `Model::assistant_open`).
pub fn chat_prompt(messages: &[serde_json::Value], open: &str) -> String {
let mut text = String::new();
for m in messages {
let role = match m["role"].as_str().unwrap_or("user") {
r @ ("system" | "user" | "assistant") => r,
"developer" => "system",
_ => "user",
};
text.push_str(&format!("<|im_start|>{role}\n{}<|im_end|>\n", message_text(&m["content"])));
}
text.push_str("<|im_start|>assistant\n");
text.push_str(open);
text
}
/// `/v1/chat/completions`: the OpenAI chat API, streamed (server-sent events) or in one reply. Any
/// API key is accepted and ignored: only programs on this machine can reach this port.
#[allow(clippy::too_many_arguments)]
fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, tokenizer: &Tokenizer, device: &Device, max_new: usize, sampling: &Sampling, session: &mut Session) -> Result<()> {
let reply = |s: &mut TcpStream, status: &str, json: serde_json::Value| -> Result<()> {
let body = json.to_string();
write!(s, "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len())?;
Ok(())
};
let bad = |m: &str| serde_json::json!({"error": {"message": m, "type": "invalid_request_error"}});
let Ok(q) = serde_json::from_slice::<serde_json::Value>(body) else {
return reply(&mut s, "400 Bad Request", bad("the body is not JSON"));
};
let Some(messages) = q["messages"].as_array().filter(|m| !m.is_empty()) else {
return reply(&mut s, "400 Bad Request", bad("`messages` must be a non-empty list"));
};
let stream = q["stream"].as_bool().unwrap_or(false);
let asked = q["max_completion_tokens"].as_u64().or(q["max_tokens"].as_u64());
let text = if model.harmony() { harmony_prompt(messages) } else { chat_prompt(messages, model.assistant_open(false)) };
let prompt = tokenizer.encode(text, true).map_err(anyhow::Error::msg)?.get_ids().to_vec();
let max_new = answer_budget(asked, max_new, prompt.len());
// The requests own temperature and top_p win over the doors defaults; 0 means the likeliest word.
// The request's own sampling wins over the door's defaults; temperature 0 means the likeliest word.
*SAMPLER_EXTRA.lock().unwrap() = (q["top_k"].as_u64().map(|k| k as usize).filter(|&k| k > 0), q["min_p"].as_f64());
*REPEAT_REQ.lock().unwrap() = q["repeat_penalty"].as_f64().filter(|&r| r > 0.0);
let door_p = match sampling { Sampling::TopP { p, .. } | Sampling::TopKThenTopP { p, .. } => *p, _ => 1.0 };
let door_t = match sampling { Sampling::ArgMax => 0.0, Sampling::TopP { temperature, .. } | Sampling::TopKThenTopP { temperature, .. } | Sampling::All { temperature } | Sampling::TopK { temperature, .. } => *temperature, #[allow(unreachable_patterns)] _ => 0.7 };
let t = q["temperature"].as_f64().unwrap_or(door_t);
let asked_sampling: Sampling = if t <= 0.0 {
Sampling::ArgMax
} else {
Sampling::TopKThenTopP { k: q["top_k"].as_u64().map(|k| k as usize).filter(|&k| k > 0).unwrap_or_else(default_top_k), p: q["top_p"].as_f64().unwrap_or(door_p), temperature: t }
};
let sampling = &asked_sampling;
let id = format!("chatcmpl-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()));
let created = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs());
let prompt_len = prompt.len();
println!(" api: {} messages, {} tokens, {}", messages.len(), prompt_len, if stream { "streamed" } else { "one reply" });
let event = |delta: serde_json::Value, finish: Option<&str>| {
serde_json::json!({"id": id, "object": "chat.completion.chunk", "created": created, "model": model_name,
"choices": [{"index": 0, "delta": delta, "finish_reason": finish}]})
};
let mut text = String::new();
let result = if stream {
write!(s, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nCache-Control: no-cache\r\n\r\n")?;
chunk(&mut s, format!("data: {}\n\n", event(serde_json::json!({"role": "assistant", "content": ""}), None)).as_bytes())?;
let mut out_s = s.try_clone()?;
let mut emit = |t: &str| chunk(&mut out_s, format!("data: {}\n\n", event(serde_json::json!({"content": t}), None)).as_bytes());
generate_full(model, tokenizer, device, session, prompt, max_new, sampling, &mut emit)
} else {
let mut emit = |t: &str| -> Result<()> {
text.push_str(t);
Ok(())
};
generate_full(model, tokenizer, device, session, prompt, max_new, sampling, &mut emit)
};
let a = match result {
Ok(a) => a,
Err(e) => {
*session = Session::default();
eprintln!(" error detail: {e:#}");
let oom = gpu_out_of_memory(&e);
let msg = if oom {
"The GPU ran out of memory. Coachwhip has stopped: close other apps and start it again, or send a shorter prompt.".to_string()
} else {
format!("{e}")
};
let too_long = format!("{e}").contains("start a new one");
let err = serde_json::json!({"error": {"message": msg, "type": if too_long { "invalid_request_error" } else { "server_error" }}});
if stream {
let _ = chunk(&mut s, format!("data: {err}\n\ndata: [DONE]\n\n").as_bytes());
let _ = s.write_all(b"0\r\n\r\n");
} else {
let status = if too_long { "400 Bad Request" } else { "500 Internal Server Error" };
let _ = reply(&mut s, status, err);
}
let _ = s.flush();
if oom {
eprintln!("\n coachwhip: {msg}\n");
std::process::exit(1);
}
return Err(e);
}
};
println!(" api: read {} new tokens at {:.1} tok/s, wrote {} at {:.1} tok/s, chat now {} tokens", a.prompt_tokens, a.read_tps, a.written, a.write_tps, a.context);
model.report();
if model.harmony() {
if let Some(i) = text.find("</think>") {
text = text[i + "</think>".len()..].trim_start().to_string();
}
}
let finish = if a.finished { "stop" } else { "length" };
let usage = serde_json::json!({"prompt_tokens": prompt_len, "completion_tokens": a.written, "total_tokens": prompt_len + a.written});
if stream {
let mut last = event(serde_json::json!({}), Some(finish));
last["usage"] = usage;
chunk(&mut s, format!("data: {last}\n\ndata: [DONE]\n\n").as_bytes())?;
s.write_all(b"0\r\n\r\n")?;
s.flush()?;
Ok(())
} else {
reply(&mut s, "200 OK", serde_json::json!({"id": id, "object": "chat.completion", "created": created, "model": model_name,
"choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": finish}], "usage": usage}))
}
}
/// Only this machine's own page, or a program on this machine, may talk to the model: the Host
/// must name this port on loopback, and an Origin, when a browser sends one, must too. Another
/// site open in the browser could otherwise post prompts here. Headers arrive lower-cased.
pub fn own_page(host: &str, origin: &str, port: u16) -> bool {
let local = |h: &str| h == format!("127.0.0.1:{port}") || h == format!("localhost:{port}");
local(host) && (origin.is_empty() || origin.strip_prefix("http://").is_some_and(local))
}
/// How many tokens an API answer may write: what the client asked for, capped by Coachwhip's own
/// limit; with no limit asked for, whatever room the 16K conversation leaves.
pub fn answer_budget(asked: Option<u64>, max_new: usize, prompt_len: usize) -> usize {
// The conversation rolls past its space, so the answer is never squeezed to fit.
let _ = prompt_len;
asked.map_or(max_new, |m| (m as usize).min(max_new))
}
pub fn html_escape(t: &str) -> String {
t.replace('&', "&").replace('<', "<").replace('>', ">")
}
pub fn chunk(s: &mut TcpStream, bytes: &[u8]) -> Result<()> {
if bytes.is_empty() {
return Ok(());
}
write!(s, "{:x}\r\n", bytes.len())?;
s.write_all(bytes)?;
s.write_all(b"\r\n")?;
s.flush()?;
Ok(())
}
fn handle(mut s: TcpStream, model: &mut Model, model_name: &str, tokenizer: &Tokenizer, device: &Device, max_new: usize, sampling: &Sampling, session: &mut Session) -> Result<()> {
// A client that stops sending must not hold the one-at-a-time server forever.
s.set_read_timeout(Some(std::time::Duration::from_secs(30)))?;
let mut r = BufReader::new(s.try_clone()?).take(64 * 1024 + MAX_BODY as u64);
let mut line = String::new();
r.read_line(&mut line)?;
let path = line.split_whitespace().nth(1).unwrap_or("/").to_string();
let mut len = 0usize;
let mut host = String::new();
let mut origin = String::new();
loop {
let mut h = String::new();
if r.read_line(&mut h)? == 0 || h.trim().is_empty() {
break;
}
let lower = h.to_lowercase();
if let Some(v) = lower.strip_prefix("content-length:") {
len = v.trim().parse().unwrap_or(0);
} else if let Some(v) = lower.strip_prefix("host:") {
host = v.trim().to_string();
} else if let Some(v) = lower.strip_prefix("origin:") {
origin = v.trim().to_string();
}
}
let own_page = own_page(&host, &origin, s.local_addr()?.port());
let path = path.split('?').next().unwrap_or("/").to_string();
let api = path.starts_with("/v1/");
if (path == "/ask" || api) && (!own_page || len > MAX_BODY) {
write!(s, "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n")?;
return Ok(());
}
if path == "/v1/models" {
let body = serde_json::json!({"object": "list", "data": [{"id": model_name, "object": "model", "owned_by": "coachwhip"}]}).to_string();
write!(s, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len())?;
return Ok(());
}
if path == "/v1/chat/completions" {
let mut body = vec![0u8; len];
r.read_exact(&mut body)?;
return openai(s, &body, model, model_name, tokenizer, device, max_new, sampling, session);
}
if api {
write!(s, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n")?;
return Ok(());
}
if path == "/ask" {
let mut body = vec![0u8; len];
r.read_exact(&mut body)?;
let Ok(q) = serde_json::from_slice::<serde_json::Value>(&body) else {
write!(s, "HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n")?;
return Ok(());
};
let prompt = q["prompt"].as_str().unwrap_or("");
let think = q["think"].as_bool().unwrap_or(false);
let new_chat = q["new_chat"].as_bool().unwrap_or(false);
let max_new = q["max_new"].as_u64().map(|m| (m as usize).min(max_new)).unwrap_or(max_new);
println!(" prompt: {} chars, thinking {}", prompt.len(), if think { "on" } else { "off" });
write!(
s,
"HTTP/1.1 200 OK\r\nContent-Type: text/plain; charset=utf-8\r\nTransfer-Encoding: chunked\r\nCache-Control: no-cache\r\nX-Accel-Buffering: no\r\n\r\n"
)?;
s.flush()?;
let mut out_s = s.try_clone()?;
let mut emit = |t: &str| chunk(&mut out_s, t.as_bytes());
let mut prog_s = s.try_clone()?;
let progress: Progress = Box::new(move |step| {
let line = match step {
Step::Layer(layer, of) => format!("{PROG_MARK}{{\"layer\":{layer},\"of\":{of}}}\n"),
Step::Summary(words) => format!("{PROG_MARK}{{\"summary\":{words}}}\n"),
};
let _ = chunk(&mut prog_s, line.as_bytes());
});
let a = match generate(model, tokenizer, device, session, new_chat, prompt, think, max_new, sampling, Some(progress), &mut emit) {
Ok(a) => a,
Err(e) => {
// A broken answer leaves the model part-way through it, so the next question starts afresh.
*session = Session::default();
eprintln!(" error detail: {e:#}");
let oom = gpu_out_of_memory(&e);
let msg = if oom {
"The GPU ran out of memory. Coachwhip has stopped: close other apps and start it again, or send a shorter prompt.".to_string()
} else {
format!("The answer could not be finished: {e}")
};
let _ = chunk(&mut s, format!("{ERROR_MARK}{msg}").as_bytes());
let _ = s.write_all(b"0\r\n\r\n");
let _ = s.flush();
if oom {
eprintln!("\n coachwhip: {msg}\n");
std::process::exit(1);
}
return Err(e);
}
};
println!(" read {} new tokens at {:.1} tok/s, wrote {} at {:.1} tok/s, chat now {} tokens", a.prompt_tokens, a.read_tps, a.written, a.write_tps, a.context);
model.report();
let stats = serde_json::json!({
"prompt_tokens": a.prompt_tokens, "written": a.written, "context": a.context, "fresh": a.fresh, "compacted": a.compacted,
"read_tps": a.read_tps, "write_tps": a.write_tps,
});
chunk(&mut s, format!("{STATS_MARK}{stats}").as_bytes())?;
s.write_all(b"0\r\n\r\n")?;
} else {
// gpt-oss is a reasoning model: its thinking switch starts ticked.
let page = if model.harmony() {
PAGE.replace(r#"id="think">"#, r#"id="think" checked>"#)
} else if model.thinks() {
PAGE.to_string()
} else {
PAGE.replace(THINK_BOX, "")
};
let page = page.replace("MODEL_NAME", &html_escape(model_name));
write!(s, "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\n\r\n", page.len())?;
s.write_all(page.as_bytes())?;
}
s.flush()?;
Ok(())
}
/// The page's thinking switch, left out for a model that has no thinking mode.
pub const THINK_BOX: &str = r#"<label><input type="checkbox" id="think"> let it think first (slower)</label>"#;
pub const PAGE: &str = r##"<!doctype html>
<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Coachwhip</title><style>
:root{--bg:#0d1117;--panel:#161b22;--line:#30363d;--text:#e6edf3;--dim:#8b949e;--on:#3fb950;--off:#6e7681}
*{box-sizing:border-box} body{margin:0;background:var(--bg);color:var(--text);
font:15px/1.55 ui-monospace,"Cascadia Code",Consolas,monospace}
.wrap{max-width:900px;margin:0 auto;padding:28px 16px 60px}
h1{font-size:22px;margin:0 0 4px} h1 b{color:var(--on)}
.sub{color:var(--dim);font-size:13px;margin-bottom:22px}
textarea{width:100%;height:160px;background:var(--panel);color:var(--text);border:1px solid var(--line);
border-radius:8px;padding:14px;font:14px/1.5 ui-monospace,Consolas,monospace;resize:vertical}
.row{display:flex;gap:14px;align-items:center;flex-wrap:wrap;margin:12px 0 22px}
button{background:var(--on);color:#08130b;border:0;border-radius:7px;padding:10px 22px;font-weight:700;
font-size:14px;cursor:pointer} button:disabled{background:var(--off);color:#0d1117;cursor:default}
label{color:var(--dim);font-size:13px;display:flex;gap:7px;align-items:center;cursor:pointer}
.card{background:var(--panel);border:1px solid var(--line);border-radius:8px;overflow:hidden}
.card h2{margin:0;padding:11px 15px;font-size:12px;letter-spacing:.09em;text-transform:uppercase;
border-bottom:1px solid var(--line);color:var(--on)}
pre{margin:0;padding:15px;white-space:pre-wrap;word-break:break-word;min-height:40px;font-size:14px}
.you{color:var(--dim);white-space:pre-wrap;word-break:break-word;margin:18px 0 8px;font-size:14px}
.you b{color:var(--on)} #log .card{margin-bottom:6px} button.ghost{background:transparent;color:var(--dim);border:1px solid var(--line)}
.ms{color:var(--dim);font-size:13px;padding:0 15px 12px} .ms b{color:var(--text)}
</style></head><body><div class="wrap">
<h1>coachwhip <b>·</b> MODEL_NAME</h1>
<div class="sub">the experts stream from the SSD; everything runs on this machine</div>
<div id="log"></div>
<textarea id="p" spellcheck="false">Write a JavaScript function debounce(fn, ms) and explain in two sentences how it works.</textarea>
<div class="row">
<button id="go">Send</button>
<button id="new" class="ghost">New chat</button>
<label><input type="checkbox" id="think"> let it think first (slower)</label>
</div>
</div><script>
const $=i=>document.getElementById(i);
const MARK = '\u0001STATS:';
const ERR = '\u0001ERROR:';
const PROG = /\u0001PROG:(\{[^\n]*\})\n/g;
let newChat = true;
const add = (tag, cls, html) => { const e = document.createElement(tag); if (cls) e.className = cls; e.innerHTML = html; $('log').appendChild(e); return e; };
$('new').onclick = () => { newChat = true; $('log').innerHTML = ''; $('p').focus(); };
$('go').onclick = async () => {
const prompt = $('p').value.trim();
if (!prompt) return;
$('go').disabled = true; $('new').disabled = true;
add('div', 'you', '<b>you ›</b> ').appendChild(document.createTextNode(prompt));
const card = add('div', 'card', '<pre></pre><div class="ms"></div>');
const A = card.querySelector('pre'), T = card.querySelector('.ms');
T.textContent = 'reading…'; $('p').value = '';
const t0 = performance.now();
let text = '', first = 0;
try{
const r = await fetch('/ask',{method:'POST',headers:{'content-type':'application/json'},
body:JSON.stringify({prompt, new_chat:newChat, think:!!($('think')&&$('think').checked)})});
newChat = false;
const reader = r.body.getReader(), dec = new TextDecoder();
while (true) {
const {value, done} = await reader.read();
if (done) break;
text += dec.decode(value, {stream:true});
let p = null;
text = text.replace(PROG, (m, j) => { p = JSON.parse(j); return ''; });
const x = text.indexOf(ERR);
if (x >= 0) { A.textContent = text.slice(0, x); T.textContent = text.slice(x + ERR.length); newChat = true; break; }
if (p && !first) {
const secs = ((performance.now()-t0)/1000).toFixed(1) + ' s';
T.textContent = p.summary !== undefined
? 'making room: summarising the chat so far · ' + p.summary + ' tokens · ' + secs
: 'reading your prompt · layer ' + p.layer + ' of ' + p.of + ' · ' + secs;
continue;
}
if (!text) continue;
if (!first) { first = performance.now(); T.textContent = 'writing…'; }
const k = text.indexOf(MARK);
A.textContent = k < 0 ? text : text.slice(0, k);
}
const k = text.indexOf(MARK);
if (k >= 0 && text.indexOf(ERR) < 0) {
const j = JSON.parse(text.slice(k + MARK.length));
T.innerHTML = 'writing <b>' + j.write_tps.toFixed(1) + ' tok/s</b> (' + j.written + ' tokens) · read <b>' +
j.prompt_tokens + ' new tokens</b> at ' + j.read_tps.toFixed(1) + ' tok/s · first word after <b>' +
((first - t0)/1000).toFixed(1) + ' s</b> · chat so far ' + j.context + ' tokens' +
(j.compacted ? ' · the chat was summarised to make room' : '');
}
}catch(e){ A.textContent = 'failed: ' + e; newChat = true; }
$('go').disabled = false; $('new').disabled = false;
card.scrollIntoView({block:'end'});
};
</script></body></html>"##;