The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.
Rust
Qwen3.5/3.6: pair value heads with key heads the way the family does, and open the turn as its template says
1picassoai84a5873f104f
diff --git a/crates/engine/src/model.rs b/crates/engine/src/model.rsindex 1131410..b18cdb6 100644--- a/crates/engine/src/model.rs+++ b/crates/engine/src/model.rs@@ -79,6 +79,33 @@ struct Layer {ffn_norm: RmsNorm,}+/// What the chat template puts right after `<|im_start|>assistant\n`. A thinking model's template+/// opens the turn with an empty think block when thinking is off, so the model answers straight+/// away; Qwen3.5 and later open a think block for it when thinking is on. Both empty for a model+/// with no thinking mode.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct ThinkOpen {+ pub off: &'static str,+ pub on: &'static str,+}++impl ThinkOpen {+ pub fn from_template(template: Option<&str>) -> Self {+ let t = template.unwrap_or("");+ if !t.contains("enable_thinking") {+ return Self { off: "", on: "" };+ }+ Self {+ off: if t.contains(r"'<think>\n\n</think>\n\n'") { "<think>\n\n</think>\n\n" } else { "" },+ on: if t.contains(r"{{- '<think>\n' }}") { "<think>\n" } else { "" },+ }+ }++ pub fn thinks(&self) -> bool {+ !self.off.is_empty()+ }+}+pub struct Qwen3Moe {embeddings: Embedding,layers: Vec<Layer>,@@ -87,8 +114,7 @@ pub struct Qwen3Moe {store: Arc<ExpertStore>,dtype: DType,device: Device,- /// Whether the model has a thinking mode (Qwen3) or answers straight away (Qwen3-Coder).- pub thinks: bool,+ pub think: ThinkOpen,pub progress: Option<Progress>,}@@ -123,8 +149,8 @@ impl Qwen3Moe {let rope_freq_base = md(&format!("{arch}.rope.freq_base")).and_then(|m| m.to_f32()).unwrap_or(10000f32);let shared = md_u("expert_shared_feed_forward_length").unwrap_or(0);let norm_topk_prob = shared == 0;- // A model with a thinking mode says so in its chat template.- let thinks = md("tokenizer.chat_template").and_then(|t| t.to_string().cloned()).map_or(false, |t| t.contains("enable_thinking"));+ let template = md("tokenizer.chat_template").ok().and_then(|t| t.to_string().ok().cloned());+ let think = ThinkOpen::from_template(template.as_deref());let store = ExpertStore::new(&ct, device, block_count, path, settings)?;let mut gg = Gguf::new(ct, &mut file, device.clone());@@ -178,7 +204,7 @@ impl Qwen3Moe {for i in ahead..layers.len() {layers[i - ahead].moe.next_gate = Some(layers[i].moe.gate.clone());}- Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, dtype, device: device.clone(), thinks, progress: None })+ Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, dtype, device: device.clone(), think, progress: None })}/// Print where each written word's time went since the last call, then reset the counters.@@ -266,9 +292,19 @@ impl Model {}pub fn thinks(&self) -> bool {+ self.think().thinks()+ }++ /// The text that opens the assistant's turn, with thinking on or off.+ pub fn assistant_open(&self, think: bool) -> &'static str {+ let t = self.think();+ if think { t.on } else { t.off }+ }++ fn think(&self) -> ThinkOpen {match self {- Self::Qwen3Moe(m) => m.thinks,- Self::Qwen3Next(m) => m.thinks,+ Self::Qwen3Moe(m) => m.think,+ Self::Qwen3Next(m) => m.think,}}diff --git a/crates/engine/src/model_next.rs b/crates/engine/src/model_next.rsindex 6b2262f..5960214 100644--- a/crates/engine/src/model_next.rs+++ b/crates/engine/src/model_next.rs@@ -191,19 +191,33 @@ mod proj_tests {ba_rows.extend(a[kh * per_group..(kh + 1) * per_group].iter().cloned());}let fused = Proj::Fused { qkvz: qm(&Tensor::cat(&fused_rows, 0).unwrap()), ba: qm(&Tensor::cat(&ba_rows, 0).unwrap()) };- // Qwen3.5: all q, all k, all v in one; the gate, betas and alphas on their own.- let qkv: Vec<Tensor> = q.iter().chain(k.iter()).chain(v.iter()).cloned().collect();+ // Qwen3.5: all q, all k, all v in one; the gate, betas and alphas on their own. Its value+ // head j pairs with key head j % n_k, so the same per-head weights sit in a different order.+ let order: Vec<usize> = (0..n_v).map(|j| (j % n_k) * per_group + j / n_k).collect();+ let pick = |t: &[Tensor]| -> Vec<Tensor> { order.iter().map(|&j| t[j].clone()).collect() };+ let qkv: Vec<Tensor> = q.iter().chain(k.iter()).cloned().chain(pick(&v)).collect();let split = Proj::Split {qkv: qm(&Tensor::cat(&qkv, 0).unwrap()),- z: qm(&Tensor::cat(&z, 0).unwrap()),- beta: qm(&Tensor::cat(&b, 0).unwrap()),- alpha: qm(&Tensor::cat(&a, 0).unwrap()),+ z: qm(&Tensor::cat(&pick(&z), 0).unwrap()),+ beta: qm(&Tensor::cat(&pick(&b), 0).unwrap()),+ alpha: qm(&Tensor::cat(&pick(&a), 0).unwrap()),};let mut shared = Lcg(99);let mut lf = layer(fused, &mut shared, n_k, n_v, dk, dv, h);let mut shared = Lcg(99);let mut ls = layer(split, &mut shared, n_k, n_v, dk, dv, h);+ // The per-value-head pieces follow the same order.+ let idx = Tensor::new(order.iter().map(|&j| j as u32).collect::<Vec<_>>(), &Device::Cpu).unwrap();+ ls.dt_bias = lf.dt_bias.index_select(&idx, 0).unwrap();+ ls.a = lf.a.index_select(&idx, 0).unwrap();+ let qk_rows = 2 * n_k * dk;+ let conv_v = lf.conv_w.narrow(0, qk_rows, n_v * dv).unwrap().reshape((n_v, dv, 4)).unwrap().index_select(&idx, 0).unwrap().reshape((n_v * dv, 4)).unwrap();+ ls.conv_w = Tensor::cat(&[&lf.conv_w.narrow(0, 0, qk_rows).unwrap(), &conv_v], 0).unwrap();+ let mut shared = Lcg(99);+ let _ = (shared.tensor(&[qk_rows + n_v * dv, 4], 0.5), shared.tensor(&[n_v], 0.5), shared.tensor(&[n_v], 0.5));+ let out_w = shared.tensor(&[h, n_v * dv], 0.3).reshape((h, n_v, dv)).unwrap().index_select(&idx, 1).unwrap().reshape((h, n_v * dv)).unwrap();+ ls.out = qm(&out_w);let x = Lcg(5).tensor(&[1, l, h], 1.0);let yf = lf.forward(&x).unwrap();let ys = ls.forward(&x).unwrap();@@ -296,8 +310,15 @@ impl LinearAttn {let qc = l2_norm(&conv.narrow(1, 0, n_k * dk)?.reshape((l, n_k, dk))?, self.eps)?;let kc = l2_norm(&conv.narrow(1, n_k * dk, n_k * dk)?.reshape((l, n_k, dk))?, self.eps)?;let vc = conv.narrow(1, 2 * n_k * dk, n_v * dv)?.reshape((l, n_v, dv))?;- // Each key head serves `per_group` value heads, side by side.- let widen = |t: Tensor| -> Result<Tensor> { t.unsqueeze(2)?.broadcast_as((l, n_k, per_group, dk))?.reshape((l, n_v, dk)) };+ // Each key head serves `per_group` value heads. Qwen3-Next lists a key head's value heads+ // side by side (value head j uses key head j / per_group); Qwen3.5 and 3.6 cycle through the+ // key heads (value head j uses key head j % n_k).+ let widen = |t: Tensor| -> Result<Tensor> {+ match &self.proj {+ Proj::Fused { .. } => t.unsqueeze(2)?.broadcast_as((l, n_k, per_group, dk))?.reshape((l, n_v, dk)),+ Proj::Split { .. } => t.unsqueeze(1)?.broadcast_as((l, per_group, n_k, dk))?.reshape((l, n_v, dk)),+ }+ };let qc = (widen(qc)? * (1.0 / (dk as f64).sqrt()))?;let kc = widen(kc)?;@@ -430,7 +451,7 @@ pub struct Qwen3Next {output: QMatMul,store: Arc<ExpertStore>,device: Device,- pub thinks: bool,+ pub think: crate::model::ThinkOpen,pub progress: Option<crate::model::Progress>,}@@ -461,7 +482,8 @@ impl Qwen3Next {let eps = md(&format!("{arch}.attention.layer_norm_rms_epsilon"))?.to_f32()? as f64;// Some conversions leave the rope base out; this family's is ten million.let freq_base = md(&format!("{arch}.rope.freq_base")).ok().and_then(|v| v.to_f32().ok()).map_or(1e7, |f| f as f64);- let thinks = md("tokenizer.chat_template").and_then(|t| t.to_string().cloned()).map_or(false, |t| t.contains("enable_thinking"));+ let template = md("tokenizer.chat_template").ok().and_then(|t| t.to_string().ok().cloned());+ let think = crate::model::ThinkOpen::from_template(template.as_deref());let store = ExpertStore::new(&ct, device, block_count, path, settings)?;let mut gg = Gguf::new(ct, &mut file, device.clone());@@ -546,7 +568,7 @@ impl Qwen3Next {});}wire_lookahead(&mut layers, settings.ahead);- Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, device: device.clone(), thinks, progress: None })+ Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, device: device.clone(), think, progress: None })}pub fn report(&self) {diff --git a/crates/engine/tests/think_open.rs b/crates/engine/tests/think_open.rsnew file mode 100644index 0000000..0a91456--- /dev/null+++ b/crates/engine/tests/think_open.rs@@ -0,0 +1,49 @@+//! How a model's chat template opens the assistant's turn, read from the template text.+//! No GPU, no model file. Run with `cargo test --release`.++use coachwhip_engine::model::ThinkOpen;++const QWEN36: &str = r#"{%- if add_generation_prompt %}+ {{- '<|im_start|>assistant\n' }}+ {%- if enable_thinking is defined and enable_thinking is false %}+ {{- '<think>\n\n</think>\n\n' }}+ {%- else %}+ {{- '<think>\n' }}+ {%- endif %}+{%- endif %}"#;++const QWEN3: &str = r#"{%- if add_generation_prompt %}+ {{- '<|im_start|>assistant\n' }}+ {%- if enable_thinking is defined and enable_thinking is false %}+ {{- '<think>\n\n</think>\n\n' }}+ {%- endif %}+{%- endif %}"#;++const CODER: &str = r#"{%- if add_generation_prompt %}+ {{- '<|im_start|>assistant\n' }}+{%- endif %}"#;++#[test]+fn qwen36_opens_and_closes_thinking_from_the_prompt() {+ let t = ThinkOpen::from_template(Some(QWEN36));+ assert!(t.thinks());+ assert_eq!(t.off, "<think>\n\n</think>\n\n");+ assert_eq!(t.on, "<think>\n");+}++#[test]+fn qwen3_only_closes_thinking_from_the_prompt() {+ let t = ThinkOpen::from_template(Some(QWEN3));+ assert!(t.thinks());+ assert_eq!(t.off, "<think>\n\n</think>\n\n");+ assert_eq!(t.on, "");+}++#[test]+fn a_model_without_thinking_adds_nothing() {+ for template in [Some(CODER), None] {+ let t = ThinkOpen::from_template(template);+ assert!(!t.thinks());+ assert_eq!((t.off, t.on), ("", ""));+ }+}diff --git a/crates/server/src/chat.rs b/crates/server/src/chat.rsindex 1522e4c..50cc602 100644--- a/crates/server/src/chat.rs+++ b/crates/server/src/chat.rs@@ -108,13 +108,14 @@ pub fn generate(emit: &mut dyn FnMut(&str) -> Result<()>,) -> Result<Answer> {let think = think && model.thinks();- let no_think = if think || !model.thinks() { "" } else { " /no_think" };+ // The template's way of switching thinking on or off: what opens the assistant's turn.+ let open = model.assistant_open(think);let encode = |text: String| -> Result<Vec<u32>> { Ok(tokenizer.encode(text, true).map_err(anyhow::Error::msg)?.get_ids().to_vec()) };let eos = *tokenizer.get_vocab(true).get("<|im_end|>").unwrap();let tell = progress.map(|p| Rc::new(RefCell::new(p)));// A follow-up closes the last answer and adds the new question; nothing earlier is read again.- let turn = format!("<|im_start|>user\n{prompt}{no_think}<|im_end|>\n<|im_start|>assistant\n");+ let turn = format!("<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n{open}");let mut tokens = Vec::new();let mut fresh = new_chat || session.pos == 0 || session.by_api;let mut summary = None;@@ -123,7 +124,7 @@ pub fn generate(tokens.extend(encode(format!("<|im_end|>\n{turn}"))?);// Too long to go on: the model summarises the chat, and the summary opens a new one.if session.pos + tokens.len() + max_new > CONTEXT_LIMIT {- summary = Some(summarise(model, tokenizer, device, session, no_think, eos, &tell)?);+ summary = Some(summarise(model, tokenizer, device, session, eos, &tell)?);fresh = true;}}@@ -131,7 +132,7 @@ pub fn generate(model.clear_kv_cache();*session = Session::default();tokens = encode(match &summary {- Some(s) => format!("<|im_start|>user\nSummary of our conversation so far, for context:\n{s}\n\nNow: {prompt}{no_think}<|im_end|>\n<|im_start|>assistant\n"),+ Some(s) => format!("<|im_start|>user\nSummary of our conversation so far, for context:\n{s}\n\nNow: {prompt}<|im_end|>\n<|im_start|>assistant\n{open}"),None => turn,})?;}@@ -287,9 +288,9 @@ fn read_prompt(model: &mut Model, device: &Device, pos: usize, tokens: &[u32], t/// Asks the model, in the chat as it stands, for a summary of the chat; the chat is then thrown/// away and the summary opens the next one. Greedy, so it is the same every time.-fn summarise(model: &mut Model, tokenizer: &Tokenizer, device: &Device, session: &Session, no_think: &str, eos: u32, tell: &Option<Rc<RefCell<Progress>>>) -> Result<String> {+fn summarise(model: &mut Model, tokenizer: &Tokenizer, device: &Device, session: &Session, eos: u32, tell: &Option<Rc<RefCell<Progress>>>) -> Result<String> {let mut tokens: Vec<u32> = session.pending.into_iter().collect();- let ask = format!("<|im_end|>\n<|im_start|>user\n{SUMMARY_ASK}{no_think}<|im_end|>\n<|im_start|>assistant\n");+ let ask = format!("<|im_end|>\n<|im_start|>user\n{SUMMARY_ASK}<|im_end|>\n<|im_start|>assistant\n{}", model.assistant_open(false));tokens.extend(tokenizer.encode(ask, true).map_err(anyhow::Error::msg)?.get_ids().to_vec());let mut sampler = LogitsProcessor::from_sampling(42, Sampling::ArgMax);let logits = read_prompt(model, device, session.pos, &tokens, tell)?;@@ -325,8 +326,9 @@ pub fn message_text(content: &serde_json::Value) -> String {}}-/// An OpenAI conversation in the model's chat format, ready for the assistant's turn.-pub fn chat_prompt(messages: &[serde_json::Value]) -> String {+/// An OpenAI conversation in the model's chat format, ready for the assistant's turn; `open` is+/// what the model's template puts at the start of that turn (see `Model::assistant_open`).+pub fn chat_prompt(messages: &[serde_json::Value], open: &str) -> String {let mut text = String::new();for m in messages {let role = match m["role"].as_str().unwrap_or("user") {@@ -337,6 +339,7 @@ pub fn chat_prompt(messages: &[serde_json::Value]) -> String {text.push_str(&format!("<|im_start|>{role}\n{}<|im_end|>\n", message_text(&m["content"])));}text.push_str("<|im_start|>assistant\n");+ text.push_str(open);text}@@ -358,7 +361,7 @@ fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, to};let stream = q["stream"].as_bool().unwrap_or(false);let asked = q["max_completion_tokens"].as_u64().or(q["max_tokens"].as_u64());- let prompt = tokenizer.encode(chat_prompt(messages), true).map_err(anyhow::Error::msg)?.get_ids().to_vec();+ let prompt = tokenizer.encode(chat_prompt(messages, model.assistant_open(false)), true).map_err(anyhow::Error::msg)?.get_ids().to_vec();let max_new = answer_budget(asked, max_new, prompt.len());let id = format!("chatcmpl-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()));let created = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs());diff --git a/crates/server/tests/api.rs b/crates/server/tests/api.rsindex f3152f3..74bc767 100644--- a/crates/server/tests/api.rs+++ b/crates/server/tests/api.rs@@ -50,13 +50,19 @@ fn the_context_limit_is_16k() {#[test]fn messages_become_the_models_chat_format() {- let p = chat_prompt(&[json!({"role": "system", "content": "Be brief."}), json!({"role": "user", "content": "Hi"})]);+ let p = chat_prompt(&[json!({"role": "system", "content": "Be brief."}), json!({"role": "user", "content": "Hi"})], "");assert_eq!(p, "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n");}+#[test]+fn a_thinking_models_turn_opens_the_way_its_template_says() {+ let p = chat_prompt(&[json!({"role": "user", "content": "Hi"})], "<think>\n\n</think>\n\n");+ assert!(p.ends_with("<|im_start|>assistant\n<think>\n\n</think>\n\n"));+}+#[test]fn unknown_roles_are_mapped_safely() {- let p = chat_prompt(&[json!({"role": "developer", "content": "a"}), json!({"role": "tool", "content": "b"}), json!({"content": "c"})]);+ let p = chat_prompt(&[json!({"role": "developer", "content": "a"}), json!({"role": "tool", "content": "b"}), json!({"content": "c"})], "");assert!(p.starts_with("<|im_start|>system\na<|im_end|>\n<|im_start|>user\nb<|im_end|>\n<|im_start|>user\nc<|im_end|>\n"));assert!(p.ends_with("<|im_start|>assistant\n"));}