Skip to content

The 80B Qwen3-Coder-Next on a 16 GB Mac, about 13 tok/s, fully offline. A mixture-of-experts inference engine for Apple silicon.

Rust

Qwen3.5/3.6: pair value heads with key heads the way the family does, and open the turn as its template says

1picassoai84a5873f104f
diff --git a/crates/engine/src/model.rs b/crates/engine/src/model.rs
index 1131410..b18cdb6 100644
--- a/crates/engine/src/model.rs
+++ b/crates/engine/src/model.rs
@@ -79,6 +79,33 @@ struct Layer {
ffn_norm: RmsNorm,
}
+/// What the chat template puts right after `<|im_start|>assistant\n`. A thinking model's template
+/// opens the turn with an empty think block when thinking is off, so the model answers straight
+/// away; Qwen3.5 and later open a think block for it when thinking is on. Both empty for a model
+/// with no thinking mode.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct ThinkOpen {
+ pub off: &'static str,
+ pub on: &'static str,
+}
+
+impl ThinkOpen {
+ pub fn from_template(template: Option<&str>) -> Self {
+ let t = template.unwrap_or("");
+ if !t.contains("enable_thinking") {
+ return Self { off: "", on: "" };
+ }
+ Self {
+ off: if t.contains(r"'<think>\n\n</think>\n\n'") { "<think>\n\n</think>\n\n" } else { "" },
+ on: if t.contains(r"{{- '<think>\n' }}") { "<think>\n" } else { "" },
+ }
+ }
+
+ pub fn thinks(&self) -> bool {
+ !self.off.is_empty()
+ }
+}
+
pub struct Qwen3Moe {
embeddings: Embedding,
layers: Vec<Layer>,
@@ -87,8 +114,7 @@ pub struct Qwen3Moe {
store: Arc<ExpertStore>,
dtype: DType,
device: Device,
- /// Whether the model has a thinking mode (Qwen3) or answers straight away (Qwen3-Coder).
- pub thinks: bool,
+ pub think: ThinkOpen,
pub progress: Option<Progress>,
}
@@ -123,8 +149,8 @@ impl Qwen3Moe {
let rope_freq_base = md(&format!("{arch}.rope.freq_base")).and_then(|m| m.to_f32()).unwrap_or(10000f32);
let shared = md_u("expert_shared_feed_forward_length").unwrap_or(0);
let norm_topk_prob = shared == 0;
- // A model with a thinking mode says so in its chat template.
- let thinks = md("tokenizer.chat_template").and_then(|t| t.to_string().cloned()).map_or(false, |t| t.contains("enable_thinking"));
+ let template = md("tokenizer.chat_template").ok().and_then(|t| t.to_string().ok().cloned());
+ let think = ThinkOpen::from_template(template.as_deref());
let store = ExpertStore::new(&ct, device, block_count, path, settings)?;
let mut gg = Gguf::new(ct, &mut file, device.clone());
@@ -178,7 +204,7 @@ impl Qwen3Moe {
for i in ahead..layers.len() {
layers[i - ahead].moe.next_gate = Some(layers[i].moe.gate.clone());
}
- Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, dtype, device: device.clone(), thinks, progress: None })
+ Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, dtype, device: device.clone(), think, progress: None })
}
/// Print where each written word's time went since the last call, then reset the counters.
@@ -266,9 +292,19 @@ impl Model {
}
pub fn thinks(&self) -> bool {
+ self.think().thinks()
+ }
+
+ /// The text that opens the assistant's turn, with thinking on or off.
+ pub fn assistant_open(&self, think: bool) -> &'static str {
+ let t = self.think();
+ if think { t.on } else { t.off }
+ }
+
+ fn think(&self) -> ThinkOpen {
match self {
- Self::Qwen3Moe(m) => m.thinks,
- Self::Qwen3Next(m) => m.thinks,
+ Self::Qwen3Moe(m) => m.think,
+ Self::Qwen3Next(m) => m.think,
}
}
diff --git a/crates/engine/src/model_next.rs b/crates/engine/src/model_next.rs
index 6b2262f..5960214 100644
--- a/crates/engine/src/model_next.rs
+++ b/crates/engine/src/model_next.rs
@@ -191,19 +191,33 @@ mod proj_tests {
ba_rows.extend(a[kh * per_group..(kh + 1) * per_group].iter().cloned());
}
let fused = Proj::Fused { qkvz: qm(&Tensor::cat(&fused_rows, 0).unwrap()), ba: qm(&Tensor::cat(&ba_rows, 0).unwrap()) };
- // Qwen3.5: all q, all k, all v in one; the gate, betas and alphas on their own.
- let qkv: Vec<Tensor> = q.iter().chain(k.iter()).chain(v.iter()).cloned().collect();
+ // Qwen3.5: all q, all k, all v in one; the gate, betas and alphas on their own. Its value
+ // head j pairs with key head j % n_k, so the same per-head weights sit in a different order.
+ let order: Vec<usize> = (0..n_v).map(|j| (j % n_k) * per_group + j / n_k).collect();
+ let pick = |t: &[Tensor]| -> Vec<Tensor> { order.iter().map(|&j| t[j].clone()).collect() };
+ let qkv: Vec<Tensor> = q.iter().chain(k.iter()).cloned().chain(pick(&v)).collect();
let split = Proj::Split {
qkv: qm(&Tensor::cat(&qkv, 0).unwrap()),
- z: qm(&Tensor::cat(&z, 0).unwrap()),
- beta: qm(&Tensor::cat(&b, 0).unwrap()),
- alpha: qm(&Tensor::cat(&a, 0).unwrap()),
+ z: qm(&Tensor::cat(&pick(&z), 0).unwrap()),
+ beta: qm(&Tensor::cat(&pick(&b), 0).unwrap()),
+ alpha: qm(&Tensor::cat(&pick(&a), 0).unwrap()),
};
let mut shared = Lcg(99);
let mut lf = layer(fused, &mut shared, n_k, n_v, dk, dv, h);
let mut shared = Lcg(99);
let mut ls = layer(split, &mut shared, n_k, n_v, dk, dv, h);
+ // The per-value-head pieces follow the same order.
+ let idx = Tensor::new(order.iter().map(|&j| j as u32).collect::<Vec<_>>(), &Device::Cpu).unwrap();
+ ls.dt_bias = lf.dt_bias.index_select(&idx, 0).unwrap();
+ ls.a = lf.a.index_select(&idx, 0).unwrap();
+ let qk_rows = 2 * n_k * dk;
+ let conv_v = lf.conv_w.narrow(0, qk_rows, n_v * dv).unwrap().reshape((n_v, dv, 4)).unwrap().index_select(&idx, 0).unwrap().reshape((n_v * dv, 4)).unwrap();
+ ls.conv_w = Tensor::cat(&[&lf.conv_w.narrow(0, 0, qk_rows).unwrap(), &conv_v], 0).unwrap();
+ let mut shared = Lcg(99);
+ let _ = (shared.tensor(&[qk_rows + n_v * dv, 4], 0.5), shared.tensor(&[n_v], 0.5), shared.tensor(&[n_v], 0.5));
+ let out_w = shared.tensor(&[h, n_v * dv], 0.3).reshape((h, n_v, dv)).unwrap().index_select(&idx, 1).unwrap().reshape((h, n_v * dv)).unwrap();
+ ls.out = qm(&out_w);
let x = Lcg(5).tensor(&[1, l, h], 1.0);
let yf = lf.forward(&x).unwrap();
let ys = ls.forward(&x).unwrap();
@@ -296,8 +310,15 @@ impl LinearAttn {
let qc = l2_norm(&conv.narrow(1, 0, n_k * dk)?.reshape((l, n_k, dk))?, self.eps)?;
let kc = l2_norm(&conv.narrow(1, n_k * dk, n_k * dk)?.reshape((l, n_k, dk))?, self.eps)?;
let vc = conv.narrow(1, 2 * n_k * dk, n_v * dv)?.reshape((l, n_v, dv))?;
- // Each key head serves `per_group` value heads, side by side.
- let widen = |t: Tensor| -> Result<Tensor> { t.unsqueeze(2)?.broadcast_as((l, n_k, per_group, dk))?.reshape((l, n_v, dk)) };
+ // Each key head serves `per_group` value heads. Qwen3-Next lists a key head's value heads
+ // side by side (value head j uses key head j / per_group); Qwen3.5 and 3.6 cycle through the
+ // key heads (value head j uses key head j % n_k).
+ let widen = |t: Tensor| -> Result<Tensor> {
+ match &self.proj {
+ Proj::Fused { .. } => t.unsqueeze(2)?.broadcast_as((l, n_k, per_group, dk))?.reshape((l, n_v, dk)),
+ Proj::Split { .. } => t.unsqueeze(1)?.broadcast_as((l, per_group, n_k, dk))?.reshape((l, n_v, dk)),
+ }
+ };
let qc = (widen(qc)? * (1.0 / (dk as f64).sqrt()))?;
let kc = widen(kc)?;
@@ -430,7 +451,7 @@ pub struct Qwen3Next {
output: QMatMul,
store: Arc<ExpertStore>,
device: Device,
- pub thinks: bool,
+ pub think: crate::model::ThinkOpen,
pub progress: Option<crate::model::Progress>,
}
@@ -461,7 +482,8 @@ impl Qwen3Next {
let eps = md(&format!("{arch}.attention.layer_norm_rms_epsilon"))?.to_f32()? as f64;
// Some conversions leave the rope base out; this family's is ten million.
let freq_base = md(&format!("{arch}.rope.freq_base")).ok().and_then(|v| v.to_f32().ok()).map_or(1e7, |f| f as f64);
- let thinks = md("tokenizer.chat_template").and_then(|t| t.to_string().cloned()).map_or(false, |t| t.contains("enable_thinking"));
+ let template = md("tokenizer.chat_template").ok().and_then(|t| t.to_string().ok().cloned());
+ let think = crate::model::ThinkOpen::from_template(template.as_deref());
let store = ExpertStore::new(&ct, device, block_count, path, settings)?;
let mut gg = Gguf::new(ct, &mut file, device.clone());
@@ -546,7 +568,7 @@ impl Qwen3Next {
});
}
wire_lookahead(&mut layers, settings.ahead);
- Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, device: device.clone(), thinks, progress: None })
+ Ok(Self { embeddings: Embedding::new(embeddings, embedding_length), layers, norm, output, store, device: device.clone(), think, progress: None })
}
pub fn report(&self) {
diff --git a/crates/engine/tests/think_open.rs b/crates/engine/tests/think_open.rs
new file mode 100644
index 0000000..0a91456
--- /dev/null
+++ b/crates/engine/tests/think_open.rs
@@ -0,0 +1,49 @@
+//! How a model's chat template opens the assistant's turn, read from the template text.
+//! No GPU, no model file. Run with `cargo test --release`.
+
+use coachwhip_engine::model::ThinkOpen;
+
+const QWEN36: &str = r#"{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '<think>\n\n</think>\n\n' }}
+ {%- else %}
+ {{- '<think>\n' }}
+ {%- endif %}
+{%- endif %}"#;
+
+const QWEN3: &str = r#"{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is false %}
+ {{- '<think>\n\n</think>\n\n' }}
+ {%- endif %}
+{%- endif %}"#;
+
+const CODER: &str = r#"{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+{%- endif %}"#;
+
+#[test]
+fn qwen36_opens_and_closes_thinking_from_the_prompt() {
+ let t = ThinkOpen::from_template(Some(QWEN36));
+ assert!(t.thinks());
+ assert_eq!(t.off, "<think>\n\n</think>\n\n");
+ assert_eq!(t.on, "<think>\n");
+}
+
+#[test]
+fn qwen3_only_closes_thinking_from_the_prompt() {
+ let t = ThinkOpen::from_template(Some(QWEN3));
+ assert!(t.thinks());
+ assert_eq!(t.off, "<think>\n\n</think>\n\n");
+ assert_eq!(t.on, "");
+}
+
+#[test]
+fn a_model_without_thinking_adds_nothing() {
+ for template in [Some(CODER), None] {
+ let t = ThinkOpen::from_template(template);
+ assert!(!t.thinks());
+ assert_eq!((t.off, t.on), ("", ""));
+ }
+}
diff --git a/crates/server/src/chat.rs b/crates/server/src/chat.rs
index 1522e4c..50cc602 100644
--- a/crates/server/src/chat.rs
+++ b/crates/server/src/chat.rs
@@ -108,13 +108,14 @@ pub fn generate(
emit: &mut dyn FnMut(&str) -> Result<()>,
) -> Result<Answer> {
let think = think && model.thinks();
- let no_think = if think || !model.thinks() { "" } else { " /no_think" };
+ // The template's way of switching thinking on or off: what opens the assistant's turn.
+ let open = model.assistant_open(think);
let encode = |text: String| -> Result<Vec<u32>> { Ok(tokenizer.encode(text, true).map_err(anyhow::Error::msg)?.get_ids().to_vec()) };
let eos = *tokenizer.get_vocab(true).get("<|im_end|>").unwrap();
let tell = progress.map(|p| Rc::new(RefCell::new(p)));
// A follow-up closes the last answer and adds the new question; nothing earlier is read again.
- let turn = format!("<|im_start|>user\n{prompt}{no_think}<|im_end|>\n<|im_start|>assistant\n");
+ let turn = format!("<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n{open}");
let mut tokens = Vec::new();
let mut fresh = new_chat || session.pos == 0 || session.by_api;
let mut summary = None;
@@ -123,7 +124,7 @@ pub fn generate(
tokens.extend(encode(format!("<|im_end|>\n{turn}"))?);
// Too long to go on: the model summarises the chat, and the summary opens a new one.
if session.pos + tokens.len() + max_new > CONTEXT_LIMIT {
- summary = Some(summarise(model, tokenizer, device, session, no_think, eos, &tell)?);
+ summary = Some(summarise(model, tokenizer, device, session, eos, &tell)?);
fresh = true;
}
}
@@ -131,7 +132,7 @@ pub fn generate(
model.clear_kv_cache();
*session = Session::default();
tokens = encode(match &summary {
- Some(s) => format!("<|im_start|>user\nSummary of our conversation so far, for context:\n{s}\n\nNow: {prompt}{no_think}<|im_end|>\n<|im_start|>assistant\n"),
+ Some(s) => format!("<|im_start|>user\nSummary of our conversation so far, for context:\n{s}\n\nNow: {prompt}<|im_end|>\n<|im_start|>assistant\n{open}"),
None => turn,
})?;
}
@@ -287,9 +288,9 @@ fn read_prompt(model: &mut Model, device: &Device, pos: usize, tokens: &[u32], t
/// Asks the model, in the chat as it stands, for a summary of the chat; the chat is then thrown
/// away and the summary opens the next one. Greedy, so it is the same every time.
-fn summarise(model: &mut Model, tokenizer: &Tokenizer, device: &Device, session: &Session, no_think: &str, eos: u32, tell: &Option<Rc<RefCell<Progress>>>) -> Result<String> {
+fn summarise(model: &mut Model, tokenizer: &Tokenizer, device: &Device, session: &Session, eos: u32, tell: &Option<Rc<RefCell<Progress>>>) -> Result<String> {
let mut tokens: Vec<u32> = session.pending.into_iter().collect();
- let ask = format!("<|im_end|>\n<|im_start|>user\n{SUMMARY_ASK}{no_think}<|im_end|>\n<|im_start|>assistant\n");
+ let ask = format!("<|im_end|>\n<|im_start|>user\n{SUMMARY_ASK}<|im_end|>\n<|im_start|>assistant\n{}", model.assistant_open(false));
tokens.extend(tokenizer.encode(ask, true).map_err(anyhow::Error::msg)?.get_ids().to_vec());
let mut sampler = LogitsProcessor::from_sampling(42, Sampling::ArgMax);
let logits = read_prompt(model, device, session.pos, &tokens, tell)?;
@@ -325,8 +326,9 @@ pub fn message_text(content: &serde_json::Value) -> String {
}
}
-/// An OpenAI conversation in the model's chat format, ready for the assistant's turn.
-pub fn chat_prompt(messages: &[serde_json::Value]) -> String {
+/// An OpenAI conversation in the model's chat format, ready for the assistant's turn; `open` is
+/// what the model's template puts at the start of that turn (see `Model::assistant_open`).
+pub fn chat_prompt(messages: &[serde_json::Value], open: &str) -> String {
let mut text = String::new();
for m in messages {
let role = match m["role"].as_str().unwrap_or("user") {
@@ -337,6 +339,7 @@ pub fn chat_prompt(messages: &[serde_json::Value]) -> String {
text.push_str(&format!("<|im_start|>{role}\n{}<|im_end|>\n", message_text(&m["content"])));
}
text.push_str("<|im_start|>assistant\n");
+ text.push_str(open);
text
}
@@ -358,7 +361,7 @@ fn openai(mut s: TcpStream, body: &[u8], model: &mut Model, model_name: &str, to
};
let stream = q["stream"].as_bool().unwrap_or(false);
let asked = q["max_completion_tokens"].as_u64().or(q["max_tokens"].as_u64());
- let prompt = tokenizer.encode(chat_prompt(messages), true).map_err(anyhow::Error::msg)?.get_ids().to_vec();
+ let prompt = tokenizer.encode(chat_prompt(messages, model.assistant_open(false)), true).map_err(anyhow::Error::msg)?.get_ids().to_vec();
let max_new = answer_budget(asked, max_new, prompt.len());
let id = format!("chatcmpl-{}", std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()));
let created = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs());
diff --git a/crates/server/tests/api.rs b/crates/server/tests/api.rs
index f3152f3..74bc767 100644
--- a/crates/server/tests/api.rs
+++ b/crates/server/tests/api.rs
@@ -50,13 +50,19 @@ fn the_context_limit_is_16k() {
#[test]
fn messages_become_the_models_chat_format() {
- let p = chat_prompt(&[json!({"role": "system", "content": "Be brief."}), json!({"role": "user", "content": "Hi"})]);
+ let p = chat_prompt(&[json!({"role": "system", "content": "Be brief."}), json!({"role": "user", "content": "Hi"})], "");
assert_eq!(p, "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n");
}
+#[test]
+fn a_thinking_models_turn_opens_the_way_its_template_says() {
+ let p = chat_prompt(&[json!({"role": "user", "content": "Hi"})], "<think>\n\n</think>\n\n");
+ assert!(p.ends_with("<|im_start|>assistant\n<think>\n\n</think>\n\n"));
+}
+
#[test]
fn unknown_roles_are_mapped_safely() {
- let p = chat_prompt(&[json!({"role": "developer", "content": "a"}), json!({"role": "tool", "content": "b"}), json!({"content": "c"})]);
+ let p = chat_prompt(&[json!({"role": "developer", "content": "a"}), json!({"role": "tool", "content": "b"}), json!({"content": "c"})], "");
assert!(p.starts_with("<|im_start|>system\na<|im_end|>\n<|im_start|>user\nb<|im_end|>\n<|im_start|>user\nc<|im_end|>\n"));
assert!(p.ends_with("<|im_start|>assistant\n"));
}