Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(llama): don't return reasoning_content as the answer when thinking is on (#2200)
voidef89b99ee735
diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.jsindex 786b9a1b3..61f4e5779 100644--- a/src/helpers/llamaServer.js+++ b/src/helpers/llamaServer.js@@ -720,9 +720,10 @@ class LlamaServerManager {stream: false,};- // Without this, Qwen chat templates leave `message.content` empty and- // route output into `reasoning_content`. Non-Qwen templates ignore it.- if (options.disableThinking !== false) {+ // Without this, Qwen chat templates think into `reasoning_content` first+ // and can spend the whole budget there. Non-Qwen templates ignore it.+ const suppressThinking = options.disableThinking !== false;+ if (suppressThinking) {requestBody.chat_template_kwargs = { enable_thinking: false };}@@ -761,18 +762,24 @@ class LlamaServerManager {try {const response = JSON.parse(data);- if (- options.requireCompleteOutput &&- ["length", "max_tokens"].includes(response.choices?.[0]?.finish_reason)- ) {+ const choice = response.choices?.[0];+ const message = choice?.message;+ const truncated = ["length", "max_tokens"].includes(choice?.finish_reason);+ // A cut-off reply with no content spent its whole budget reasoning+ // (#2187): there is no answer to return, whichever field the+ // reasoning sits in. Lenient callers still accept a partial answer.+ if (truncated && (options.requireCompleteOutput || !message?.content?.trim())) {// The renderer maps the code to the cleanup toast's wording (#2091).const error = new Error("Model output was truncated");error.code = "OUTPUT_TRUNCATED";reject(error);return;}- const message = response.choices?.[0]?.message;- const text = message?.content || message?.reasoning_content || "";+ // Some builds still route a suppressed-thinking answer into+ // `reasoning_content` (#809). With thinking on, that field is the+ // reasoning itself, which must never stand in for the answer.+ const text =+ message?.content || (suppressThinking && message?.reasoning_content) || "";resolve(text.trim());} catch (e) {reject(new Error(`Failed to parse llama-server response: ${e.message}`));diff --git a/test/helpers/llamaServerContext.test.js b/test/helpers/llamaServerContext.test.jsindex d17f974b8..f0c7b7b0c 100644--- a/test/helpers/llamaServerContext.test.js+++ b/test/helpers/llamaServerContext.test.js@@ -280,6 +280,83 @@ test("countPromptTokens returns null when the server cannot measure", async () =);});+// --- reading the reply ---------------------------------------------------+//+// With thinking on, `reasoning_content` is the reasoning, not the answer (#2187).++const MESSAGES = [{ role: "user", content: "hi" }];++// Stubs one chat completion and records the request bodies it receives.+function withReply(message, finishReason = "stop") {+ const requests = [];+ const run = withStubServer({+ "/v1/chat/completions": (body) => {+ requests.push(JSON.parse(body));+ return { payload: { choices: [{ message, finish_reason: finishReason }] } };+ },+ });+ return { run, requests };+}++const TRUNCATED = { code: "OUTPUT_TRUNCATED" };++test("reasoning never stands in for the answer when thinking was requested", async () => {+ const reasoningOnly = { content: "", reasoning_content: "Thinking Process:\n\n1. Analyze" };++ // Cut off mid-reasoning: there is no answer to return, so every caller gets+ // the truncation error rather than an empty string it would have to explain.+ const truncated = withReply(reasoningOnly, "length");+ await truncated.run(async (manager) => {+ await assert.rejects(manager.inference(MESSAGES, { disableThinking: false }), TRUNCATED);+ const strict = { disableThinking: false, requireCompleteOutput: true };+ await assert.rejects(manager.inference(MESSAGES, strict), TRUNCATED);+ });+ assert.equal(truncated.requests[0].chat_template_kwargs, undefined);++ // Finished with no answer: reaches strict callers such as dictation cleanup too.+ const finished = withReply(reasoningOnly, "stop");+ await finished.run(async (manager) => {+ const options = { disableThinking: false, requireCompleteOutput: true };+ assert.equal(await manager.inference(MESSAGES, options), "");+ });+});++test("a cut-off reply with no content is truncated even when thinking was suppressed", async () => {+ // gpt-oss ignores enable_thinking and reasons into reasoning_content anyway;+ // a budget spent entirely there is a cut-off reply, not a misrouted answer.+ const analysisOnly = withReply({ content: "", reasoning_content: "analysis..." }, "length");+ await analysisOnly.run(async (manager) => {+ await assert.rejects(manager.inference(MESSAGES, { disableThinking: true }), TRUNCATED);+ });+});++test("a cut-off reply that has content is still returned to lenient callers", async () => {+ // Only requireCompleteOutput callers reject a partial answer (#2130).+ const partial = withReply({ content: "Cleaned te", reasoning_content: "" }, "length");+ await partial.run(async (manager) => {+ assert.equal(await manager.inference(MESSAGES, { disableThinking: true }), "Cleaned te");+ await assert.rejects(manager.inference(MESSAGES, { requireCompleteOutput: true }), TRUNCATED);+ });+});++test("an answer routed into reasoning_content with thinking suppressed is still returned", async () => {+ // Some llama-server builds do this despite enable_thinking: false (#809).+ const misrouted = withReply({ content: "", reasoning_content: "Cleaned text." });+ await misrouted.run(async (manager) => {+ assert.equal(await manager.inference(MESSAGES, { disableThinking: true }), "Cleaned text.");+ assert.equal(await manager.inference(MESSAGES, {}), "Cleaned text.");+ });+ assert.deepEqual(misrouted.requests[0].chat_template_kwargs, { enable_thinking: false });+});++test("content is returned over reasoning_content whether or not thinking was on", async () => {+ const both = withReply({ content: "Cleaned text.", reasoning_content: "Thinking Process:" });+ await both.run(async (manager) => {+ assert.equal(await manager.inference(MESSAGES, { disableThinking: false }), "Cleaned text.");+ assert.equal(await manager.inference(MESSAGES, {}), "Cleaned text.");+ });+});+// --- Vulkan context step-down --------------------------------------------//// Sizing the context to the request is bounded by SYSTEM RAM, but a discrete