Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(reasoning): track think-tag depth in streamed chat deltas (#1644)

Chad Piha5abb00444567
diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.ts
index 9b943f370..0f0429500 100644
--- a/src/services/ReasoningService.ts
+++ b/src/services/ReasoningService.ts
@@ -32,6 +32,7 @@ import {
} from "./ai/chatRequestBody";
import { getModelFamilyConstraints } from "./ai/modelFamilyConstraints";
import { detectEndpointDialect } from "./ai/thinkingSuppressionDialects";
+import { createStreamingThinkFilter } from "./ai/streamingThinkFilter";
import { extractApiErrorMessage } from "./ai/apiErrorMessage";
import { clearTinfoilClientCache } from "./ai/tinfoilClient";
import { resolveChatRoute } from "../helpers/chatRouting";
@@ -628,7 +629,8 @@ class ReasoningService extends BaseReasoningService {
const decoder = new TextDecoder();
let buffer = "";
- let insideThinkBlock = false;
+ const stripThinking = (isLocalProvider || isLanChat) && config.disableThinking !== false;
+ const filterThinkTags = stripThinking ? createStreamingThinkFilter() : null;
try {
while (true) {
@@ -644,37 +646,19 @@ class ReasoningService extends BaseReasoningService {
if (!trimmed || !trimmed.startsWith("data: ")) continue;
const data = trimmed.slice(6);
- if (data === "[DONE]") return;
+ if (data === "[DONE]") {
+ const trailing = filterThinkTags?.finish();
+ if (trailing) yield trailing;
+ return;
+ }
try {
const parsed = JSON.parse(data);
let content = parsed.choices?.[0]?.delta?.content;
if (!content) continue;
- const stripThinking =
- (isLocalProvider || isLanChat) && config.disableThinking !== false;
- if (stripThinking) {
- if (insideThinkBlock) {
- const endIdx = content.indexOf("</think>");
- if (endIdx !== -1) {
- insideThinkBlock = false;
- content = content.slice(endIdx + 8);
- } else {
- continue;
- }
- }
- const startIdx = content.indexOf("<think>");
- if (startIdx !== -1) {
- const before = content.slice(0, startIdx);
- const after = content.slice(startIdx + 7);
- const endIdx = after.indexOf("</think>");
- if (endIdx !== -1) {
- content = before + after.slice(endIdx + 8);
- } else {
- insideThinkBlock = true;
- content = before;
- }
- }
+ if (filterThinkTags) {
+ content = filterThinkTags(content);
if (!content) continue;
}
@@ -684,6 +668,9 @@ class ReasoningService extends BaseReasoningService {
}
}
}
+
+ const trailing = filterThinkTags?.finish();
+ if (trailing) yield trailing;
} finally {
clearTimeout(timeoutId);
this.streamAbortController = null;
@@ -732,6 +719,10 @@ class ReasoningService extends BaseReasoningService {
return;
}
+ const filterThinkTags =
+ (isLocalProvider || isLanChat) && config.disableThinking !== false
+ ? createStreamingThinkFilter()
+ : null;
let apiKey = "";
let baseURL: string | undefined;
@@ -812,10 +803,20 @@ class ReasoningService extends BaseReasoningService {
...(hasProviderOptions ? { providerOptions } : {}),
});
+ let canFlushFilteredText = true;
+ const finishFilteredText = (): string => {
+ const trailing = filterThinkTags?.finish() ?? "";
+ return canFlushFilteredText ? trailing : "";
+ };
+
try {
for await (const chunk of result.fullStream) {
if (chunk.type === "text-delta") {
- yield { type: "content", text: chunk.text };
+ const text = filterThinkTags ? filterThinkTags(chunk.text) : chunk.text;
+ if (text) yield { type: "content", text };
+ } else if (chunk.type === "text-end" || chunk.type === "finish-step") {
+ const trailing = finishFilteredText();
+ if (trailing) yield { type: "content", text: trailing };
} else if (chunk.type === "tool-call") {
yield {
type: "tool_calls",
@@ -837,11 +838,18 @@ class ReasoningService extends BaseReasoningService {
toolName: chunk.toolName,
displayText,
};
+ } else if (chunk.type === "abort" || chunk.type === "error") {
+ canFlushFilteredText = false;
+ finishFilteredText();
} else if (chunk.type === "finish") {
+ const trailing = finishFilteredText();
+ if (trailing) yield { type: "content", text: trailing };
yield { type: "done", finishReason: chunk.finishReason };
}
}
} catch (error) {
+ canFlushFilteredText = false;
+ finishFilteredText();
if (abortController.signal.aborted) {
yield { type: "done", finishReason: "stop" };
return;
diff --git a/src/services/ai/streamingThinkFilter.ts b/src/services/ai/streamingThinkFilter.ts
new file mode 100644
index 000000000..33f979adc
--- /dev/null
+++ b/src/services/ai/streamingThinkFilter.ts
@@ -0,0 +1,75 @@
+const OPEN_TAG = "<think>";
+const CLOSE_TAG = "</think>";
+const OPEN_TAGS = [OPEN_TAG] as const;
+const THINK_TAGS = [OPEN_TAG, CLOSE_TAG] as const;
+const MAX_PARTIAL_TAG_LENGTH = CLOSE_TAG.length - 1;
+
+export interface StreamingThinkFilter {
+ (chunk: string): string;
+ finish: () => string;
+}
+
+function getPartialTagSuffix(value: string, tags: readonly string[]): string {
+ const maxLength = Math.min(value.length, MAX_PARTIAL_TAG_LENGTH);
+ for (let length = maxLength; length > 0; length -= 1) {
+ const suffix = value.slice(-length);
+ if (tags.some((tag) => tag.startsWith(suffix))) return suffix;
+ }
+ return "";
+}
+
+// Filters <think> blocks out of streamed chat deltas, keeping state across
+// chunks, including tags split between deltas. Depth is a counter because
+// the first </think> in a nested block only closes the inner block. finish()
+// preserves a trailing tag-like fragment only when it was outside reasoning.
+export function createStreamingThinkFilter(): StreamingThinkFilter {
+ let depth = 0;
+ let pending = "";
+
+ const filter = (chunk: string): string => {
+ const input = pending + chunk;
+ pending = "";
+ let visible = "";
+ let cursor = 0;
+
+ while (cursor < input.length) {
+ const open = input.indexOf(OPEN_TAG, cursor);
+
+ if (depth === 0) {
+ if (open === -1) {
+ const remaining = input.slice(cursor);
+ pending = getPartialTagSuffix(remaining, OPEN_TAGS);
+ visible += remaining.slice(0, remaining.length - pending.length);
+ break;
+ }
+ visible += input.slice(cursor, open);
+ depth = 1;
+ cursor = open + OPEN_TAG.length;
+ continue;
+ }
+
+ const close = input.indexOf(CLOSE_TAG, cursor);
+ if (close !== -1 && (open === -1 || close < open)) {
+ depth -= 1;
+ cursor = close + CLOSE_TAG.length;
+ } else if (open !== -1) {
+ depth += 1;
+ cursor = open + OPEN_TAG.length;
+ } else {
+ pending = getPartialTagSuffix(input.slice(cursor), THINK_TAGS);
+ break;
+ }
+ }
+
+ return visible;
+ };
+
+ filter.finish = (): string => {
+ const trailing = depth === 0 ? pending : "";
+ depth = 0;
+ pending = "";
+ return trailing;
+ };
+
+ return filter;
+}
diff --git a/test/services/reasoningServiceStreamingThink.test.js b/test/services/reasoningServiceStreamingThink.test.js
new file mode 100644
index 000000000..9a8082d71
--- /dev/null
+++ b/test/services/reasoningServiceStreamingThink.test.js
@@ -0,0 +1,319 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");
+
+function createRawSseResponse(chunks, { includeDone = true } = {}) {
+ const events = chunks
+ .map((content) => `data: ${JSON.stringify({ choices: [{ delta: { content } }] })}\n\n`)
+ .join("");
+ const done = includeDone ? "data: [DONE]\n\n" : "";
+ return new Response(`${events}${done}`, {
+ status: 200,
+ headers: { "content-type": "text/event-stream" },
+ });
+}
+
+function createOpenAiChunk(delta, finishReason = null) {
+ return {
+ id: "chatcmpl-streaming-think-test",
+ object: "chat.completion.chunk",
+ created: 1,
+ model: "qwen3-4b-q4_k_m",
+ choices: [{ index: 0, delta, finish_reason: finishReason }],
+ };
+}
+
+function createOpenAiSseResponse(chunks, { finishReason = "stop", toolCall } = {}) {
+ const events = chunks
+ .map((content) => `data: ${JSON.stringify(createOpenAiChunk({ content }))}\n\n`)
+ .join("");
+ const toolEvent = toolCall
+ ? `data: ${JSON.stringify(
+ createOpenAiChunk({
+ tool_calls: [
+ {
+ index: 0,
+ id: toolCall.id,
+ type: "function",
+ function: { name: toolCall.name, arguments: toolCall.arguments },
+ },
+ ],
+ })
+ )}\n\n`
+ : "";
+ const finishEvent = `data: ${JSON.stringify(createOpenAiChunk({}, finishReason))}\n\n`;
+ return new Response(`${events}${toolEvent}${finishEvent}data: [DONE]\n\n`, {
+ status: 200,
+ headers: { "content-type": "text/event-stream" },
+ });
+}
+
+async function loadReasoningService(t, cachePrefix, { window = {} } = {}) {
+ installBrowserGlobals(t, { window });
+ const vite = await createRendererServer(t, { cachePrefix });
+ const reasoningService = (await vite.ssrLoadModule("/services/ReasoningService.ts")).default;
+ const { usePolicyStore } = await vite.ssrLoadModule("/stores/policyStore.ts");
+ usePolicyStore.setState({ status: "unmanaged", appVersion: "1.8.3", policy: null });
+ t.after(() => reasoningService.destroy());
+ return { reasoningService, vite };
+}
+
+async function collectRawText(stream) {
+ let output = "";
+ for await (const chunk of stream) output += chunk;
+ return output;
+}
+
+async function collectAgentText(stream) {
+ let output = "";
+ for await (const chunk of stream) {
+ if (chunk.type === "content") output += chunk.text;
+ }
+ return output;
+}
+
+test("raw self-hosted streaming filters split tags and flushes visible trailing text", async (t) => {
+ const { reasoningService } = await loadReasoningService(
+ t,
+ "openwhispr-raw-streaming-think-test-"
+ );
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ globalThis.fetch = async () =>
+ createRawSseResponse(["<thi", "nk>hidden</thi", "nk>Answer<"]);
+
+ const stream = reasoningService.processTextStreaming(
+ [{ role: "user", content: "hello" }],
+ "qwen3-4b-q4_k_m",
+ "lan",
+ {
+ systemPrompt: "Answer the user.",
+ lanUrl: "http://127.0.0.1:11434/v1",
+ disableThinking: true,
+ }
+ );
+
+ assert.equal(await collectRawText(stream), "Answer<");
+});
+
+test("raw self-hosted streaming flushes visible trailing text at body EOF", async (t) => {
+ const { reasoningService } = await loadReasoningService(
+ t,
+ "openwhispr-raw-streaming-think-eof-test-"
+ );
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ globalThis.fetch = async () => createRawSseResponse(["Answer<"], { includeDone: false });
+
+ const stream = reasoningService.processTextStreaming(
+ [{ role: "user", content: "hello" }],
+ "qwen3-4b-q4_k_m",
+ "lan",
+ {
+ systemPrompt: "Answer the user.",
+ lanUrl: "http://127.0.0.1:11434/v1",
+ disableThinking: true,
+ }
+ );
+
+ assert.equal(await collectRawText(stream), "Answer<");
+});
+
+test("tool-enabled self-hosted streaming filters nested tags", async (t) => {
+ const { reasoningService, vite } = await loadReasoningService(
+ t,
+ "openwhispr-tool-streaming-think-test-"
+ );
+ const { ToolRegistry } = await vite.ssrLoadModule("/services/tools/ToolRegistry.ts");
+ const registry = new ToolRegistry();
+ registry.register({
+ name: "noop",
+ description: "No-op test tool",
+ parameters: { type: "object", properties: {}, additionalProperties: false },
+ readOnly: true,
+ execute: async () => ({ success: true, data: "ok", displayText: "ok" }),
+ });
+
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ globalThis.fetch = async () =>
+ createOpenAiSseResponse(["<thi", "nk>a<think>b</think>c</thi", "nk>Answer"]);
+
+ const stream = reasoningService.processTextStreamingAI(
+ [{ role: "user", content: "hello" }],
+ "qwen3-4b-q4_k_m",
+ "lan",
+ {
+ systemPrompt: "Answer the user.",
+ lanUrl: "http://127.0.0.1:11434/v1",
+ disableThinking: true,
+ },
+ registry.toAISDKFormat()
+ );
+
+ assert.equal(await collectAgentText(stream), "Answer");
+});
+
+test("tool-loop filtering resets after an unterminated reasoning block", async (t) => {
+ const { reasoningService, vite } = await loadReasoningService(
+ t,
+ "openwhispr-tool-step-streaming-think-test-"
+ );
+ const { ToolRegistry } = await vite.ssrLoadModule("/services/tools/ToolRegistry.ts");
+ const registry = new ToolRegistry();
+ registry.register({
+ name: "noop",
+ description: "No-op test tool",
+ parameters: { type: "object", properties: {}, additionalProperties: false },
+ readOnly: true,
+ execute: async () => ({ success: true, data: "ok", displayText: "ok" }),
+ });
+
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ let fetchCalls = 0;
+ globalThis.fetch = async () => {
+ fetchCalls += 1;
+ if (fetchCalls === 1) {
+ return createOpenAiSseResponse(["<think>secret"], {
+ finishReason: "tool_calls",
+ toolCall: { id: "call-noop", name: "noop", arguments: "{}" },
+ });
+ }
+ return createOpenAiSseResponse(["Answer"]);
+ };
+
+ const stream = reasoningService.processTextStreamingAI(
+ [{ role: "user", content: "hello" }],
+ "qwen3-4b-q4_k_m",
+ "lan",
+ {
+ systemPrompt: "Answer the user.",
+ lanUrl: "http://127.0.0.1:11434/v1",
+ disableThinking: true,
+ },
+ registry.toAISDKFormat()
+ );
+
+ assert.equal(await collectAgentText(stream), "Answer");
+ assert.equal(fetchCalls, 2);
+});
+
+for (const terminalType of ["abort", "error"]) {
+ test(`tool-enabled streaming does not flush buffered text after ${terminalType}`, async (t) => {
+ const { reasoningService } = await loadReasoningService(
+ t,
+ `openwhispr-${terminalType}-streaming-think-test-`
+ );
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ let failResponse;
+ globalThis.fetch = async (_input, init) => {
+ const encoder = new TextEncoder();
+ const body = new ReadableStream({
+ start(controller) {
+ controller.enqueue(
+ encoder.encode(
+ `data: ${JSON.stringify(createOpenAiChunk({ content: "Answer<thi" }))}\n\n`
+ )
+ );
+ failResponse = () => {
+ controller.enqueue(
+ encoder.encode(
+ `data: ${JSON.stringify({ error: { message: "stream failed" } })}\n\n`
+ )
+ );
+ controller.close();
+ };
+ init?.signal?.addEventListener(
+ "abort",
+ () => controller.error(new DOMException("aborted", "AbortError")),
+ { once: true }
+ );
+ },
+ });
+ return new Response(body, {
+ status: 200,
+ headers: { "content-type": "text/event-stream" },
+ });
+ };
+
+ const stream = reasoningService.processTextStreamingAI(
+ [{ role: "user", content: "hello" }],
+ "qwen3-4b-q4_k_m",
+ "lan",
+ {
+ systemPrompt: "Answer the user.",
+ lanUrl: "http://127.0.0.1:11434/v1",
+ disableThinking: true,
+ },
+ {}
+ );
+
+ const first = await stream.next();
+ assert.deepEqual(first.value, { type: "content", text: "Answer" });
+ if (terminalType === "abort") reasoningService.cancelActiveStream();
+ else failResponse();
+ assert.equal(await collectAgentText(stream), "");
+ });
+}
+
+test("self-hosted streaming preserves think tags when thinking is enabled", async (t) => {
+ const { reasoningService } = await loadReasoningService(
+ t,
+ "openwhispr-enabled-streaming-think-test-"
+ );
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ globalThis.fetch = async () => createOpenAiSseResponse(["<think>visible</think>Answer"]);
+
+ const stream = reasoningService.processTextStreamingAI(
+ [{ role: "user", content: "hello" }],
+ "qwen3-4b-q4_k_m",
+ "lan",
+ {
+ systemPrompt: "Answer the user.",
+ lanUrl: "http://127.0.0.1:11434/v1",
+ disableThinking: false,
+ },
+ {}
+ );
+
+ assert.equal(await collectAgentText(stream), "<think>visible</think>Answer");
+});
+
+test("non-local streaming remains unfiltered", async (t) => {
+ const { reasoningService } = await loadReasoningService(
+ t,
+ "openwhispr-cloud-streaming-think-test-",
+ { window: { electronAPI: { getGroqKey: async () => "test-key" } } }
+ );
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ globalThis.fetch = async () => createOpenAiSseResponse(["<think>visible</think>Answer"]);
+
+ const stream = reasoningService.processTextStreamingAI(
+ [{ role: "user", content: "hello" }],
+ "llama-3.3-70b-versatile",
+ "groq",
+ { systemPrompt: "Answer the user.", disableThinking: true },
+ {}
+ );
+
+ assert.equal(await collectAgentText(stream), "<think>visible</think>Answer");
+});
diff --git a/test/services/streamingThinkFilter.test.js b/test/services/streamingThinkFilter.test.js
new file mode 100644
index 000000000..c785218b1
--- /dev/null
+++ b/test/services/streamingThinkFilter.test.js
@@ -0,0 +1,94 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+
+const load = () => import("../../src/services/ai/streamingThinkFilter.ts");
+
+const filterAll = (filter, chunks) => chunks.map((chunk) => filter(chunk)).join("");
+
+test("plain text passes through unchanged", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filter("Hello world"), "Hello world");
+});
+
+test("strips a complete think block within one chunk", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filter("<think>reasoning</think>Answer"), "Answer");
+});
+
+test("strips a nested think block within one chunk (#1617 streaming twin)", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filter("<think>a<think>b</think>c</think>Answer"), "Answer");
+});
+
+test("strips a nested think block split across chunks", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filterAll(filter, ["<think>a<think>b", "</think>c</think>Answer"]), "Answer");
+});
+
+test("strips multiple sibling think blocks within one chunk", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filter("<think>a</think>X<think>b</think>Y"), "XY");
+});
+
+test("suppresses chunks that are entirely inside a block", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filter("<think>start"), "");
+ assert.equal(filter("still reasoning"), "");
+ assert.equal(filter("done</think>The answer"), "The answer");
+});
+
+test("preserves text before the opening tag", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filter("Intro<think>hidden</think> outro"), "Intro outro");
+});
+
+test("an unterminated block suppresses everything after it", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filterAll(filter, ["Answer<think>never", " closes"]), "Answer");
+ assert.equal(filter.finish(), "");
+});
+
+test("a stray close tag with no open block passes through", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(filter("plain</think>text"), "plain</think>text");
+});
+
+test("resumes normal output after a closed nested block", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+ assert.equal(
+ filterAll(filter, ["<think>a<think>b</think>", "c</think>First", " second"]),
+ "First second"
+ );
+});
+
+test("strips nested think blocks at every two-chunk boundary", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const input = "<think>a<think>b</think>c</think>Answer";
+
+ for (let split = 1; split < input.length; split += 1) {
+ const filter = createStreamingThinkFilter();
+ assert.equal(
+ filterAll(filter, [input.slice(0, split), input.slice(split)]),
+ "Answer",
+ `split at character ${split}`
+ );
+ }
+});
+
+test("buffers a possible opening tag and flushes it when the stream ends", async () => {
+ const { createStreamingThinkFilter } = await load();
+ const filter = createStreamingThinkFilter();
+
+ assert.equal(filter("Answer<thi"), "Answer");
+ assert.equal(filter.finish(), "<thi");
+});