Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(reasoning): track think-tag depth in streamed chat deltas (#1644)
Chad Piha5abb00444567
diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.tsindex 9b943f370..0f0429500 100644--- a/src/services/ReasoningService.ts+++ b/src/services/ReasoningService.ts@@ -32,6 +32,7 @@ import {} from "./ai/chatRequestBody";import { getModelFamilyConstraints } from "./ai/modelFamilyConstraints";import { detectEndpointDialect } from "./ai/thinkingSuppressionDialects";+import { createStreamingThinkFilter } from "./ai/streamingThinkFilter";import { extractApiErrorMessage } from "./ai/apiErrorMessage";import { clearTinfoilClientCache } from "./ai/tinfoilClient";import { resolveChatRoute } from "../helpers/chatRouting";@@ -628,7 +629,8 @@ class ReasoningService extends BaseReasoningService {const decoder = new TextDecoder();let buffer = "";- let insideThinkBlock = false;+ const stripThinking = (isLocalProvider || isLanChat) && config.disableThinking !== false;+ const filterThinkTags = stripThinking ? createStreamingThinkFilter() : null;try {while (true) {@@ -644,37 +646,19 @@ class ReasoningService extends BaseReasoningService {if (!trimmed || !trimmed.startsWith("data: ")) continue;const data = trimmed.slice(6);- if (data === "[DONE]") return;+ if (data === "[DONE]") {+ const trailing = filterThinkTags?.finish();+ if (trailing) yield trailing;+ return;+ }try {const parsed = JSON.parse(data);let content = parsed.choices?.[0]?.delta?.content;if (!content) continue;- const stripThinking =- (isLocalProvider || isLanChat) && config.disableThinking !== false;- if (stripThinking) {- if (insideThinkBlock) {- const endIdx = content.indexOf("</think>");- if (endIdx !== -1) {- insideThinkBlock = false;- content = content.slice(endIdx + 8);- } else {- continue;- }- }- const startIdx = content.indexOf("<think>");- if (startIdx !== -1) {- const before = content.slice(0, startIdx);- const after = content.slice(startIdx + 7);- const endIdx = after.indexOf("</think>");- if (endIdx !== -1) {- content = before + after.slice(endIdx + 8);- } else {- insideThinkBlock = true;- content = before;- }- }+ if (filterThinkTags) {+ content = filterThinkTags(content);if (!content) continue;}@@ -684,6 +668,9 @@ class ReasoningService extends BaseReasoningService {}}}++ const trailing = filterThinkTags?.finish();+ if (trailing) yield trailing;} finally {clearTimeout(timeoutId);this.streamAbortController = null;@@ -732,6 +719,10 @@ class ReasoningService extends BaseReasoningService {return;}+ const filterThinkTags =+ (isLocalProvider || isLanChat) && config.disableThinking !== false+ ? createStreamingThinkFilter()+ : null;let apiKey = "";let baseURL: string | undefined;@@ -812,10 +803,20 @@ class ReasoningService extends BaseReasoningService {...(hasProviderOptions ? { providerOptions } : {}),});+ let canFlushFilteredText = true;+ const finishFilteredText = (): string => {+ const trailing = filterThinkTags?.finish() ?? "";+ return canFlushFilteredText ? trailing : "";+ };+try {for await (const chunk of result.fullStream) {if (chunk.type === "text-delta") {- yield { type: "content", text: chunk.text };+ const text = filterThinkTags ? filterThinkTags(chunk.text) : chunk.text;+ if (text) yield { type: "content", text };+ } else if (chunk.type === "text-end" || chunk.type === "finish-step") {+ const trailing = finishFilteredText();+ if (trailing) yield { type: "content", text: trailing };} else if (chunk.type === "tool-call") {yield {type: "tool_calls",@@ -837,11 +838,18 @@ class ReasoningService extends BaseReasoningService {toolName: chunk.toolName,displayText,};+ } else if (chunk.type === "abort" || chunk.type === "error") {+ canFlushFilteredText = false;+ finishFilteredText();} else if (chunk.type === "finish") {+ const trailing = finishFilteredText();+ if (trailing) yield { type: "content", text: trailing };yield { type: "done", finishReason: chunk.finishReason };}}} catch (error) {+ canFlushFilteredText = false;+ finishFilteredText();if (abortController.signal.aborted) {yield { type: "done", finishReason: "stop" };return;diff --git a/src/services/ai/streamingThinkFilter.ts b/src/services/ai/streamingThinkFilter.tsnew file mode 100644index 000000000..33f979adc--- /dev/null+++ b/src/services/ai/streamingThinkFilter.ts@@ -0,0 +1,75 @@+const OPEN_TAG = "<think>";+const CLOSE_TAG = "</think>";+const OPEN_TAGS = [OPEN_TAG] as const;+const THINK_TAGS = [OPEN_TAG, CLOSE_TAG] as const;+const MAX_PARTIAL_TAG_LENGTH = CLOSE_TAG.length - 1;++export interface StreamingThinkFilter {+ (chunk: string): string;+ finish: () => string;+}++function getPartialTagSuffix(value: string, tags: readonly string[]): string {+ const maxLength = Math.min(value.length, MAX_PARTIAL_TAG_LENGTH);+ for (let length = maxLength; length > 0; length -= 1) {+ const suffix = value.slice(-length);+ if (tags.some((tag) => tag.startsWith(suffix))) return suffix;+ }+ return "";+}++// Filters <think> blocks out of streamed chat deltas, keeping state across+// chunks, including tags split between deltas. Depth is a counter because+// the first </think> in a nested block only closes the inner block. finish()+// preserves a trailing tag-like fragment only when it was outside reasoning.+export function createStreamingThinkFilter(): StreamingThinkFilter {+ let depth = 0;+ let pending = "";++ const filter = (chunk: string): string => {+ const input = pending + chunk;+ pending = "";+ let visible = "";+ let cursor = 0;++ while (cursor < input.length) {+ const open = input.indexOf(OPEN_TAG, cursor);++ if (depth === 0) {+ if (open === -1) {+ const remaining = input.slice(cursor);+ pending = getPartialTagSuffix(remaining, OPEN_TAGS);+ visible += remaining.slice(0, remaining.length - pending.length);+ break;+ }+ visible += input.slice(cursor, open);+ depth = 1;+ cursor = open + OPEN_TAG.length;+ continue;+ }++ const close = input.indexOf(CLOSE_TAG, cursor);+ if (close !== -1 && (open === -1 || close < open)) {+ depth -= 1;+ cursor = close + CLOSE_TAG.length;+ } else if (open !== -1) {+ depth += 1;+ cursor = open + OPEN_TAG.length;+ } else {+ pending = getPartialTagSuffix(input.slice(cursor), THINK_TAGS);+ break;+ }+ }++ return visible;+ };++ filter.finish = (): string => {+ const trailing = depth === 0 ? pending : "";+ depth = 0;+ pending = "";+ return trailing;+ };++ return filter;+}diff --git a/test/services/reasoningServiceStreamingThink.test.js b/test/services/reasoningServiceStreamingThink.test.jsnew file mode 100644index 000000000..9a8082d71--- /dev/null+++ b/test/services/reasoningServiceStreamingThink.test.js@@ -0,0 +1,319 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");++function createRawSseResponse(chunks, { includeDone = true } = {}) {+ const events = chunks+ .map((content) => `data: ${JSON.stringify({ choices: [{ delta: { content } }] })}\n\n`)+ .join("");+ const done = includeDone ? "data: [DONE]\n\n" : "";+ return new Response(`${events}${done}`, {+ status: 200,+ headers: { "content-type": "text/event-stream" },+ });+}++function createOpenAiChunk(delta, finishReason = null) {+ return {+ id: "chatcmpl-streaming-think-test",+ object: "chat.completion.chunk",+ created: 1,+ model: "qwen3-4b-q4_k_m",+ choices: [{ index: 0, delta, finish_reason: finishReason }],+ };+}++function createOpenAiSseResponse(chunks, { finishReason = "stop", toolCall } = {}) {+ const events = chunks+ .map((content) => `data: ${JSON.stringify(createOpenAiChunk({ content }))}\n\n`)+ .join("");+ const toolEvent = toolCall+ ? `data: ${JSON.stringify(+ createOpenAiChunk({+ tool_calls: [+ {+ index: 0,+ id: toolCall.id,+ type: "function",+ function: { name: toolCall.name, arguments: toolCall.arguments },+ },+ ],+ })+ )}\n\n`+ : "";+ const finishEvent = `data: ${JSON.stringify(createOpenAiChunk({}, finishReason))}\n\n`;+ return new Response(`${events}${toolEvent}${finishEvent}data: [DONE]\n\n`, {+ status: 200,+ headers: { "content-type": "text/event-stream" },+ });+}++async function loadReasoningService(t, cachePrefix, { window = {} } = {}) {+ installBrowserGlobals(t, { window });+ const vite = await createRendererServer(t, { cachePrefix });+ const reasoningService = (await vite.ssrLoadModule("/services/ReasoningService.ts")).default;+ const { usePolicyStore } = await vite.ssrLoadModule("/stores/policyStore.ts");+ usePolicyStore.setState({ status: "unmanaged", appVersion: "1.8.3", policy: null });+ t.after(() => reasoningService.destroy());+ return { reasoningService, vite };+}++async function collectRawText(stream) {+ let output = "";+ for await (const chunk of stream) output += chunk;+ return output;+}++async function collectAgentText(stream) {+ let output = "";+ for await (const chunk of stream) {+ if (chunk.type === "content") output += chunk.text;+ }+ return output;+}++test("raw self-hosted streaming filters split tags and flushes visible trailing text", async (t) => {+ const { reasoningService } = await loadReasoningService(+ t,+ "openwhispr-raw-streaming-think-test-"+ );+ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ globalThis.fetch = async () =>+ createRawSseResponse(["<thi", "nk>hidden</thi", "nk>Answer<"]);++ const stream = reasoningService.processTextStreaming(+ [{ role: "user", content: "hello" }],+ "qwen3-4b-q4_k_m",+ "lan",+ {+ systemPrompt: "Answer the user.",+ lanUrl: "http://127.0.0.1:11434/v1",+ disableThinking: true,+ }+ );++ assert.equal(await collectRawText(stream), "Answer<");+});++test("raw self-hosted streaming flushes visible trailing text at body EOF", async (t) => {+ const { reasoningService } = await loadReasoningService(+ t,+ "openwhispr-raw-streaming-think-eof-test-"+ );+ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ globalThis.fetch = async () => createRawSseResponse(["Answer<"], { includeDone: false });++ const stream = reasoningService.processTextStreaming(+ [{ role: "user", content: "hello" }],+ "qwen3-4b-q4_k_m",+ "lan",+ {+ systemPrompt: "Answer the user.",+ lanUrl: "http://127.0.0.1:11434/v1",+ disableThinking: true,+ }+ );++ assert.equal(await collectRawText(stream), "Answer<");+});++test("tool-enabled self-hosted streaming filters nested tags", async (t) => {+ const { reasoningService, vite } = await loadReasoningService(+ t,+ "openwhispr-tool-streaming-think-test-"+ );+ const { ToolRegistry } = await vite.ssrLoadModule("/services/tools/ToolRegistry.ts");+ const registry = new ToolRegistry();+ registry.register({+ name: "noop",+ description: "No-op test tool",+ parameters: { type: "object", properties: {}, additionalProperties: false },+ readOnly: true,+ execute: async () => ({ success: true, data: "ok", displayText: "ok" }),+ });++ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ globalThis.fetch = async () =>+ createOpenAiSseResponse(["<thi", "nk>a<think>b</think>c</thi", "nk>Answer"]);++ const stream = reasoningService.processTextStreamingAI(+ [{ role: "user", content: "hello" }],+ "qwen3-4b-q4_k_m",+ "lan",+ {+ systemPrompt: "Answer the user.",+ lanUrl: "http://127.0.0.1:11434/v1",+ disableThinking: true,+ },+ registry.toAISDKFormat()+ );++ assert.equal(await collectAgentText(stream), "Answer");+});++test("tool-loop filtering resets after an unterminated reasoning block", async (t) => {+ const { reasoningService, vite } = await loadReasoningService(+ t,+ "openwhispr-tool-step-streaming-think-test-"+ );+ const { ToolRegistry } = await vite.ssrLoadModule("/services/tools/ToolRegistry.ts");+ const registry = new ToolRegistry();+ registry.register({+ name: "noop",+ description: "No-op test tool",+ parameters: { type: "object", properties: {}, additionalProperties: false },+ readOnly: true,+ execute: async () => ({ success: true, data: "ok", displayText: "ok" }),+ });++ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ let fetchCalls = 0;+ globalThis.fetch = async () => {+ fetchCalls += 1;+ if (fetchCalls === 1) {+ return createOpenAiSseResponse(["<think>secret"], {+ finishReason: "tool_calls",+ toolCall: { id: "call-noop", name: "noop", arguments: "{}" },+ });+ }+ return createOpenAiSseResponse(["Answer"]);+ };++ const stream = reasoningService.processTextStreamingAI(+ [{ role: "user", content: "hello" }],+ "qwen3-4b-q4_k_m",+ "lan",+ {+ systemPrompt: "Answer the user.",+ lanUrl: "http://127.0.0.1:11434/v1",+ disableThinking: true,+ },+ registry.toAISDKFormat()+ );++ assert.equal(await collectAgentText(stream), "Answer");+ assert.equal(fetchCalls, 2);+});++for (const terminalType of ["abort", "error"]) {+ test(`tool-enabled streaming does not flush buffered text after ${terminalType}`, async (t) => {+ const { reasoningService } = await loadReasoningService(+ t,+ `openwhispr-${terminalType}-streaming-think-test-`+ );+ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ let failResponse;+ globalThis.fetch = async (_input, init) => {+ const encoder = new TextEncoder();+ const body = new ReadableStream({+ start(controller) {+ controller.enqueue(+ encoder.encode(+ `data: ${JSON.stringify(createOpenAiChunk({ content: "Answer<thi" }))}\n\n`+ )+ );+ failResponse = () => {+ controller.enqueue(+ encoder.encode(+ `data: ${JSON.stringify({ error: { message: "stream failed" } })}\n\n`+ )+ );+ controller.close();+ };+ init?.signal?.addEventListener(+ "abort",+ () => controller.error(new DOMException("aborted", "AbortError")),+ { once: true }+ );+ },+ });+ return new Response(body, {+ status: 200,+ headers: { "content-type": "text/event-stream" },+ });+ };++ const stream = reasoningService.processTextStreamingAI(+ [{ role: "user", content: "hello" }],+ "qwen3-4b-q4_k_m",+ "lan",+ {+ systemPrompt: "Answer the user.",+ lanUrl: "http://127.0.0.1:11434/v1",+ disableThinking: true,+ },+ {}+ );++ const first = await stream.next();+ assert.deepEqual(first.value, { type: "content", text: "Answer" });+ if (terminalType === "abort") reasoningService.cancelActiveStream();+ else failResponse();+ assert.equal(await collectAgentText(stream), "");+ });+}++test("self-hosted streaming preserves think tags when thinking is enabled", async (t) => {+ const { reasoningService } = await loadReasoningService(+ t,+ "openwhispr-enabled-streaming-think-test-"+ );+ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ globalThis.fetch = async () => createOpenAiSseResponse(["<think>visible</think>Answer"]);++ const stream = reasoningService.processTextStreamingAI(+ [{ role: "user", content: "hello" }],+ "qwen3-4b-q4_k_m",+ "lan",+ {+ systemPrompt: "Answer the user.",+ lanUrl: "http://127.0.0.1:11434/v1",+ disableThinking: false,+ },+ {}+ );++ assert.equal(await collectAgentText(stream), "<think>visible</think>Answer");+});++test("non-local streaming remains unfiltered", async (t) => {+ const { reasoningService } = await loadReasoningService(+ t,+ "openwhispr-cloud-streaming-think-test-",+ { window: { electronAPI: { getGroqKey: async () => "test-key" } } }+ );+ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ globalThis.fetch = async () => createOpenAiSseResponse(["<think>visible</think>Answer"]);++ const stream = reasoningService.processTextStreamingAI(+ [{ role: "user", content: "hello" }],+ "llama-3.3-70b-versatile",+ "groq",+ { systemPrompt: "Answer the user.", disableThinking: true },+ {}+ );++ assert.equal(await collectAgentText(stream), "<think>visible</think>Answer");+});diff --git a/test/services/streamingThinkFilter.test.js b/test/services/streamingThinkFilter.test.jsnew file mode 100644index 000000000..c785218b1--- /dev/null+++ b/test/services/streamingThinkFilter.test.js@@ -0,0 +1,94 @@+const test = require("node:test");+const assert = require("node:assert/strict");++const load = () => import("../../src/services/ai/streamingThinkFilter.ts");++const filterAll = (filter, chunks) => chunks.map((chunk) => filter(chunk)).join("");++test("plain text passes through unchanged", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filter("Hello world"), "Hello world");+});++test("strips a complete think block within one chunk", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filter("<think>reasoning</think>Answer"), "Answer");+});++test("strips a nested think block within one chunk (#1617 streaming twin)", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filter("<think>a<think>b</think>c</think>Answer"), "Answer");+});++test("strips a nested think block split across chunks", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filterAll(filter, ["<think>a<think>b", "</think>c</think>Answer"]), "Answer");+});++test("strips multiple sibling think blocks within one chunk", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filter("<think>a</think>X<think>b</think>Y"), "XY");+});++test("suppresses chunks that are entirely inside a block", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filter("<think>start"), "");+ assert.equal(filter("still reasoning"), "");+ assert.equal(filter("done</think>The answer"), "The answer");+});++test("preserves text before the opening tag", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filter("Intro<think>hidden</think> outro"), "Intro outro");+});++test("an unterminated block suppresses everything after it", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filterAll(filter, ["Answer<think>never", " closes"]), "Answer");+ assert.equal(filter.finish(), "");+});++test("a stray close tag with no open block passes through", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(filter("plain</think>text"), "plain</think>text");+});++test("resumes normal output after a closed nested block", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();+ assert.equal(+ filterAll(filter, ["<think>a<think>b</think>", "c</think>First", " second"]),+ "First second"+ );+});++test("strips nested think blocks at every two-chunk boundary", async () => {+ const { createStreamingThinkFilter } = await load();+ const input = "<think>a<think>b</think>c</think>Answer";++ for (let split = 1; split < input.length; split += 1) {+ const filter = createStreamingThinkFilter();+ assert.equal(+ filterAll(filter, [input.slice(0, split), input.slice(split)]),+ "Answer",+ `split at character ${split}`+ );+ }+});++test("buffers a possible opening tag and flushes it when the stream ends", async () => {+ const { createStreamingThinkFilter } = await load();+ const filter = createStreamingThinkFilter();++ assert.equal(filter("Answer<thi"), "Answer");+ assert.equal(filter.finish(), "<thi");+});