Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(parakeet): fall back to full decode when the streaming commit loses audio (#1285)
Alcahest Malus Dienheim5ffe6ddeed19
diff --git a/.github/workflows/build-and-notarize.yml b/.github/workflows/build-and-notarize.ymlindex df4e1c386..6ef1c6a58 100644--- a/.github/workflows/build-and-notarize.yml+++ b/.github/workflows/build-and-notarize.yml@@ -25,10 +25,6 @@ jobs:node-version-file: .nvmrc- name: Install dependenciesrun: npm ci --ignore-scripts- - name: Build better-sqlite3 for the runner's node- # --ignore-scripts skips the native build; DB-backed tests silently- # skip without it. This job is the only test gate on push to main.- run: npm rebuild better-sqlite3- name: Run testsrun: npm testenv:diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 3c5e9d7f5..25dcf1679 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -13,12 +13,6 @@ jobs:node-version-file: .nvmrc- name: Install dependenciesrun: npm ci --ignore-scripts- - name: Build better-sqlite3 for the runner's node- # --ignore-scripts skips the native build, so every DB-backed test- # (snippetsDatabase, noteCloudUpsertGuard, …) silently skips without this.- run: npm rebuild better-sqlite3- - name: Typecheck- run: npm run typecheck- name: Run testsrun: npm testenv:diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 28b5c4c32..ac48beaeb 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -58,6 +58,8 @@ import { getDictionaryHintWords } from "../utils/snippets";const REASONING_CACHE_TTL = 30000; // 30 secondsconst RECORDING_TIMESLICE_MS = 250; // flush chunks periodically so short recordings still carry audio frames. See #871.+// Failure detector only: fires when the worklet or audio graph is dead and never flushes.+const PREVIEW_FLUSH_WATCHDOG_MS = 1000;const REALTIME_MODELS = new Set(["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]);function dictationAgentReachable(settings) {@@ -1439,11 +1441,10 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);try {let result;- if (typeof metadata.streamedText === "string") {- const streamedText = metadata.streamedText.trim();- if (!streamedText) {- throw new Error("No audio detected");- }+ const streamedText =+ typeof metadata.streamedText === "string" ? metadata.streamedText.trim() : null;+ // An empty stream is indistinguishable from silence; let the offline decode settle it.+ if (streamedText) {logger.debug("Parakeet using committed streaming transcript", { model }, "performance");timings.transcriptionProcessingDurationMs = 0;result = { success: true, text: streamedText };@@ -4027,17 +4028,22 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this._previewSource = null;this._previewAudioContext = null;+ let flushed = true;if (processor) {- // Wait for the worklet to flush its partial buffer so the final PCM- // chunk reaches the stream before it is finished.+ // The worklet posts all PCM before "flushed", and the PCM sends share the+ // renderer->main pipe with the stop invoke (FIFO), so the final chunk precedes finish.let resolveFlush;- const flushed = new Promise((resolve) => {- resolveFlush = resolve;- setTimeout(resolve, 150);+ const flushSentinel = new Promise((resolve) => {+ resolveFlush = () => resolve(true);+ });+ let watchdogTimer;+ const watchdogFired = new Promise((resolve) => {+ watchdogTimer = setTimeout(() => resolve(false), PREVIEW_FLUSH_WATCHDOG_MS);});this._previewFlushResolve = resolveFlush;processor.port.postMessage("stop");- await flushed;+ flushed = await Promise.race([flushSentinel, watchdogFired]);+ clearTimeout(watchdogTimer);if (this._previewFlushResolve === resolveFlush) this._previewFlushResolve = null;processor.disconnect();}@@ -4047,7 +4053,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);window.electronAPI?.dismissDictationPreview?.();return null;}- return (await window.electronAPI?.stopDictationPreview?.({ showCleanup })) || null;+ return (await window.electronAPI?.stopDictationPreview?.({ showCleanup, flushed })) || null;}cleanupStreamingAudio() {diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 1e09027a7..3983b384f 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -6738,6 +6738,8 @@ class IPCHandlers {clearInterval(dictationPreviewTimer);dictationPreviewTimer = null;const display = dictationPreviewDisplay;+ // Missing flag defaults to trusted so non-streaming callers never regress.+ const rendererFlushOk = options.flushed !== false;let streamed = false;let streamedText = "";if (dictationPreviewStream) {@@ -6750,8 +6752,8 @@ class IPCHandlers {}if (result) {streamedText = result.text || "";- // Only a clean flush is trustworthy as the final transcript.- streamed = !result.truncated;+ // Trust the streamed transcript only on a clean server flush and a clean renderer flush.+ streamed = !result.truncated && rendererFlushOk;}if (streamedText && display && dictationPreviewSessionActive) {this.windowManager.showTranscriptionPreview(streamedText);diff --git a/src/helpers/parakeetWsResult.js b/src/helpers/parakeetWsResult.jsindex e95940e27..2a1666483 100644--- a/src/helpers/parakeetWsResult.js+++ b/src/helpers/parakeetWsResult.js@@ -8,16 +8,20 @@ function parseOfflineMessage(message) {}}-// Dedupes finalized segments by id; text() is finalized text plus the trailing partial.+// Latest-wins per finalized segment id (the server refines a segment before its+// endpoint); text() joins segments in first-arrival order plus the trailing partial.function createOnlineAccumulator() {- const finalizedSegments = new Set();- let finalizedText = "";+ const finalizedSegments = new Map();let partialText = "";let partialSegment = null;let fallbackKey = 0;- const text = () =>- finalizedText && partialText ? `${finalizedText} ${partialText}` : finalizedText || partialText;+ const text = () => {+ const finalizedText = Array.from(finalizedSegments.values()).join(" ");+ return finalizedText && partialText+ ? `${finalizedText} ${partialText}`+ : finalizedText || partialText;+ };return {push(message) {@@ -39,10 +43,7 @@ function createOnlineAccumulator() {}const segment = parsed.segment ?? `fallback:${fallbackKey++}`;- if (!finalizedSegments.has(segment)) {- finalizedSegments.add(segment);- finalizedText = finalizedText ? `${finalizedText} ${messageText}` : messageText;- }+ finalizedSegments.set(segment, messageText);if (partialSegment === null || partialSegment === segment) {partialText = "";partialSegment = null;diff --git a/src/helpers/parakeetWsServer.js b/src/helpers/parakeetWsServer.jsindex 3b2f1629f..9b7de4e42 100644--- a/src/helpers/parakeetWsServer.js+++ b/src/helpers/parakeetWsServer.js@@ -429,10 +429,17 @@ class ParakeetWsServer {};const sendFloat32 = (float32Samples) => {- if (closed || finishResolve) return;+ if (closed) return;+ // A chunk after finish() means the renderer flushed late (contract broke) so the+ // tail is lost; when only closed is set, settle already happened and a plain drop is fine.+ if (finishResolve) {+ truncated = true;+ return;+ }if (ws.readyState === WebSocket.OPEN) {ws.send(float32Samples, (err) => {if (err) {+ truncated = true;debugLogger.error("parakeet-ws online stream send error", { error: err.message });}});@@ -443,7 +450,12 @@ class ParakeetWsServer {ws.on("open", () => {for (const chunk of pendingChunks) {- ws.send(chunk);+ ws.send(chunk, (err) => {+ if (err) {+ truncated = true;+ debugLogger.error("parakeet-ws online stream send error", { error: err.message });+ }+ });}pendingChunks.length = 0;if (finishResolve) ws.send("Done");diff --git a/src/types/electron.ts b/src/types/electron.tsindex 12a41cea5..a6e216b9b 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -2028,6 +2028,7 @@ declare global {}) => Promise<{ success: boolean }>;stopDictationPreview?: (opts?: {showCleanup?: boolean;+ flushed?: boolean;}) => Promise<{ success: boolean; streamed?: boolean; text?: string }>;dismissDictationPreview?: () => Promise<{ success: boolean }>;completeDictationPreview?: (payload: { text?: string }) => Promise<{ success: boolean }>;diff --git a/test/helpers/parakeetOnlineStream.test.js b/test/helpers/parakeetOnlineStream.test.jsindex 79c35c4e4..2a77652a5 100644--- a/test/helpers/parakeetOnlineStream.test.js+++ b/test/helpers/parakeetOnlineStream.test.js@@ -179,6 +179,57 @@ test("unexpected close before Done! reports an error and truncation", async () =}});+test("a clean finish reports no truncation", async () => {+ const mock = await startMockOnlineServer({});+ try {+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});+ stream.sendPcm16(Buffer.alloc(3200));+ const { text, truncated } = await stream.finish();+ assert.equal(truncated, false);+ assert.equal(text, "final after 1 frames");+ } finally {+ await mock.close();+ }+});++test("a send error during the queued flush flags truncation", async () => {+ const mock = await startMockOnlineServer({});+ const WebSocket = require("ws");+ const realSend = WebSocket.prototype.send;+ // Only client audio frames are binary; the mock server sends strings, so gate on type.+ WebSocket.prototype.send = function (data, cb) {+ if (typeof data !== "string") {+ if (typeof cb === "function") cb(new Error("forced send failure"));+ return;+ }+ return realSend.call(this, data, cb);+ };+ try {+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});+ stream.sendPcm16(Buffer.alloc(3200));+ const { truncated } = await stream.finish({ idleTimeoutMs: 1000 });+ assert.equal(truncated, true);+ } finally {+ WebSocket.prototype.send = realSend;+ await mock.close();+ }+});++test("a chunk arriving after finish flags truncation", async () => {+ const mock = await startMockOnlineServer({});+ try {+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});+ stream.sendPcm16(Buffer.alloc(3200));+ const finishPromise = stream.finish({ idleTimeoutMs: 1000 });+ // Renderer flushed a chunk after we already told the server we were done.+ stream.sendPcm16(Buffer.alloc(3200));+ const { truncated } = await finishPromise;+ assert.equal(truncated, true);+ } finally {+ await mock.close();+ }+});+test("finish is idempotent and returns the same result", async () => {const mock = await startMockOnlineServer({});try {diff --git a/test/helpers/parakeetWsResult.test.js b/test/helpers/parakeetWsResult.test.jsindex b0b71d3d6..68b77f015 100644--- a/test/helpers/parakeetWsResult.test.js+++ b/test/helpers/parakeetWsResult.test.js@@ -75,3 +75,35 @@ test("online sherpa parser tolerates non-string text payloads", () => {"2026");});++test("online sherpa parser keeps the latest text when a segment is refined", () => {+ assert.equal(+ parseOnlineMessages([+ JSON.stringify({ text: "the quick", is_final: true, segment: 0 }),+ JSON.stringify({ text: "brown", is_final: true, segment: 1 }),+ JSON.stringify({ text: "the quick fox", is_final: true, segment: 0 }),+ ]),+ "the quick fox brown"+ );+});++test("online sherpa parser treats an identical duplicate final as a no-op", () => {+ assert.equal(+ parseOnlineMessages([+ JSON.stringify({ text: "hello", is_final: true, segment: 0 }),+ JSON.stringify({ text: "world", is_final: true, segment: 1 }),+ JSON.stringify({ text: "hello", is_final: true, segment: 0 }),+ ]),+ "hello world"+ );+});++test("online sherpa parser suppresses a partial for an already-finalized segment", () => {+ assert.equal(+ parseOnlineMessages([+ JSON.stringify({ text: "done", is_final: true, segment: 0 }),+ JSON.stringify({ text: "late partial", is_final: false, segment: 0 }),+ ]),+ "done"+ );+});