Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(parakeet): fall back to full decode when the streaming commit loses audio (#1285)

Alcahest Malus Dienheim5ffe6ddeed19
diff --git a/.github/workflows/build-and-notarize.yml b/.github/workflows/build-and-notarize.yml
index df4e1c386..6ef1c6a58 100644
--- a/.github/workflows/build-and-notarize.yml
+++ b/.github/workflows/build-and-notarize.yml
@@ -25,10 +25,6 @@ jobs:
node-version-file: .nvmrc
- name: Install dependencies
run: npm ci --ignore-scripts
- - name: Build better-sqlite3 for the runner's node
- # --ignore-scripts skips the native build; DB-backed tests silently
- # skip without it. This job is the only test gate on push to main.
- run: npm rebuild better-sqlite3
- name: Run tests
run: npm test
env:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 3c5e9d7f5..25dcf1679 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -13,12 +13,6 @@ jobs:
node-version-file: .nvmrc
- name: Install dependencies
run: npm ci --ignore-scripts
- - name: Build better-sqlite3 for the runner's node
- # --ignore-scripts skips the native build, so every DB-backed test
- # (snippetsDatabase, noteCloudUpsertGuard, …) silently skips without this.
- run: npm rebuild better-sqlite3
- - name: Typecheck
- run: npm run typecheck
- name: Run tests
run: npm test
env:
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 28b5c4c32..ac48beaeb 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -58,6 +58,8 @@ import { getDictionaryHintWords } from "../utils/snippets";
const REASONING_CACHE_TTL = 30000; // 30 seconds
const RECORDING_TIMESLICE_MS = 250; // flush chunks periodically so short recordings still carry audio frames. See #871.
+// Failure detector only: fires when the worklet or audio graph is dead and never flushes.
+const PREVIEW_FLUSH_WATCHDOG_MS = 1000;
const REALTIME_MODELS = new Set(["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]);
function dictationAgentReachable(settings) {
@@ -1439,11 +1441,10 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
try {
let result;
- if (typeof metadata.streamedText === "string") {
- const streamedText = metadata.streamedText.trim();
- if (!streamedText) {
- throw new Error("No audio detected");
- }
+ const streamedText =
+ typeof metadata.streamedText === "string" ? metadata.streamedText.trim() : null;
+ // An empty stream is indistinguishable from silence; let the offline decode settle it.
+ if (streamedText) {
logger.debug("Parakeet using committed streaming transcript", { model }, "performance");
timings.transcriptionProcessingDurationMs = 0;
result = { success: true, text: streamedText };
@@ -4027,17 +4028,22 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this._previewSource = null;
this._previewAudioContext = null;
+ let flushed = true;
if (processor) {
- // Wait for the worklet to flush its partial buffer so the final PCM
- // chunk reaches the stream before it is finished.
+ // The worklet posts all PCM before "flushed", and the PCM sends share the
+ // renderer->main pipe with the stop invoke (FIFO), so the final chunk precedes finish.
let resolveFlush;
- const flushed = new Promise((resolve) => {
- resolveFlush = resolve;
- setTimeout(resolve, 150);
+ const flushSentinel = new Promise((resolve) => {
+ resolveFlush = () => resolve(true);
+ });
+ let watchdogTimer;
+ const watchdogFired = new Promise((resolve) => {
+ watchdogTimer = setTimeout(() => resolve(false), PREVIEW_FLUSH_WATCHDOG_MS);
});
this._previewFlushResolve = resolveFlush;
processor.port.postMessage("stop");
- await flushed;
+ flushed = await Promise.race([flushSentinel, watchdogFired]);
+ clearTimeout(watchdogTimer);
if (this._previewFlushResolve === resolveFlush) this._previewFlushResolve = null;
processor.disconnect();
}
@@ -4047,7 +4053,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
window.electronAPI?.dismissDictationPreview?.();
return null;
}
- return (await window.electronAPI?.stopDictationPreview?.({ showCleanup })) || null;
+ return (await window.electronAPI?.stopDictationPreview?.({ showCleanup, flushed })) || null;
}
cleanupStreamingAudio() {
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 1e09027a7..3983b384f 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -6738,6 +6738,8 @@ class IPCHandlers {
clearInterval(dictationPreviewTimer);
dictationPreviewTimer = null;
const display = dictationPreviewDisplay;
+ // Missing flag defaults to trusted so non-streaming callers never regress.
+ const rendererFlushOk = options.flushed !== false;
let streamed = false;
let streamedText = "";
if (dictationPreviewStream) {
@@ -6750,8 +6752,8 @@ class IPCHandlers {
}
if (result) {
streamedText = result.text || "";
- // Only a clean flush is trustworthy as the final transcript.
- streamed = !result.truncated;
+ // Trust the streamed transcript only on a clean server flush and a clean renderer flush.
+ streamed = !result.truncated && rendererFlushOk;
}
if (streamedText && display && dictationPreviewSessionActive) {
this.windowManager.showTranscriptionPreview(streamedText);
diff --git a/src/helpers/parakeetWsResult.js b/src/helpers/parakeetWsResult.js
index e95940e27..2a1666483 100644
--- a/src/helpers/parakeetWsResult.js
+++ b/src/helpers/parakeetWsResult.js
@@ -8,16 +8,20 @@ function parseOfflineMessage(message) {
}
}
-// Dedupes finalized segments by id; text() is finalized text plus the trailing partial.
+// Latest-wins per finalized segment id (the server refines a segment before its
+// endpoint); text() joins segments in first-arrival order plus the trailing partial.
function createOnlineAccumulator() {
- const finalizedSegments = new Set();
- let finalizedText = "";
+ const finalizedSegments = new Map();
let partialText = "";
let partialSegment = null;
let fallbackKey = 0;
- const text = () =>
- finalizedText && partialText ? `${finalizedText} ${partialText}` : finalizedText || partialText;
+ const text = () => {
+ const finalizedText = Array.from(finalizedSegments.values()).join(" ");
+ return finalizedText && partialText
+ ? `${finalizedText} ${partialText}`
+ : finalizedText || partialText;
+ };
return {
push(message) {
@@ -39,10 +43,7 @@ function createOnlineAccumulator() {
}
const segment = parsed.segment ?? `fallback:${fallbackKey++}`;
- if (!finalizedSegments.has(segment)) {
- finalizedSegments.add(segment);
- finalizedText = finalizedText ? `${finalizedText} ${messageText}` : messageText;
- }
+ finalizedSegments.set(segment, messageText);
if (partialSegment === null || partialSegment === segment) {
partialText = "";
partialSegment = null;
diff --git a/src/helpers/parakeetWsServer.js b/src/helpers/parakeetWsServer.js
index 3b2f1629f..9b7de4e42 100644
--- a/src/helpers/parakeetWsServer.js
+++ b/src/helpers/parakeetWsServer.js
@@ -429,10 +429,17 @@ class ParakeetWsServer {
};
const sendFloat32 = (float32Samples) => {
- if (closed || finishResolve) return;
+ if (closed) return;
+ // A chunk after finish() means the renderer flushed late (contract broke) so the
+ // tail is lost; when only closed is set, settle already happened and a plain drop is fine.
+ if (finishResolve) {
+ truncated = true;
+ return;
+ }
if (ws.readyState === WebSocket.OPEN) {
ws.send(float32Samples, (err) => {
if (err) {
+ truncated = true;
debugLogger.error("parakeet-ws online stream send error", { error: err.message });
}
});
@@ -443,7 +450,12 @@ class ParakeetWsServer {
ws.on("open", () => {
for (const chunk of pendingChunks) {
- ws.send(chunk);
+ ws.send(chunk, (err) => {
+ if (err) {
+ truncated = true;
+ debugLogger.error("parakeet-ws online stream send error", { error: err.message });
+ }
+ });
}
pendingChunks.length = 0;
if (finishResolve) ws.send("Done");
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 12a41cea5..a6e216b9b 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -2028,6 +2028,7 @@ declare global {
}) => Promise<{ success: boolean }>;
stopDictationPreview?: (opts?: {
showCleanup?: boolean;
+ flushed?: boolean;
}) => Promise<{ success: boolean; streamed?: boolean; text?: string }>;
dismissDictationPreview?: () => Promise<{ success: boolean }>;
completeDictationPreview?: (payload: { text?: string }) => Promise<{ success: boolean }>;
diff --git a/test/helpers/parakeetOnlineStream.test.js b/test/helpers/parakeetOnlineStream.test.js
index 79c35c4e4..2a77652a5 100644
--- a/test/helpers/parakeetOnlineStream.test.js
+++ b/test/helpers/parakeetOnlineStream.test.js
@@ -179,6 +179,57 @@ test("unexpected close before Done! reports an error and truncation", async () =
}
});
+test("a clean finish reports no truncation", async () => {
+ const mock = await startMockOnlineServer({});
+ try {
+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});
+ stream.sendPcm16(Buffer.alloc(3200));
+ const { text, truncated } = await stream.finish();
+ assert.equal(truncated, false);
+ assert.equal(text, "final after 1 frames");
+ } finally {
+ await mock.close();
+ }
+});
+
+test("a send error during the queued flush flags truncation", async () => {
+ const mock = await startMockOnlineServer({});
+ const WebSocket = require("ws");
+ const realSend = WebSocket.prototype.send;
+ // Only client audio frames are binary; the mock server sends strings, so gate on type.
+ WebSocket.prototype.send = function (data, cb) {
+ if (typeof data !== "string") {
+ if (typeof cb === "function") cb(new Error("forced send failure"));
+ return;
+ }
+ return realSend.call(this, data, cb);
+ };
+ try {
+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});
+ stream.sendPcm16(Buffer.alloc(3200));
+ const { truncated } = await stream.finish({ idleTimeoutMs: 1000 });
+ assert.equal(truncated, true);
+ } finally {
+ WebSocket.prototype.send = realSend;
+ await mock.close();
+ }
+});
+
+test("a chunk arriving after finish flags truncation", async () => {
+ const mock = await startMockOnlineServer({});
+ try {
+ const stream = onlineWsServerAt(mock.port).createOnlineStream({});
+ stream.sendPcm16(Buffer.alloc(3200));
+ const finishPromise = stream.finish({ idleTimeoutMs: 1000 });
+ // Renderer flushed a chunk after we already told the server we were done.
+ stream.sendPcm16(Buffer.alloc(3200));
+ const { truncated } = await finishPromise;
+ assert.equal(truncated, true);
+ } finally {
+ await mock.close();
+ }
+});
+
test("finish is idempotent and returns the same result", async () => {
const mock = await startMockOnlineServer({});
try {
diff --git a/test/helpers/parakeetWsResult.test.js b/test/helpers/parakeetWsResult.test.js
index b0b71d3d6..68b77f015 100644
--- a/test/helpers/parakeetWsResult.test.js
+++ b/test/helpers/parakeetWsResult.test.js
@@ -75,3 +75,35 @@ test("online sherpa parser tolerates non-string text payloads", () => {
"2026"
);
});
+
+test("online sherpa parser keeps the latest text when a segment is refined", () => {
+ assert.equal(
+ parseOnlineMessages([
+ JSON.stringify({ text: "the quick", is_final: true, segment: 0 }),
+ JSON.stringify({ text: "brown", is_final: true, segment: 1 }),
+ JSON.stringify({ text: "the quick fox", is_final: true, segment: 0 }),
+ ]),
+ "the quick fox brown"
+ );
+});
+
+test("online sherpa parser treats an identical duplicate final as a no-op", () => {
+ assert.equal(
+ parseOnlineMessages([
+ JSON.stringify({ text: "hello", is_final: true, segment: 0 }),
+ JSON.stringify({ text: "world", is_final: true, segment: 1 }),
+ JSON.stringify({ text: "hello", is_final: true, segment: 0 }),
+ ]),
+ "hello world"
+ );
+});
+
+test("online sherpa parser suppresses a partial for an already-finalized segment", () => {
+ assert.equal(
+ parseOnlineMessages([
+ JSON.stringify({ text: "done", is_final: true, segment: 0 }),
+ JSON.stringify({ text: "late partial", is_final: false, segment: 0 }),
+ ]),
+ "done"
+ );
+});