Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat: stream chunked local transcription to preview during recording

Gabriel Stein6830ff45aec4
diff --git a/preload.js b/preload.js
index ba116b7d5..d520e5508 100644
--- a/preload.js
+++ b/preload.js
@@ -602,9 +602,15 @@ contextBridge.exposeInMainWorld("electronAPI", {
getAgentWindowBounds: () => ipcRenderer.invoke("get-agent-window-bounds"),
setAgentWindowBounds: (x, y, width, height) =>
ipcRenderer.invoke("set-agent-window-bounds", x, y, width, height),
- showTranscriptionPreview: (text) => ipcRenderer.invoke("show-transcription-preview", text),
onPreviewText: registerListener("preview-text", (callback) => (_event, text) => callback(text)),
+ onPreviewAppend: registerListener(
+ "preview-append",
+ (callback) => (_event, text) => callback(text)
+ ),
onPreviewHide: registerListener("preview-hide", (callback) => () => callback()),
+ startDictationPreview: (opts) => ipcRenderer.invoke("start-dictation-preview", opts),
+ stopDictationPreview: () => ipcRenderer.invoke("stop-dictation-preview"),
+ sendDictationPreviewAudio: (data) => ipcRenderer.send("dictation-preview-audio", data),
acquireRecordingLock: (pipeline) => ipcRenderer.invoke("acquire-recording-lock", pipeline),
releaseRecordingLock: (pipeline) => ipcRenderer.invoke("release-recording-lock", pipeline),
diff --git a/src/components/TranscriptionPreviewOverlay.tsx b/src/components/TranscriptionPreviewOverlay.tsx
index 520f8bcff..685ac72bf 100644
--- a/src/components/TranscriptionPreviewOverlay.tsx
+++ b/src/components/TranscriptionPreviewOverlay.tsx
@@ -1,8 +1,6 @@
import { useState, useEffect } from "react";
-import { useTranslation } from "react-i18next";
export default function TranscriptionPreviewOverlay() {
- const { t } = useTranslation();
const [text, setText] = useState("");
const [isVisible, setIsVisible] = useState(false);
@@ -12,17 +10,23 @@ export default function TranscriptionPreviewOverlay() {
setTimeout(() => setIsVisible(true), 50);
});
+ const cleanupAppend = window.electronAPI?.onPreviewAppend?.((chunk: string) => {
+ setText((prev) => (prev ? prev + " " + chunk : chunk));
+ });
+
const cleanupHide = window.electronAPI?.onPreviewHide?.(() => {
setIsVisible(false);
+ setTimeout(() => setText(""), 200);
});
return () => {
cleanupText?.();
+ cleanupAppend?.();
cleanupHide?.();
};
}, []);
- if (!text) {
+ if (!isVisible) {
return <div className="w-full h-full bg-transparent" />;
}
@@ -37,10 +41,14 @@ export default function TranscriptionPreviewOverlay() {
isVisible ? "translate-y-0 opacity-100" : "translate-y-2 opacity-0",
].join(" ")}
>
- <p className="text-[11px] text-muted-foreground font-medium">
- {t("transcriptionPreview.label")}
- </p>
- <p className="text-[12px] text-foreground leading-snug line-clamp-3 mt-0.5">{text}</p>
+ {text ? (
+ <p className="text-[12px] text-foreground leading-snug line-clamp-3">{text}</p>
+ ) : (
+ <div className="flex items-center gap-1.5">
+ <div className="w-1.5 h-1.5 rounded-full bg-primary animate-pulse" />
+ <p className="text-[11px] text-muted-foreground">Listening...</p>
+ </div>
+ )}
</div>
</div>
);
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 96920ead4..d21018c0c 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -344,6 +344,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this._silenceCtx = null;
this._silenceAnalyser = null;
+ this.cleanupPreview();
+
this.isRecording = false;
this.isProcessing = true;
this.onStateChange?.({ isRecording: false, isProcessing: true });
@@ -374,6 +376,37 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.isRecording = true;
this.onStateChange?.({ isRecording: true, isProcessing: false });
+ const {
+ showTranscriptionPreview,
+ useLocalWhisper,
+ localTranscriptionProvider,
+ whisperModel,
+ parakeetModel,
+ } = getSettings();
+ if (showTranscriptionPreview && useLocalWhisper) {
+ try {
+ const previewCtx = await this.getOrCreateAudioContext();
+ this._previewSource = previewCtx.createMediaStreamSource(micStream);
+
+ if (!this.workletModuleLoaded) {
+ await previewCtx.audioWorklet.addModule(this.getWorkletBlobUrl());
+ this.workletModuleLoaded = true;
+ }
+
+ this._previewProcessor = new AudioWorkletNode(previewCtx, "pcm-streaming-processor");
+ this._previewProcessor.port.onmessage = (event) => {
+ window.electronAPI?.sendDictationPreviewAudio?.(event.data);
+ };
+ this._previewSource.connect(this._previewProcessor);
+
+ const provider = localTranscriptionProvider === "nvidia" ? "nvidia" : "whisper";
+ const model = provider === "nvidia" ? parakeetModel : whisperModel;
+ window.electronAPI?.startDictationPreview?.({ provider, model });
+ } catch (e) {
+ logger.warn("Preview worklet setup failed", { error: e.message }, "audio");
+ }
+ }
+
return true;
} catch (error) {
let errorTitle = "Recording Error";
@@ -411,6 +444,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
cancelRecording() {
if (this.mediaRecorder && this.mediaRecorder.state === "recording") {
this.mediaRecorder.onstop = () => {
+ this.cleanupPreview();
this.isRecording = false;
this.isProcessing = false;
this.audioChunks = [];
@@ -617,9 +651,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
if (result.success && result.text) {
const rawText = result.text;
- if (getSettings().showTranscriptionPreview) {
- window.electronAPI?.showTranscriptionPreview?.(rawText);
- }
const reasoningStart = performance.now();
const text = await this.processTranscription(result.text, "local");
timings.reasoningProcessingDurationMs = Math.round(performance.now() - reasoningStart);
@@ -694,9 +725,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
if (result.success && result.text) {
const rawText = result.text;
- if (getSettings().showTranscriptionPreview) {
- window.electronAPI?.showTranscriptionPreview?.(rawText);
- }
const reasoningStart = performance.now();
const text = await this.processTranscription(result.text, "local-parakeet");
timings.reasoningProcessingDurationMs = Math.round(performance.now() - reasoningStart);
@@ -2533,6 +2561,19 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
return true;
}
+ cleanupPreview() {
+ if (this._previewProcessor) {
+ this._previewProcessor.port.postMessage("stop");
+ this._previewProcessor.disconnect();
+ this._previewProcessor = null;
+ }
+ if (this._previewSource) {
+ this._previewSource.disconnect();
+ this._previewSource = null;
+ }
+ window.electronAPI?.stopDictationPreview?.();
+ }
+
cleanupStreamingAudio() {
if (this.streamingFallbackRecorder?.state === "recording") {
try {
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index fe641c8ac..248507dad 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -2885,24 +2885,6 @@ class IPCHandlers {
let meetingSendCounts = { mic: 0, system: 0 };
- // Meeting mic captures at 24kHz (for OpenAI Realtime), but local engines
- // expect 16kHz. Downsample with linear interpolation (3:2 ratio).
- const downsample24kTo16k = (pcmBuffer) => {
- const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);
- const ratio = 1.5; // 24000 / 16000
- const outputLength = Math.floor(input.length / ratio);
- const output = new Int16Array(outputLength);
- for (let i = 0; i < outputLength; i++) {
- const srcIdx = i * ratio;
- const idx = Math.floor(srcIdx);
- const frac = srcIdx - idx;
- const s0 = input[idx];
- const s1 = idx + 1 < input.length ? input[idx + 1] : s0;
- output[i] = Math.round(s0 + frac * (s1 - s0));
- }
- return Buffer.from(output.buffer);
- };
-
const pcm16ToWav = (pcmBuffer, sampleRate = 16000, channels = 1) => {
const dataSize = pcmBuffer.length;
const header = Buffer.alloc(44);
@@ -2922,6 +2904,24 @@ class IPCHandlers {
return Buffer.concat([header, pcmBuffer]);
};
+ // Meeting mic captures at 24kHz (for OpenAI Realtime), but local engines
+ // expect 16kHz. Downsample with linear interpolation (3:2 ratio).
+ const downsample24kTo16k = (pcmBuffer) => {
+ const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);
+ const ratio = 1.5; // 24000 / 16000
+ const outputLength = Math.floor(input.length / ratio);
+ const output = new Int16Array(outputLength);
+ for (let i = 0; i < outputLength; i++) {
+ const srcIdx = i * ratio;
+ const idx = Math.floor(srcIdx);
+ const frac = srcIdx - idx;
+ const s0 = input[idx];
+ const s1 = idx + 1 < input.length ? input[idx + 1] : s0;
+ output[i] = Math.round(s0 + frac * (s1 - s0));
+ }
+ return Buffer.from(output.buffer);
+ };
+
let meetingLocalMode = false;
let meetingLocalBuffers = { mic: [], system: [] };
let meetingLocalTimer = null;
@@ -3013,6 +3013,56 @@ class IPCHandlers {
meetingLocalTranscribing = false;
};
+ let dictationPreviewMode = false;
+ let dictationPreviewBuffer = [];
+ let dictationPreviewTimer = null;
+ let dictationPreviewTranscribing = false;
+ let dictationPreviewProvider = null;
+ let dictationPreviewModel = null;
+
+ const transcribeDictationPreviewChunk = async () => {
+ if (dictationPreviewTranscribing) return;
+ if (!dictationPreviewBuffer.length) return;
+
+ dictationPreviewTranscribing = true;
+ try {
+ const pcm = Buffer.concat(dictationPreviewBuffer);
+ dictationPreviewBuffer = [];
+
+ const samples = new Int16Array(pcm.buffer, pcm.byteOffset, pcm.length / 2);
+ let sumSq = 0;
+ for (let i = 0; i < samples.length; i++) {
+ const n = samples[i] / 0x7fff;
+ sumSq += n * n;
+ }
+ const rms = Math.sqrt(sumSq / samples.length);
+ if (rms < 0.005) return;
+
+ const wav = pcm16ToWav(pcm);
+
+ let result;
+ if (dictationPreviewProvider === "nvidia") {
+ result = await this.parakeetManager.transcribeLocalParakeet(wav, {
+ model: dictationPreviewModel,
+ });
+ } else {
+ result = await this.whisperManager.transcribeLocalWhisper(wav, {
+ model: dictationPreviewModel,
+ });
+ }
+
+ if (result?.success && result.text?.trim()) {
+ this.windowManager.appendTranscriptionPreview(result.text.trim());
+ }
+ } catch (error) {
+ debugLogger.error("Dictation preview transcription chunk failed", {
+ error: error.message,
+ });
+ } finally {
+ dictationPreviewTranscribing = false;
+ }
+ };
+
const resetMeetingStreamingState = () => {
this._meetingMicStreaming = null;
this._meetingSystemStreaming = null;
@@ -3286,6 +3336,38 @@ class IPCHandlers {
return { success: true, text: result.text || "" };
});
+ ipcMain.handle("start-dictation-preview", async (_event, { provider, model }) => {
+ dictationPreviewMode = true;
+ dictationPreviewProvider = provider;
+ dictationPreviewModel = model;
+ dictationPreviewBuffer = [];
+ this.windowManager.showTranscriptionPreview("");
+ dictationPreviewTimer = setInterval(() => transcribeDictationPreviewChunk(), 1500);
+ return { success: true };
+ });
+
+ ipcMain.on("dictation-preview-audio", (_event, audioBuffer) => {
+ if (!dictationPreviewMode) return;
+ dictationPreviewBuffer.push(
+ Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer)
+ );
+ });
+
+ ipcMain.handle("stop-dictation-preview", async () => {
+ if (dictationPreviewTimer) {
+ clearInterval(dictationPreviewTimer);
+ dictationPreviewTimer = null;
+ }
+ await transcribeDictationPreviewChunk();
+ this.windowManager.hideTranscriptionPreview();
+ dictationPreviewMode = false;
+ dictationPreviewBuffer = [];
+ dictationPreviewTranscribing = false;
+ dictationPreviewProvider = null;
+ dictationPreviewModel = null;
+ return { success: true };
+ });
+
ipcMain.handle("update-transcription-text", async (_event, id, text, rawText) => {
try {
this.databaseManager.updateTranscriptionText(id, text, rawText);
@@ -4709,10 +4791,6 @@ class IPCHandlers {
return { success: true };
});
- ipcMain.handle("show-transcription-preview", (_, text) =>
- this.windowManager.showTranscriptionPreview(text)
- );
-
ipcMain.handle("resize-agent-window", async (_event, width, height) => {
this.windowManager.resizeAgentWindow(width, height);
return { success: true };
diff --git a/src/helpers/windowManager.js b/src/helpers/windowManager.js
index aca8848ea..a3253a5fa 100644
--- a/src/helpers/windowManager.js
+++ b/src/helpers/windowManager.js
@@ -25,7 +25,6 @@ class WindowManager {
this.notificationWindow = null;
this._notificationTimeout = null;
this.transcriptionPreviewWindow = null;
- this._previewTimeout = null;
this.updateNotificationWindow = null;
this._updateNotificationDismissed = false;
this.tray = null;
@@ -770,21 +769,14 @@ class WindowManager {
this.transcriptionPreviewWindow.webContents.send("preview-text", text);
this.transcriptionPreviewWindow.showInactive();
WindowPositionUtil.setupAlwaysOnTop(this.transcriptionPreviewWindow);
+ }
- if (this._previewTimeout) {
- clearTimeout(this._previewTimeout);
- }
- this._previewTimeout = setTimeout(() => {
- this.hideTranscriptionPreview();
- }, 5000);
+ appendTranscriptionPreview(text) {
+ if (!this.transcriptionPreviewWindow || this.transcriptionPreviewWindow.isDestroyed()) return;
+ this.transcriptionPreviewWindow.webContents.send("preview-append", text);
}
hideTranscriptionPreview() {
- if (this._previewTimeout) {
- clearTimeout(this._previewTimeout);
- this._previewTimeout = null;
- }
-
if (!this.transcriptionPreviewWindow || this.transcriptionPreviewWindow.isDestroyed()) return;
this.transcriptionPreviewWindow.webContents.send("preview-hide");
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 0ab3083d0..4ca8419f2 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -1358,9 +1358,15 @@ declare global {
} | null>;
updateNotificationReady?: () => Promise<void>;
updateNotificationRespond?: (action: string) => Promise<{ success: boolean }>;
- showTranscriptionPreview?: (text: string) => Promise<void>;
onPreviewText?: (callback: (text: string) => void) => () => void;
+ onPreviewAppend?: (callback: (text: string) => void) => () => void;
onPreviewHide?: (callback: () => void) => () => void;
+ startDictationPreview?: (opts: {
+ provider: string;
+ model: string;
+ }) => Promise<{ success: boolean }>;
+ stopDictationPreview?: () => Promise<{ success: boolean }>;
+ sendDictationPreviewAudio?: (data: ArrayBuffer) => void;
};
api?: {