Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat: stream chunked local transcription to preview during recording
Gabriel Stein6830ff45aec4
diff --git a/preload.js b/preload.jsindex ba116b7d5..d520e5508 100644--- a/preload.js+++ b/preload.js@@ -602,9 +602,15 @@ contextBridge.exposeInMainWorld("electronAPI", {getAgentWindowBounds: () => ipcRenderer.invoke("get-agent-window-bounds"),setAgentWindowBounds: (x, y, width, height) =>ipcRenderer.invoke("set-agent-window-bounds", x, y, width, height),- showTranscriptionPreview: (text) => ipcRenderer.invoke("show-transcription-preview", text),onPreviewText: registerListener("preview-text", (callback) => (_event, text) => callback(text)),+ onPreviewAppend: registerListener(+ "preview-append",+ (callback) => (_event, text) => callback(text)+ ),onPreviewHide: registerListener("preview-hide", (callback) => () => callback()),+ startDictationPreview: (opts) => ipcRenderer.invoke("start-dictation-preview", opts),+ stopDictationPreview: () => ipcRenderer.invoke("stop-dictation-preview"),+ sendDictationPreviewAudio: (data) => ipcRenderer.send("dictation-preview-audio", data),acquireRecordingLock: (pipeline) => ipcRenderer.invoke("acquire-recording-lock", pipeline),releaseRecordingLock: (pipeline) => ipcRenderer.invoke("release-recording-lock", pipeline),diff --git a/src/components/TranscriptionPreviewOverlay.tsx b/src/components/TranscriptionPreviewOverlay.tsxindex 520f8bcff..685ac72bf 100644--- a/src/components/TranscriptionPreviewOverlay.tsx+++ b/src/components/TranscriptionPreviewOverlay.tsx@@ -1,8 +1,6 @@import { useState, useEffect } from "react";-import { useTranslation } from "react-i18next";export default function TranscriptionPreviewOverlay() {- const { t } = useTranslation();const [text, setText] = useState("");const [isVisible, setIsVisible] = useState(false);@@ -12,17 +10,23 @@ export default function TranscriptionPreviewOverlay() {setTimeout(() => setIsVisible(true), 50);});+ const cleanupAppend = window.electronAPI?.onPreviewAppend?.((chunk: string) => {+ setText((prev) => (prev ? prev + " " + chunk : chunk));+ });+const cleanupHide = window.electronAPI?.onPreviewHide?.(() => {setIsVisible(false);+ setTimeout(() => setText(""), 200);});return () => {cleanupText?.();+ cleanupAppend?.();cleanupHide?.();};}, []);- if (!text) {+ if (!isVisible) {return <div className="w-full h-full bg-transparent" />;}@@ -37,10 +41,14 @@ export default function TranscriptionPreviewOverlay() {isVisible ? "translate-y-0 opacity-100" : "translate-y-2 opacity-0",].join(" ")}>- <p className="text-[11px] text-muted-foreground font-medium">- {t("transcriptionPreview.label")}- </p>- <p className="text-[12px] text-foreground leading-snug line-clamp-3 mt-0.5">{text}</p>+ {text ? (+ <p className="text-[12px] text-foreground leading-snug line-clamp-3">{text}</p>+ ) : (+ <div className="flex items-center gap-1.5">+ <div className="w-1.5 h-1.5 rounded-full bg-primary animate-pulse" />+ <p className="text-[11px] text-muted-foreground">Listening...</p>+ </div>+ )}</div></div>);diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 96920ead4..d21018c0c 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -344,6 +344,8 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this._silenceCtx = null;this._silenceAnalyser = null;+ this.cleanupPreview();+this.isRecording = false;this.isProcessing = true;this.onStateChange?.({ isRecording: false, isProcessing: true });@@ -374,6 +376,37 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.isRecording = true;this.onStateChange?.({ isRecording: true, isProcessing: false });+ const {+ showTranscriptionPreview,+ useLocalWhisper,+ localTranscriptionProvider,+ whisperModel,+ parakeetModel,+ } = getSettings();+ if (showTranscriptionPreview && useLocalWhisper) {+ try {+ const previewCtx = await this.getOrCreateAudioContext();+ this._previewSource = previewCtx.createMediaStreamSource(micStream);++ if (!this.workletModuleLoaded) {+ await previewCtx.audioWorklet.addModule(this.getWorkletBlobUrl());+ this.workletModuleLoaded = true;+ }++ this._previewProcessor = new AudioWorkletNode(previewCtx, "pcm-streaming-processor");+ this._previewProcessor.port.onmessage = (event) => {+ window.electronAPI?.sendDictationPreviewAudio?.(event.data);+ };+ this._previewSource.connect(this._previewProcessor);++ const provider = localTranscriptionProvider === "nvidia" ? "nvidia" : "whisper";+ const model = provider === "nvidia" ? parakeetModel : whisperModel;+ window.electronAPI?.startDictationPreview?.({ provider, model });+ } catch (e) {+ logger.warn("Preview worklet setup failed", { error: e.message }, "audio");+ }+ }+return true;} catch (error) {let errorTitle = "Recording Error";@@ -411,6 +444,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);cancelRecording() {if (this.mediaRecorder && this.mediaRecorder.state === "recording") {this.mediaRecorder.onstop = () => {+ this.cleanupPreview();this.isRecording = false;this.isProcessing = false;this.audioChunks = [];@@ -617,9 +651,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);if (result.success && result.text) {const rawText = result.text;- if (getSettings().showTranscriptionPreview) {- window.electronAPI?.showTranscriptionPreview?.(rawText);- }const reasoningStart = performance.now();const text = await this.processTranscription(result.text, "local");timings.reasoningProcessingDurationMs = Math.round(performance.now() - reasoningStart);@@ -694,9 +725,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);if (result.success && result.text) {const rawText = result.text;- if (getSettings().showTranscriptionPreview) {- window.electronAPI?.showTranscriptionPreview?.(rawText);- }const reasoningStart = performance.now();const text = await this.processTranscription(result.text, "local-parakeet");timings.reasoningProcessingDurationMs = Math.round(performance.now() - reasoningStart);@@ -2533,6 +2561,19 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);return true;}+ cleanupPreview() {+ if (this._previewProcessor) {+ this._previewProcessor.port.postMessage("stop");+ this._previewProcessor.disconnect();+ this._previewProcessor = null;+ }+ if (this._previewSource) {+ this._previewSource.disconnect();+ this._previewSource = null;+ }+ window.electronAPI?.stopDictationPreview?.();+ }+cleanupStreamingAudio() {if (this.streamingFallbackRecorder?.state === "recording") {try {diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex fe641c8ac..248507dad 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -2885,24 +2885,6 @@ class IPCHandlers {let meetingSendCounts = { mic: 0, system: 0 };- // Meeting mic captures at 24kHz (for OpenAI Realtime), but local engines- // expect 16kHz. Downsample with linear interpolation (3:2 ratio).- const downsample24kTo16k = (pcmBuffer) => {- const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);- const ratio = 1.5; // 24000 / 16000- const outputLength = Math.floor(input.length / ratio);- const output = new Int16Array(outputLength);- for (let i = 0; i < outputLength; i++) {- const srcIdx = i * ratio;- const idx = Math.floor(srcIdx);- const frac = srcIdx - idx;- const s0 = input[idx];- const s1 = idx + 1 < input.length ? input[idx + 1] : s0;- output[i] = Math.round(s0 + frac * (s1 - s0));- }- return Buffer.from(output.buffer);- };-const pcm16ToWav = (pcmBuffer, sampleRate = 16000, channels = 1) => {const dataSize = pcmBuffer.length;const header = Buffer.alloc(44);@@ -2922,6 +2904,24 @@ class IPCHandlers {return Buffer.concat([header, pcmBuffer]);};+ // Meeting mic captures at 24kHz (for OpenAI Realtime), but local engines+ // expect 16kHz. Downsample with linear interpolation (3:2 ratio).+ const downsample24kTo16k = (pcmBuffer) => {+ const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);+ const ratio = 1.5; // 24000 / 16000+ const outputLength = Math.floor(input.length / ratio);+ const output = new Int16Array(outputLength);+ for (let i = 0; i < outputLength; i++) {+ const srcIdx = i * ratio;+ const idx = Math.floor(srcIdx);+ const frac = srcIdx - idx;+ const s0 = input[idx];+ const s1 = idx + 1 < input.length ? input[idx + 1] : s0;+ output[i] = Math.round(s0 + frac * (s1 - s0));+ }+ return Buffer.from(output.buffer);+ };+let meetingLocalMode = false;let meetingLocalBuffers = { mic: [], system: [] };let meetingLocalTimer = null;@@ -3013,6 +3013,56 @@ class IPCHandlers {meetingLocalTranscribing = false;};+ let dictationPreviewMode = false;+ let dictationPreviewBuffer = [];+ let dictationPreviewTimer = null;+ let dictationPreviewTranscribing = false;+ let dictationPreviewProvider = null;+ let dictationPreviewModel = null;++ const transcribeDictationPreviewChunk = async () => {+ if (dictationPreviewTranscribing) return;+ if (!dictationPreviewBuffer.length) return;++ dictationPreviewTranscribing = true;+ try {+ const pcm = Buffer.concat(dictationPreviewBuffer);+ dictationPreviewBuffer = [];++ const samples = new Int16Array(pcm.buffer, pcm.byteOffset, pcm.length / 2);+ let sumSq = 0;+ for (let i = 0; i < samples.length; i++) {+ const n = samples[i] / 0x7fff;+ sumSq += n * n;+ }+ const rms = Math.sqrt(sumSq / samples.length);+ if (rms < 0.005) return;++ const wav = pcm16ToWav(pcm);++ let result;+ if (dictationPreviewProvider === "nvidia") {+ result = await this.parakeetManager.transcribeLocalParakeet(wav, {+ model: dictationPreviewModel,+ });+ } else {+ result = await this.whisperManager.transcribeLocalWhisper(wav, {+ model: dictationPreviewModel,+ });+ }++ if (result?.success && result.text?.trim()) {+ this.windowManager.appendTranscriptionPreview(result.text.trim());+ }+ } catch (error) {+ debugLogger.error("Dictation preview transcription chunk failed", {+ error: error.message,+ });+ } finally {+ dictationPreviewTranscribing = false;+ }+ };+const resetMeetingStreamingState = () => {this._meetingMicStreaming = null;this._meetingSystemStreaming = null;@@ -3286,6 +3336,38 @@ class IPCHandlers {return { success: true, text: result.text || "" };});+ ipcMain.handle("start-dictation-preview", async (_event, { provider, model }) => {+ dictationPreviewMode = true;+ dictationPreviewProvider = provider;+ dictationPreviewModel = model;+ dictationPreviewBuffer = [];+ this.windowManager.showTranscriptionPreview("");+ dictationPreviewTimer = setInterval(() => transcribeDictationPreviewChunk(), 1500);+ return { success: true };+ });++ ipcMain.on("dictation-preview-audio", (_event, audioBuffer) => {+ if (!dictationPreviewMode) return;+ dictationPreviewBuffer.push(+ Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer)+ );+ });++ ipcMain.handle("stop-dictation-preview", async () => {+ if (dictationPreviewTimer) {+ clearInterval(dictationPreviewTimer);+ dictationPreviewTimer = null;+ }+ await transcribeDictationPreviewChunk();+ this.windowManager.hideTranscriptionPreview();+ dictationPreviewMode = false;+ dictationPreviewBuffer = [];+ dictationPreviewTranscribing = false;+ dictationPreviewProvider = null;+ dictationPreviewModel = null;+ return { success: true };+ });+ipcMain.handle("update-transcription-text", async (_event, id, text, rawText) => {try {this.databaseManager.updateTranscriptionText(id, text, rawText);@@ -4709,10 +4791,6 @@ class IPCHandlers {return { success: true };});- ipcMain.handle("show-transcription-preview", (_, text) =>- this.windowManager.showTranscriptionPreview(text)- );-ipcMain.handle("resize-agent-window", async (_event, width, height) => {this.windowManager.resizeAgentWindow(width, height);return { success: true };diff --git a/src/helpers/windowManager.js b/src/helpers/windowManager.jsindex aca8848ea..a3253a5fa 100644--- a/src/helpers/windowManager.js+++ b/src/helpers/windowManager.js@@ -25,7 +25,6 @@ class WindowManager {this.notificationWindow = null;this._notificationTimeout = null;this.transcriptionPreviewWindow = null;- this._previewTimeout = null;this.updateNotificationWindow = null;this._updateNotificationDismissed = false;this.tray = null;@@ -770,21 +769,14 @@ class WindowManager {this.transcriptionPreviewWindow.webContents.send("preview-text", text);this.transcriptionPreviewWindow.showInactive();WindowPositionUtil.setupAlwaysOnTop(this.transcriptionPreviewWindow);+ }- if (this._previewTimeout) {- clearTimeout(this._previewTimeout);- }- this._previewTimeout = setTimeout(() => {- this.hideTranscriptionPreview();- }, 5000);+ appendTranscriptionPreview(text) {+ if (!this.transcriptionPreviewWindow || this.transcriptionPreviewWindow.isDestroyed()) return;+ this.transcriptionPreviewWindow.webContents.send("preview-append", text);}hideTranscriptionPreview() {- if (this._previewTimeout) {- clearTimeout(this._previewTimeout);- this._previewTimeout = null;- }-if (!this.transcriptionPreviewWindow || this.transcriptionPreviewWindow.isDestroyed()) return;this.transcriptionPreviewWindow.webContents.send("preview-hide");diff --git a/src/types/electron.ts b/src/types/electron.tsindex 0ab3083d0..4ca8419f2 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -1358,9 +1358,15 @@ declare global {} | null>;updateNotificationReady?: () => Promise<void>;updateNotificationRespond?: (action: string) => Promise<{ success: boolean }>;- showTranscriptionPreview?: (text: string) => Promise<void>;onPreviewText?: (callback: (text: string) => void) => () => void;+ onPreviewAppend?: (callback: (text: string) => void) => () => void;onPreviewHide?: (callback: () => void) => () => void;+ startDictationPreview?: (opts: {+ provider: string;+ model: string;+ }) => Promise<{ success: boolean }>;+ stopDictationPreview?: () => Promise<{ success: boolean }>;+ sendDictationPreviewAudio?: (data: ArrayBuffer) => void;};api?: {