Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat: batch meeting transcription via BaseTen chain
Gabriel Stein21cf3402f345
diff --git a/package-lock.json b/package-lock.jsonindex a59071e08..b794dbdc5 100644--- a/package-lock.json+++ b/package-lock.json@@ -20,6 +20,7 @@"@radix-ui/react-select": "^2.2.5","@radix-ui/react-slot": "^1.2.3","@radix-ui/react-tabs": "^1.1.12",+ "@vercel/blob": "^2.3.0","better-sqlite3": "^11.10.0","class-variance-authority": "^0.7.1","clsx": "^2.1.1",@@ -6612,6 +6613,22 @@"integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==","license": "ISC"},+ "node_modules/@vercel/blob": {+ "version": "2.3.0",+ "resolved": "https://registry.npmjs.org/@vercel/blob/-/blob-2.3.0.tgz",+ "integrity": "sha512-oYWiJbWRQ7gz9Mj0X/NHFJ3OcLMOBzq/2b3j6zeNrQmtFo6dHwU8FAwNpxVIYddVMd+g8eqEi7iRueYx8FtM0Q==",+ "license": "Apache-2.0",+ "dependencies": {+ "async-retry": "^1.3.3",+ "is-buffer": "^2.0.5",+ "is-node-process": "^1.2.0",+ "throttleit": "^2.1.0",+ "undici": "^6.23.0"+ },+ "engines": {+ "node": ">=20.0.0"+ }+ },"node_modules/@vitejs/plugin-react": {"version": "4.7.0","resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz",@@ -7247,6 +7264,24 @@"node": ">=0.12.0"}},+ "node_modules/async-retry": {+ "version": "1.3.3",+ "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz",+ "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==",+ "license": "MIT",+ "dependencies": {+ "retry": "0.13.1"+ }+ },+ "node_modules/async-retry/node_modules/retry": {+ "version": "0.13.1",+ "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",+ "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",+ "license": "MIT",+ "engines": {+ "node": ">= 4"+ }+ },"node_modules/asynckit": {"version": "0.4.0","resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",@@ -10985,6 +11020,29 @@"url": "https://github.com/sponsors/wooorm"}},+ "node_modules/is-buffer": {+ "version": "2.0.5",+ "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",+ "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==",+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/feross"+ },+ {+ "type": "patreon",+ "url": "https://www.patreon.com/feross"+ },+ {+ "type": "consulting",+ "url": "https://feross.org/support"+ }+ ],+ "license": "MIT",+ "engines": {+ "node": ">=4"+ }+ },"node_modules/is-core-module": {"version": "2.16.1","resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz",@@ -11063,6 +11121,12 @@"node": ">=8"}},+ "node_modules/is-node-process": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz",+ "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==",+ "license": "MIT"+ },"node_modules/is-plain-obj": {"version": "4.1.0","resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz",@@ -15577,6 +15641,18 @@"real-require": "^0.2.0"}},+ "node_modules/throttleit": {+ "version": "2.1.0",+ "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz",+ "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==",+ "license": "MIT",+ "engines": {+ "node": ">=18"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },"node_modules/through": {"version": "2.3.8","resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",@@ -15897,6 +15973,15 @@"node": "*"}},+ "node_modules/undici": {+ "version": "6.23.0",+ "resolved": "https://registry.npmjs.org/undici/-/undici-6.23.0.tgz",+ "integrity": "sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==",+ "license": "MIT",+ "engines": {+ "node": ">=18.17"+ }+ },"node_modules/undici-types": {"version": "6.21.0","resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",diff --git a/package.json b/package.jsonindex 06b0ba6ef..37e1efdf1 100644--- a/package.json+++ b/package.json@@ -99,6 +99,7 @@"@radix-ui/react-select": "^2.2.5","@radix-ui/react-slot": "^1.2.3","@radix-ui/react-tabs": "^1.1.12",+ "@vercel/blob": "^2.3.0","better-sqlite3": "^11.10.0","class-variance-authority": "^0.7.1","clsx": "^2.1.1",diff --git a/preload.js b/preload.jsindex d02a7ede8..44f15c2f2 100644--- a/preload.js+++ b/preload.js@@ -395,6 +395,10 @@ contextBridge.exposeInMainWorld("electronAPI", {(callback) => (_event, data) => callback(data)),+ // Meeting chain transcription (BaseTen)+ meetingTranscribeChain: (blobUrl, opts) =>+ ipcRenderer.invoke("meeting-transcribe-chain", blobUrl, opts),+// Usage limit events (for showing UpgradePrompt in ControlPanel)notifyLimitReached: (data) => ipcRenderer.send("limit-reached", data),onLimitReached: registerListener("limit-reached", (callback) => (_event, data) => callback(data)),diff --git a/src/components/notes/NoteEditor.tsx b/src/components/notes/NoteEditor.tsxindex d1c14a996..bb2c4e2f9 100644--- a/src/components/notes/NoteEditor.tsx+++ b/src/components/notes/NoteEditor.tsx@@ -53,9 +53,9 @@ interface NoteEditorProps {actionPicker?: React.ReactNode;actionProcessingState?: ActionProcessingState;actionName?: string | null;- isMeetingTranscribing?: boolean;+ isMeetingRecording?: boolean;+ isMeetingProcessing?: boolean;meetingTranscript?: string;- meetingPartialTranscript?: string;meetingEvent?: { summary: string };onStopMeetingRecording?: () => void;}@@ -150,9 +150,9 @@ export default function NoteEditor({actionPicker,actionProcessingState,actionName,- isMeetingTranscribing,+ isMeetingRecording,+ isMeetingProcessing,meetingTranscript,- meetingPartialTranscript,meetingEvent,onStopMeetingRecording,}: NoteEditorProps) {@@ -763,15 +763,15 @@ export default function NoteEditor({style={{ background: "linear-gradient(to bottom, transparent, var(--color-background))" }}/><DictationWidget- isRecording={isRecording || !!isMeetingTranscribing}- isProcessing={isProcessing}+ isRecording={isRecording || !!isMeetingRecording}+ isProcessing={isProcessing || !!isMeetingProcessing}onStart={handleStartRecording}- onStop={isMeetingTranscribing ? onStopMeetingRecording! : onStopRecording}- actionPicker={isMeetingTranscribing ? undefined : actionPicker}+ onStop={isMeetingRecording ? onStopMeetingRecording! : onStopRecording}+ actionPicker={isMeetingRecording ? undefined : actionPicker}/></div>- {(isMeetingTranscribing || meetingTranscript) && (+ {(isMeetingRecording || isMeetingProcessing || meetingTranscript) && (<div className="border-t border-border/50 bg-surface-0/50 dark:bg-surface-1/30"><div className="px-4 py-1.5 flex items-center gap-1.5"><span className="text-[10px] font-semibold uppercase tracking-wider text-muted-foreground/50">@@ -779,12 +779,13 @@ export default function NoteEditor({</span></div><div className="px-4 pb-3 max-h-32 overflow-y-auto">- <p className="text-xs text-muted-foreground/70 leading-relaxed whitespace-pre-wrap">- {meetingTranscript}- {meetingPartialTranscript && (- <span className="text-muted-foreground/40">{meetingPartialTranscript}</span>- )}- </p>+ {isMeetingProcessing && !meetingTranscript ? (+ <p className="text-xs text-muted-foreground/50 italic">Transcribing meeting audio...</p>+ ) : (+ <p className="text-xs text-muted-foreground/70 leading-relaxed whitespace-pre-wrap">+ {meetingTranscript}+ </p>+ )}</div></div>)}diff --git a/src/components/notes/PersonalNotesView.tsx b/src/components/notes/PersonalNotesView.tsxindex 0d01bdaa1..07512bb6e 100644--- a/src/components/notes/PersonalNotesView.tsx+++ b/src/components/notes/PersonalNotesView.tsx@@ -84,12 +84,14 @@ export default function PersonalNotesView({} = useNotesOnboarding();const {- isTranscribing: isMeetingTranscribing,+ isRecording: isMeetingRecording,+ isProcessing: isMeetingProcessing,transcript: meetingTranscript,- partialTranscript: meetingPartialTranscript,+ error: meetingError,startTranscription: startMeetingTranscription,stopTranscription: stopMeetingTranscription,} = useMeetingTranscription();+ const isMeetingActive = isMeetingRecording || isMeetingProcessing;const [activeMeetingEvent, setActiveMeetingEvent] = useState<any>(null);const meetingNoteIdRef = useRef<number | null>(null);@@ -346,12 +348,12 @@ export default function PersonalNotesView({onMeetingRecordingRequestHandled,]);- const prevTranscribingRef = useRef(false);+ const prevProcessingRef = useRef(false);useEffect(() => {if (- prevTranscribingRef.current &&- !isMeetingTranscribing &&+ prevProcessingRef.current &&+ !isMeetingProcessing &&meetingNoteIdRef.current &&meetingTranscript) {@@ -363,8 +365,8 @@ export default function PersonalNotesView({meetingNoteIdRef.current = null;setActiveMeetingEvent(null);}- prevTranscribingRef.current = isMeetingTranscribing;- }, [isMeetingTranscribing, meetingTranscript]);+ prevProcessingRef.current = isMeetingProcessing;+ }, [isMeetingProcessing, meetingTranscript]);const editorNote = activeNote? { ...activeNote, title: localTitle, content: localContent }@@ -704,9 +706,9 @@ export default function PersonalNotesView({}: undefined}- isMeetingTranscribing={isMeetingTranscribing}+ isMeetingRecording={isMeetingRecording}+ isMeetingProcessing={isMeetingProcessing}meetingTranscript={meetingTranscript}- meetingPartialTranscript={meetingPartialTranscript}meetingEvent={activeMeetingEvent}onStopMeetingRecording={stopMeetingTranscription}actionProcessingState={actionProcessingState}diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 59aeeada4..0f66bf19f 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -1712,6 +1712,54 @@ class IPCHandlers {}});+ ipcMain.handle("meeting-transcribe-chain", async (event, blobUrl, opts = {}) => {+ try {+ const apiUrl = getApiUrl();+ if (!apiUrl) throw new Error("OpenWhispr API URL not configured");++ const cookieHeader = await getSessionCookies(event);+ if (!cookieHeader) throw new Error("No session cookies available");++ const response = await fetch(`${apiUrl}/api/transcribe-chain`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ Cookie: cookieHeader,+ },+ body: JSON.stringify({+ mediaUrl: blobUrl,+ skipCleanup: opts.skipCleanup ?? false,+ agentName: opts.agentName,+ customDictionary: opts.customDictionary,+ }),+ });++ const data = await response.json();++ if (!response.ok) {+ throw new Error(data.error || `Chain failed: ${response.status}`);+ }++ fetch(`${apiUrl}/api/delete-audio`, {+ method: "POST",+ headers: { "Content-Type": "application/json", Cookie: cookieHeader },+ body: JSON.stringify({ url: blobUrl }),+ }).catch((err) => debugLogger.warn("Blob cleanup failed", { error: err.message }));++ return {+ success: true,+ text: data.text,+ rawText: data.rawText,+ cleanedText: data.cleanedText,+ processingDurationSec: data.processingDurationSec,+ speedupFactor: data.speedupFactor,+ };+ } catch (error) {+ debugLogger.error("Meeting chain transcription error", { error: error.message });+ return { success: false, error: error.message };+ }+ });+ipcMain.handle("cloud-reason", async (event, text, opts = {}) => {try {const apiUrl = getApiUrl();diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.tsindex afbba221c..5baf1cc5d 100644--- a/src/hooks/useMeetingTranscription.ts+++ b/src/hooks/useMeetingTranscription.ts@@ -1,10 +1,13 @@import { useState, useEffect, useRef, useCallback } from "react";import logger from "../utils/logger";+import { buildWav } from "../utils/wavBuilder";+import { OPENWHISPR_API_URL } from "../config/constants";interface UseMeetingTranscriptionReturn {- isTranscribing: boolean;+ isRecording: boolean;+ isProcessing: boolean;transcript: string;- partialTranscript: string;+ error: string | null;startTranscription: () => Promise<void>;stopTranscription: () => Promise<void>;}@@ -81,22 +84,20 @@ const getSystemAudioStream = async (): Promise<MediaStream | null> => {};export function useMeetingTranscription(): UseMeetingTranscriptionReturn {- const [isTranscribing, setIsTranscribing] = useState(false);+ const [isRecording, setIsRecording] = useState(false);+ const [isProcessing, setIsProcessing] = useState(false);const [transcript, setTranscript] = useState("");- const [partialTranscript, setPartialTranscript] = useState("");+ const [error, setError] = useState<string | null>(null);const audioContextRef = useRef<AudioContext | null>(null);const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);const processorRef = useRef<AudioWorkletNode | null>(null);const streamRef = useRef<MediaStream | null>(null);- const ipcCleanupsRef = useRef<Array<() => void>>([]);- const isTranscribingRef = useRef(false);+ const isRecordingRef = useRef(false);const workletBlobUrlRef = useRef<string | null>(null);+ const chunksRef = useRef<ArrayBuffer[]>([]);const cleanup = useCallback(async () => {- ipcCleanupsRef.current.forEach((fn) => fn());- ipcCleanupsRef.current = [];-if (processorRef.current) {processorRef.current.port.postMessage("stop");processorRef.current.disconnect();@@ -126,20 +127,63 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {URL.revokeObjectURL(workletBlobUrlRef.current);workletBlobUrlRef.current = null;}-- await window.electronAPI?.deepgramStreamingStop?.();}, []);const stopTranscription = useCallback(async () => {- if (!isTranscribingRef.current) return;- isTranscribingRef.current = false;- setIsTranscribing(false);+ if (!isRecordingRef.current) return;+ isRecordingRef.current = false;+ setIsRecording(false);+await cleanup();++ const chunks = chunksRef.current;+ chunksRef.current = [];++ if (chunks.length === 0) {+ logger.info("Meeting transcription stopped (no audio captured)", {}, "meeting");+ return;+ }++ setIsProcessing(true);+ setError(null);++ try {+ const totalLength = chunks.reduce((sum, buf) => sum + buf.byteLength / 2, 0);+ const allSamples = new Int16Array(totalLength);+ let offset = 0;+ for (const buf of chunks) {+ const chunk = new Int16Array(buf);+ allSamples.set(chunk, offset);+ offset += chunk.length;+ }++ const wavBlob = buildWav(allSamples, 16000);++ const { upload } = await import("@vercel/blob/client");+ const blob = await upload(`meeting-${Date.now()}.wav`, wavBlob, {+ access: "public",+ handleUploadUrl: `${OPENWHISPR_API_URL}/api/upload-audio`,+ });++ const result = await window.electronAPI?.meetingTranscribeChain(blob.url);++ if (result?.success) {+ setTranscript(result.text);+ } else {+ setError(result?.error || "Transcription failed");+ }+ } catch (err) {+ setError((err as Error).message);+ logger.error("Meeting transcription failed", { error: (err as Error).message }, "meeting");+ } finally {+ setIsProcessing(false);+ }+logger.info("Meeting transcription stopped", {}, "meeting");}, [cleanup]);const startTranscription = useCallback(async () => {- if (isTranscribingRef.current) return;+ if (isRecordingRef.current) return;logger.info("Meeting transcription starting...", {}, "meeting");@@ -171,66 +215,28 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {const processor = new AudioWorkletNode(audioContext, "pcm-meeting-processor");processorRef.current = processor;+ chunksRef.current = [];+ setTranscript("");+ setError(null);+let chunkCount = 0;processor.port.onmessage = (event) => {- if (!isTranscribingRef.current) return;+ if (!isRecordingRef.current) return;chunkCount++;if (chunkCount <= 3 || chunkCount % 50 === 0) {logger.debug(- "Audio chunk sent",+ "Audio chunk buffered",{ chunk: chunkCount, bytes: event.data.byteLength },"meeting");}- window.electronAPI?.deepgramStreamingSend?.(event.data);+ chunksRef.current.push(event.data);};source.connect(processor);- const partialCleanup = window.electronAPI?.onDeepgramPartialTranscript?.((text) => {- logger.debug(- "Meeting partial transcript",- { length: text.length, preview: text.slice(-80) },- "meeting"- );- setPartialTranscript(text);- });- if (partialCleanup) ipcCleanupsRef.current.push(partialCleanup);-- const finalCleanup = window.electronAPI?.onDeepgramFinalTranscript?.((text) => {- logger.info(- "Meeting final transcript",- { length: text.length, preview: text.slice(-80) },- "meeting"- );- setTranscript(text);- setPartialTranscript("");- });- if (finalCleanup) ipcCleanupsRef.current.push(finalCleanup);-- const result = await window.electronAPI?.deepgramStreamingStart?.({- sampleRate: 16000,- forceNew: true,- });-- logger.info(- "Deepgram streaming start result",- { success: result?.success, error: result?.error },- "meeting"- );-- if (!result?.success) {- logger.error(- "Failed to start Deepgram streaming",- { error: result?.error, code: result?.code },- "meeting"- );- await cleanup();- return;- }-- isTranscribingRef.current = true;- setIsTranscribing(true);+ isRecordingRef.current = true;+ setIsRecording(true);logger.info("Meeting transcription started successfully", {}, "meeting");} catch (err) {logger.error(@@ -244,17 +250,18 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {useEffect(() => {return () => {- if (isTranscribingRef.current) {- isTranscribingRef.current = false;+ if (isRecordingRef.current) {+ isRecordingRef.current = false;cleanup();}};}, [cleanup]);return {- isTranscribing,+ isRecording,+ isProcessing,transcript,- partialTranscript,+ error,startTranscription,stopTranscription,};diff --git a/src/types/electron.ts b/src/types/electron.tsindex 54c90eb11..73e4c6745 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -911,6 +911,24 @@ declare global {windowMinutes?: number) => Promise<{ success: boolean; events: any[] }>;+ // Meeting chain transcription (BaseTen)+ meetingTranscribeChain?: (+ blobUrl: string,+ opts?: {+ skipCleanup?: boolean;+ agentName?: string;+ customDictionary?: string[];+ }+ ) => Promise<{+ success: boolean;+ text?: string;+ rawText?: string;+ cleanedText?: string;+ processingDurationSec?: number;+ speedupFactor?: number;+ error?: string;+ }>;+// Desktop audio capturegetDesktopSources?: (types: string[]) => Promise<Array<{ id: string; name: string }>>;diff --git a/src/utils/wavBuilder.ts b/src/utils/wavBuilder.tsnew file mode 100644index 000000000..2bb74b35b--- /dev/null+++ b/src/utils/wavBuilder.ts@@ -0,0 +1,30 @@+export function buildWav(samples: Int16Array, sampleRate: number): Blob {+ const dataSize = samples.length * 2;+ const buffer = new ArrayBuffer(44 + dataSize);+ const view = new DataView(buffer);++ const writeString = (offset: number, str: string) => {+ for (let i = 0; i < str.length; i++) {+ view.setUint8(offset + i, str.charCodeAt(i));+ }+ };++ writeString(0, "RIFF");+ view.setUint32(4, 36 + dataSize, true);+ writeString(8, "WAVE");+ writeString(12, "fmt ");+ view.setUint32(16, 16, true);+ view.setUint16(20, 1, true);+ view.setUint16(22, 1, true);+ view.setUint32(24, sampleRate, true);+ view.setUint32(28, sampleRate * 2, true);+ view.setUint16(32, 2, true);+ view.setUint16(34, 16, true);+ writeString(36, "data");+ view.setUint32(40, dataSize, true);++ const output = new Int16Array(buffer, 44);+ output.set(samples);++ return new Blob([buffer], { type: "audio/wav" });+}