Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
refactor(audio): Finalize native-only system audio capture
Gabriel Steinb0632f360a2d
diff --git a/main.js b/main.jsindex b9798e2b0..eab7ef4fb 100644--- a/main.js+++ b/main.js@@ -90,16 +90,6 @@ if (process.platform === "linux" && process.env.XDG_SESSION_TYPE === "wayland")app.commandLine.appendSwitch("enable-features", "UseOzonePlatform,WaylandWindowDecorations");}-// Enable system audio loopback capture on macOS via CoreAudio Taps (Electron 39+).-// Without these flags, getDisplayMedia({ audio: "loopback" }) returns a live audio track-// that produces only silence (all-zero samples).-if (process.platform === "darwin") {- app.commandLine.appendSwitch(- "enable-features",- "MacLoopbackAudioForScreenShare,MacSckSystemAudioLoopbackOverride,MacCatapSystemAudioLoopbackCapture"- );-}-// Set desktop filename so Wayland compositors can match windows to the .desktop entry.// This allows XDG portals (e.g. PipeWire) to persist permissions across sessions.if (process.platform === "linux") {@@ -539,28 +529,6 @@ async function startApp() {});- // Handle getDisplayMedia() calls from the renderer — auto-select the first- // screen source with loopback audio so no system picker dialog is shown.- const { desktopCapturer } = require("electron");- session.defaultSession.setDisplayMediaRequestHandler(async (_request, callback) => {- try {- const sources = await desktopCapturer.getSources({ types: ["screen"] });- debugLogger.debug("Display media sources", {- count: sources.length,- names: sources.map((s) => s.name),- });- if (!sources.length) {- debugLogger.error("No screen sources available — System Audio permission may be denied");- callback({});- return;- }- callback({ video: sources[0], audio: "loopback" });- } catch (err) {- debugLogger.error("Display media handler error", { error: err.message });- callback({});- }- });-windowManager.setActivationModeCache(environmentManager.getActivationMode());windowManager.setFloatingIconAutoHide(environmentManager.getFloatingIconAutoHide());windowManager.setPanelStartPosition(environmentManager.getPanelStartPosition());diff --git a/preload.js b/preload.jsindex 1b6e75c53..e60562040 100644--- a/preload.js+++ b/preload.js@@ -170,8 +170,7 @@ contextBridge.exposeInMainWorld("electronAPI", {// Clipboard functionscheckAccessibilityPermission: (silent) =>ipcRenderer.invoke("check-accessibility-permission", silent),- promptAccessibilityPermission: () =>- ipcRenderer.invoke("prompt-accessibility-permission"),+ promptAccessibilityPermission: () => ipcRenderer.invoke("prompt-accessibility-permission"),readClipboard: () => ipcRenderer.invoke("read-clipboard"),writeClipboard: (text) => ipcRenderer.invoke("write-clipboard", text),checkPasteTools: () => ipcRenderer.invoke("check-paste-tools"),@@ -608,9 +607,6 @@ contextBridge.exposeInMainWorld("electronAPI", {gcalGetUpcomingEvents: (windowMinutes) =>ipcRenderer.invoke("gcal-get-upcoming-events", windowMinutes),- // Desktop audio capture- getDesktopSources: (types) => ipcRenderer.invoke("get-desktop-sources", types),-// Google Calendar event listenersonGcalMeetingStarting: registerListener("gcal-meeting-starting",diff --git a/src/components/IntegrationsView.tsx b/src/components/IntegrationsView.tsxindex 0f7ce68ba..764e848a0 100644--- a/src/components/IntegrationsView.tsx+++ b/src/components/IntegrationsView.tsx@@ -36,12 +36,12 @@ export default function IntegrationsView() {}, [setGcalAccounts]);const handleConnect = useCallback(async () => {- if (systemAudio.isMacOS && !systemAudio.granted) {+ if (systemAudio.isMacOS && systemAudio.status === "denied") {setShowPermissionDialog(true);return;}await startOAuth();- }, [systemAudio.isMacOS, systemAudio.granted, startOAuth]);+ }, [systemAudio.isMacOS, systemAudio.status, startOAuth]);const handleDisconnect = useCallback(async (email: string) => {diff --git a/src/components/SettingsPage.tsx b/src/components/SettingsPage.tsxindex 15fef8963..494a984ec 100644--- a/src/components/SettingsPage.tsx+++ b/src/components/SettingsPage.tsx@@ -3324,16 +3324,18 @@ EOF`,buttonText={t("settingsPage.permissions.testAndGrant")}onOpenSettings={permissionsHook.openAccessibilitySettings}/>- <PermissionCard- icon={Monitor}- title={t("settingsPage.permissions.systemAudioTitle")}- description={t("settingsPage.permissions.systemAudioDescription")}- granted={systemAudio.granted}- onRequest={systemAudio.request}- buttonText={t("settingsPage.permissions.test")}- onOpenSettings={systemAudio.openSettings}- badge={t("settingsPage.permissions.optional")}- />+ {systemAudio.mode === "native" && (+ <PermissionCard+ icon={Monitor}+ title={t("settingsPage.permissions.systemAudioTitle")}+ description={t("settingsPage.permissions.systemAudioDescription")}+ granted={systemAudio.granted}+ onRequest={systemAudio.request}+ buttonText={t("settingsPage.permissions.test")}+ onOpenSettings={systemAudio.openSettings}+ badge={t("settingsPage.permissions.optional")}+ />+ )}</>)}</div>diff --git a/src/components/UpcomingMeetings.tsx b/src/components/UpcomingMeetings.tsxindex e9a5baf20..ac5599a50 100644--- a/src/components/UpcomingMeetings.tsx+++ b/src/components/UpcomingMeetings.tsx@@ -104,7 +104,7 @@ export default function UpcomingMeetings({ events, isLoading }: UpcomingMeetings{/* Empty state */}{!isLoading && events.length === 0 && (<div className="flex flex-col items-center justify-center py-8 px-3">- {systemAudio.isMacOS && !systemAudio.granted ? (+ {systemAudio.isMacOS && systemAudio.status === "denied" ? (<><Monitor size={24} className="text-muted-foreground/30 mb-2.5" /><p className="text-xs text-muted-foreground/60 text-center mb-3">diff --git a/src/components/notes/NotesOnboarding.tsx b/src/components/notes/NotesOnboarding.tsxindex 804422a23..b262a2d68 100644--- a/src/components/notes/NotesOnboarding.tsx+++ b/src/components/notes/NotesOnboarding.tsx@@ -53,6 +53,7 @@ export default function NotesOnboarding({ onComplete }: NotesOnboardingProps) {const { alertDialog, hideAlertDialog } = useDialogs();const {granted: systemAudioGranted,+ mode: systemAudioMode,request: requestSystemAudio,isMacOS,} = useSystemAudioPermission();@@ -185,7 +186,7 @@ export default function NotesOnboarding({ onComplete }: NotesOnboardingProps) {)}{/* System Audio Permission — macOS only */}- {isMacOS && (+ {isMacOS && systemAudioMode === "native" && (<divclassName={cn("rounded-lg border transition-colors duration-200",diff --git a/src/components/notes/PersonalNotesView.tsx b/src/components/notes/PersonalNotesView.tsxindex 534c7fdda..a536dd215 100644--- a/src/components/notes/PersonalNotesView.tsx+++ b/src/components/notes/PersonalNotesView.tsx@@ -111,7 +111,7 @@ export default function PersonalNotesView({const activeNote = notes.find((n) => n.id === activeNoteId) ?? null;- // Note recording uses the same dual-stream transcription as meeting mode.+ // Note recording uses the same meeting transcription pipeline.// The `isMeetingMode` ref distinguishes whether the recording was triggered// by the meeting hotkey (creates a separate note) or the note record button.const isMeetingModeRef = useRef(false);diff --git a/src/components/ui/PermissionsSection.tsx b/src/components/ui/PermissionsSection.tsxindex 1f7d44e2b..607375a13 100644--- a/src/components/ui/PermissionsSection.tsx+++ b/src/components/ui/PermissionsSection.tsx@@ -9,6 +9,7 @@ interface PermissionsSectionProps {permissions: UsePermissionsReturn;systemAudio: {granted: boolean;+ mode: "native" | "unsupported";request: () => Promise<boolean>;openSettings: () => Promise<void>;};@@ -49,16 +50,18 @@ export default function PermissionsSection({ permissions, systemAudio }: Permiss: undefined}/>- <PermissionCard- icon={Monitor}- title={t("onboarding.permissions.systemAudioTitle")}- description={t("onboarding.permissions.systemAudioDescription")}- granted={systemAudio.granted}- onRequest={systemAudio.request}- buttonText={t("onboarding.permissions.grant")}- onOpenSettings={systemAudio.openSettings}- badge={t("onboarding.permissions.optional")}- />+ {systemAudio.mode === "native" && (+ <PermissionCard+ icon={Monitor}+ title={t("onboarding.permissions.systemAudioTitle")}+ description={t("onboarding.permissions.systemAudioDescription")}+ granted={systemAudio.granted}+ onRequest={systemAudio.request}+ buttonText={t("onboarding.permissions.grant")}+ onOpenSettings={systemAudio.openSettings}+ badge={t("onboarding.permissions.optional")}+ />+ )}</>)}</div>diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex e4ff3fdfa..2b11eece5 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -2,7 +2,6 @@ import ReasoningService from "../services/ReasoningService";import { API_ENDPOINTS, buildApiUrl, normalizeBaseUrl } from "../config/constants";import logger from "../utils/logger";import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";-import { getSystemAudioStream, stopSystemAudioStream } from "../utils/systemAudio";import { isSecureEndpoint } from "../utils/urlUtils";import { withSessionRefresh } from "../lib/neonAuth";import { getBaseLanguageCode, validateLanguageForModel } from "../utils/languageSupport";@@ -112,13 +111,6 @@ class AudioManager {this.sttConfig = null;this.lastAudioBlob = null;this.lastAudioMetadata = null;-- // System audio capture- this.systemAudioEnabled = false;- this.systemAudioStream = null;- this._mixingContext = null;- this._mixingDestination = null;- this._systemAudioSource = null;}getWorkletBlobUrl() {@@ -196,30 +188,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.sttConfig = config;}- setSystemAudioEnabled(enabled) {- this.systemAudioEnabled = enabled;- }-- cleanupSystemAudio() {- stopSystemAudioStream(this.systemAudioStream);- this.systemAudioStream = null;-- if (this._systemAudioSource) {- try {- this._systemAudioSource.disconnect();- } catch {}- this._systemAudioSource = null;- }-- if (this._mixingContext) {- try {- this._mixingContext.close();- } catch {}- this._mixingContext = null;- }- this._mixingDestination = null;- }-getStreamingProvider() {const { cloudTranscriptionModel } = getSettings();if (REALTIME_MODELS.has(cloudTranscriptionModel)) {@@ -334,31 +302,12 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor););}- // Mix in system audio if enabled- let recordingStream = micStream;- if (this.systemAudioEnabled) {- const sysStream = await getSystemAudioStream();- if (sysStream) {- this.systemAudioStream = sysStream;- this._mixingContext = new AudioContext();- this._mixingDestination = this._mixingContext.createMediaStreamDestination();- const micSource = this._mixingContext.createMediaStreamSource(micStream);- this._systemAudioSource = this._mixingContext.createMediaStreamSource(sysStream);- micSource.connect(this._mixingDestination);- this._systemAudioSource.connect(this._mixingDestination);- recordingStream = this._mixingDestination.stream;- logger.info("System audio mixed into batch recording", {}, "audio");- } else {- logger.warn("System audio unavailable, recording mic only", {}, "audio");- }- }-// Silence detection: observe audio energy via AnalyserNodetry {this._silenceCtx = new AudioContext();this._silenceAnalyser = this._silenceCtx.createAnalyser();this._silenceAnalyser.fftSize = 2048;- const sourceNode = this._silenceCtx.createMediaStreamSource(recordingStream);+ const sourceNode = this._silenceCtx.createMediaStreamSource(micStream);sourceNode.connect(this._silenceAnalyser);this._peakRms = 0;const dataArray = new Uint8Array(this._silenceAnalyser.fftSize);@@ -377,7 +326,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this._peakRms = 1; // assume speech if detection fails}- this.mediaRecorder = new MediaRecorder(recordingStream);+ this.mediaRecorder = new MediaRecorder(micStream);this.audioChunks = [];this.recordingStartTime = Date.now();this.recordingMimeType = this.mediaRecorder.mimeType || "audio/webm";@@ -420,7 +369,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);await this.processAudio(audioBlob, { durationSeconds });micStream.getTracks().forEach((track) => track.stop());- this.cleanupSystemAudio();};this.mediaRecorder.start();@@ -476,7 +424,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);if (this.mediaRecorder.stream) {this.mediaRecorder.stream.getTracks().forEach((track) => track.stop());}- this.cleanupSystemAudio();return true;}@@ -2226,20 +2173,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.isStreaming = true;this.streamingSource.connect(this.streamingProcessor);- // Mix in system audio if enabled — connecting a second source to the same- // AudioWorkletNode sums the signals automatically via Web Audio API.- if (this.systemAudioEnabled) {- const sysStream = await getSystemAudioStream();- if (sysStream) {- this.systemAudioStream = sysStream;- this._systemAudioSource = audioContext.createMediaStreamSource(sysStream);- this._systemAudioSource.connect(this.streamingProcessor);- logger.info("System audio mixed into streaming recording", {}, "audio");- } else {- logger.warn("System audio unavailable, streaming mic only", {}, "audio");- }- }-const tPipeline = performance.now();// 3. Register IPC event listeners BEFORE connecting, so no transcript@@ -2717,8 +2650,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);this.streamingStream = null;}- this.cleanupSystemAudio();-this.isStreaming = false;}diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 2179ebd01..bd2a529a2 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -107,7 +107,6 @@ class IPCHandlers {this.sessionId = crypto.randomUUID();this.assemblyAiStreaming = null;this.deepgramStreaming = null;- this.openaiRealtimeStreaming = null;this._dictationStreaming = null;this._autoLearnEnabled = true; // Default on, synced from rendererthis._autoLearnDebounceTimer = null;@@ -2057,18 +2056,12 @@ class IPCHandlers {return { granted: true, status: "granted", mode: "unsupported" };}- if (this.audioTapManager?.isAvailable()) {+ if (this.audioTapManager?.isSupported()) {const status = this.audioTapManager.getPermissionStatus();return { granted: status === "granted", status, mode: "native" };}- const { systemPreferences } = require("electron");- const status = systemPreferences.getMediaAccessStatus("screen");- return {- granted: status === "granted",- status: status === "granted" ? "granted" : status === "denied" ? "denied" : "unknown",- mode: "legacy",- };+ return { granted: false, status: "unsupported", mode: "unsupported" };});ipcMain.handle("request-system-audio-access", async () => {@@ -2076,12 +2069,17 @@ class IPCHandlers {return { granted: true, status: "granted", mode: "unsupported" };}- if (!this.audioTapManager?.isAvailable()) {- return { granted: false, status: "unknown", mode: "legacy" };+ if (!this.audioTapManager?.isSupported()) {+ return { granted: false, status: "unsupported", mode: "unsupported" };}- const result = await this.audioTapManager.requestAccess();- return { ...result, mode: "native" };+ try {+ const result = await this.audioTapManager.requestAccess();+ return { ...result, mode: "native" };+ } catch (error) {+ debugLogger.error("System audio permission request failed", { error: error.message });+ return { granted: false, status: "unknown", mode: "native", error: error.message };+ }});// Auth: clear all session cookies for sign-out.@@ -2446,6 +2444,15 @@ class IPCHandlers {return data.clientSecret;};+ const getMeetingSystemAudioMode = () =>+ this.audioTapManager?.isSupported() ? "native" : "unsupported";++ const hasNativeMeetingSystemAudio = () => getMeetingSystemAudioMode() === "native";++ const isMeetingStreamingConnected = () =>+ !!this._meetingMicStreaming?.isConnected &&+ (!hasNativeMeetingSystemAudio() || !!this._meetingSystemStreaming?.isConnected);+const connectRealtimeStreaming = async (event, options) => {if (this._meetingMicStreaming?.isConnected) {await this._meetingMicStreaming.disconnect();@@ -2453,23 +2460,28 @@ class IPCHandlers {if (this._meetingSystemStreaming?.isConnected) {await this._meetingSystemStreaming.disconnect();}- if (this.openaiRealtimeStreaming?.isConnected) {- await this.openaiRealtimeStreaming.disconnect();- }-+ this._meetingMicStreaming = null;+ this._meetingSystemStreaming = null;const win = BrowserWindow.fromWebContents(event.sender);- const [micSecret, systemSecret] = await fetchRealtimeToken(event, options, { streams: 2 });-const connectOpts = {model: options.model,language: options.language,preconfigured: options.mode !== "byok",};- const pairs = [- { ref: "_meetingMicStreaming", secret: micSecret, source: "mic" },- { ref: "_meetingSystemStreaming", secret: systemSecret, source: "system" },- ];+ const pairs = hasNativeMeetingSystemAudio()+ ? (await fetchRealtimeToken(event, options, { streams: 2 })).map((secret, index) => ({+ ref: index === 0 ? "_meetingMicStreaming" : "_meetingSystemStreaming",+ secret,+ source: index === 0 ? "mic" : "system",+ }))+ : [+ {+ ref: "_meetingMicStreaming",+ secret: await fetchRealtimeToken(event, options),+ source: "mic",+ },+ ];for (const { ref, source } of pairs) {this[ref] = new OpenAIRealtimeStreaming();@@ -2480,12 +2492,38 @@ class IPCHandlers {pairs.map(({ ref, secret }) => this[ref].connect({ apiKey: secret, ...connectOpts })));- // Keep legacy reference for backward compat with prepare check- this.openaiRealtimeStreaming = this._meetingMicStreaming;-return win;};+ let meetingSendCounts = { mic: 0, system: 0 };++ const resetMeetingStreamingState = () => {+ this._meetingMicStreaming = null;+ this._meetingSystemStreaming = null;+ meetingSendCounts = { mic: 0, system: 0 };+ };++ const disconnectMeetingStreaming = async () => {+ const results = await Promise.all([+ this._meetingMicStreaming+ ? this._meetingMicStreaming.disconnect().catch(() => ({ text: "" }))+ : Promise.resolve({ text: "" }),+ this._meetingSystemStreaming+ ? this._meetingSystemStreaming.disconnect().catch(() => ({ text: "" }))+ : Promise.resolve({ text: "" }),+ ]);++ resetMeetingStreamingState();+ return results;+ };++ const rollbackMeetingTranscriptionStart = async () => {+ if (this.audioTapManager) {+ await this.audioTapManager.stop().catch(() => {});+ }+ await disconnectMeetingStreaming().catch(() => {});+ };+const setupDictationCallbacks = (streaming, event) => {streaming.onPartialTranscript = (text) =>event.sender.send("dictation-realtime-partial", text);@@ -2519,7 +2557,7 @@ class IPCHandlers {return { success: false, error: "Operation in progress" };}- if (this._meetingMicStreaming?.isConnected && this._meetingSystemStreaming?.isConnected) {+ if (isMeetingStreamingConnected()) {debugLogger.debug("Meeting transcription already prepared (warm connections)");return { success: true, alreadyPrepared: true };}@@ -2532,7 +2570,7 @@ class IPCHandlers {meetingTranscriptionPreparePromise = (async () => {try {await connectRealtimeStreaming(event, options);- debugLogger.debug("Meeting transcription prepared (dual WebSockets warm)");+ debugLogger.debug("Meeting transcription prepared (meeting streams warm)");return { success: true };} catch (error) {debugLogger.error("Meeting transcription prepare error", { error: error.message });@@ -2563,12 +2601,12 @@ class IPCHandlers {const systemAudioMode = getMeetingSystemAudioMode();// If already prepared (warm connections from prepare), just re-attach handlers- if (this._meetingMicStreaming?.isConnected && this._meetingSystemStreaming?.isConnected) {+ if (isMeetingStreamingConnected()) {debugLogger.debug("Meeting transcription start: reusing warm connections");const win = BrowserWindow.fromWebContents(event.sender);attachMeetingStreamingHandlers(this._meetingMicStreaming, win, "mic");- attachMeetingStreamingHandlers(this._meetingSystemStreaming, win, "system");if (systemAudioMode === "native") {+ attachMeetingStreamingHandlers(this._meetingSystemStreaming, win, "system");await startNativeMeetingSystemAudio(event);}return { success: true, systemAudioMode };@@ -2584,6 +2622,7 @@ class IPCHandlers {}return { success: true, systemAudioMode };} catch (error) {+ await rollbackMeetingTranscriptionStart();debugLogger.error("Meeting transcription start error", { error: error.message });return { success: false, error: error.message };} finally {@@ -2591,7 +2630,6 @@ class IPCHandlers {}});- let meetingSendCounts = { mic: 0, system: 0 };const sendMeetingAudio = (audioBuffer, source) => {const streaming = source === "mic" ? this._meetingMicStreaming : this._meetingSystemStreaming;if (!streaming) {@@ -2615,9 +2653,6 @@ class IPCHandlers {}};- const getMeetingSystemAudioMode = () =>- this.audioTapManager?.isAvailable() ? "native" : "legacy";-const startNativeMeetingSystemAudio = async (event) => {const win = BrowserWindow.fromWebContents(event.sender);await this.audioTapManager.start({@@ -2642,19 +2677,7 @@ class IPCHandlers {await this.audioTapManager.stop();}- const results = await Promise.all([- this._meetingMicStreaming- ? this._meetingMicStreaming.disconnect()- : Promise.resolve({ text: "" }),- this._meetingSystemStreaming- ? this._meetingSystemStreaming.disconnect()- : Promise.resolve({ text: "" }),- ]);-- this._meetingMicStreaming = null;- this._meetingSystemStreaming = null;- this.openaiRealtimeStreaming = null;- meetingSendCounts = { mic: 0, system: 0 };+ const results = await disconnectMeetingStreaming();return {success: true,@@ -4203,17 +4226,6 @@ class IPCHandlers {}return { success: true };});-- ipcMain.handle("get-desktop-sources", async (_event, types) => {- try {- const { desktopCapturer } = require("electron");- const sources = await desktopCapturer.getSources({ types: types || ["screen"] });- return sources.map((s) => ({ id: s.id, name: s.name }));- } catch (error) {- debugLogger.error("Failed to get desktop sources", { error: error.message }, "calendar");- return [];- }- });}broadcastToWindows(channel, payload) {diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.tsindex fadc3f9d2..dbe302932 100644--- a/src/hooks/useMeetingTranscription.ts+++ b/src/hooks/useMeetingTranscription.ts@@ -1,7 +1,6 @@import { useState, useEffect, useRef, useCallback } from "react";import { getSettings } from "../stores/settingsStore";import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";-import { getSystemAudioStream } from "../utils/systemAudio";import logger from "../utils/logger";export interface TranscriptSegment {@@ -221,13 +220,9 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {const [systemPartial, setSystemPartial] = useState("");const [error, setError] = useState<string | null>(null);- const audioContextRef = useRef<AudioContext | null>(null);const micContextRef = useRef<AudioContext | null>(null);- const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);- const processorRef = useRef<AudioWorkletNode | null>(null);const micSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);const micProcessorRef = useRef<AudioWorkletNode | null>(null);- const streamRef = useRef<MediaStream | null>(null);const micStreamRef = useRef<MediaStream | null>(null);const isRecordingRef = useRef(false);const isStartingRef = useRef(false);@@ -237,10 +232,10 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {const pendingCleanupRef = useRef<ReturnType<typeof setTimeout> | null>(null);const cleanup = useCallback(async () => {- const processors = [processorRef, micProcessorRef];- const sources = [sourceRef, micSourceRef];- const streams = [streamRef, micStreamRef];- const contexts = [audioContextRef, micContextRef];+ const processors = [micProcessorRef];+ const sources = [micSourceRef];+ const streams = [micStreamRef];+ const contexts = [micContextRef];for (const ref of processors) {await flushAndDisconnectProcessor(ref.current);@@ -376,13 +371,9 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {]);const streamsMs = performance.now() - startTime;- const systemAudioMode = startResult?.systemAudioMode || "legacy";- const stream = systemAudioMode === "legacy" ? await getSystemAudioStream() : null;-// Abort if stop was called during setupif (!isRecordingRef.current) {logger.info("Meeting transcription aborted during setup (stop called)", {}, "meeting");- stream?.getTracks().forEach((t) => t.stop());micResult?.getTracks().forEach((t) => t.stop());isStartingRef.current = false;return;@@ -394,7 +385,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {{ error: startResult?.error },"meeting");- stream?.getTracks().forEach((track) => track.stop());+ setError(startResult?.error || "Failed to start meeting transcription");micResult?.getTracks().forEach((track) => track.stop());isRecordingRef.current = false;isStartingRef.current = false;@@ -402,18 +393,19 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {return;}- if (systemAudioMode === "legacy" && !stream) {- logger.error("Could not capture system audio for meeting transcription", {}, "meeting");- micResult?.getTracks().forEach((track) => track.stop());+ const systemAudioMode = startResult.systemAudioMode || "unsupported";++ if (!micResult && systemAudioMode !== "native") {+ logger.error("Meeting transcription has no available audio source", {}, "meeting");+ setError(+ "No microphone is available and system audio capture is unsupported on this device."+ );await window.electronAPI?.meetingTranscriptionStop?.();isRecordingRef.current = false;isStartingRef.current = false;setIsRecording(false);return;}- if (stream) {- streamRef.current = stream;- }const partialSetters = { mic: setMicPartial, system: setSystemPartial };@@ -468,55 +460,9 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {});if (errorCleanup) ipcCleanupsRef.current.push(errorCleanup);- const pendingSystemChunks: ArrayBuffer[] = [];const pendingMicChunks: ArrayBuffer[] = [];let socketReady = false;- let systemPipelinePromise: Promise<{- source: MediaStreamAudioSourceNode;- processor: AudioWorkletNode;- }> | null = null;- if (stream) {- const loopbackRate = stream.getAudioTracks()[0]?.getSettings()?.sampleRate || 48000;- const systemNeedsResample = loopbackRate !== 24000;- const audioContext = new AudioContext({ sampleRate: loopbackRate });- await detachFromOutputDevice(audioContext);- audioContextRef.current = audioContext;-- logger.debug(- "System audio context created",- { loopbackRate, needsResample: systemNeedsResample },- "meeting"- );-- systemPipelinePromise = createAudioPipeline({- stream,- context: audioContext,- label: "Meeting system",- onChunk: (chunk) => {- if (!isRecordingRef.current) return;- let samples = new Int16Array(chunk);-- if (systemNeedsResample) {- const ratio = loopbackRate / 24000;- const outLen = Math.floor(samples.length / ratio);- const out = new Int16Array(outLen);- for (let i = 0; i < outLen; i++) {- out[i] = samples[Math.round(i * ratio)];- }- samples = out;- }-- const buf = samples.buffer;- if (socketReady) {- window.electronAPI?.meetingTranscriptionSend?.(buf, "system");- return;- }- pendingSystemChunks.push(buf.slice(0) as ArrayBuffer);- },- });- }-let micPipelinePromise: Promise<void> | null = null;if (micResult) {micStreamRef.current = micResult;@@ -552,13 +498,8 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {});}- const [systemPipeline] = await Promise.all(- [systemPipelinePromise, micPipelinePromise].filter(Boolean)- );-- if (systemPipeline) {- sourceRef.current = systemPipeline.source;- processorRef.current = systemPipeline.processor;+ if (micPipelinePromise) {+ await micPipelinePromise;}// Abort if stop was called during pipeline setup@@ -576,9 +517,6 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {isStartingRef.current = false;socketReady = true;- for (const chunk of pendingSystemChunks) {- window.electronAPI?.meetingTranscriptionSend?.(chunk, "system");- }for (const chunk of pendingMicChunks) {window.electronAPI?.meetingTranscriptionSend?.(chunk, "mic");}@@ -588,7 +526,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {"Meeting transcription started successfully",{systemAudioMode,- bufferedChunks: pendingSystemChunks.length + pendingMicChunks.length,+ bufferedChunks: pendingMicChunks.length,streamsMs: Math.round(streamsMs),totalMs: Math.round(totalMs),wasPrepared: isPreparedRef.current,@@ -601,6 +539,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {{ error: (err as Error).message },"meeting");+ setError((err as Error).message);isRecordingRef.current = false;isStartingRef.current = false;setIsRecording(false);diff --git a/src/hooks/useNoteRecording.ts b/src/hooks/useNoteRecording.tsdeleted file mode 100644index 5acb8e6dd..000000000--- a/src/hooks/useNoteRecording.ts+++ /dev/null@@ -1,196 +0,0 @@-import { useState, useEffect, useRef, useCallback } from "react";-import { useTranslation } from "react-i18next";-import AudioManager from "../helpers/audioManager";-import logger from "../utils/logger";-import { getRecordingErrorTitle } from "../utils/recordingErrors";--interface UseNoteRecordingOptions {- onTranscriptionComplete: (text: string) => void;- onPartialTranscript?: (text: string) => void;- onStreamingCommit?: (text: string) => void;- onError?: (error: { title: string; description: string }) => void;- systemAudioEnabled?: boolean;-}--interface UseNoteRecordingReturn {- isRecording: boolean;- isProcessing: boolean;- isStreaming: boolean;- partialTranscript: string;- streamingCommit: string | null;- consumeStreamingCommit: () => void;- startRecording: () => Promise<void>;- stopRecording: () => Promise<void>;- cancelRecording: () => void;-}--export function useNoteRecording({- onTranscriptionComplete,- onPartialTranscript,- onStreamingCommit: onStreamingCommitCb,- onError,- systemAudioEnabled = false,-}: UseNoteRecordingOptions): UseNoteRecordingReturn {- const { t } = useTranslation();- const [isRecording, setIsRecording] = useState(false);- const [isProcessing, setIsProcessing] = useState(false);- const [isStreaming, setIsStreaming] = useState(false);- const [partialTranscript, setPartialTranscript] = useState("");- const [streamingCommits, setStreamingCommits] = useState<string[]>([]);- const audioManagerRef = useRef<InstanceType<typeof AudioManager> | null>(null);-- const callbacksRef = useRef({- onTranscriptionComplete,- onPartialTranscript,- onStreamingCommitCb,- onError,- });- callbacksRef.current = {- onTranscriptionComplete,- onPartialTranscript,- onStreamingCommitCb,- onError,- };-- useEffect(() => {- const manager = new AudioManager();- audioManagerRef.current = manager;- manager.setSkipReasoning(true);- manager.setSystemAudioEnabled(systemAudioEnabled);-- manager.setCallbacks({- onStateChange: ({- isRecording,- isProcessing,- isStreaming,- }: {- isRecording: boolean;- isProcessing: boolean;- isStreaming?: boolean;- }) => {- setIsRecording(isRecording);- setIsProcessing(isProcessing);- setIsStreaming(isStreaming ?? false);- if (!isStreaming) {- setPartialTranscript("");- setStreamingCommits([]);- }- },- onError: (error: { title: string; description: string; code?: string }) => {- const title = getRecordingErrorTitle(error, t);- callbacksRef.current.onError?.({ title, description: error.description });- },- onPartialTranscript: (text: string) => {- setPartialTranscript(text);- callbacksRef.current.onPartialTranscript?.(text);- },- onStreamingCommit: (text: string) => {- setStreamingCommits((pending) => [...pending, text]);- callbacksRef.current.onStreamingCommitCb?.(text);- },- onTranscriptionComplete: (result: {- success: boolean;- text: string;- source?: string;- limitReached?: boolean;- wordsUsed?: number;- wordsRemaining?: number;- }) => {- if (result.success) {- callbacksRef.current.onTranscriptionComplete(result.text);- if (manager.shouldUseStreaming()) {- manager.warmupStreamingConnection();- }- }- },- });-- manager.setContext("notes");- window.electronAPI.getSttConfig?.().then((config) => {- if (config?.success && audioManagerRef.current) {- audioManagerRef.current.setSttConfig(config);- if (manager.shouldUseStreaming()) {- manager.warmupStreamingConnection();- }- }- });-- return () => {- manager.cleanup();- audioManagerRef.current = null;- };- }, []);-- useEffect(() => {- audioManagerRef.current?.setSystemAudioEnabled(systemAudioEnabled);- }, [systemAudioEnabled]);-- const startRecording = useCallback(async () => {- const manager = audioManagerRef.current;- if (!manager) return;-- const state = manager.getState();- if (state.isRecording || state.isProcessing) return;-- // Retry STT config fetch if it wasn't loaded on mount (e.g. auth wasn't ready)- if (!manager.sttConfig) {- const config = await window.electronAPI.getSttConfig?.();- if (config?.success) {- manager.setSttConfig(config);- }- }-- const didStart = manager.shouldUseStreaming()- ? await manager.startStreamingRecording()- : await manager.startRecording();-- if (!didStart) {- logger.debug("Note recording failed to start", {}, "notes");- }- }, []);-- const stopRecording = useCallback(async () => {- const manager = audioManagerRef.current;- if (!manager) return;-- const state = manager.getState();- if (!state.isRecording) return;-- if (state.isStreaming) {- await manager.stopStreamingRecording();- } else {- manager.stopRecording();- }- }, []);-- const cancelRecording = useCallback(() => {- const manager = audioManagerRef.current;- if (!manager) return;-- const state = manager.getState();- if (state.isStreaming) {- manager.stopStreamingRecording();- } else {- manager.cancelRecording();- }- }, []);-- const consumeStreamingCommit = useCallback(- () => setStreamingCommits((pending) => pending.slice(1)),- []- );-- const streamingCommit = streamingCommits[0] ?? null;-- return {- isRecording,- isProcessing,- isStreaming,- partialTranscript,- streamingCommit,- consumeStreamingCommit,- startRecording,- stopRecording,- cancelRecording,- };-}diff --git a/src/hooks/useSystemAudioPermission.ts b/src/hooks/useSystemAudioPermission.tsindex 42dc5eb3c..05073f141 100644--- a/src/hooks/useSystemAudioPermission.ts+++ b/src/hooks/useSystemAudioPermission.ts@@ -1,23 +1,31 @@import { useState, useCallback, useEffect, useRef } from "react";import { getCachedPlatform } from "../utils/platform";+import type { SystemAudioAccessResult } from "../types/electron";++const NON_MACOS_ACCESS: SystemAudioAccessResult = {+ granted: true,+ status: "granted",+ mode: "unsupported",+};export function useSystemAudioPermission() {const isMacOS = getCachedPlatform() === "darwin";- const [granted, setGranted] = useState(!isMacOS);- const [accessMode, setAccessMode] = useState<"native" | "legacy" | "unsupported">(- isMacOS ? "legacy" : "unsupported"+ const [access, setAccess] = useState<SystemAudioAccessResult | null>(+ isMacOS ? null : NON_MACOS_ACCESS);+ const [isChecking, setIsChecking] = useState(false);const checkingRef = useRef(false);const check = useCallback(async () => {if (!isMacOS || checkingRef.current) return;checkingRef.current = true;+ setIsChecking(true);try {const result = await window.electronAPI?.checkSystemAudioAccess?.();- setGranted(result?.granted ?? false);- setAccessMode(result?.mode ?? "legacy");+ setAccess(result ?? { granted: false, status: "unsupported", mode: "unsupported" });} finally {checkingRef.current = false;+ setIsChecking(false);}}, [isMacOS]);@@ -38,26 +46,30 @@ export function useSystemAudioPermission() {await window.electronAPI?.openSystemAudioSettings?.();}, []);- // Trigger the native macOS permission prompt via getDisplayMedia (used in onboarding)const request = useCallback(async (): Promise<boolean> => {if (!isMacOS) return true;- if (accessMode === "native") {- const result = await window.electronAPI?.requestSystemAudioAccess?.();- const isGranted = result?.granted ?? false;- setGranted(isGranted);- setAccessMode(result?.mode ?? "native");- return isGranted;- }- try {- const stream = await navigator.mediaDevices.getDisplayMedia({ audio: true, video: true });- stream.getTracks().forEach((t) => t.stop());- setGranted(true);- return true;- } catch {+ const currentAccess = access ??+ (await window.electronAPI?.checkSystemAudioAccess?.()) ?? {+ granted: false,+ status: "unsupported" as const,+ mode: "unsupported" as const,+ };++ if (currentAccess.mode !== "native") {+ setAccess(currentAccess);return false;}- }, [accessMode, isMacOS]);- return { granted, request, openSettings, check, isMacOS };+ const result = await window.electronAPI?.requestSystemAudioAccess?.();+ const nextAccess = result ?? currentAccess;+ setAccess(nextAccess);+ return nextAccess.granted;+ }, [access, isMacOS]);++ const granted = access?.granted ?? false;+ const status = access?.status ?? "unknown";+ const mode = access?.mode ?? "unsupported";++ return { granted, status, mode, isChecking, request, openSettings, check, isMacOS };}diff --git a/src/types/electron.ts b/src/types/electron.tsindex c138b51a7..b90a0f4c4 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -118,7 +118,7 @@ export interface AudioDiagnosticsResult {export interface SystemAudioAccessResult {granted: boolean;status: "granted" | "denied" | "unknown" | "unsupported";- mode: "native" | "legacy" | "unsupported";+ mode: "native" | "unsupported";error?: string;}@@ -1134,7 +1134,11 @@ declare global {provider?: string;model?: string;language?: string;- }) => Promise<{ success: boolean; error?: string; systemAudioMode?: "native" | "legacy" }>;+ }) => Promise<{+ success: boolean;+ error?: string;+ systemAudioMode?: "native" | "unsupported";+ }>;meetingTranscriptionSend?: (buffer: ArrayBuffer, source: "mic" | "system") => void;meetingTranscriptionStop?: () => Promise<{success: boolean;@@ -1166,9 +1170,6 @@ declare global {onDictationRealtimeError?: (callback: (error: string) => void) => () => void;onDictationRealtimeSessionEnd?: (callback: (data: { text: string }) => void) => () => void;- // Desktop audio capture- getDesktopSources?: (types: string[]) => Promise<Array<{ id: string; name: string }>>;-// Google Calendar event listenersonGcalMeetingStarting?: (callback: (data: any) => void) => () => void;onGcalMeetingEnded?: (callback: (data: any) => void) => () => void;diff --git a/src/utils/systemAudio.ts b/src/utils/systemAudio.tsdeleted file mode 100644index 2ce538cf6..000000000--- a/src/utils/systemAudio.ts+++ /dev/null@@ -1,97 +0,0 @@-import logger from "./logger";--// Module-level reference prevents the hidden video element from being garbage-// collected while the capture session is active. getDisplayMedia requires-// active video frame consumption to keep the loopback audio pump alive.-let activeVideoSink: HTMLVideoElement | null = null;--/**- * Capture system audio via getDisplayMedia (CoreAudio Taps on macOS / Electron 39+).- * Returns a MediaStream with audio (and video tracks that must stay alive).- * Returns null if capture fails or no audio tracks are available.- */-export async function getSystemAudioStream(): Promise<MediaStream | null> {- try {- // Use getDisplayMedia (handled by setDisplayMediaRequestHandler in main process)- // which properly captures system audio via macOS CoreAudio Taps.- // Disable all audio processing — echo cancellation and noise suppression- // on loopback audio can cancel out the captured system audio entirely,- // especially when Bluetooth headphones change the audio routing.- const stream = await navigator.mediaDevices.getDisplayMedia({- audio: {- echoCancellation: false,- noiseSuppression: false,- autoGainControl: false,- },- video: true,- });-- const audioTracks = stream.getAudioTracks();- const videoTracks = stream.getVideoTracks();-- // Belt-and-suspenders: also apply constraints directly on the track- // in case getDisplayMedia ignored the initial constraints.- if (audioTracks[0]) {- try {- await audioTracks[0].applyConstraints({- echoCancellation: false,- noiseSuppression: false,- autoGainControl: false,- });- } catch {}- }-- logger.debug(- "Display media stream obtained",- {- audioTracks: audioTracks.length,- videoTracks: videoTracks.length,- audioSettings: audioTracks[0]?.getSettings(),- },- "audio"- );-- if (!audioTracks.length) {- logger.error("No audio track in display media stream", {}, "audio");- videoTracks.forEach((t) => t.stop());- return null;- }-- // getDisplayMedia requires active video frame consumption to pump the capture- // session. Without a consumer, the loopback audio track may produce silence.- // Attach the stream to a hidden <video> element so Chromium pulls frames and- // keeps the session active.- // IMPORTANT: Store in module-level variable to prevent garbage collection.- activeVideoSink = document.createElement("video");- activeVideoSink.srcObject = new MediaStream(videoTracks);- activeVideoSink.muted = true;- activeVideoSink.play().catch(() => {});-- audioTracks[0].addEventListener("ended", () => {- logger.error("System audio track ended unexpectedly", {}, "audio");- if (activeVideoSink) {- activeVideoSink.srcObject = null;- activeVideoSink = null;- }- });-- return stream;- } catch (err) {- logger.error("Failed to capture system audio", { error: (err as Error).message }, "audio");- return null;- }-}--/**- * Stop all tracks (audio + video) on a system audio stream.- */-export function stopSystemAudioStream(stream: MediaStream | null): void {- if (!stream) return;- try {- stream.getTracks().forEach((t) => t.stop());- } catch {}- if (activeVideoSink) {- activeVideoSink.srcObject = null;- activeVideoSink = null;- }-}