Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

refactor(audio): Finalize native-only system audio capture

Gabriel Steinb0632f360a2d
diff --git a/main.js b/main.js
index b9798e2b0..eab7ef4fb 100644
--- a/main.js
+++ b/main.js
@@ -90,16 +90,6 @@ if (process.platform === "linux" && process.env.XDG_SESSION_TYPE === "wayland")
app.commandLine.appendSwitch("enable-features", "UseOzonePlatform,WaylandWindowDecorations");
}
-// Enable system audio loopback capture on macOS via CoreAudio Taps (Electron 39+).
-// Without these flags, getDisplayMedia({ audio: "loopback" }) returns a live audio track
-// that produces only silence (all-zero samples).
-if (process.platform === "darwin") {
- app.commandLine.appendSwitch(
- "enable-features",
- "MacLoopbackAudioForScreenShare,MacSckSystemAudioLoopbackOverride,MacCatapSystemAudioLoopbackCapture"
- );
-}
-
// Set desktop filename so Wayland compositors can match windows to the .desktop entry.
// This allows XDG portals (e.g. PipeWire) to persist permissions across sessions.
if (process.platform === "linux") {
@@ -539,28 +529,6 @@ async function startApp() {
}
);
- // Handle getDisplayMedia() calls from the renderer — auto-select the first
- // screen source with loopback audio so no system picker dialog is shown.
- const { desktopCapturer } = require("electron");
- session.defaultSession.setDisplayMediaRequestHandler(async (_request, callback) => {
- try {
- const sources = await desktopCapturer.getSources({ types: ["screen"] });
- debugLogger.debug("Display media sources", {
- count: sources.length,
- names: sources.map((s) => s.name),
- });
- if (!sources.length) {
- debugLogger.error("No screen sources available — System Audio permission may be denied");
- callback({});
- return;
- }
- callback({ video: sources[0], audio: "loopback" });
- } catch (err) {
- debugLogger.error("Display media handler error", { error: err.message });
- callback({});
- }
- });
-
windowManager.setActivationModeCache(environmentManager.getActivationMode());
windowManager.setFloatingIconAutoHide(environmentManager.getFloatingIconAutoHide());
windowManager.setPanelStartPosition(environmentManager.getPanelStartPosition());
diff --git a/preload.js b/preload.js
index 1b6e75c53..e60562040 100644
--- a/preload.js
+++ b/preload.js
@@ -170,8 +170,7 @@ contextBridge.exposeInMainWorld("electronAPI", {
// Clipboard functions
checkAccessibilityPermission: (silent) =>
ipcRenderer.invoke("check-accessibility-permission", silent),
- promptAccessibilityPermission: () =>
- ipcRenderer.invoke("prompt-accessibility-permission"),
+ promptAccessibilityPermission: () => ipcRenderer.invoke("prompt-accessibility-permission"),
readClipboard: () => ipcRenderer.invoke("read-clipboard"),
writeClipboard: (text) => ipcRenderer.invoke("write-clipboard", text),
checkPasteTools: () => ipcRenderer.invoke("check-paste-tools"),
@@ -608,9 +607,6 @@ contextBridge.exposeInMainWorld("electronAPI", {
gcalGetUpcomingEvents: (windowMinutes) =>
ipcRenderer.invoke("gcal-get-upcoming-events", windowMinutes),
- // Desktop audio capture
- getDesktopSources: (types) => ipcRenderer.invoke("get-desktop-sources", types),
-
// Google Calendar event listeners
onGcalMeetingStarting: registerListener(
"gcal-meeting-starting",
diff --git a/src/components/IntegrationsView.tsx b/src/components/IntegrationsView.tsx
index 0f7ce68ba..764e848a0 100644
--- a/src/components/IntegrationsView.tsx
+++ b/src/components/IntegrationsView.tsx
@@ -36,12 +36,12 @@ export default function IntegrationsView() {
}, [setGcalAccounts]);
const handleConnect = useCallback(async () => {
- if (systemAudio.isMacOS && !systemAudio.granted) {
+ if (systemAudio.isMacOS && systemAudio.status === "denied") {
setShowPermissionDialog(true);
return;
}
await startOAuth();
- }, [systemAudio.isMacOS, systemAudio.granted, startOAuth]);
+ }, [systemAudio.isMacOS, systemAudio.status, startOAuth]);
const handleDisconnect = useCallback(
async (email: string) => {
diff --git a/src/components/SettingsPage.tsx b/src/components/SettingsPage.tsx
index 15fef8963..494a984ec 100644
--- a/src/components/SettingsPage.tsx
+++ b/src/components/SettingsPage.tsx
@@ -3324,16 +3324,18 @@ EOF`,
buttonText={t("settingsPage.permissions.testAndGrant")}
onOpenSettings={permissionsHook.openAccessibilitySettings}
/>
- <PermissionCard
- icon={Monitor}
- title={t("settingsPage.permissions.systemAudioTitle")}
- description={t("settingsPage.permissions.systemAudioDescription")}
- granted={systemAudio.granted}
- onRequest={systemAudio.request}
- buttonText={t("settingsPage.permissions.test")}
- onOpenSettings={systemAudio.openSettings}
- badge={t("settingsPage.permissions.optional")}
- />
+ {systemAudio.mode === "native" && (
+ <PermissionCard
+ icon={Monitor}
+ title={t("settingsPage.permissions.systemAudioTitle")}
+ description={t("settingsPage.permissions.systemAudioDescription")}
+ granted={systemAudio.granted}
+ onRequest={systemAudio.request}
+ buttonText={t("settingsPage.permissions.test")}
+ onOpenSettings={systemAudio.openSettings}
+ badge={t("settingsPage.permissions.optional")}
+ />
+ )}
</>
)}
</div>
diff --git a/src/components/UpcomingMeetings.tsx b/src/components/UpcomingMeetings.tsx
index e9a5baf20..ac5599a50 100644
--- a/src/components/UpcomingMeetings.tsx
+++ b/src/components/UpcomingMeetings.tsx
@@ -104,7 +104,7 @@ export default function UpcomingMeetings({ events, isLoading }: UpcomingMeetings
{/* Empty state */}
{!isLoading && events.length === 0 && (
<div className="flex flex-col items-center justify-center py-8 px-3">
- {systemAudio.isMacOS && !systemAudio.granted ? (
+ {systemAudio.isMacOS && systemAudio.status === "denied" ? (
<>
<Monitor size={24} className="text-muted-foreground/30 mb-2.5" />
<p className="text-xs text-muted-foreground/60 text-center mb-3">
diff --git a/src/components/notes/NotesOnboarding.tsx b/src/components/notes/NotesOnboarding.tsx
index 804422a23..b262a2d68 100644
--- a/src/components/notes/NotesOnboarding.tsx
+++ b/src/components/notes/NotesOnboarding.tsx
@@ -53,6 +53,7 @@ export default function NotesOnboarding({ onComplete }: NotesOnboardingProps) {
const { alertDialog, hideAlertDialog } = useDialogs();
const {
granted: systemAudioGranted,
+ mode: systemAudioMode,
request: requestSystemAudio,
isMacOS,
} = useSystemAudioPermission();
@@ -185,7 +186,7 @@ export default function NotesOnboarding({ onComplete }: NotesOnboardingProps) {
)}
{/* System Audio Permission — macOS only */}
- {isMacOS && (
+ {isMacOS && systemAudioMode === "native" && (
<div
className={cn(
"rounded-lg border transition-colors duration-200",
diff --git a/src/components/notes/PersonalNotesView.tsx b/src/components/notes/PersonalNotesView.tsx
index 534c7fdda..a536dd215 100644
--- a/src/components/notes/PersonalNotesView.tsx
+++ b/src/components/notes/PersonalNotesView.tsx
@@ -111,7 +111,7 @@ export default function PersonalNotesView({
const activeNote = notes.find((n) => n.id === activeNoteId) ?? null;
- // Note recording uses the same dual-stream transcription as meeting mode.
+ // Note recording uses the same meeting transcription pipeline.
// The `isMeetingMode` ref distinguishes whether the recording was triggered
// by the meeting hotkey (creates a separate note) or the note record button.
const isMeetingModeRef = useRef(false);
diff --git a/src/components/ui/PermissionsSection.tsx b/src/components/ui/PermissionsSection.tsx
index 1f7d44e2b..607375a13 100644
--- a/src/components/ui/PermissionsSection.tsx
+++ b/src/components/ui/PermissionsSection.tsx
@@ -9,6 +9,7 @@ interface PermissionsSectionProps {
permissions: UsePermissionsReturn;
systemAudio: {
granted: boolean;
+ mode: "native" | "unsupported";
request: () => Promise<boolean>;
openSettings: () => Promise<void>;
};
@@ -49,16 +50,18 @@ export default function PermissionsSection({ permissions, systemAudio }: Permiss
: undefined
}
/>
- <PermissionCard
- icon={Monitor}
- title={t("onboarding.permissions.systemAudioTitle")}
- description={t("onboarding.permissions.systemAudioDescription")}
- granted={systemAudio.granted}
- onRequest={systemAudio.request}
- buttonText={t("onboarding.permissions.grant")}
- onOpenSettings={systemAudio.openSettings}
- badge={t("onboarding.permissions.optional")}
- />
+ {systemAudio.mode === "native" && (
+ <PermissionCard
+ icon={Monitor}
+ title={t("onboarding.permissions.systemAudioTitle")}
+ description={t("onboarding.permissions.systemAudioDescription")}
+ granted={systemAudio.granted}
+ onRequest={systemAudio.request}
+ buttonText={t("onboarding.permissions.grant")}
+ onOpenSettings={systemAudio.openSettings}
+ badge={t("onboarding.permissions.optional")}
+ />
+ )}
</>
)}
</div>
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index e4ff3fdfa..2b11eece5 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -2,7 +2,6 @@ import ReasoningService from "../services/ReasoningService";
import { API_ENDPOINTS, buildApiUrl, normalizeBaseUrl } from "../config/constants";
import logger from "../utils/logger";
import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";
-import { getSystemAudioStream, stopSystemAudioStream } from "../utils/systemAudio";
import { isSecureEndpoint } from "../utils/urlUtils";
import { withSessionRefresh } from "../lib/neonAuth";
import { getBaseLanguageCode, validateLanguageForModel } from "../utils/languageSupport";
@@ -112,13 +111,6 @@ class AudioManager {
this.sttConfig = null;
this.lastAudioBlob = null;
this.lastAudioMetadata = null;
-
- // System audio capture
- this.systemAudioEnabled = false;
- this.systemAudioStream = null;
- this._mixingContext = null;
- this._mixingDestination = null;
- this._systemAudioSource = null;
}
getWorkletBlobUrl() {
@@ -196,30 +188,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.sttConfig = config;
}
- setSystemAudioEnabled(enabled) {
- this.systemAudioEnabled = enabled;
- }
-
- cleanupSystemAudio() {
- stopSystemAudioStream(this.systemAudioStream);
- this.systemAudioStream = null;
-
- if (this._systemAudioSource) {
- try {
- this._systemAudioSource.disconnect();
- } catch {}
- this._systemAudioSource = null;
- }
-
- if (this._mixingContext) {
- try {
- this._mixingContext.close();
- } catch {}
- this._mixingContext = null;
- }
- this._mixingDestination = null;
- }
-
getStreamingProvider() {
const { cloudTranscriptionModel } = getSettings();
if (REALTIME_MODELS.has(cloudTranscriptionModel)) {
@@ -334,31 +302,12 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
);
}
- // Mix in system audio if enabled
- let recordingStream = micStream;
- if (this.systemAudioEnabled) {
- const sysStream = await getSystemAudioStream();
- if (sysStream) {
- this.systemAudioStream = sysStream;
- this._mixingContext = new AudioContext();
- this._mixingDestination = this._mixingContext.createMediaStreamDestination();
- const micSource = this._mixingContext.createMediaStreamSource(micStream);
- this._systemAudioSource = this._mixingContext.createMediaStreamSource(sysStream);
- micSource.connect(this._mixingDestination);
- this._systemAudioSource.connect(this._mixingDestination);
- recordingStream = this._mixingDestination.stream;
- logger.info("System audio mixed into batch recording", {}, "audio");
- } else {
- logger.warn("System audio unavailable, recording mic only", {}, "audio");
- }
- }
-
// Silence detection: observe audio energy via AnalyserNode
try {
this._silenceCtx = new AudioContext();
this._silenceAnalyser = this._silenceCtx.createAnalyser();
this._silenceAnalyser.fftSize = 2048;
- const sourceNode = this._silenceCtx.createMediaStreamSource(recordingStream);
+ const sourceNode = this._silenceCtx.createMediaStreamSource(micStream);
sourceNode.connect(this._silenceAnalyser);
this._peakRms = 0;
const dataArray = new Uint8Array(this._silenceAnalyser.fftSize);
@@ -377,7 +326,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this._peakRms = 1; // assume speech if detection fails
}
- this.mediaRecorder = new MediaRecorder(recordingStream);
+ this.mediaRecorder = new MediaRecorder(micStream);
this.audioChunks = [];
this.recordingStartTime = Date.now();
this.recordingMimeType = this.mediaRecorder.mimeType || "audio/webm";
@@ -420,7 +369,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
await this.processAudio(audioBlob, { durationSeconds });
micStream.getTracks().forEach((track) => track.stop());
- this.cleanupSystemAudio();
};
this.mediaRecorder.start();
@@ -476,7 +424,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
if (this.mediaRecorder.stream) {
this.mediaRecorder.stream.getTracks().forEach((track) => track.stop());
}
- this.cleanupSystemAudio();
return true;
}
@@ -2226,20 +2173,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.isStreaming = true;
this.streamingSource.connect(this.streamingProcessor);
- // Mix in system audio if enabled — connecting a second source to the same
- // AudioWorkletNode sums the signals automatically via Web Audio API.
- if (this.systemAudioEnabled) {
- const sysStream = await getSystemAudioStream();
- if (sysStream) {
- this.systemAudioStream = sysStream;
- this._systemAudioSource = audioContext.createMediaStreamSource(sysStream);
- this._systemAudioSource.connect(this.streamingProcessor);
- logger.info("System audio mixed into streaming recording", {}, "audio");
- } else {
- logger.warn("System audio unavailable, streaming mic only", {}, "audio");
- }
- }
-
const tPipeline = performance.now();
// 3. Register IPC event listeners BEFORE connecting, so no transcript
@@ -2717,8 +2650,6 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
this.streamingStream = null;
}
- this.cleanupSystemAudio();
-
this.isStreaming = false;
}
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 2179ebd01..bd2a529a2 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -107,7 +107,6 @@ class IPCHandlers {
this.sessionId = crypto.randomUUID();
this.assemblyAiStreaming = null;
this.deepgramStreaming = null;
- this.openaiRealtimeStreaming = null;
this._dictationStreaming = null;
this._autoLearnEnabled = true; // Default on, synced from renderer
this._autoLearnDebounceTimer = null;
@@ -2057,18 +2056,12 @@ class IPCHandlers {
return { granted: true, status: "granted", mode: "unsupported" };
}
- if (this.audioTapManager?.isAvailable()) {
+ if (this.audioTapManager?.isSupported()) {
const status = this.audioTapManager.getPermissionStatus();
return { granted: status === "granted", status, mode: "native" };
}
- const { systemPreferences } = require("electron");
- const status = systemPreferences.getMediaAccessStatus("screen");
- return {
- granted: status === "granted",
- status: status === "granted" ? "granted" : status === "denied" ? "denied" : "unknown",
- mode: "legacy",
- };
+ return { granted: false, status: "unsupported", mode: "unsupported" };
});
ipcMain.handle("request-system-audio-access", async () => {
@@ -2076,12 +2069,17 @@ class IPCHandlers {
return { granted: true, status: "granted", mode: "unsupported" };
}
- if (!this.audioTapManager?.isAvailable()) {
- return { granted: false, status: "unknown", mode: "legacy" };
+ if (!this.audioTapManager?.isSupported()) {
+ return { granted: false, status: "unsupported", mode: "unsupported" };
}
- const result = await this.audioTapManager.requestAccess();
- return { ...result, mode: "native" };
+ try {
+ const result = await this.audioTapManager.requestAccess();
+ return { ...result, mode: "native" };
+ } catch (error) {
+ debugLogger.error("System audio permission request failed", { error: error.message });
+ return { granted: false, status: "unknown", mode: "native", error: error.message };
+ }
});
// Auth: clear all session cookies for sign-out.
@@ -2446,6 +2444,15 @@ class IPCHandlers {
return data.clientSecret;
};
+ const getMeetingSystemAudioMode = () =>
+ this.audioTapManager?.isSupported() ? "native" : "unsupported";
+
+ const hasNativeMeetingSystemAudio = () => getMeetingSystemAudioMode() === "native";
+
+ const isMeetingStreamingConnected = () =>
+ !!this._meetingMicStreaming?.isConnected &&
+ (!hasNativeMeetingSystemAudio() || !!this._meetingSystemStreaming?.isConnected);
+
const connectRealtimeStreaming = async (event, options) => {
if (this._meetingMicStreaming?.isConnected) {
await this._meetingMicStreaming.disconnect();
@@ -2453,23 +2460,28 @@ class IPCHandlers {
if (this._meetingSystemStreaming?.isConnected) {
await this._meetingSystemStreaming.disconnect();
}
- if (this.openaiRealtimeStreaming?.isConnected) {
- await this.openaiRealtimeStreaming.disconnect();
- }
-
+ this._meetingMicStreaming = null;
+ this._meetingSystemStreaming = null;
const win = BrowserWindow.fromWebContents(event.sender);
- const [micSecret, systemSecret] = await fetchRealtimeToken(event, options, { streams: 2 });
-
const connectOpts = {
model: options.model,
language: options.language,
preconfigured: options.mode !== "byok",
};
- const pairs = [
- { ref: "_meetingMicStreaming", secret: micSecret, source: "mic" },
- { ref: "_meetingSystemStreaming", secret: systemSecret, source: "system" },
- ];
+ const pairs = hasNativeMeetingSystemAudio()
+ ? (await fetchRealtimeToken(event, options, { streams: 2 })).map((secret, index) => ({
+ ref: index === 0 ? "_meetingMicStreaming" : "_meetingSystemStreaming",
+ secret,
+ source: index === 0 ? "mic" : "system",
+ }))
+ : [
+ {
+ ref: "_meetingMicStreaming",
+ secret: await fetchRealtimeToken(event, options),
+ source: "mic",
+ },
+ ];
for (const { ref, source } of pairs) {
this[ref] = new OpenAIRealtimeStreaming();
@@ -2480,12 +2492,38 @@ class IPCHandlers {
pairs.map(({ ref, secret }) => this[ref].connect({ apiKey: secret, ...connectOpts }))
);
- // Keep legacy reference for backward compat with prepare check
- this.openaiRealtimeStreaming = this._meetingMicStreaming;
-
return win;
};
+ let meetingSendCounts = { mic: 0, system: 0 };
+
+ const resetMeetingStreamingState = () => {
+ this._meetingMicStreaming = null;
+ this._meetingSystemStreaming = null;
+ meetingSendCounts = { mic: 0, system: 0 };
+ };
+
+ const disconnectMeetingStreaming = async () => {
+ const results = await Promise.all([
+ this._meetingMicStreaming
+ ? this._meetingMicStreaming.disconnect().catch(() => ({ text: "" }))
+ : Promise.resolve({ text: "" }),
+ this._meetingSystemStreaming
+ ? this._meetingSystemStreaming.disconnect().catch(() => ({ text: "" }))
+ : Promise.resolve({ text: "" }),
+ ]);
+
+ resetMeetingStreamingState();
+ return results;
+ };
+
+ const rollbackMeetingTranscriptionStart = async () => {
+ if (this.audioTapManager) {
+ await this.audioTapManager.stop().catch(() => {});
+ }
+ await disconnectMeetingStreaming().catch(() => {});
+ };
+
const setupDictationCallbacks = (streaming, event) => {
streaming.onPartialTranscript = (text) =>
event.sender.send("dictation-realtime-partial", text);
@@ -2519,7 +2557,7 @@ class IPCHandlers {
return { success: false, error: "Operation in progress" };
}
- if (this._meetingMicStreaming?.isConnected && this._meetingSystemStreaming?.isConnected) {
+ if (isMeetingStreamingConnected()) {
debugLogger.debug("Meeting transcription already prepared (warm connections)");
return { success: true, alreadyPrepared: true };
}
@@ -2532,7 +2570,7 @@ class IPCHandlers {
meetingTranscriptionPreparePromise = (async () => {
try {
await connectRealtimeStreaming(event, options);
- debugLogger.debug("Meeting transcription prepared (dual WebSockets warm)");
+ debugLogger.debug("Meeting transcription prepared (meeting streams warm)");
return { success: true };
} catch (error) {
debugLogger.error("Meeting transcription prepare error", { error: error.message });
@@ -2563,12 +2601,12 @@ class IPCHandlers {
const systemAudioMode = getMeetingSystemAudioMode();
// If already prepared (warm connections from prepare), just re-attach handlers
- if (this._meetingMicStreaming?.isConnected && this._meetingSystemStreaming?.isConnected) {
+ if (isMeetingStreamingConnected()) {
debugLogger.debug("Meeting transcription start: reusing warm connections");
const win = BrowserWindow.fromWebContents(event.sender);
attachMeetingStreamingHandlers(this._meetingMicStreaming, win, "mic");
- attachMeetingStreamingHandlers(this._meetingSystemStreaming, win, "system");
if (systemAudioMode === "native") {
+ attachMeetingStreamingHandlers(this._meetingSystemStreaming, win, "system");
await startNativeMeetingSystemAudio(event);
}
return { success: true, systemAudioMode };
@@ -2584,6 +2622,7 @@ class IPCHandlers {
}
return { success: true, systemAudioMode };
} catch (error) {
+ await rollbackMeetingTranscriptionStart();
debugLogger.error("Meeting transcription start error", { error: error.message });
return { success: false, error: error.message };
} finally {
@@ -2591,7 +2630,6 @@ class IPCHandlers {
}
});
- let meetingSendCounts = { mic: 0, system: 0 };
const sendMeetingAudio = (audioBuffer, source) => {
const streaming = source === "mic" ? this._meetingMicStreaming : this._meetingSystemStreaming;
if (!streaming) {
@@ -2615,9 +2653,6 @@ class IPCHandlers {
}
};
- const getMeetingSystemAudioMode = () =>
- this.audioTapManager?.isAvailable() ? "native" : "legacy";
-
const startNativeMeetingSystemAudio = async (event) => {
const win = BrowserWindow.fromWebContents(event.sender);
await this.audioTapManager.start({
@@ -2642,19 +2677,7 @@ class IPCHandlers {
await this.audioTapManager.stop();
}
- const results = await Promise.all([
- this._meetingMicStreaming
- ? this._meetingMicStreaming.disconnect()
- : Promise.resolve({ text: "" }),
- this._meetingSystemStreaming
- ? this._meetingSystemStreaming.disconnect()
- : Promise.resolve({ text: "" }),
- ]);
-
- this._meetingMicStreaming = null;
- this._meetingSystemStreaming = null;
- this.openaiRealtimeStreaming = null;
- meetingSendCounts = { mic: 0, system: 0 };
+ const results = await disconnectMeetingStreaming();
return {
success: true,
@@ -4203,17 +4226,6 @@ class IPCHandlers {
}
return { success: true };
});
-
- ipcMain.handle("get-desktop-sources", async (_event, types) => {
- try {
- const { desktopCapturer } = require("electron");
- const sources = await desktopCapturer.getSources({ types: types || ["screen"] });
- return sources.map((s) => ({ id: s.id, name: s.name }));
- } catch (error) {
- debugLogger.error("Failed to get desktop sources", { error: error.message }, "calendar");
- return [];
- }
- });
}
broadcastToWindows(channel, payload) {
diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.ts
index fadc3f9d2..dbe302932 100644
--- a/src/hooks/useMeetingTranscription.ts
+++ b/src/hooks/useMeetingTranscription.ts
@@ -1,7 +1,6 @@
import { useState, useEffect, useRef, useCallback } from "react";
import { getSettings } from "../stores/settingsStore";
import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";
-import { getSystemAudioStream } from "../utils/systemAudio";
import logger from "../utils/logger";
export interface TranscriptSegment {
@@ -221,13 +220,9 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
const [systemPartial, setSystemPartial] = useState("");
const [error, setError] = useState<string | null>(null);
- const audioContextRef = useRef<AudioContext | null>(null);
const micContextRef = useRef<AudioContext | null>(null);
- const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
- const processorRef = useRef<AudioWorkletNode | null>(null);
const micSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
const micProcessorRef = useRef<AudioWorkletNode | null>(null);
- const streamRef = useRef<MediaStream | null>(null);
const micStreamRef = useRef<MediaStream | null>(null);
const isRecordingRef = useRef(false);
const isStartingRef = useRef(false);
@@ -237,10 +232,10 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
const pendingCleanupRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const cleanup = useCallback(async () => {
- const processors = [processorRef, micProcessorRef];
- const sources = [sourceRef, micSourceRef];
- const streams = [streamRef, micStreamRef];
- const contexts = [audioContextRef, micContextRef];
+ const processors = [micProcessorRef];
+ const sources = [micSourceRef];
+ const streams = [micStreamRef];
+ const contexts = [micContextRef];
for (const ref of processors) {
await flushAndDisconnectProcessor(ref.current);
@@ -376,13 +371,9 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
]);
const streamsMs = performance.now() - startTime;
- const systemAudioMode = startResult?.systemAudioMode || "legacy";
- const stream = systemAudioMode === "legacy" ? await getSystemAudioStream() : null;
-
// Abort if stop was called during setup
if (!isRecordingRef.current) {
logger.info("Meeting transcription aborted during setup (stop called)", {}, "meeting");
- stream?.getTracks().forEach((t) => t.stop());
micResult?.getTracks().forEach((t) => t.stop());
isStartingRef.current = false;
return;
@@ -394,7 +385,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
{ error: startResult?.error },
"meeting"
);
- stream?.getTracks().forEach((track) => track.stop());
+ setError(startResult?.error || "Failed to start meeting transcription");
micResult?.getTracks().forEach((track) => track.stop());
isRecordingRef.current = false;
isStartingRef.current = false;
@@ -402,18 +393,19 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
return;
}
- if (systemAudioMode === "legacy" && !stream) {
- logger.error("Could not capture system audio for meeting transcription", {}, "meeting");
- micResult?.getTracks().forEach((track) => track.stop());
+ const systemAudioMode = startResult.systemAudioMode || "unsupported";
+
+ if (!micResult && systemAudioMode !== "native") {
+ logger.error("Meeting transcription has no available audio source", {}, "meeting");
+ setError(
+ "No microphone is available and system audio capture is unsupported on this device."
+ );
await window.electronAPI?.meetingTranscriptionStop?.();
isRecordingRef.current = false;
isStartingRef.current = false;
setIsRecording(false);
return;
}
- if (stream) {
- streamRef.current = stream;
- }
const partialSetters = { mic: setMicPartial, system: setSystemPartial };
@@ -468,55 +460,9 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
});
if (errorCleanup) ipcCleanupsRef.current.push(errorCleanup);
- const pendingSystemChunks: ArrayBuffer[] = [];
const pendingMicChunks: ArrayBuffer[] = [];
let socketReady = false;
- let systemPipelinePromise: Promise<{
- source: MediaStreamAudioSourceNode;
- processor: AudioWorkletNode;
- }> | null = null;
- if (stream) {
- const loopbackRate = stream.getAudioTracks()[0]?.getSettings()?.sampleRate || 48000;
- const systemNeedsResample = loopbackRate !== 24000;
- const audioContext = new AudioContext({ sampleRate: loopbackRate });
- await detachFromOutputDevice(audioContext);
- audioContextRef.current = audioContext;
-
- logger.debug(
- "System audio context created",
- { loopbackRate, needsResample: systemNeedsResample },
- "meeting"
- );
-
- systemPipelinePromise = createAudioPipeline({
- stream,
- context: audioContext,
- label: "Meeting system",
- onChunk: (chunk) => {
- if (!isRecordingRef.current) return;
- let samples = new Int16Array(chunk);
-
- if (systemNeedsResample) {
- const ratio = loopbackRate / 24000;
- const outLen = Math.floor(samples.length / ratio);
- const out = new Int16Array(outLen);
- for (let i = 0; i < outLen; i++) {
- out[i] = samples[Math.round(i * ratio)];
- }
- samples = out;
- }
-
- const buf = samples.buffer;
- if (socketReady) {
- window.electronAPI?.meetingTranscriptionSend?.(buf, "system");
- return;
- }
- pendingSystemChunks.push(buf.slice(0) as ArrayBuffer);
- },
- });
- }
-
let micPipelinePromise: Promise<void> | null = null;
if (micResult) {
micStreamRef.current = micResult;
@@ -552,13 +498,8 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
});
}
- const [systemPipeline] = await Promise.all(
- [systemPipelinePromise, micPipelinePromise].filter(Boolean)
- );
-
- if (systemPipeline) {
- sourceRef.current = systemPipeline.source;
- processorRef.current = systemPipeline.processor;
+ if (micPipelinePromise) {
+ await micPipelinePromise;
}
// Abort if stop was called during pipeline setup
@@ -576,9 +517,6 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
isStartingRef.current = false;
socketReady = true;
- for (const chunk of pendingSystemChunks) {
- window.electronAPI?.meetingTranscriptionSend?.(chunk, "system");
- }
for (const chunk of pendingMicChunks) {
window.electronAPI?.meetingTranscriptionSend?.(chunk, "mic");
}
@@ -588,7 +526,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
"Meeting transcription started successfully",
{
systemAudioMode,
- bufferedChunks: pendingSystemChunks.length + pendingMicChunks.length,
+ bufferedChunks: pendingMicChunks.length,
streamsMs: Math.round(streamsMs),
totalMs: Math.round(totalMs),
wasPrepared: isPreparedRef.current,
@@ -601,6 +539,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
{ error: (err as Error).message },
"meeting"
);
+ setError((err as Error).message);
isRecordingRef.current = false;
isStartingRef.current = false;
setIsRecording(false);
diff --git a/src/hooks/useNoteRecording.ts b/src/hooks/useNoteRecording.ts
deleted file mode 100644
index 5acb8e6dd..000000000
--- a/src/hooks/useNoteRecording.ts
+++ /dev/null
@@ -1,196 +0,0 @@
-import { useState, useEffect, useRef, useCallback } from "react";
-import { useTranslation } from "react-i18next";
-import AudioManager from "../helpers/audioManager";
-import logger from "../utils/logger";
-import { getRecordingErrorTitle } from "../utils/recordingErrors";
-
-interface UseNoteRecordingOptions {
- onTranscriptionComplete: (text: string) => void;
- onPartialTranscript?: (text: string) => void;
- onStreamingCommit?: (text: string) => void;
- onError?: (error: { title: string; description: string }) => void;
- systemAudioEnabled?: boolean;
-}
-
-interface UseNoteRecordingReturn {
- isRecording: boolean;
- isProcessing: boolean;
- isStreaming: boolean;
- partialTranscript: string;
- streamingCommit: string | null;
- consumeStreamingCommit: () => void;
- startRecording: () => Promise<void>;
- stopRecording: () => Promise<void>;
- cancelRecording: () => void;
-}
-
-export function useNoteRecording({
- onTranscriptionComplete,
- onPartialTranscript,
- onStreamingCommit: onStreamingCommitCb,
- onError,
- systemAudioEnabled = false,
-}: UseNoteRecordingOptions): UseNoteRecordingReturn {
- const { t } = useTranslation();
- const [isRecording, setIsRecording] = useState(false);
- const [isProcessing, setIsProcessing] = useState(false);
- const [isStreaming, setIsStreaming] = useState(false);
- const [partialTranscript, setPartialTranscript] = useState("");
- const [streamingCommits, setStreamingCommits] = useState<string[]>([]);
- const audioManagerRef = useRef<InstanceType<typeof AudioManager> | null>(null);
-
- const callbacksRef = useRef({
- onTranscriptionComplete,
- onPartialTranscript,
- onStreamingCommitCb,
- onError,
- });
- callbacksRef.current = {
- onTranscriptionComplete,
- onPartialTranscript,
- onStreamingCommitCb,
- onError,
- };
-
- useEffect(() => {
- const manager = new AudioManager();
- audioManagerRef.current = manager;
- manager.setSkipReasoning(true);
- manager.setSystemAudioEnabled(systemAudioEnabled);
-
- manager.setCallbacks({
- onStateChange: ({
- isRecording,
- isProcessing,
- isStreaming,
- }: {
- isRecording: boolean;
- isProcessing: boolean;
- isStreaming?: boolean;
- }) => {
- setIsRecording(isRecording);
- setIsProcessing(isProcessing);
- setIsStreaming(isStreaming ?? false);
- if (!isStreaming) {
- setPartialTranscript("");
- setStreamingCommits([]);
- }
- },
- onError: (error: { title: string; description: string; code?: string }) => {
- const title = getRecordingErrorTitle(error, t);
- callbacksRef.current.onError?.({ title, description: error.description });
- },
- onPartialTranscript: (text: string) => {
- setPartialTranscript(text);
- callbacksRef.current.onPartialTranscript?.(text);
- },
- onStreamingCommit: (text: string) => {
- setStreamingCommits((pending) => [...pending, text]);
- callbacksRef.current.onStreamingCommitCb?.(text);
- },
- onTranscriptionComplete: (result: {
- success: boolean;
- text: string;
- source?: string;
- limitReached?: boolean;
- wordsUsed?: number;
- wordsRemaining?: number;
- }) => {
- if (result.success) {
- callbacksRef.current.onTranscriptionComplete(result.text);
- if (manager.shouldUseStreaming()) {
- manager.warmupStreamingConnection();
- }
- }
- },
- });
-
- manager.setContext("notes");
- window.electronAPI.getSttConfig?.().then((config) => {
- if (config?.success && audioManagerRef.current) {
- audioManagerRef.current.setSttConfig(config);
- if (manager.shouldUseStreaming()) {
- manager.warmupStreamingConnection();
- }
- }
- });
-
- return () => {
- manager.cleanup();
- audioManagerRef.current = null;
- };
- }, []);
-
- useEffect(() => {
- audioManagerRef.current?.setSystemAudioEnabled(systemAudioEnabled);
- }, [systemAudioEnabled]);
-
- const startRecording = useCallback(async () => {
- const manager = audioManagerRef.current;
- if (!manager) return;
-
- const state = manager.getState();
- if (state.isRecording || state.isProcessing) return;
-
- // Retry STT config fetch if it wasn't loaded on mount (e.g. auth wasn't ready)
- if (!manager.sttConfig) {
- const config = await window.electronAPI.getSttConfig?.();
- if (config?.success) {
- manager.setSttConfig(config);
- }
- }
-
- const didStart = manager.shouldUseStreaming()
- ? await manager.startStreamingRecording()
- : await manager.startRecording();
-
- if (!didStart) {
- logger.debug("Note recording failed to start", {}, "notes");
- }
- }, []);
-
- const stopRecording = useCallback(async () => {
- const manager = audioManagerRef.current;
- if (!manager) return;
-
- const state = manager.getState();
- if (!state.isRecording) return;
-
- if (state.isStreaming) {
- await manager.stopStreamingRecording();
- } else {
- manager.stopRecording();
- }
- }, []);
-
- const cancelRecording = useCallback(() => {
- const manager = audioManagerRef.current;
- if (!manager) return;
-
- const state = manager.getState();
- if (state.isStreaming) {
- manager.stopStreamingRecording();
- } else {
- manager.cancelRecording();
- }
- }, []);
-
- const consumeStreamingCommit = useCallback(
- () => setStreamingCommits((pending) => pending.slice(1)),
- []
- );
-
- const streamingCommit = streamingCommits[0] ?? null;
-
- return {
- isRecording,
- isProcessing,
- isStreaming,
- partialTranscript,
- streamingCommit,
- consumeStreamingCommit,
- startRecording,
- stopRecording,
- cancelRecording,
- };
-}
diff --git a/src/hooks/useSystemAudioPermission.ts b/src/hooks/useSystemAudioPermission.ts
index 42dc5eb3c..05073f141 100644
--- a/src/hooks/useSystemAudioPermission.ts
+++ b/src/hooks/useSystemAudioPermission.ts
@@ -1,23 +1,31 @@
import { useState, useCallback, useEffect, useRef } from "react";
import { getCachedPlatform } from "../utils/platform";
+import type { SystemAudioAccessResult } from "../types/electron";
+
+const NON_MACOS_ACCESS: SystemAudioAccessResult = {
+ granted: true,
+ status: "granted",
+ mode: "unsupported",
+};
export function useSystemAudioPermission() {
const isMacOS = getCachedPlatform() === "darwin";
- const [granted, setGranted] = useState(!isMacOS);
- const [accessMode, setAccessMode] = useState<"native" | "legacy" | "unsupported">(
- isMacOS ? "legacy" : "unsupported"
+ const [access, setAccess] = useState<SystemAudioAccessResult | null>(
+ isMacOS ? null : NON_MACOS_ACCESS
);
+ const [isChecking, setIsChecking] = useState(false);
const checkingRef = useRef(false);
const check = useCallback(async () => {
if (!isMacOS || checkingRef.current) return;
checkingRef.current = true;
+ setIsChecking(true);
try {
const result = await window.electronAPI?.checkSystemAudioAccess?.();
- setGranted(result?.granted ?? false);
- setAccessMode(result?.mode ?? "legacy");
+ setAccess(result ?? { granted: false, status: "unsupported", mode: "unsupported" });
} finally {
checkingRef.current = false;
+ setIsChecking(false);
}
}, [isMacOS]);
@@ -38,26 +46,30 @@ export function useSystemAudioPermission() {
await window.electronAPI?.openSystemAudioSettings?.();
}, []);
- // Trigger the native macOS permission prompt via getDisplayMedia (used in onboarding)
const request = useCallback(async (): Promise<boolean> => {
if (!isMacOS) return true;
- if (accessMode === "native") {
- const result = await window.electronAPI?.requestSystemAudioAccess?.();
- const isGranted = result?.granted ?? false;
- setGranted(isGranted);
- setAccessMode(result?.mode ?? "native");
- return isGranted;
- }
- try {
- const stream = await navigator.mediaDevices.getDisplayMedia({ audio: true, video: true });
- stream.getTracks().forEach((t) => t.stop());
- setGranted(true);
- return true;
- } catch {
+ const currentAccess = access ??
+ (await window.electronAPI?.checkSystemAudioAccess?.()) ?? {
+ granted: false,
+ status: "unsupported" as const,
+ mode: "unsupported" as const,
+ };
+
+ if (currentAccess.mode !== "native") {
+ setAccess(currentAccess);
return false;
}
- }, [accessMode, isMacOS]);
- return { granted, request, openSettings, check, isMacOS };
+ const result = await window.electronAPI?.requestSystemAudioAccess?.();
+ const nextAccess = result ?? currentAccess;
+ setAccess(nextAccess);
+ return nextAccess.granted;
+ }, [access, isMacOS]);
+
+ const granted = access?.granted ?? false;
+ const status = access?.status ?? "unknown";
+ const mode = access?.mode ?? "unsupported";
+
+ return { granted, status, mode, isChecking, request, openSettings, check, isMacOS };
}
diff --git a/src/types/electron.ts b/src/types/electron.ts
index c138b51a7..b90a0f4c4 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -118,7 +118,7 @@ export interface AudioDiagnosticsResult {
export interface SystemAudioAccessResult {
granted: boolean;
status: "granted" | "denied" | "unknown" | "unsupported";
- mode: "native" | "legacy" | "unsupported";
+ mode: "native" | "unsupported";
error?: string;
}
@@ -1134,7 +1134,11 @@ declare global {
provider?: string;
model?: string;
language?: string;
- }) => Promise<{ success: boolean; error?: string; systemAudioMode?: "native" | "legacy" }>;
+ }) => Promise<{
+ success: boolean;
+ error?: string;
+ systemAudioMode?: "native" | "unsupported";
+ }>;
meetingTranscriptionSend?: (buffer: ArrayBuffer, source: "mic" | "system") => void;
meetingTranscriptionStop?: () => Promise<{
success: boolean;
@@ -1166,9 +1170,6 @@ declare global {
onDictationRealtimeError?: (callback: (error: string) => void) => () => void;
onDictationRealtimeSessionEnd?: (callback: (data: { text: string }) => void) => () => void;
- // Desktop audio capture
- getDesktopSources?: (types: string[]) => Promise<Array<{ id: string; name: string }>>;
-
// Google Calendar event listeners
onGcalMeetingStarting?: (callback: (data: any) => void) => () => void;
onGcalMeetingEnded?: (callback: (data: any) => void) => () => void;
diff --git a/src/utils/systemAudio.ts b/src/utils/systemAudio.ts
deleted file mode 100644
index 2ce538cf6..000000000
--- a/src/utils/systemAudio.ts
+++ /dev/null
@@ -1,97 +0,0 @@
-import logger from "./logger";
-
-// Module-level reference prevents the hidden video element from being garbage
-// collected while the capture session is active. getDisplayMedia requires
-// active video frame consumption to keep the loopback audio pump alive.
-let activeVideoSink: HTMLVideoElement | null = null;
-
-/**
- * Capture system audio via getDisplayMedia (CoreAudio Taps on macOS / Electron 39+).
- * Returns a MediaStream with audio (and video tracks that must stay alive).
- * Returns null if capture fails or no audio tracks are available.
- */
-export async function getSystemAudioStream(): Promise<MediaStream | null> {
- try {
- // Use getDisplayMedia (handled by setDisplayMediaRequestHandler in main process)
- // which properly captures system audio via macOS CoreAudio Taps.
- // Disable all audio processing — echo cancellation and noise suppression
- // on loopback audio can cancel out the captured system audio entirely,
- // especially when Bluetooth headphones change the audio routing.
- const stream = await navigator.mediaDevices.getDisplayMedia({
- audio: {
- echoCancellation: false,
- noiseSuppression: false,
- autoGainControl: false,
- },
- video: true,
- });
-
- const audioTracks = stream.getAudioTracks();
- const videoTracks = stream.getVideoTracks();
-
- // Belt-and-suspenders: also apply constraints directly on the track
- // in case getDisplayMedia ignored the initial constraints.
- if (audioTracks[0]) {
- try {
- await audioTracks[0].applyConstraints({
- echoCancellation: false,
- noiseSuppression: false,
- autoGainControl: false,
- });
- } catch {}
- }
-
- logger.debug(
- "Display media stream obtained",
- {
- audioTracks: audioTracks.length,
- videoTracks: videoTracks.length,
- audioSettings: audioTracks[0]?.getSettings(),
- },
- "audio"
- );
-
- if (!audioTracks.length) {
- logger.error("No audio track in display media stream", {}, "audio");
- videoTracks.forEach((t) => t.stop());
- return null;
- }
-
- // getDisplayMedia requires active video frame consumption to pump the capture
- // session. Without a consumer, the loopback audio track may produce silence.
- // Attach the stream to a hidden <video> element so Chromium pulls frames and
- // keeps the session active.
- // IMPORTANT: Store in module-level variable to prevent garbage collection.
- activeVideoSink = document.createElement("video");
- activeVideoSink.srcObject = new MediaStream(videoTracks);
- activeVideoSink.muted = true;
- activeVideoSink.play().catch(() => {});
-
- audioTracks[0].addEventListener("ended", () => {
- logger.error("System audio track ended unexpectedly", {}, "audio");
- if (activeVideoSink) {
- activeVideoSink.srcObject = null;
- activeVideoSink = null;
- }
- });
-
- return stream;
- } catch (err) {
- logger.error("Failed to capture system audio", { error: (err as Error).message }, "audio");
- return null;
- }
-}
-
-/**
- * Stop all tracks (audio + video) on a system audio stream.
- */
-export function stopSystemAudioStream(stream: MediaStream | null): void {
- if (!stream) return;
- try {
- stream.getTracks().forEach((t) => t.stop());
- } catch {}
- if (activeVideoSink) {
- activeVideoSink.srcObject = null;
- activeVideoSink = null;
- }
-}