Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat(audio): add native macOS system audio tap
Gabriel Steindf5a234821a1
diff --git a/electron-builder.json b/electron-builder.jsonindex ca991b9aa..232c41adf 100644--- a/electron-builder.json+++ b/electron-builder.json@@ -86,6 +86,7 @@"extraResources": [".env","src/assets/**/*",+ "resources/bin/macos-audio-tap","resources/bin/macos-globe-listener","resources/bin/macos-fast-paste","resources/bin/macos-mic-listener",diff --git a/main.js b/main.jsindex 271623a4c..b9798e2b0 100644--- a/main.js+++ b/main.js@@ -198,6 +198,7 @@ const WhisperCudaManager = require("./src/helpers/whisperCudaManager");const GoogleCalendarManager = require("./src/helpers/googleCalendarManager");const MeetingProcessDetector = require("./src/helpers/meetingProcessDetector");const AudioActivityDetector = require("./src/helpers/audioActivityDetector");+const AudioTapManager = require("./src/helpers/audioTapManager");const MeetingDetectionEngine = require("./src/helpers/meetingDetectionEngine");const { i18nMain, changeLanguage } = require("./src/helpers/i18nMain");const { ensureYdotool } = require("./src/helpers/ensureYdotool");@@ -219,6 +220,7 @@ let textEditMonitor = null;let whisperCudaManager = null;let googleCalendarManager = null;let meetingDetectionEngine = null;+let audioTapManager = null;let ipcHandlers = null;let globeKeyAlertShown = false;let authBridgeServer = null;@@ -295,6 +297,7 @@ function initializeCoreManagers() {updateManager.setWindowManager(windowManager);windowsKeyManager = new WindowsKeyManager();textEditMonitor = new TextEditMonitor();+ audioTapManager = new AudioTapManager();windowManager.textEditMonitor = textEditMonitor;// IPC handlers must be registered before window content loads@@ -311,6 +314,7 @@ function initializeCoreManagers() {whisperCudaManager,googleCalendarManager,meetingDetectionEngine,+ audioTapManager,getTrayManager: () => trayManager,});}@@ -1151,6 +1155,9 @@ if (gotSingleInstanceLock) {if (googleCalendarManager) {googleCalendarManager.stop();}+ if (audioTapManager) {+ audioTapManager.stop().catch(() => {});+ }if (ipcHandlers) {ipcHandlers._cleanupTextEditMonitor();}diff --git a/package.json b/package.jsonindex 636d1cbf6..125fc0ead 100644--- a/package.json+++ b/package.json@@ -16,7 +16,8 @@"compile:linux-paste": "node scripts/build-linux-fast-paste.js","compile:media-remote": "node scripts/build-media-remote.js","compile:mic-listener": "node scripts/build-macos-mic-listener.js",- "compile:native": "npm run compile:globe && npm run compile:fast-paste && npm run compile:winkeys && npm run compile:winpaste && npm run compile:linux-paste && npm run compile:text-monitor && npm run compile:media-remote && npm run compile:mic-listener",+ "compile:audio-tap": "node scripts/build-macos-audio-tap.js",+ "compile:native": "npm run compile:globe && npm run compile:fast-paste && npm run compile:winkeys && npm run compile:winpaste && npm run compile:linux-paste && npm run compile:text-monitor && npm run compile:media-remote && npm run compile:mic-listener && npm run compile:audio-tap","prestart": "npm run compile:native","start": "electron .","predev": "npm run compile:native",diff --git a/preload.js b/preload.jsindex 26a2c0ca6..1b6e75c53 100644--- a/preload.js+++ b/preload.js@@ -363,6 +363,7 @@ contextBridge.exposeInMainWorld("electronAPI", {// System settings helpers for microphone/audio permissionsrequestMicrophoneAccess: () => ipcRenderer.invoke("request-microphone-access"),checkSystemAudioAccess: () => ipcRenderer.invoke("check-system-audio-access"),+ requestSystemAudioAccess: () => ipcRenderer.invoke("request-system-audio-access"),openMicrophoneSettings: () => ipcRenderer.invoke("open-microphone-settings"),openSoundInputSettings: () => ipcRenderer.invoke("open-sound-input-settings"),openAccessibilitySettings: () => ipcRenderer.invoke("open-accessibility-settings"),diff --git a/resources/macos-audio-tap.swift b/resources/macos-audio-tap.swiftnew file mode 100644index 000000000..c525ce388--- /dev/null+++ b/resources/macos-audio-tap.swift@@ -0,0 +1,423 @@+import AVFoundation+import AudioToolbox+import CoreAudio+import Foundation++@available(macOS 14.2, *)+struct Config {+ let sampleRate: Double+ let chunkMilliseconds: Int+}++@available(macOS 14.2, *)+final class AudioTapCapture {+ private let config: Config+ private let targetFormat: AVAudioFormat+ private let chunkBytes: Int+ private let ioQueue = DispatchQueue(label: "com.openwhispr.audio-tap")++ private var tapID: AudioObjectID = 0+ private var aggregateDeviceID: AudioObjectID = 0+ private var ioProcID: AudioDeviceIOProcID?+ private var converter: AVAudioConverter?+ private var sourceFormat: AVAudioFormat?+ private var pendingPCM = Data()+ private var stopping = false++ init(config: Config) {+ self.config = config+ self.targetFormat = AVAudioFormat(+ commonFormat: .pcmFormatInt16,+ sampleRate: config.sampleRate,+ channels: 1,+ interleaved: true+ )!+ self.chunkBytes = max(2, Int(config.sampleRate * Double(config.chunkMilliseconds) / 1000.0) * 2)+ }++ func start() throws {+ let tapDescription = CATapDescription()+ tapDescription.name = "openwhispr-audio-tap"+ tapDescription.uuid = UUID()+ tapDescription.processes = []+ tapDescription.isMono = true+ tapDescription.isExclusive = true+ tapDescription.isMixdown = true+ tapDescription.isPrivate = true+ tapDescription.muteBehavior = .unmuted++ var newTapID = AudioObjectID()+ var status = AudioHardwareCreateProcessTap(tapDescription, &newTapID)+ guard status == noErr else {+ throw makeError("Failed to create process tap", status: status, operation: "create_process_tap")+ }+ tapID = newTapID++ let tapUID = try getTapUID()+ try createAggregateDevice(tapUID: tapUID)+ try waitForAggregateDeviceReady()+ try configureConverter()+ try registerIOProc()++ status = AudioDeviceStart(aggregateDeviceID, ioProcID)+ guard status == noErr else {+ throw makeError("Failed to start aggregate device", status: status, operation: "start_device")+ }++ emit(event: [+ "type": "start",+ "sampleRate": Int(config.sampleRate),+ "channels": 1,+ "bitsPerChannel": 16,+ ])+ }++ func stop() {+ if stopping {+ return+ }+ stopping = true++ if aggregateDeviceID != 0 {+ AudioDeviceStop(aggregateDeviceID, ioProcID)+ }+ if let ioProcID {+ AudioDeviceDestroyIOProcID(aggregateDeviceID, ioProcID)+ self.ioProcID = nil+ }+ if aggregateDeviceID != 0 {+ AudioHardwareDestroyAggregateDevice(aggregateDeviceID)+ aggregateDeviceID = 0+ }+ if tapID != 0 {+ AudioHardwareDestroyProcessTap(tapID)+ tapID = 0+ }++ flushPendingPCM()+ emit(event: ["type": "stop"])+ }++ private func createAggregateDevice(tapUID: String) throws {+ let description: [String: Any] = [+ kAudioAggregateDeviceNameKey: "OpenWhispr Audio Tap",+ kAudioAggregateDeviceUIDKey: "com.openwhispr.audio-tap.\(UUID().uuidString)",+ kAudioAggregateDeviceSubDeviceListKey: [],+ kAudioAggregateDeviceTapListKey: [[kAudioSubTapUIDKey: tapUID]],+ kAudioAggregateDeviceTapAutoStartKey: false,+ kAudioAggregateDeviceIsPrivateKey: true,+ kAudioAggregateDeviceIsStackedKey: false,+ ]++ var deviceID = AudioObjectID()+ let status = AudioHardwareCreateAggregateDevice(description as CFDictionary, &deviceID)+ guard status == noErr else {+ throw makeError("Failed to create aggregate device", status: status, operation: "create_aggregate_device")+ }+ aggregateDeviceID = deviceID+ }++ private func waitForAggregateDeviceReady() throws {+ var address = AudioObjectPropertyAddress(+ mSelector: kAudioDevicePropertyDeviceIsAlive,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain+ )++ for _ in 0..<20 {+ var isAlive: UInt32 = 0+ var dataSize = UInt32(MemoryLayout<UInt32>.size)+ let status = AudioObjectGetPropertyData(+ aggregateDeviceID,+ &address,+ 0,+ nil,+ &dataSize,+ &isAlive+ )+ if status == noErr, isAlive != 0 {+ return+ }+ Thread.sleep(forTimeInterval: 0.1)+ }++ throw makeError("Aggregate device did not become ready", operation: "wait_for_device")+ }++ private func configureConverter() throws {+ var asbd = AudioStreamBasicDescription()+ var dataSize = UInt32(MemoryLayout<AudioStreamBasicDescription>.size)+ var address = AudioObjectPropertyAddress(+ mSelector: kAudioTapPropertyFormat,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain+ )++ let status = AudioObjectGetPropertyData(tapID, &address, 0, nil, &dataSize, &asbd)+ guard status == noErr else {+ throw makeError("Failed to get tap format", status: status, operation: "get_tap_format")+ }++ guard let sourceFormat = AVAudioFormat(streamDescription: &asbd) else {+ throw makeError("Failed to build source audio format", operation: "source_format")+ }+ guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {+ throw makeError("Failed to create audio converter", operation: "create_converter")+ }++ self.sourceFormat = sourceFormat+ self.converter = converter+ }++ private func registerIOProc() throws {+ guard let sourceFormat, let converter else {+ throw makeError("Audio converter is not configured", operation: "register_ioproc")+ }++ var procID: AudioDeviceIOProcID?+ let status = AudioDeviceCreateIOProcIDWithBlock(+ &procID,+ aggregateDeviceID,+ ioQueue+ ) { [weak self] _, inputData, _, _, _ in+ guard let self, !self.stopping else {+ return+ }+ self.processInput(inputData, sourceFormat: sourceFormat, converter: converter)+ }++ guard status == noErr, let procID else {+ throw makeError("Failed to create IO proc", status: status, operation: "create_ioproc")+ }++ ioProcID = procID+ }++ private func processInput(+ _ inputData: UnsafePointer<AudioBufferList>,+ sourceFormat: AVAudioFormat,+ converter: AVAudioConverter+ ) {+ let inputBufferList = UnsafeMutablePointer(mutating: inputData)+ guard+ let sourceBuffer = AVAudioPCMBuffer(+ pcmFormat: sourceFormat,+ bufferListNoCopy: inputBufferList,+ deallocator: nil+ )+ else {+ return+ }++ let sourceRate = max(sourceFormat.sampleRate, 1)+ let targetCapacity = AVAudioFrameCount(+ ceil(Double(sourceBuffer.frameLength) * targetFormat.sampleRate / sourceRate)+ ) + 32++ guard+ let outputBuffer = AVAudioPCMBuffer(+ pcmFormat: targetFormat,+ frameCapacity: max(targetCapacity, 32)+ )+ else {+ return+ }++ var didProvideInput = false+ var error: NSError?+ let status = converter.convert(to: outputBuffer, error: &error) { _, outStatus in+ if didProvideInput {+ outStatus.pointee = .noDataNow+ return nil+ }+ didProvideInput = true+ outStatus.pointee = .haveData+ return sourceBuffer+ }++ if let error {+ emit(event: [+ "type": "error",+ "code": "convert_failed",+ "message": error.localizedDescription,+ ])+ return+ }++ guard status == .haveData || status == .inputRanDry else {+ return+ }++ let audioBuffer = outputBuffer.audioBufferList.pointee.mBuffers+ guard let data = audioBuffer.mData, audioBuffer.mDataByteSize > 0 else {+ return+ }++ pendingPCM.append(data.assumingMemoryBound(to: UInt8.self), count: Int(audioBuffer.mDataByteSize))+ flushFullChunks()+ }++ private func flushFullChunks() {+ while pendingPCM.count >= chunkBytes {+ let chunk = pendingPCM.prefix(chunkBytes)+ chunk.withUnsafeBytes { rawBuffer in+ guard let baseAddress = rawBuffer.baseAddress else {+ return+ }+ writeAll(fd: STDOUT_FILENO, buffer: baseAddress, count: rawBuffer.count)+ }+ pendingPCM.removeFirst(chunkBytes)+ }+ }++ private func flushPendingPCM() {+ guard !pendingPCM.isEmpty else {+ return+ }++ pendingPCM.withUnsafeBytes { rawBuffer in+ guard let baseAddress = rawBuffer.baseAddress else {+ return+ }+ writeAll(fd: STDOUT_FILENO, buffer: baseAddress, count: rawBuffer.count)+ }+ pendingPCM.removeAll(keepingCapacity: false)+ }++ private func getTapUID() throws -> String {+ var address = AudioObjectPropertyAddress(+ mSelector: kAudioTapPropertyUID,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain+ )+ var unmanagedUID: Unmanaged<CFString>?+ var dataSize = UInt32(MemoryLayout<Unmanaged<CFString>?>.size)+ let status = withUnsafeMutablePointer(to: &unmanagedUID) { pointer in+ AudioObjectGetPropertyData(tapID, &address, 0, nil, &dataSize, pointer)+ }+ guard status == noErr, let unmanagedUID else {+ throw makeError("Failed to get tap UID", status: status, operation: "get_tap_uid")+ }+ return unmanagedUID.takeRetainedValue() as String+ }+}++@available(macOS 14.2, *)+func parseConfig() -> Config {+ var sampleRate = 24_000.0+ var chunkMilliseconds = 100+ let args = Array(CommandLine.arguments.dropFirst())+ var index = 0++ while index < args.count {+ switch args[index] {+ case "--sample-rate":+ if index + 1 < args.count, let value = Double(args[index + 1]), value > 0 {+ sampleRate = value+ }+ index += 2+ case "--chunk-ms":+ if index + 1 < args.count, let value = Int(args[index + 1]), value > 0 {+ chunkMilliseconds = value+ }+ index += 2+ default:+ index += 1+ }+ }++ return Config(sampleRate: sampleRate, chunkMilliseconds: chunkMilliseconds)+}++func emit(event: [String: Any]) {+ guard JSONSerialization.isValidJSONObject(event) else {+ return+ }++ guard let data = try? JSONSerialization.data(withJSONObject: event) else {+ return+ }++ FileHandle.standardError.write(data)+ FileHandle.standardError.write(Data([0x0a]))+}++func writeAll(fd: Int32, buffer: UnsafeRawPointer, count: Int) {+ var written = 0+ while written < count {+ let result = Darwin.write(fd, buffer.advanced(by: written), count - written)+ if result <= 0 {+ break+ }+ written += result+ }+}++func makeError(+ _ message: String,+ status: OSStatus? = nil,+ operation: String,+ code: String? = nil+) -> NSError {+ var userInfo: [String: Any] = [NSLocalizedDescriptionKey: message]+ let resolvedCode = code ?? inferErrorCode(status: status, operation: operation)+ userInfo["AudioTapErrorCode"] = resolvedCode+ userInfo["AudioTapOperation"] = operation+ if let status {+ userInfo["AudioTapStatus"] = Int(status)+ userInfo["NSLocalizedFailureReasonErrorKey"] = "\(message): \(Int(status))"+ }+ return NSError(domain: "OpenWhisprAudioTap", code: Int(status ?? -1), userInfo: userInfo)+}++func inferErrorCode(status: OSStatus?, operation: String) -> String {+ if status == kAudioHardwareIllegalOperationError {+ return "permission_denied"+ }+ return operation+}++if #available(macOS 14.2, *) {+ let capture = AudioTapCapture(config: parseConfig())+ var signalSources: [DispatchSourceSignal] = []++ func stopAndExit(_ code: Int32) -> Never {+ capture.stop()+ exit(code)+ }++ signal(SIGINT, SIG_IGN)+ signal(SIGTERM, SIG_IGN)++ for signalValue in [SIGINT, SIGTERM] {+ let source = DispatchSource.makeSignalSource(signal: signalValue, queue: .main)+ source.setEventHandler {+ stopAndExit(0)+ }+ source.resume()+ signalSources.append(source)+ }++ do {+ try capture.start()+ dispatchMain()+ } catch {+ let nsError = error as NSError+ emit(event: [+ "type": "error",+ "code": nsError.userInfo["AudioTapErrorCode"] as? String ?? "start_failed",+ "message": nsError.localizedDescription,+ "operation": nsError.userInfo["AudioTapOperation"] as? String ?? "start",+ "status": nsError.userInfo["AudioTapStatus"] as? Int ?? nsError.code,+ ])+ capture.stop()+ exit(1)+ }+} else {+ emit(event: [+ "type": "error",+ "code": "unsupported_os",+ "message": "macOS 14.2 or later is required",+ ])+ exit(1)+}diff --git a/scripts/build-macos-audio-tap.js b/scripts/build-macos-audio-tap.jsnew file mode 100644index 000000000..2056f2504--- /dev/null+++ b/scripts/build-macos-audio-tap.js@@ -0,0 +1,169 @@+#!/usr/bin/env node++const { spawnSync } = require("child_process");+const crypto = require("crypto");+const fs = require("fs");+const path = require("path");++if (process.platform !== "darwin") {+ process.exit(0);+}++const archIndex = process.argv.indexOf("--arch");+const targetArch =+ (archIndex !== -1 && process.argv[archIndex + 1]) || process.env.TARGET_ARCH || process.arch;++const ARCH_TO_TARGET = {+ arm64: "arm64-apple-macosx14.2",+ x64: "x86_64-apple-macosx14.2",+};++const ARCH_CPU_TYPE = {+ arm64: 0x0100000c,+ x64: 0x01000007,+};++const swiftTarget = ARCH_TO_TARGET[targetArch];+if (!swiftTarget) {+ console.error(`[audio-tap] Unsupported architecture: ${targetArch}`);+ process.exit(1);+}++const projectRoot = path.resolve(__dirname, "..");+const swiftSource = path.join(projectRoot, "resources", "macos-audio-tap.swift");+const outputDir = path.join(projectRoot, "resources", "bin");+const outputBinary = path.join(outputDir, "macos-audio-tap");+const hashFile = path.join(outputDir, `.macos-audio-tap.${targetArch}.hash`);+const moduleCacheDir = path.join(outputDir, ".swift-module-cache");++function log(message) {+ console.log(`[audio-tap] ${message}`);+}++function ensureDir(dirPath) {+ if (!fs.existsSync(dirPath)) {+ fs.mkdirSync(dirPath, { recursive: true });+ }+}++function verifyBinaryArch(binaryPath, expectedArch) {+ try {+ const fd = fs.openSync(binaryPath, "r");+ const header = Buffer.alloc(8);+ fs.readSync(fd, header, 0, 8, 0);+ fs.closeSync(fd);++ if (header.readUInt32LE(0) !== 0xfeedfacf) {+ return false;+ }++ return header.readInt32LE(4) === ARCH_CPU_TYPE[expectedArch];+ } catch {+ return false;+ }+}++if (!fs.existsSync(swiftSource)) {+ console.error(`[audio-tap] Swift source not found at ${swiftSource}`);+ process.exit(1);+}++ensureDir(outputDir);+ensureDir(moduleCacheDir);++let needsBuild = true;+if (fs.existsSync(outputBinary)) {+ if (!verifyBinaryArch(outputBinary, targetArch)) {+ log(`Existing binary is wrong architecture (expected ${targetArch}), rebuild needed`);+ } else {+ try {+ const binaryStat = fs.statSync(outputBinary);+ const sourceStat = fs.statSync(swiftSource);+ needsBuild = binaryStat.mtimeMs < sourceStat.mtimeMs;+ } catch {+ needsBuild = true;+ }+ }+}++if (!needsBuild && fs.existsSync(outputBinary)) {+ try {+ const sourceContent = fs.readFileSync(swiftSource, "utf8");+ const currentHash = crypto.createHash("sha256").update(sourceContent).digest("hex");+ const savedHash = fs.existsSync(hashFile) ? fs.readFileSync(hashFile, "utf8").trim() : "";+ if (savedHash !== currentHash) {+ log("Source hash changed, rebuild needed");+ needsBuild = true;+ }+ } catch (error) {+ log(`Hash check failed: ${error.message}, forcing rebuild`);+ needsBuild = true;+ }+}++if (!needsBuild) {+ process.exit(0);+}++function attemptCompile(command, args) {+ log(`Compiling with ${[command, ...args].join(" ")}`);+ return spawnSync(command, args, {+ stdio: "inherit",+ env: {+ ...process.env,+ SWIFT_MODULE_CACHE_PATH: moduleCacheDir,+ },+ });+}++const compileArgs = [+ swiftSource,+ "-O",+ "-target",+ swiftTarget,+ "-module-cache-path",+ moduleCacheDir,+ "-o",+ outputBinary,+ "-framework",+ "CoreAudio",+ "-framework",+ "AudioToolbox",+ "-framework",+ "AVFoundation",+ "-framework",+ "Foundation",+];++let result = attemptCompile("xcrun", ["swiftc", ...compileArgs]);+if (result.status !== 0) {+ result = attemptCompile("swiftc", compileArgs);+}++if (result.status !== 0) {+ console.error("[audio-tap] Failed to compile macOS audio tap binary.");+ process.exit(result.status ?? 1);+}++try {+ fs.chmodSync(outputBinary, 0o755);+} catch (error) {+ console.warn(`[audio-tap] Unable to set executable permissions: ${error.message}`);+}++if (!verifyBinaryArch(outputBinary, targetArch)) {+ console.error(+ `[audio-tap] FATAL: Compiled binary architecture does not match target (${targetArch}).`+ );+ process.exit(1);+}++try {+ const sourceContent = fs.readFileSync(swiftSource, "utf8");+ const hash = crypto.createHash("sha256").update(sourceContent).digest("hex");+ fs.writeFileSync(hashFile, hash);+} catch (error) {+ log(`Warning: Could not save source hash: ${error.message}`);+}++log(`Successfully built macOS audio tap binary (${targetArch}).`);diff --git a/src/helpers/audioTapManager.js b/src/helpers/audioTapManager.jsnew file mode 100644index 000000000..25f7950cd--- /dev/null+++ b/src/helpers/audioTapManager.js@@ -0,0 +1,437 @@+const { spawn } = require("child_process");+const fs = require("fs");+const path = require("path");+const debugLogger = require("./debugLogger");++const ARCH_CPU_TYPE = {+ arm64: 0x0100000c,+ x64: 0x01000007,+};++const DEFAULT_SAMPLE_RATE = 24000;+const DEFAULT_CHUNK_MS = 100;+const START_TIMEOUT_MS = 3000;+const REQUEST_TIMEOUT_MS = 15000;+const STOP_TIMEOUT_MS = 5000;++function compareVersions(left, right) {+ const leftParts = String(left)+ .split(".")+ .map((part) => parseInt(part, 10) || 0);+ const rightParts = String(right)+ .split(".")+ .map((part) => parseInt(part, 10) || 0);+ const length = Math.max(leftParts.length, rightParts.length);++ for (let index = 0; index < length; index += 1) {+ const leftPart = leftParts[index] || 0;+ const rightPart = rightParts[index] || 0;+ if (leftPart !== rightPart) {+ return leftPart > rightPart ? 1 : -1;+ }+ }++ return 0;+}++class AudioTapManager {+ constructor() {+ this.process = null;+ this.stderrBuffer = "";+ this.onChunk = null;+ this.onError = null;+ this.isStopping = false;+ this.permissionStatus = "unknown";+ this._requestPromise = null;+ }++ isSupported() {+ return (+ process.platform === "darwin" && compareVersions(process.getSystemVersion(), "14.2") >= 0+ );+ }++ isAvailable() {+ if (!this.isSupported()) {+ return false;+ }+ const binaryPath = this.resolveBinary();+ if (!binaryPath) {+ return false;+ }+ return !this._checkArchMismatch(binaryPath);+ }++ getPermissionStatus() {+ if (this.process) {+ return "granted";+ }+ return this.permissionStatus;+ }++ async requestAccess() {+ if (!this.isSupported()) {+ return { granted: false, status: "unsupported" };+ }+ if (this.process) {+ this.permissionStatus = "granted";+ return { granted: true, status: "granted" };+ }+ if (this._requestPromise) {+ return this._requestPromise;+ }++ this._requestPromise = this._probeForAccess()+ .catch((error) => {+ const status = error.code === "permission_denied" ? "denied" : "unknown";+ this.permissionStatus = status;+ return { granted: false, status, error: error.message };+ })+ .finally(() => {+ this._requestPromise = null;+ });++ return this._requestPromise;+ }++ async start({ onChunk, onError } = {}) {+ if (!this.isSupported()) {+ throw new Error("macOS 14.2 or later is required for native system audio capture.");+ }+ if (this.process) {+ this.onChunk = onChunk || null;+ this.onError = onError || null;+ this.permissionStatus = "granted";+ return;+ }+ if (this._requestPromise) {+ await this._requestPromise.catch(() => {});+ }++ const binaryPath = this._prepareBinary();+ this.onChunk = onChunk || null;+ this.onError = onError || null;+ this.isStopping = false;+ this.stderrBuffer = "";++ const child = spawn(+ binaryPath,+ ["--sample-rate", String(DEFAULT_SAMPLE_RATE), "--chunk-ms", String(DEFAULT_CHUNK_MS)],+ { stdio: ["ignore", "pipe", "pipe"] }+ );+ this.process = child;++ await new Promise((resolve, reject) => {+ let settled = false;+ const timeout = setTimeout(() => {+ finish(+ reject,+ new Error("Timed out starting macOS audio tap. Check System Audio permissions."),+ true+ );+ }, START_TIMEOUT_MS);++ const finish = (callback, value, shouldStop = false) => {+ if (settled) {+ return;+ }+ settled = true;+ clearTimeout(timeout);+ if (shouldStop) {+ void this.stop();+ }+ callback(value);+ };++ child.stdout.on("data", (chunk) => {+ if (this.process !== child) {+ return;+ }+ this.onChunk?.(chunk);+ });++ child.stderr.on("data", (chunk) => {+ if (this.process !== child) {+ return;+ }+ this._consumeStderr(chunk, (message) => {+ if (message.type === "start") {+ this.permissionStatus = "granted";+ finish(resolve);+ return;+ }++ if (message.type === "error") {+ const error = this._buildProcessError(message);+ if (error.code === "permission_denied") {+ this.permissionStatus = "denied";+ }+ if (!settled) {+ finish(reject, error, true);+ } else {+ this.onError?.(error);+ }+ }+ });+ });++ child.on("error", (error) => {+ if (this.process === child) {+ this.process = null;+ }+ finish(reject, error);+ });++ child.on("exit", (code, signal) => {+ const wasStopping = this.isStopping;+ if (this.process === child) {+ this.process = null;+ }++ if (!settled) {+ finish(+ reject,+ new Error(+ `macOS audio tap exited before start (code ${code ?? "null"}, signal ${signal ?? "null"}).`+ )+ );+ return;+ }++ if (!wasStopping) {+ this.onError?.(+ new Error(+ `macOS audio tap exited unexpectedly (code ${code ?? "null"}, signal ${signal ?? "null"}).`+ )+ );+ }+ });+ });+ }++ async stop() {+ if (!this.process) {+ return;+ }++ const child = this.process;+ this.isStopping = true;++ await new Promise((resolve) => {+ const timeout = setTimeout(() => {+ try {+ child.kill("SIGKILL");+ } catch {+ resolve();+ }+ }, STOP_TIMEOUT_MS);++ child.once("exit", () => {+ clearTimeout(timeout);+ resolve();+ });++ try {+ child.kill("SIGTERM");+ } catch {+ clearTimeout(timeout);+ resolve();+ }+ });++ if (this.process === child) {+ this.process = null;+ }+ this.stderrBuffer = "";+ this.onChunk = null;+ this.onError = null;+ this.isStopping = false;+ }++ async _probeForAccess() {+ const binaryPath = this._prepareBinary();+ const child = spawn(+ binaryPath,+ ["--sample-rate", String(DEFAULT_SAMPLE_RATE), "--chunk-ms", String(DEFAULT_CHUNK_MS)],+ { stdio: ["ignore", "ignore", "pipe"] }+ );++ let stderrBuffer = "";++ return new Promise((resolve, reject) => {+ let settled = false;+ const timeout = setTimeout(() => {+ finish(resolve, { granted: false, status: "unknown" }, true);+ }, REQUEST_TIMEOUT_MS);++ const finish = (callback, value, shouldStop = false) => {+ if (settled) {+ return;+ }+ settled = true;+ clearTimeout(timeout);+ if (shouldStop) {+ try {+ child.kill("SIGTERM");+ } catch {}+ }+ callback(value);+ };++ child.stderr.on("data", (chunk) => {+ stderrBuffer += chunk.toString();+ let newlineIndex = stderrBuffer.indexOf("\n");+ while (newlineIndex !== -1) {+ const line = stderrBuffer.slice(0, newlineIndex).trim();+ stderrBuffer = stderrBuffer.slice(newlineIndex + 1);+ if (line) {+ try {+ const message = JSON.parse(line);+ if (message.type === "start") {+ this.permissionStatus = "granted";+ finish(resolve, { granted: true, status: "granted" }, true);+ return;+ }+ if (message.type === "error") {+ const error = this._buildProcessError(message);+ if (error.code === "permission_denied") {+ this.permissionStatus = "denied";+ }+ finish(reject, error, true);+ return;+ }+ } catch {+ debugLogger.warn("[AudioTapManager] Non-JSON stderr output", { line }, "meeting");+ }+ }+ newlineIndex = stderrBuffer.indexOf("\n");+ }+ });++ child.on("error", (error) => {+ finish(reject, error);+ });++ child.on("exit", (code, signal) => {+ if (settled) {+ return;+ }+ finish(resolve, {+ granted: false,+ status: this.permissionStatus === "denied" ? "denied" : "unknown",+ error:+ code && code !== 0+ ? `macOS audio tap exited (code ${code}, signal ${signal ?? "null"})`+ : undefined,+ });+ });+ });+ }++ _prepareBinary() {+ const binaryPath = this.resolveBinary();+ if (!binaryPath) {+ throw new Error(+ "macOS audio tap binary not found. Run `npm run compile:audio-tap` before packaging."+ );+ }++ const archMismatch = this._checkArchMismatch(binaryPath);+ if (archMismatch) {+ throw new Error(archMismatch);+ }++ try {+ fs.accessSync(binaryPath, fs.constants.X_OK);+ } catch {+ fs.chmodSync(binaryPath, 0o755);+ }++ return binaryPath;+ }++ resolveBinary() {+ const candidates = new Set([+ path.join(__dirname, "..", "..", "resources", "bin", "macos-audio-tap"),+ path.join(__dirname, "..", "..", "resources", "macos-audio-tap"),+ ]);++ if (process.resourcesPath) {+ candidates.add(path.join(process.resourcesPath, "macos-audio-tap"));+ candidates.add(path.join(process.resourcesPath, "bin", "macos-audio-tap"));+ candidates.add(path.join(process.resourcesPath, "resources", "bin", "macos-audio-tap"));+ candidates.add(+ path.join(process.resourcesPath, "app.asar.unpacked", "resources", "bin", "macos-audio-tap")+ );+ }++ for (const candidate of candidates) {+ if (fs.existsSync(candidate)) {+ return candidate;+ }+ }++ return null;+ }++ _consumeStderr(chunk, onMessage) {+ this.stderrBuffer += chunk.toString();+ let newlineIndex = this.stderrBuffer.indexOf("\n");++ while (newlineIndex !== -1) {+ const line = this.stderrBuffer.slice(0, newlineIndex).trim();+ this.stderrBuffer = this.stderrBuffer.slice(newlineIndex + 1);++ if (line) {+ try {+ onMessage(JSON.parse(line));+ } catch {+ debugLogger.warn("[AudioTapManager] Non-JSON stderr output", { line }, "meeting");+ }+ }++ newlineIndex = this.stderrBuffer.indexOf("\n");+ }+ }++ _buildProcessError(message) {+ const error = new Error(message.message || "macOS audio tap failed");+ error.code = message.code;+ error.status = message.status;+ error.operation = message.operation;+ return error;+ }++ _checkArchMismatch(binaryPath) {+ try {+ const fd = fs.openSync(binaryPath, "r");+ const header = Buffer.alloc(8);+ fs.readSync(fd, header, 0, 8, 0);+ fs.closeSync(fd);++ if (header.readUInt32LE(0) !== 0xfeedfacf) {+ return "macOS audio tap binary is not a valid 64-bit Mach-O file.";+ }++ const cpuType = header.readInt32LE(4);+ const expectedCpu = ARCH_CPU_TYPE[process.arch];+ if (expectedCpu && cpuType !== expectedCpu) {+ return (+ `macOS audio tap binary architecture mismatch: binary does not match ${process.arch}. ` ++ `Try reinstalling or run \`TARGET_ARCH=${process.arch} npm run compile:audio-tap\`.`+ );+ }++ return null;+ } catch (error) {+ debugLogger.warn(+ "[AudioTapManager] Could not verify binary architecture",+ { error: error.message },+ "meeting"+ );+ return null;+ }+ }+}++module.exports = AudioTapManager;diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 8e67af09d..2179ebd01 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -103,6 +103,7 @@ class IPCHandlers {this.whisperCudaManager = managers.whisperCudaManager;this.googleCalendarManager = managers.googleCalendarManager;this.meetingDetectionEngine = managers.meetingDetectionEngine;+ this.audioTapManager = managers.audioTapManager;this.sessionId = crypto.randomUUID();this.assemblyAiStreaming = null;this.deepgramStreaming = null;@@ -2053,11 +2054,34 @@ class IPCHandlers {ipcMain.handle("check-system-audio-access", () => {if (process.platform !== "darwin") {- return { granted: true };+ return { granted: true, status: "granted", mode: "unsupported" };+ }++ if (this.audioTapManager?.isAvailable()) {+ const status = this.audioTapManager.getPermissionStatus();+ return { granted: status === "granted", status, mode: "native" };}+const { systemPreferences } = require("electron");const status = systemPreferences.getMediaAccessStatus("screen");- return { granted: status === "granted" };+ return {+ granted: status === "granted",+ status: status === "granted" ? "granted" : status === "denied" ? "denied" : "unknown",+ mode: "legacy",+ };+ });++ ipcMain.handle("request-system-audio-access", async () => {+ if (process.platform !== "darwin") {+ return { granted: true, status: "granted", mode: "unsupported" };+ }++ if (!this.audioTapManager?.isAvailable()) {+ return { granted: false, status: "unknown", mode: "legacy" };+ }++ const result = await this.audioTapManager.requestAccess();+ return { ...result, mode: "native" };});// Auth: clear all session cookies for sign-out.@@ -2536,13 +2560,18 @@ class IPCHandlers {meetingTranscriptionStartInProgress = true;try {+ const systemAudioMode = getMeetingSystemAudioMode();+// If already prepared (warm connections from prepare), just re-attach handlersif (this._meetingMicStreaming?.isConnected && this._meetingSystemStreaming?.isConnected) {debugLogger.debug("Meeting transcription start: reusing warm connections");const win = BrowserWindow.fromWebContents(event.sender);attachMeetingStreamingHandlers(this._meetingMicStreaming, win, "mic");attachMeetingStreamingHandlers(this._meetingSystemStreaming, win, "system");- return { success: true };+ if (systemAudioMode === "native") {+ await startNativeMeetingSystemAudio(event);+ }+ return { success: true, systemAudioMode };}if (options.provider !== "openai-realtime") {@@ -2550,7 +2579,10 @@ class IPCHandlers {}await connectRealtimeStreaming(event, options);- return { success: true };+ if (systemAudioMode === "native") {+ await startNativeMeetingSystemAudio(event);+ }+ return { success: true, systemAudioMode };} catch (error) {debugLogger.error("Meeting transcription start error", { error: error.message });return { success: false, error: error.message };@@ -2560,7 +2592,7 @@ class IPCHandlers {});let meetingSendCounts = { mic: 0, system: 0 };- ipcMain.on("meeting-transcription-send", (_event, audioBuffer, source) => {+ const sendMeetingAudio = (audioBuffer, source) => {const streaming = source === "mic" ? this._meetingMicStreaming : this._meetingSystemStreaming;if (!streaming) {if (meetingSendCounts[source] === 0) {@@ -2568,7 +2600,7 @@ class IPCHandlers {}return;}- const buf = Buffer.from(audioBuffer);+ const buf = Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer);const sent = streaming.sendAudio(buf);meetingSendCounts[source]++;if (meetingSendCounts[source] <= 5 || meetingSendCounts[source] % 100 === 0) {@@ -2581,10 +2613,35 @@ class IPCHandlers {count: meetingSendCounts[source],});}+ };++ const getMeetingSystemAudioMode = () =>+ this.audioTapManager?.isAvailable() ? "native" : "legacy";++ const startNativeMeetingSystemAudio = async (event) => {+ const win = BrowserWindow.fromWebContents(event.sender);+ await this.audioTapManager.start({+ onChunk: (chunk) => {+ sendMeetingAudio(chunk, "system");+ },+ onError: (error) => {+ if (win && !win.isDestroyed()) {+ win.webContents.send("meeting-transcription-error", error.message);+ }+ },+ });+ };++ ipcMain.on("meeting-transcription-send", (_event, audioBuffer, source) => {+ sendMeetingAudio(audioBuffer, source);});ipcMain.handle("meeting-transcription-stop", async () => {try {+ if (this.audioTapManager) {+ await this.audioTapManager.stop();+ }+const results = await Promise.all([this._meetingMicStreaming? this._meetingMicStreaming.disconnect()diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.tsindex 414a7ae2e..fadc3f9d2 100644--- a/src/hooks/useMeetingTranscription.ts+++ b/src/hooks/useMeetingTranscription.ts@@ -361,9 +361,8 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {try {const startTime = performance.now();- const [startResult, stream, micResult] = await Promise.all([+ const [startResult, micResult] = await Promise.all([window.electronAPI?.meetingTranscriptionStart?.(getMeetingTranscriptionOptions()),- getSystemAudioStream(),getMeetingMicConstraints().then((constraints) =>navigator.mediaDevices.getUserMedia(constraints).catch((err) => {logger.error(@@ -377,6 +376,8 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {]);const streamsMs = performance.now() - startTime;+ const systemAudioMode = startResult?.systemAudioMode || "legacy";+ const stream = systemAudioMode === "legacy" ? await getSystemAudioStream() : null;// Abort if stop was called during setupif (!isRecordingRef.current) {@@ -401,7 +402,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {return;}- if (!stream) {+ if (systemAudioMode === "legacy" && !stream) {logger.error("Could not capture system audio for meeting transcription", {}, "meeting");micResult?.getTracks().forEach((track) => track.stop());await window.electronAPI?.meetingTranscriptionStop?.();@@ -410,7 +411,9 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {setIsRecording(false);return;}- streamRef.current = stream;+ if (stream) {+ streamRef.current = stream;+ }const partialSetters = { mic: setMicPartial, system: setSystemPartial };@@ -469,49 +472,50 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {const pendingMicChunks: ArrayBuffer[] = [];let socketReady = false;- // Use the loopback stream's native sample rate for the system AudioContext.- // System audio loopback produces silence when forced through a 24kHz- // AudioContext — the resampling path differs from getUserMedia and doesn't- // work correctly for loopback sources.- const loopbackRate = stream.getAudioTracks()[0]?.getSettings()?.sampleRate || 48000;- const systemNeedsResample = loopbackRate !== 24000;- const audioContext = new AudioContext({ sampleRate: loopbackRate });- await detachFromOutputDevice(audioContext);- audioContextRef.current = audioContext;-- logger.debug(- "System audio context created",- { loopbackRate, needsResample: systemNeedsResample },- "meeting"- );+ let systemPipelinePromise: Promise<{+ source: MediaStreamAudioSourceNode;+ processor: AudioWorkletNode;+ }> | null = null;+ if (stream) {+ const loopbackRate = stream.getAudioTracks()[0]?.getSettings()?.sampleRate || 48000;+ const systemNeedsResample = loopbackRate !== 24000;+ const audioContext = new AudioContext({ sampleRate: loopbackRate });+ await detachFromOutputDevice(audioContext);+ audioContextRef.current = audioContext;++ logger.debug(+ "System audio context created",+ { loopbackRate, needsResample: systemNeedsResample },+ "meeting"+ );- const systemPipelinePromise = createAudioPipeline({- stream,- context: audioContext,- label: "Meeting system",- onChunk: (chunk) => {- if (!isRecordingRef.current) return;- let samples = new Int16Array(chunk);-- // Downsample to 24kHz for the OpenAI Realtime API- if (systemNeedsResample) {- const ratio = loopbackRate / 24000;- const outLen = Math.floor(samples.length / ratio);- const out = new Int16Array(outLen);- for (let i = 0; i < outLen; i++) {- out[i] = samples[Math.round(i * ratio)];+ systemPipelinePromise = createAudioPipeline({+ stream,+ context: audioContext,+ label: "Meeting system",+ onChunk: (chunk) => {+ if (!isRecordingRef.current) return;+ let samples = new Int16Array(chunk);++ if (systemNeedsResample) {+ const ratio = loopbackRate / 24000;+ const outLen = Math.floor(samples.length / ratio);+ const out = new Int16Array(outLen);+ for (let i = 0; i < outLen; i++) {+ out[i] = samples[Math.round(i * ratio)];+ }+ samples = out;}- samples = out;- }- const buf = samples.buffer;- if (socketReady) {- window.electronAPI?.meetingTranscriptionSend?.(buf, "system");- return;- }- pendingSystemChunks.push(buf.slice(0) as ArrayBuffer);- },- });+ const buf = samples.buffer;+ if (socketReady) {+ window.electronAPI?.meetingTranscriptionSend?.(buf, "system");+ return;+ }+ pendingSystemChunks.push(buf.slice(0) as ArrayBuffer);+ },+ });+ }let micPipelinePromise: Promise<void> | null = null;if (micResult) {@@ -583,6 +587,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {logger.info("Meeting transcription started successfully",{+ systemAudioMode,bufferedChunks: pendingSystemChunks.length + pendingMicChunks.length,streamsMs: Math.round(streamsMs),totalMs: Math.round(totalMs),diff --git a/src/hooks/useSystemAudioPermission.ts b/src/hooks/useSystemAudioPermission.tsindex 81c07136f..42dc5eb3c 100644--- a/src/hooks/useSystemAudioPermission.ts+++ b/src/hooks/useSystemAudioPermission.ts@@ -4,6 +4,9 @@ import { getCachedPlatform } from "../utils/platform";export function useSystemAudioPermission() {const isMacOS = getCachedPlatform() === "darwin";const [granted, setGranted] = useState(!isMacOS);+ const [accessMode, setAccessMode] = useState<"native" | "legacy" | "unsupported">(+ isMacOS ? "legacy" : "unsupported"+ );const checkingRef = useRef(false);const check = useCallback(async () => {@@ -12,6 +15,7 @@ export function useSystemAudioPermission() {try {const result = await window.electronAPI?.checkSystemAudioAccess?.();setGranted(result?.granted ?? false);+ setAccessMode(result?.mode ?? "legacy");} finally {checkingRef.current = false;}@@ -37,6 +41,14 @@ export function useSystemAudioPermission() {// Trigger the native macOS permission prompt via getDisplayMedia (used in onboarding)const request = useCallback(async (): Promise<boolean> => {if (!isMacOS) return true;+ if (accessMode === "native") {+ const result = await window.electronAPI?.requestSystemAudioAccess?.();+ const isGranted = result?.granted ?? false;+ setGranted(isGranted);+ setAccessMode(result?.mode ?? "native");+ return isGranted;+ }+try {const stream = await navigator.mediaDevices.getDisplayMedia({ audio: true, video: true });stream.getTracks().forEach((t) => t.stop());@@ -45,7 +57,7 @@ export function useSystemAudioPermission() {} catch {return false;}- }, [isMacOS]);+ }, [accessMode, isMacOS]);return { granted, request, openSettings, check, isMacOS };}diff --git a/src/types/electron.ts b/src/types/electron.tsindex 27e2a4c86..c138b51a7 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -115,6 +115,13 @@ export interface AudioDiagnosticsResult {models: string[];}+export interface SystemAudioAccessResult {+ granted: boolean;+ status: "granted" | "denied" | "unknown" | "unsupported";+ mode: "native" | "legacy" | "unsupported";+ error?: string;+}+export interface UpdateCheckResult {updateAvailable: boolean;version?: string;@@ -732,7 +739,8 @@ declare global {// System settings helpersrequestMicrophoneAccess?: () => Promise<{ granted: boolean }>;- checkSystemAudioAccess?: () => Promise<{ granted: boolean }>;+ checkSystemAudioAccess?: () => Promise<SystemAudioAccessResult>;+ requestSystemAudioAccess?: () => Promise<SystemAudioAccessResult>;openMicrophoneSettings?: () => Promise<{ success: boolean; error?: string }>;openSoundInputSettings?: () => Promise<{ success: boolean; error?: string }>;openAccessibilitySettings?: () => Promise<{ success: boolean; error?: string }>;@@ -1126,7 +1134,7 @@ declare global {provider?: string;model?: string;language?: string;- }) => Promise<{ success: boolean; error?: string }>;+ }) => Promise<{ success: boolean; error?: string; systemAudioMode?: "native" | "legacy" }>;meetingTranscriptionSend?: (buffer: ArrayBuffer, source: "mic" | "system") => void;meetingTranscriptionStop?: () => Promise<{success: boolean;@@ -1188,6 +1196,15 @@ declare global {onNavigateToMeetingNote?: (callback: (data: { noteId: number; folderId: number; event: any }) => void) => () => void;+ onUpdateNotificationData?: (+ callback: (data: { version: string; releaseDate?: string }) => void+ ) => () => void;+ getUpdateNotificationData?: () => Promise<{+ version: string;+ releaseDate?: string;+ } | null>;+ updateNotificationReady?: () => Promise<void>;+ updateNotificationRespond?: (action: string) => Promise<{ success: boolean }>;};api?: {