Skip to content

Doubao-powered voice input for Linux, Wayland and Omarchy

9.1 KB
Raw Blame
import Foundation
import AVFoundation
import AudioToolbox

/// Captures microphone audio using AVAudioEngine, resamples to 16kHz mono,
/// and delivers Int16 little-endian PCM data via the `onAudioData` callback.
class AudioCaptureManager {
    private var audioEngine: AVAudioEngine?
    private let lock = NSLock()
    private var callback: ((Data) -> Void)?
    private var isCapturing = false
    private var captureGeneration = UUID()
    private var preferredDeviceID = ""
    private(set) var activeDevice: AudioInputDevice?
    private var deviceMonitor: AudioDeviceMonitor?
    private var recoveryWork: DispatchWorkItem?
    private var recoveryAttempts = 0
    private var configurationObserver: NSObjectProtocol?
    private let inputDevices: () -> [AudioInputDevice]
    private let defaultInputUID: () -> String?
    init(inputDevices: @escaping () -> [AudioInputDevice] = AudioDevices.inputs,
         defaultInputUID: @escaping () -> String? = AudioDevices.defaultInputUID) {
        self.inputDevices = inputDevices
        self.defaultInputUID = defaultInputUID
    }
    var onError: (() -> Void)?
    private var rmsCallback: ((Double) -> Void)?
    var onRMS: ((Double) -> Void)? {
        get { lock.lock(); defer { lock.unlock() }; return rmsCallback }
        set { lock.lock(); rmsCallback = newValue; lock.unlock() }
    }

    /// Called on the audio thread with raw Int16 LE PCM data ready to send over WebSocket.
    var onAudioData: ((Data) -> Void)? {
        get { lock.lock(); defer { lock.unlock() }; return callback }
        set { lock.lock(); callback = newValue; lock.unlock() }
    }

    func startCapture(deviceID: String = "") throws {
        guard !isCapturing else { return }
        preferredDeviceID = deviceID
        recoveryAttempts = 0
        try beginCapture()
        isCapturing = true
        deviceMonitor = AudioDeviceMonitor { [weak self] in self?.hardwareChanged() }
    }

    private func resolvedDevice() -> AudioInputDevice? {
        AudioDevices.resolve(preferred: preferredDeviceID, inputs: inputDevices(),
                             defaultUID: defaultInputUID())
    }

    private func beginCapture() throws {
        guard let device = resolvedDevice() else {
            throw NSError(domain: "AudioCapture", code: -3,
                          userInfo: [NSLocalizedDescriptionKey: "No audio input available"])
        }
        let token = UUID()
        lock.lock(); captureGeneration = token; lock.unlock()
        let engine = AVAudioEngine()
        let inputNode = engine.inputNode
        // Explicitly resolve even System default: a running engine does not reliably
        // rebind when the default changes to a device with the same audio format.
        guard let unit = inputNode.audioUnit else { throw NSError(domain: "AudioCapture", code: -4) }
        var id = device.audioID
        guard AudioUnitSetProperty(unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0,
                                   &id, UInt32(MemoryLayout.size(ofValue: id))) == noErr else {
            throw NSError(domain: "AudioCapture", code: -4)
        }
        let inputFormat = inputNode.outputFormat(forBus: 0)

        guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else {
            throw NSError(domain: "AudioCapture", code: -1,
                          userInfo: [NSLocalizedDescriptionKey: "No audio input available"])
        }

        // Target: 16kHz mono Float32 (converted to Int16 manually per doubao spec)
        let targetFormat = AVAudioFormat(
            commonFormat: .pcmFormatFloat32,
            sampleRate: 16000,
            channels: 1,
            interleaved: false
        )!

        guard let conv = AVAudioConverter(from: inputFormat, to: targetFormat) else {
            throw NSError(domain: "AudioCapture", code: -2,
                          userInfo: [NSLocalizedDescriptionKey: "Cannot create audio format converter"])
        }
        var reportedInput = false
        inputNode.installTap(onBus: 0, bufferSize: 4096, format: inputFormat) {
            [weak self] buffer, _ in
            guard let self = self, self.isCurrent(token) else { return }
            if !reportedInput && buffer.frameLength > 0 {
                reportedInput = true
                DispatchQueue.main.async { [weak self] in
                    guard let self, self.isCapturing, self.isCurrent(token) else { return }
                    self.recoveryAttempts = 0
                }
            }
            self.processAudioBuffer(buffer, converter: conv, token: token, onAudioData: self.onAudioData)
        }

        engine.prepare()
        do { try engine.start() }
        catch {
            inputNode.removeTap(onBus: 0)
            throw error
        }

        self.audioEngine = engine
        self.activeDevice = device
        configurationObserver = NotificationCenter.default.addObserver(forName: .AVAudioEngineConfigurationChange, object: engine, queue: nil) { [weak self, weak engine] _ in
            // Never tear down an engine on the notification's internal audio queue.
            DispatchQueue.main.async {
                guard let self, let engine, self.isCurrent(token), self.isCapturing else { return }
                let format = engine.inputNode.outputFormat(forBus: 0)
                if self.resolvedDevice() == self.activeDevice && engine.isRunning &&
                    format.sampleRate == inputFormat.sampleRate && format.channelCount == inputFormat.channelCount { return }
                self.scheduleRecovery()
            }
        }
    }

    func hardwareChanged() {
        guard isCapturing, resolvedDevice() != activeDevice || audioEngine?.isRunning != true else { return }
        scheduleRecovery()
    }

    private func scheduleRecovery() {
        guard isCapturing, recoveryWork == nil else { return }
        // Coalesce CoreAudio's device/default/configuration notifications. Retain the
        // callbacks and ASR session while replacing only the microphone stream.
        if recoveryAttempts >= 4 {
            stopCapture()
            onError?()
            return
        }
        stopEngine()
        let token = captureGeneration
        let work = DispatchWorkItem { [weak self] in
            guard let self, self.isCapturing, self.isCurrent(token) else { return }
            self.recoveryWork = nil
            self.recoveryAttempts += 1
            do { try self.beginCapture() }
            catch { self.scheduleRecovery() }
        }
        recoveryWork = work
        DispatchQueue.main.asyncAfter(deadline: .now() + 0.25, execute: work)
    }
    private func isCurrent(_ token: UUID) -> Bool {
        lock.lock(); defer { lock.unlock() }; return captureGeneration == token
    }

    func stopCapture() {
        isCapturing = false
        recoveryWork?.cancel(); recoveryWork = nil
        deviceMonitor = nil
        stopEngine()
    }

    private func stopEngine() {
        lock.lock(); captureGeneration = UUID(); lock.unlock()
        if let observer = configurationObserver { NotificationCenter.default.removeObserver(observer); configurationObserver = nil }
        audioEngine?.inputNode.removeTap(onBus: 0)
        audioEngine?.stop()
        audioEngine = nil
        activeDevice = nil
    }

    // MARK: - Audio Processing

    /// Resample native audio to 16kHz mono Float32, then convert to Int16 LE PCM.
    private func processAudioBuffer(_ buffer: AVAudioPCMBuffer, converter: AVAudioConverter, token: UUID,
                                    onAudioData: ((Data) -> Void)?) {
        let ratio = 16000.0 / converter.inputFormat.sampleRate
        let outputFrameCapacity = AVAudioFrameCount(ceil(Double(buffer.frameLength) * ratio))
        guard outputFrameCapacity > 0,
              let outputBuffer = AVAudioPCMBuffer(
                  pcmFormat: converter.outputFormat,
                  frameCapacity: outputFrameCapacity
              ) else { return }

        var error: NSError?
        var hasInput = true
        let status = converter.convert(to: outputBuffer, error: &error) { _, outStatus in
            if hasInput {
                hasInput = false
                outStatus.pointee = .haveData
                return buffer
            }
            outStatus.pointee = .noDataNow
            return nil
        }

        guard status != .error, error == nil, outputBuffer.frameLength > 0 else { return }

        // Float32 → Int16 LE using doubao's non-symmetric scaling:
        //   negative samples × 32768, positive samples × 32767
        let floats = outputBuffer.floatChannelData![0]
        let count = Int(outputBuffer.frameLength)
        let energy = (0..<count).reduce(0.0) { $0 + Double(floats[$1] * floats[$1]) }
        guard isCurrent(token) else { return }
        onRMS?(sqrt(energy / Double(count)))
        var pcm = Data(count: count * 2)
        pcm.withUnsafeMutableBytes { raw in
            let int16 = raw.bindMemory(to: Int16.self)
            for i in 0..<count {
                let s = floats[i]
                let v = s < 0 ? s * 32768.0 : s * 32767.0
                int16[i] = Int16(max(-32768, min(32767, v)))
            }
        }

        guard isCurrent(token) else { return }
        onAudioData?(pcm)
    }
}