Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
downmix stereo audio from both channels
grohith327d8990906aefe
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxprojindex a2cb0a9f..14b46cb9 100644--- a/Fluid.xcodeproj/project.pbxproj+++ b/Fluid.xcodeproj/project.pbxproj@@ -16,6 +16,7 @@7CDB0A2E2F3C4D5600FB7CAD /* AudioFixtureLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */; };86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; };+ A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; };7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; };@@ -38,6 +39,7 @@7CDB0A292F3C4D5600FB7CAD /* DictationE2ETests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DictationE2ETests.swift; sourceTree = "<group>"; };343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMClientRequestBodyTests.swift; sourceTree = "<group>"; };980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };+ A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = "<group>"; };7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };@@ -110,6 +112,7 @@7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */,343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */,980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,+ A62300000000000000000001 /* AudioBufferConverterTests.swift */,);path = FluidDictationIntegrationTests;sourceTree = "<group>";@@ -266,6 +269,7 @@7C91B0012F42AA0100C0DEF0 /* HotkeyShortcutTests.swift in Sources */,86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */,272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,+ A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */,);runOnlyForDeploymentPostprocessing = 0;};diff --git a/Sources/Fluid/Services/AudioBufferConverter.swift b/Sources/Fluid/Services/AudioBufferConverter.swiftnew file mode 100644index 00000000..8847cb7a--- /dev/null+++ b/Sources/Fluid/Services/AudioBufferConverter.swift@@ -0,0 +1,99 @@+import AVFoundation+import Foundation++enum AudioBufferConverter {+ enum ConversionError: LocalizedError {+ case invalidSourceFormat+ case invalidTargetSampleRate+ case inaccessibleChannelData+ case targetFormatCreationFailed+ case converterCreationFailed+ case outputBufferAllocationFailed++ var errorDescription: String? {+ switch self {+ case .invalidSourceFormat:+ return "Audio buffer has an invalid sample rate or channel count."+ case .invalidTargetSampleRate:+ return "Target audio sample rate must be greater than zero."+ case .inaccessibleChannelData:+ return "Could not access Float32 audio channel data."+ case .targetFormatCreationFailed:+ return "Could not create the target mono audio format."+ case .converterCreationFailed:+ return "Could not create an audio converter."+ case .outputBufferAllocationFailed:+ return "Could not allocate the converted audio buffer."+ }+ }+ }++ static func monoSamples(+ from buffer: AVAudioPCMBuffer,+ targetSampleRate: Double+ ) throws -> [Float] {+ let sourceFormat = buffer.format+ guard sourceFormat.sampleRate > 0, sourceFormat.channelCount > 0 else {+ throw ConversionError.invalidSourceFormat+ }+ guard targetSampleRate.isFinite, targetSampleRate > 0 else {+ throw ConversionError.invalidTargetSampleRate+ }+ guard buffer.frameLength > 0 else { return [] }++ if sourceFormat.sampleRate == targetSampleRate,+ sourceFormat.channelCount == 1,+ sourceFormat.commonFormat == .pcmFormatFloat32+ {+ guard let channelData = buffer.floatChannelData else {+ throw ConversionError.inaccessibleChannelData+ }+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)))+ }++ guard let targetFormat = AVAudioFormat(+ commonFormat: .pcmFormatFloat32,+ sampleRate: targetSampleRate,+ channels: 1,+ interleaved: false+ ) else {+ throw ConversionError.targetFormatCreationFailed+ }+ guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {+ throw ConversionError.converterCreationFailed+ }++ converter.downmix = sourceFormat.channelCount > 1++ let ratio = targetSampleRate / sourceFormat.sampleRate+ let estimatedFrameCount = AVAudioFrameCount(+ (Double(buffer.frameLength) * ratio).rounded(.up)+ )+ guard let outputBuffer = AVAudioPCMBuffer(+ pcmFormat: targetFormat,+ frameCapacity: max(estimatedFrameCount, 1) + 1024+ ) else {+ throw ConversionError.outputBufferAllocationFailed+ }++ var conversionError: NSError?+ var inputConsumed = false+ converter.convert(to: outputBuffer, error: &conversionError) { _, status in+ if inputConsumed {+ status.pointee = .endOfStream+ return nil+ }+ inputConsumed = true+ status.pointee = .haveData+ return buffer+ }++ if let conversionError {+ throw conversionError+ }+ guard let channelData = outputBuffer.floatChannelData else {+ throw ConversionError.inaccessibleChannelData+ }+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(outputBuffer.frameLength)))+ }+}diff --git a/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift b/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swiftindex 83b0b5a4..f13638f8 100644--- a/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift+++ b/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift@@ -20,7 +20,10 @@ enum LocalAPIAudioDecoder {}try file.read(into: sourceBuffer, frameCount: AVAudioFrameCount(framesToRead))- return try self.convertToMono16k(sourceBuffer)+ return try AudioBufferConverter.monoSamples(+ from: sourceBuffer,+ targetSampleRate: self.sampleRate+ )}static func samples(fromAudioData data: Data, suggestedExtension: String) throws -> [Float] {@@ -53,49 +56,4 @@ enum LocalAPIAudioDecoder {return Int((Double(file.length) * self.sampleRate / sourceFormat.sampleRate).rounded())}-- private static func convertToMono16k(_ sourceBuffer: AVAudioPCMBuffer) throws -> [Float] {- guard let targetFormat = AVAudioFormat(- commonFormat: .pcmFormatFloat32,- sampleRate: self.sampleRate,- channels: 1,- interleaved: false- ) else {- throw NSError(domain: "LocalAPIAudioDecoder", code: -4, userInfo: [NSLocalizedDescriptionKey: "Unable to create target audio format."])- }-- guard sourceBuffer.format != targetFormat else {- guard let channel = sourceBuffer.floatChannelData?[0] else { return [] }- return Array(UnsafeBufferPointer(start: channel, count: Int(sourceBuffer.frameLength)))- }-- guard let converter = AVAudioConverter(from: sourceBuffer.format, to: targetFormat) else {- throw NSError(domain: "LocalAPIAudioDecoder", code: -2, userInfo: [NSLocalizedDescriptionKey: "Unable to convert audio to 16 kHz mono."])- }-- let ratio = self.sampleRate / sourceBuffer.format.sampleRate- let capacity = AVAudioFrameCount((Double(sourceBuffer.frameLength) * ratio).rounded(.up)) + 1024- guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {- throw NSError(domain: "LocalAPIAudioDecoder", code: -3, userInfo: [NSLocalizedDescriptionKey: "Unable to allocate converted audio buffer."])- }-- var consumedInput = false- var conversionError: NSError?- converter.convert(to: outputBuffer, error: &conversionError) { _, status in- if consumedInput {- status.pointee = .noDataNow- return nil- }- consumedInput = true- status.pointee = .haveData- return sourceBuffer- }-- if let conversionError {- throw conversionError- }-- guard let channel = outputBuffer.floatChannelData?[0] else { return [] }- return Array(UnsafeBufferPointer(start: channel, count: Int(outputBuffer.frameLength)))- }}diff --git a/Sources/Fluid/Services/MeetingTranscriptionService.swift b/Sources/Fluid/Services/MeetingTranscriptionService.swiftindex be7391e4..67d1eede 100644--- a/Sources/Fluid/Services/MeetingTranscriptionService.swift+++ b/Sources/Fluid/Services/MeetingTranscriptionService.swift@@ -302,7 +302,10 @@ final class MeetingTranscriptionService: ObservableObject {// Convert buffer to 16kHz mono Float32 sampleslet samples: [Float]do {- samples = try self.resampleBuffer(buffer, targetSampleRate: sampleRate)+ samples = try AudioBufferConverter.monoSamples(+ from: buffer,+ targetSampleRate: sampleRate+ )} catch {throw TranscriptionError.audioConversionFailed("Could not resample audio: \(error.localizedDescription)")}@@ -430,99 +433,4 @@ final class MeetingTranscriptionService: ObservableObject {self.currentStatus = ""self.progress = 0.0}-- // MARK: - Audio Resampling Helpers-- /// Resample an audio buffer to 16kHz mono Float32 samples- /// - Parameters:- /// - buffer: Source audio buffer- /// - targetSampleRate: Target sample rate (default 16000 Hz)- /// - Returns: Array of Float32 samples at target sample rate- private nonisolated func resampleBuffer(_ buffer: AVAudioPCMBuffer, targetSampleRate: Double = 16_000) throws -> [Float] {- let sourceFormat = buffer.format- let sourceSampleRate = sourceFormat.sampleRate- let sourceChannels = sourceFormat.channelCount-- // Create target format (16kHz mono Float32)- guard let targetFormat = AVAudioFormat(- commonFormat: .pcmFormatFloat32,- sampleRate: targetSampleRate,- channels: 1,- interleaved: false- ) else {- throw NSError(- domain: "MeetingTranscriptionService",- code: -1,- userInfo: [NSLocalizedDescriptionKey: "Could not create target audio format"]- )- }-- // If already in correct format, just extract samples- // Must also verify the format is Float32 - if source is Float64, floatChannelData returns nil- if sourceSampleRate == targetSampleRate,- sourceChannels == 1,- sourceFormat.commonFormat == .pcmFormatFloat32- {- guard let channelData = buffer.floatChannelData else {- throw NSError(- domain: "MeetingTranscriptionService",- code: -2,- userInfo: [NSLocalizedDescriptionKey: "Could not access audio channel data"]- )- }- let frameLength = Int(buffer.frameLength)- return Array(UnsafeBufferPointer(start: channelData[0], count: frameLength))- }-- // Create converter- guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {- throw NSError(- domain: "MeetingTranscriptionService",- code: -3,- userInfo: [NSLocalizedDescriptionKey: "Could not create audio converter"]- )- }-- // Calculate output buffer size- let ratio = targetSampleRate / sourceSampleRate- let estimatedFrameCount = AVAudioFrameCount(Double(buffer.frameLength) * ratio)-- guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: estimatedFrameCount + 1024) else {- throw NSError(- domain: "MeetingTranscriptionService",- code: -4,- userInfo: [NSLocalizedDescriptionKey: "Could not create output buffer"]- )- }-- // Convert- var error: NSError?- var inputConsumed = false-- converter.convert(to: outputBuffer, error: &error) { _, outStatus in- if inputConsumed {- outStatus.pointee = .endOfStream- return nil- }- inputConsumed = true- outStatus.pointee = .haveData- return buffer- }-- if let error = error {- throw error- }-- // Extract samples- guard let channelData = outputBuffer.floatChannelData else {- throw NSError(- domain: "MeetingTranscriptionService",- code: -5,- userInfo: [NSLocalizedDescriptionKey: "Could not access converted audio data"]- )- }-- let frameLength = Int(outputBuffer.frameLength)- return Array(UnsafeBufferPointer(start: channelData[0], count: frameLength))- }}diff --git a/Sources/Fluid/Services/NemotronProvider.swift b/Sources/Fluid/Services/NemotronProvider.swiftindex 05a3dabb..185e8280 100644--- a/Sources/Fluid/Services/NemotronProvider.swift+++ b/Sources/Fluid/Services/NemotronProvider.swift@@ -299,7 +299,10 @@ final class NemotronProvider: TranscriptionProvider {audioFile.framePosition = currentFrametry audioFile.read(into: buffer, frameCount: framesToRead)- try pendingSamples.append(contentsOf: Self.resampleBuffer(buffer, targetSampleRate: targetSampleRate))+ try pendingSamples.append(contentsOf: AudioBufferConverter.monoSamples(+ from: buffer,+ targetSampleRate: targetSampleRate+ ))while pendingSamples.count > self.maxTranscriptionSamples {let end = self.chunkEnd(in: pendingSamples, offset: 0)let text = try await self.transcribeSinglePass(Array(pendingSamples[..<end]))@@ -526,57 +529,6 @@ final class NemotronProvider: TranscriptionProvider {Int((seconds * 1000).rounded())}- private static func resampleBuffer(_ buffer: AVAudioPCMBuffer, targetSampleRate: Double) throws -> [Float] {- let sourceFormat = buffer.format- guard let targetFormat = AVAudioFormat(- commonFormat: .pcmFormatFloat32,- sampleRate: targetSampleRate,- channels: 1,- interleaved: false- ) else {- throw Self.makeError("Failed to create target audio format.")- }-- if sourceFormat.sampleRate == targetSampleRate,- sourceFormat.channelCount == 1,- sourceFormat.commonFormat == .pcmFormatFloat32- {- guard let channelData = buffer.floatChannelData else {- throw Self.makeError("Failed to access audio channel data.")- }- return Array(UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)))- }-- guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {- throw Self.makeError("Failed to create audio converter.")- }- let ratio = targetSampleRate / sourceFormat.sampleRate- let capacity = AVAudioFrameCount((Double(buffer.frameLength) * ratio).rounded(.up)) + 1024- guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {- throw Self.makeError("Failed to allocate converted audio buffer.")- }-- var consumedInput = false- var conversionError: NSError?- converter.convert(to: outputBuffer, error: &conversionError) { _, status in- if consumedInput {- status.pointee = .noDataNow- return nil- }- consumedInput = true- status.pointee = .haveData- return buffer- }-- if let conversionError {- throw conversionError- }- guard let channelData = outputBuffer.floatChannelData else {- throw Self.makeError("Failed to access converted audio channel data.")- }- return Array(UnsafeBufferPointer(start: channelData[0], count: Int(outputBuffer.frameLength)))- }-private static func loadMaxAudioSamples(from dir: URL) -> Int? {let url = dir.appendingPathComponent("metadata.json")guarddiff --git a/Tests/FluidDictationIntegrationTests/AudioBufferConverterTests.swift b/Tests/FluidDictationIntegrationTests/AudioBufferConverterTests.swiftnew file mode 100644index 00000000..ff7a78cb--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/AudioBufferConverterTests.swift@@ -0,0 +1,104 @@+import AVFoundation+@testable import FluidVoice_Debug+import XCTest++final class AudioBufferConverterTests: XCTestCase {+ func testStereoDownmixIncludesRightChannel() throws {+ let buffer = try self.makeFloatBuffer(sampleRate: 16_000, channels: 2, frameCount: 16)+ self.fill(buffer, channel: 1, with: 1)++ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)++ XCTAssertEqual(samples.count, 16)+ self.assertAllSamples(samples, equalTo: 0.5)+ }++ func testStereoDownmixIncludesLeftChannel() throws {+ let buffer = try self.makeFloatBuffer(sampleRate: 16_000, channels: 2, frameCount: 16)+ self.fill(buffer, channel: 0, with: 1)++ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)++ XCTAssertEqual(samples.count, 16)+ self.assertAllSamples(samples, equalTo: 0.5)+ }++ func testStereoDownmixPreservesMatchingChannels() throws {+ let buffer = try self.makeFloatBuffer(sampleRate: 16_000, channels: 2, frameCount: 16)+ self.fill(buffer, channel: 0, with: 1)+ self.fill(buffer, channel: 1, with: 1)++ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)++ XCTAssertEqual(samples.count, 16)+ self.assertAllSamples(samples, equalTo: 1)+ }++ func testMonoFloatFastPathPreservesSamples() throws {+ let expected: [Float] = [0.25, -0.5, 0.75, -1]+ let buffer = try self.makeFloatBuffer(+ sampleRate: 16_000,+ channels: 1,+ frameCount: expected.count+ )+ let channelData = try XCTUnwrap(buffer.floatChannelData)+ for (index, sample) in expected.enumerated() {+ channelData[0][index] = sample+ }++ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)++ XCTAssertEqual(samples, expected)+ }++ func testStereoDownmixAndResampleIncludesRightChannel() throws {+ let buffer = try self.makeFloatBuffer(sampleRate: 48_000, channels: 2, frameCount: 480)+ self.fill(buffer, channel: 1, with: 1)++ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)++ XCTAssertEqual(samples.count, 160)+ let average = samples.reduce(0, +) / Float(samples.count)+ XCTAssertEqual(average, 0.5, accuracy: 0.01)+ XCTAssertGreaterThan(samples.map(abs).max() ?? 0, 0.4)+ }++ private func makeFloatBuffer(+ sampleRate: Double,+ channels: AVAudioChannelCount,+ frameCount: Int+ ) throws -> AVAudioPCMBuffer {+ let format = try XCTUnwrap(AVAudioFormat(+ commonFormat: .pcmFormatFloat32,+ sampleRate: sampleRate,+ channels: channels,+ interleaved: false+ ))+ let buffer = try XCTUnwrap(AVAudioPCMBuffer(+ pcmFormat: format,+ frameCapacity: AVAudioFrameCount(frameCount)+ ))+ buffer.frameLength = AVAudioFrameCount(frameCount)+ return buffer+ }++ private func fill(_ buffer: AVAudioPCMBuffer, channel: Int, with value: Float) {+ guard let channelData = buffer.floatChannelData else {+ XCTFail("Expected Float32 channel data")+ return+ }+ for frame in 0..<Int(buffer.frameLength) {+ channelData[channel][frame] = value+ }+ }++ private func assertAllSamples(+ _ samples: [Float],+ equalTo expected: Float,+ accuracy: Float = 0.00_001+ ) {+ for sample in samples {+ XCTAssertEqual(sample, expected, accuracy: accuracy)+ }+ }+}