Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

downmix stereo audio from both channels

grohith327d8990906aefe
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj
index a2cb0a9f..14b46cb9 100644
--- a/Fluid.xcodeproj/project.pbxproj
+++ b/Fluid.xcodeproj/project.pbxproj
@@ -16,6 +16,7 @@
7CDB0A2E2F3C4D5600FB7CAD /* AudioFixtureLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */; };
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; };
+ A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; };
7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; };
@@ -38,6 +39,7 @@
7CDB0A292F3C4D5600FB7CAD /* DictationE2ETests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DictationE2ETests.swift; sourceTree = "<group>"; };
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMClientRequestBodyTests.swift; sourceTree = "<group>"; };
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };
+ A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = "<group>"; };
7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };
7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };
7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };
@@ -110,6 +112,7 @@
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */,
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */,
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,
+ A62300000000000000000001 /* AudioBufferConverterTests.swift */,
);
path = FluidDictationIntegrationTests;
sourceTree = "<group>";
@@ -266,6 +269,7 @@
7C91B0012F42AA0100C0DEF0 /* HotkeyShortcutTests.swift in Sources */,
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */,
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,
+ A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
diff --git a/Sources/Fluid/Services/AudioBufferConverter.swift b/Sources/Fluid/Services/AudioBufferConverter.swift
new file mode 100644
index 00000000..8847cb7a
--- /dev/null
+++ b/Sources/Fluid/Services/AudioBufferConverter.swift
@@ -0,0 +1,99 @@
+import AVFoundation
+import Foundation
+
+enum AudioBufferConverter {
+ enum ConversionError: LocalizedError {
+ case invalidSourceFormat
+ case invalidTargetSampleRate
+ case inaccessibleChannelData
+ case targetFormatCreationFailed
+ case converterCreationFailed
+ case outputBufferAllocationFailed
+
+ var errorDescription: String? {
+ switch self {
+ case .invalidSourceFormat:
+ return "Audio buffer has an invalid sample rate or channel count."
+ case .invalidTargetSampleRate:
+ return "Target audio sample rate must be greater than zero."
+ case .inaccessibleChannelData:
+ return "Could not access Float32 audio channel data."
+ case .targetFormatCreationFailed:
+ return "Could not create the target mono audio format."
+ case .converterCreationFailed:
+ return "Could not create an audio converter."
+ case .outputBufferAllocationFailed:
+ return "Could not allocate the converted audio buffer."
+ }
+ }
+ }
+
+ static func monoSamples(
+ from buffer: AVAudioPCMBuffer,
+ targetSampleRate: Double
+ ) throws -> [Float] {
+ let sourceFormat = buffer.format
+ guard sourceFormat.sampleRate > 0, sourceFormat.channelCount > 0 else {
+ throw ConversionError.invalidSourceFormat
+ }
+ guard targetSampleRate.isFinite, targetSampleRate > 0 else {
+ throw ConversionError.invalidTargetSampleRate
+ }
+ guard buffer.frameLength > 0 else { return [] }
+
+ if sourceFormat.sampleRate == targetSampleRate,
+ sourceFormat.channelCount == 1,
+ sourceFormat.commonFormat == .pcmFormatFloat32
+ {
+ guard let channelData = buffer.floatChannelData else {
+ throw ConversionError.inaccessibleChannelData
+ }
+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)))
+ }
+
+ guard let targetFormat = AVAudioFormat(
+ commonFormat: .pcmFormatFloat32,
+ sampleRate: targetSampleRate,
+ channels: 1,
+ interleaved: false
+ ) else {
+ throw ConversionError.targetFormatCreationFailed
+ }
+ guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
+ throw ConversionError.converterCreationFailed
+ }
+
+ converter.downmix = sourceFormat.channelCount > 1
+
+ let ratio = targetSampleRate / sourceFormat.sampleRate
+ let estimatedFrameCount = AVAudioFrameCount(
+ (Double(buffer.frameLength) * ratio).rounded(.up)
+ )
+ guard let outputBuffer = AVAudioPCMBuffer(
+ pcmFormat: targetFormat,
+ frameCapacity: max(estimatedFrameCount, 1) + 1024
+ ) else {
+ throw ConversionError.outputBufferAllocationFailed
+ }
+
+ var conversionError: NSError?
+ var inputConsumed = false
+ converter.convert(to: outputBuffer, error: &conversionError) { _, status in
+ if inputConsumed {
+ status.pointee = .endOfStream
+ return nil
+ }
+ inputConsumed = true
+ status.pointee = .haveData
+ return buffer
+ }
+
+ if let conversionError {
+ throw conversionError
+ }
+ guard let channelData = outputBuffer.floatChannelData else {
+ throw ConversionError.inaccessibleChannelData
+ }
+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(outputBuffer.frameLength)))
+ }
+}
diff --git a/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift b/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift
index 83b0b5a4..f13638f8 100644
--- a/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift
+++ b/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift
@@ -20,7 +20,10 @@ enum LocalAPIAudioDecoder {
}
try file.read(into: sourceBuffer, frameCount: AVAudioFrameCount(framesToRead))
- return try self.convertToMono16k(sourceBuffer)
+ return try AudioBufferConverter.monoSamples(
+ from: sourceBuffer,
+ targetSampleRate: self.sampleRate
+ )
}
static func samples(fromAudioData data: Data, suggestedExtension: String) throws -> [Float] {
@@ -53,49 +56,4 @@ enum LocalAPIAudioDecoder {
return Int((Double(file.length) * self.sampleRate / sourceFormat.sampleRate).rounded())
}
-
- private static func convertToMono16k(_ sourceBuffer: AVAudioPCMBuffer) throws -> [Float] {
- guard let targetFormat = AVAudioFormat(
- commonFormat: .pcmFormatFloat32,
- sampleRate: self.sampleRate,
- channels: 1,
- interleaved: false
- ) else {
- throw NSError(domain: "LocalAPIAudioDecoder", code: -4, userInfo: [NSLocalizedDescriptionKey: "Unable to create target audio format."])
- }
-
- guard sourceBuffer.format != targetFormat else {
- guard let channel = sourceBuffer.floatChannelData?[0] else { return [] }
- return Array(UnsafeBufferPointer(start: channel, count: Int(sourceBuffer.frameLength)))
- }
-
- guard let converter = AVAudioConverter(from: sourceBuffer.format, to: targetFormat) else {
- throw NSError(domain: "LocalAPIAudioDecoder", code: -2, userInfo: [NSLocalizedDescriptionKey: "Unable to convert audio to 16 kHz mono."])
- }
-
- let ratio = self.sampleRate / sourceBuffer.format.sampleRate
- let capacity = AVAudioFrameCount((Double(sourceBuffer.frameLength) * ratio).rounded(.up)) + 1024
- guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
- throw NSError(domain: "LocalAPIAudioDecoder", code: -3, userInfo: [NSLocalizedDescriptionKey: "Unable to allocate converted audio buffer."])
- }
-
- var consumedInput = false
- var conversionError: NSError?
- converter.convert(to: outputBuffer, error: &conversionError) { _, status in
- if consumedInput {
- status.pointee = .noDataNow
- return nil
- }
- consumedInput = true
- status.pointee = .haveData
- return sourceBuffer
- }
-
- if let conversionError {
- throw conversionError
- }
-
- guard let channel = outputBuffer.floatChannelData?[0] else { return [] }
- return Array(UnsafeBufferPointer(start: channel, count: Int(outputBuffer.frameLength)))
- }
}
diff --git a/Sources/Fluid/Services/MeetingTranscriptionService.swift b/Sources/Fluid/Services/MeetingTranscriptionService.swift
index be7391e4..67d1eede 100644
--- a/Sources/Fluid/Services/MeetingTranscriptionService.swift
+++ b/Sources/Fluid/Services/MeetingTranscriptionService.swift
@@ -302,7 +302,10 @@ final class MeetingTranscriptionService: ObservableObject {
// Convert buffer to 16kHz mono Float32 samples
let samples: [Float]
do {
- samples = try self.resampleBuffer(buffer, targetSampleRate: sampleRate)
+ samples = try AudioBufferConverter.monoSamples(
+ from: buffer,
+ targetSampleRate: sampleRate
+ )
} catch {
throw TranscriptionError.audioConversionFailed("Could not resample audio: \(error.localizedDescription)")
}
@@ -430,99 +433,4 @@ final class MeetingTranscriptionService: ObservableObject {
self.currentStatus = ""
self.progress = 0.0
}
-
- // MARK: - Audio Resampling Helpers
-
- /// Resample an audio buffer to 16kHz mono Float32 samples
- /// - Parameters:
- /// - buffer: Source audio buffer
- /// - targetSampleRate: Target sample rate (default 16000 Hz)
- /// - Returns: Array of Float32 samples at target sample rate
- private nonisolated func resampleBuffer(_ buffer: AVAudioPCMBuffer, targetSampleRate: Double = 16_000) throws -> [Float] {
- let sourceFormat = buffer.format
- let sourceSampleRate = sourceFormat.sampleRate
- let sourceChannels = sourceFormat.channelCount
-
- // Create target format (16kHz mono Float32)
- guard let targetFormat = AVAudioFormat(
- commonFormat: .pcmFormatFloat32,
- sampleRate: targetSampleRate,
- channels: 1,
- interleaved: false
- ) else {
- throw NSError(
- domain: "MeetingTranscriptionService",
- code: -1,
- userInfo: [NSLocalizedDescriptionKey: "Could not create target audio format"]
- )
- }
-
- // If already in correct format, just extract samples
- // Must also verify the format is Float32 - if source is Float64, floatChannelData returns nil
- if sourceSampleRate == targetSampleRate,
- sourceChannels == 1,
- sourceFormat.commonFormat == .pcmFormatFloat32
- {
- guard let channelData = buffer.floatChannelData else {
- throw NSError(
- domain: "MeetingTranscriptionService",
- code: -2,
- userInfo: [NSLocalizedDescriptionKey: "Could not access audio channel data"]
- )
- }
- let frameLength = Int(buffer.frameLength)
- return Array(UnsafeBufferPointer(start: channelData[0], count: frameLength))
- }
-
- // Create converter
- guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
- throw NSError(
- domain: "MeetingTranscriptionService",
- code: -3,
- userInfo: [NSLocalizedDescriptionKey: "Could not create audio converter"]
- )
- }
-
- // Calculate output buffer size
- let ratio = targetSampleRate / sourceSampleRate
- let estimatedFrameCount = AVAudioFrameCount(Double(buffer.frameLength) * ratio)
-
- guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: estimatedFrameCount + 1024) else {
- throw NSError(
- domain: "MeetingTranscriptionService",
- code: -4,
- userInfo: [NSLocalizedDescriptionKey: "Could not create output buffer"]
- )
- }
-
- // Convert
- var error: NSError?
- var inputConsumed = false
-
- converter.convert(to: outputBuffer, error: &error) { _, outStatus in
- if inputConsumed {
- outStatus.pointee = .endOfStream
- return nil
- }
- inputConsumed = true
- outStatus.pointee = .haveData
- return buffer
- }
-
- if let error = error {
- throw error
- }
-
- // Extract samples
- guard let channelData = outputBuffer.floatChannelData else {
- throw NSError(
- domain: "MeetingTranscriptionService",
- code: -5,
- userInfo: [NSLocalizedDescriptionKey: "Could not access converted audio data"]
- )
- }
-
- let frameLength = Int(outputBuffer.frameLength)
- return Array(UnsafeBufferPointer(start: channelData[0], count: frameLength))
- }
}
diff --git a/Sources/Fluid/Services/NemotronProvider.swift b/Sources/Fluid/Services/NemotronProvider.swift
index 05a3dabb..185e8280 100644
--- a/Sources/Fluid/Services/NemotronProvider.swift
+++ b/Sources/Fluid/Services/NemotronProvider.swift
@@ -299,7 +299,10 @@ final class NemotronProvider: TranscriptionProvider {
audioFile.framePosition = currentFrame
try audioFile.read(into: buffer, frameCount: framesToRead)
- try pendingSamples.append(contentsOf: Self.resampleBuffer(buffer, targetSampleRate: targetSampleRate))
+ try pendingSamples.append(contentsOf: AudioBufferConverter.monoSamples(
+ from: buffer,
+ targetSampleRate: targetSampleRate
+ ))
while pendingSamples.count > self.maxTranscriptionSamples {
let end = self.chunkEnd(in: pendingSamples, offset: 0)
let text = try await self.transcribeSinglePass(Array(pendingSamples[..<end]))
@@ -526,57 +529,6 @@ final class NemotronProvider: TranscriptionProvider {
Int((seconds * 1000).rounded())
}
- private static func resampleBuffer(_ buffer: AVAudioPCMBuffer, targetSampleRate: Double) throws -> [Float] {
- let sourceFormat = buffer.format
- guard let targetFormat = AVAudioFormat(
- commonFormat: .pcmFormatFloat32,
- sampleRate: targetSampleRate,
- channels: 1,
- interleaved: false
- ) else {
- throw Self.makeError("Failed to create target audio format.")
- }
-
- if sourceFormat.sampleRate == targetSampleRate,
- sourceFormat.channelCount == 1,
- sourceFormat.commonFormat == .pcmFormatFloat32
- {
- guard let channelData = buffer.floatChannelData else {
- throw Self.makeError("Failed to access audio channel data.")
- }
- return Array(UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)))
- }
-
- guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
- throw Self.makeError("Failed to create audio converter.")
- }
- let ratio = targetSampleRate / sourceFormat.sampleRate
- let capacity = AVAudioFrameCount((Double(buffer.frameLength) * ratio).rounded(.up)) + 1024
- guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
- throw Self.makeError("Failed to allocate converted audio buffer.")
- }
-
- var consumedInput = false
- var conversionError: NSError?
- converter.convert(to: outputBuffer, error: &conversionError) { _, status in
- if consumedInput {
- status.pointee = .noDataNow
- return nil
- }
- consumedInput = true
- status.pointee = .haveData
- return buffer
- }
-
- if let conversionError {
- throw conversionError
- }
- guard let channelData = outputBuffer.floatChannelData else {
- throw Self.makeError("Failed to access converted audio channel data.")
- }
- return Array(UnsafeBufferPointer(start: channelData[0], count: Int(outputBuffer.frameLength)))
- }
-
private static func loadMaxAudioSamples(from dir: URL) -> Int? {
let url = dir.appendingPathComponent("metadata.json")
guard
diff --git a/Tests/FluidDictationIntegrationTests/AudioBufferConverterTests.swift b/Tests/FluidDictationIntegrationTests/AudioBufferConverterTests.swift
new file mode 100644
index 00000000..ff7a78cb
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/AudioBufferConverterTests.swift
@@ -0,0 +1,104 @@
+import AVFoundation
+@testable import FluidVoice_Debug
+import XCTest
+
+final class AudioBufferConverterTests: XCTestCase {
+ func testStereoDownmixIncludesRightChannel() throws {
+ let buffer = try self.makeFloatBuffer(sampleRate: 16_000, channels: 2, frameCount: 16)
+ self.fill(buffer, channel: 1, with: 1)
+
+ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)
+
+ XCTAssertEqual(samples.count, 16)
+ self.assertAllSamples(samples, equalTo: 0.5)
+ }
+
+ func testStereoDownmixIncludesLeftChannel() throws {
+ let buffer = try self.makeFloatBuffer(sampleRate: 16_000, channels: 2, frameCount: 16)
+ self.fill(buffer, channel: 0, with: 1)
+
+ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)
+
+ XCTAssertEqual(samples.count, 16)
+ self.assertAllSamples(samples, equalTo: 0.5)
+ }
+
+ func testStereoDownmixPreservesMatchingChannels() throws {
+ let buffer = try self.makeFloatBuffer(sampleRate: 16_000, channels: 2, frameCount: 16)
+ self.fill(buffer, channel: 0, with: 1)
+ self.fill(buffer, channel: 1, with: 1)
+
+ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)
+
+ XCTAssertEqual(samples.count, 16)
+ self.assertAllSamples(samples, equalTo: 1)
+ }
+
+ func testMonoFloatFastPathPreservesSamples() throws {
+ let expected: [Float] = [0.25, -0.5, 0.75, -1]
+ let buffer = try self.makeFloatBuffer(
+ sampleRate: 16_000,
+ channels: 1,
+ frameCount: expected.count
+ )
+ let channelData = try XCTUnwrap(buffer.floatChannelData)
+ for (index, sample) in expected.enumerated() {
+ channelData[0][index] = sample
+ }
+
+ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)
+
+ XCTAssertEqual(samples, expected)
+ }
+
+ func testStereoDownmixAndResampleIncludesRightChannel() throws {
+ let buffer = try self.makeFloatBuffer(sampleRate: 48_000, channels: 2, frameCount: 480)
+ self.fill(buffer, channel: 1, with: 1)
+
+ let samples = try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000)
+
+ XCTAssertEqual(samples.count, 160)
+ let average = samples.reduce(0, +) / Float(samples.count)
+ XCTAssertEqual(average, 0.5, accuracy: 0.01)
+ XCTAssertGreaterThan(samples.map(abs).max() ?? 0, 0.4)
+ }
+
+ private func makeFloatBuffer(
+ sampleRate: Double,
+ channels: AVAudioChannelCount,
+ frameCount: Int
+ ) throws -> AVAudioPCMBuffer {
+ let format = try XCTUnwrap(AVAudioFormat(
+ commonFormat: .pcmFormatFloat32,
+ sampleRate: sampleRate,
+ channels: channels,
+ interleaved: false
+ ))
+ let buffer = try XCTUnwrap(AVAudioPCMBuffer(
+ pcmFormat: format,
+ frameCapacity: AVAudioFrameCount(frameCount)
+ ))
+ buffer.frameLength = AVAudioFrameCount(frameCount)
+ return buffer
+ }
+
+ private func fill(_ buffer: AVAudioPCMBuffer, channel: Int, with value: Float) {
+ guard let channelData = buffer.floatChannelData else {
+ XCTFail("Expected Float32 channel data")
+ return
+ }
+ for frame in 0..<Int(buffer.frameLength) {
+ channelData[channel][frame] = value
+ }
+ }
+
+ private func assertAllSamples(
+ _ samples: [Float],
+ equalTo expected: Float,
+ accuracy: Float = 0.00_001
+ ) {
+ for sample in samples {
+ XCTAssertEqual(sample, expected, accuracy: accuracy)
+ }
+ }
+}