Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
fix(transcription): preserve subsecond file audio (#1162)
Alex Efremovc981bc84e3d3
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxprojindex 6cefe3fb..3164f65c 100644--- a/Fluid.xcodeproj/project.pbxproj+++ b/Fluid.xcodeproj/project.pbxproj@@ -99,6 +99,7 @@7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */; };803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 803000000000000000000001 /* MediaPlaybackServiceTests.swift */; };0B05F11E000000000000A002 /* SupportedFileExtensionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */; };+ BF1510100000000000000002 /* FileTranscriptionAudioTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1510100000000000000001 /* FileTranscriptionAudioTests.swift */; };CD0E00000000000000000002 /* MeetingOverlayModelsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD0E00000000000000000001 /* MeetingOverlayModelsTests.swift */; };CD0F00000000000000000002 /* MeetingProcessingConfigurationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD0F00000000000000000001 /* MeetingProcessingConfigurationTests.swift */; };CD100000000000000000000002 /* MeetingProviderOptionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD100000000000000000000001 /* MeetingProviderOptionsTests.swift */; };@@ -294,6 +295,7 @@7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypingServiceTransientPasteboardTests.swift; sourceTree = "<group>"; };803000000000000000000001 /* MediaPlaybackServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPlaybackServiceTests.swift; sourceTree = "<group>"; };0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SupportedFileExtensionsTests.swift; sourceTree = "<group>"; };+ BF1510100000000000000001 /* FileTranscriptionAudioTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTranscriptionAudioTests.swift; sourceTree = "<group>"; };CD0E00000000000000000001 /* MeetingOverlayModelsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingOverlayModelsTests.swift; sourceTree = "<group>"; };CD0F00000000000000000001 /* MeetingProcessingConfigurationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingProcessingConfigurationTests.swift; sourceTree = "<group>"; };CD100000000000000000000001 /* MeetingProviderOptionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingProviderOptionsTests.swift; sourceTree = "<group>"; };@@ -553,6 +555,7 @@C0DE63800000000000000001 /* AppleSpeechLocaleSelectionTests.swift */,CACEC0010000000000000001 /* KeychainServiceCacheTests.swift */,0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */,+ BF1510100000000000000001 /* FileTranscriptionAudioTests.swift */,FEED67600000000000000001 /* PasteDeliveryCoordinatorTests.swift */,CD0E00000000000000000001 /* MeetingOverlayModelsTests.swift */,CD0F00000000000000000001 /* MeetingProcessingConfigurationTests.swift */,@@ -960,6 +963,7 @@C0DE63800000000000000002 /* AppleSpeechLocaleSelectionTests.swift in Sources */,CACEC0010000000000000002 /* KeychainServiceCacheTests.swift in Sources */,0B05F11E000000000000A002 /* SupportedFileExtensionsTests.swift in Sources */,+ BF1510100000000000000002 /* FileTranscriptionAudioTests.swift in Sources */,FEED67600000000000000002 /* PasteDeliveryCoordinatorTests.swift in Sources */,CD0E00000000000000000002 /* MeetingOverlayModelsTests.swift in Sources */,CD0F00000000000000000002 /* MeetingProcessingConfigurationTests.swift in Sources */,diff --git a/Sources/Fluid/Services/FileTranscriptionService.swift b/Sources/Fluid/Services/FileTranscriptionService.swiftindex 53252c0a..5d030ede 100644--- a/Sources/Fluid/Services/FileTranscriptionService.swift+++ b/Sources/Fluid/Services/FileTranscriptionService.swift@@ -587,7 +587,7 @@ final class FileTranscriptionService: ObservableObject {}// Convert buffer to 16kHz mono Float32 samples- let samples: [Float]+ var samples: [Float]do {samples = try AudioBufferConverter.monoSamples(from: buffer,@@ -597,11 +597,14 @@ final class FileTranscriptionService: ObservableObject {throw TranscriptionError.audioConversionFailed("Could not resample audio: \(error.localizedDescription)")}- // Skip if chunk is too short (< 1 second)- guard samples.count >= Int(sampleRate) else {+ guard !samples.isEmpty else {currentFrame += AVAudioFramePosition(framesToRead)continue}+ // Match dictation: preserve short speech while meeting the ASR minimum.+ if samples.count < Int(sampleRate) {+ samples.append(contentsOf: repeatElement(Float(0), count: Int(sampleRate) - samples.count))+ }// Transcribe this chunk using the provider (works for both Parakeet and Whisper)let chunkResult = try await provider.transcribe(samples)@@ -622,7 +625,7 @@ final class FileTranscriptionService: ObservableObject {if allTranscriptions.isEmpty {DebugLogger.shared.warning(- "No audio chunks were long enough to transcribe (minimum 1 second required)",+ "No transcription text was produced for this file",source: "FileTranscriptionService")}diff --git a/Sources/Fluid/Services/NemotronProvider.swift b/Sources/Fluid/Services/NemotronProvider.swiftindex 7c7ab75e..54ec17fa 100644--- a/Sources/Fluid/Services/NemotronProvider.swift+++ b/Sources/Fluid/Services/NemotronProvider.swift@@ -279,6 +279,13 @@ final class NemotronProvider: TranscriptionProvider {throw Self.makeError("Nemotron provider is not ready.")}+ return try await self.transcribeFile(at: fileURL, transcribeChunk: self.transcribeSinglePass)+ }++ func transcribeFile(+ at fileURL: URL,+ transcribeChunk: ([Float]) async throws -> String+ ) async throws -> ASRTranscriptionResult {let audioFile = try AVAudioFile(forReading: fileURL)let sourceFormat = audioFile.processingFormatlet targetSampleRate = 16_000.0@@ -305,7 +312,7 @@ final class NemotronProvider: TranscriptionProvider {))while pendingSamples.count > self.maxTranscriptionSamples {let end = self.chunkEnd(in: pendingSamples, offset: 0)- let text = try await self.transcribeSinglePass(Array(pendingSamples[..<end]))+ let text = try await transcribeChunk(Array(pendingSamples[..<end])).trimmingCharacters(in: .whitespacesAndNewlines)if text.isEmpty == false {transcriptions.append(text)@@ -315,9 +322,12 @@ final class NemotronProvider: TranscriptionProvider {currentFrame += AVAudioFramePosition(framesToRead)}- if pendingSamples.count >= Int(targetSampleRate) {- let result = try await self.transcribeBatched(pendingSamples)- let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines)+ if pendingSamples.isEmpty == false {+ if pendingSamples.count < self.minimumFinalChunkSamples {+ pendingSamples.append(contentsOf: repeatElement(Float(0), count: self.minimumFinalChunkSamples - pendingSamples.count))+ }+ let text = try await transcribeChunk(pendingSamples)+ .trimmingCharacters(in: .whitespacesAndNewlines)if text.isEmpty == false {transcriptions.append(text)}diff --git a/Tests/FluidDictationIntegrationTests/FileTranscriptionAudioTests.swift b/Tests/FluidDictationIntegrationTests/FileTranscriptionAudioTests.swiftnew file mode 100644index 00000000..9b277797--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/FileTranscriptionAudioTests.swift@@ -0,0 +1,214 @@+import AVFoundation+@testable import FluidVoice_Debug+import XCTest++@MainActor+final class FileTranscriptionAudioTests: XCTestCase {+ private final class RecordingProvider: TranscriptionProvider {+ let name = "Buffered file test provider"+ let isAvailable = true+ let isReady = true+ private(set) var inputs: [[Float]] = []++ func prepare(progressHandler: ((ModelPreparationProgress) -> Void)?) async throws {+ XCTFail("An already-ready provider must not prepare models")+ }++ func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult {+ self.inputs.append(samples)+ // Observe the real file path without adding synthetic text to shared history.+ return ASRTranscriptionResult(text: "", confidence: 0)+ }+ }++ func testSubsecondFilesPreserveSamplesAndAddOnlyTrailingSilence() async throws {+ for frameCount in [1, 8000, 15_999] {+ try await self.withTranscription(frameCount: frameCount) { result, provider in+ XCTAssertEqual(provider.inputs.count, 1, "Source frames: \(frameCount)")+ let input = try XCTUnwrap(provider.inputs.first)+ XCTAssertEqual(input.count, 16_000)+ XCTAssertEqual(Array(input.prefix(frameCount)), Array(repeating: 0.25, count: frameCount))+ XCTAssertEqual(Array(input.dropFirst(frameCount)), Array(repeating: 0, count: 16_000 - frameCount))+ XCTAssertEqual(result.duration, Double(frameCount) / 16_000, accuracy: 0.0001)+ }+ }+ }++ func testFilesAtOrAboveOneSecondAreNotPadded() async throws {+ for frameCount in [16_000, 16_001] {+ try await self.withTranscription(frameCount: frameCount) { result, provider in+ XCTAssertEqual(provider.inputs.count, 1, "Source frames: \(frameCount)")+ let input = try XCTUnwrap(provider.inputs.first)+ XCTAssertEqual(input, Array(repeating: 0.25, count: frameCount))+ XCTAssertEqual(result.duration, Double(frameCount) / 16_000, accuracy: 0.0001)+ }+ }+ }++ func testSubsecondTailAfterFullChunkStillReachesProvider() async throws {+ let fullChunkFrames = 20 * 60 * 16_000+ try await self.withTranscription(frameCount: fullChunkFrames + 4000) { result, provider in+ XCTAssertEqual(provider.inputs.count, 2)+ guard provider.inputs.count == 2 else { return }+ XCTAssertEqual(provider.inputs[0].count, fullChunkFrames)+ XCTAssertTrue(provider.inputs[0].allSatisfy { $0 == 0.25 })+ XCTAssertEqual(+ provider.inputs[1],+ Array(repeating: 0.25, count: 4000) + Array(repeating: 0, count: 12_000)+ )+ XCTAssertEqual(result.duration, 1200.25, accuracy: 0.0001)+ }+ }++ func testEmptyFileDoesNotCreatePaddedInference() async throws {+ try await self.withTranscription(frameCount: 0) { result, provider in+ XCTAssertTrue(provider.inputs.isEmpty)+ XCTAssertTrue(result.text.isEmpty)+ XCTAssertEqual(result.duration, 0)+ }+ }++ func testSubsecondSilenceIsPassedThroughWithoutChangingDuration() async throws {+ try await self.withTranscription(frameCount: 8000, sample: 0) { result, provider in+ XCTAssertEqual(provider.inputs.count, 1)+ let input = try XCTUnwrap(provider.inputs.first)+ XCTAssertEqual(input, Array(repeating: 0, count: 16_000))+ XCTAssertEqual(result.duration, 0.5, accuracy: 0.0001)+ }+ }++ #if arch(arm64)+ func testNemotronNativeFilePadsSubsecondInput() async throws {+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {+ for frameCount in [1, 8000, 15_999] {+ try await self.withNemotronFile(mode: mode, frameCount: frameCount) { result, inputs in+ XCTAssertEqual(inputs.count, 1, "Mode: \(mode), frames: \(frameCount)")+ let input = try XCTUnwrap(inputs.first)+ XCTAssertEqual(input, Array(repeating: 0.25, count: frameCount) + Array(repeating: 0, count: 16_000 - frameCount))+ XCTAssertEqual(result.text, "chunk-1")+ }+ }+ }+ }++ func testNemotronNativeFileHandlesEmptyAndBoundaryInputs() async throws {+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {+ for frameCount in [0, 16_000, 16_001] {+ try await self.withNemotronFile(mode: mode, frameCount: frameCount) { result, inputs in+ XCTAssertEqual(inputs.count, frameCount == 0 ? 0 : 1)+ XCTAssertEqual(inputs.flatMap { $0 }, Array(repeating: 0.25, count: frameCount))+ XCTAssertEqual(result.text, frameCount == 0 ? "" : "chunk-1")+ }+ }+ }+ }++ func testNemotronNativeFileKeepsAudioAcrossBatches() async throws {+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {+ try await self.withNemotronFile(mode: mode, frameCount: 244_000) { result, inputs in+ XCTAssertGreaterThan(inputs.count, 1)+ XCTAssertTrue(inputs.allSatisfy { $0.count >= 16_000 && $0.count <= 240_000 })+ XCTAssertEqual(inputs.flatMap { $0 }, Array(repeating: 0.25, count: 244_000))+ XCTAssertEqual(result.text, inputs.indices.map { "chunk-\($0 + 1)" }.joined(separator: " "))+ }+ }+ }++ func testNemotronNativeFileRequiresReadyProvider() async throws {+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {+ let provider = NemotronProvider(mode: mode)+ do {+ _ = try await provider.transcribeFile(at: URL(fileURLWithPath: "/unused.wav"))+ XCTFail("A provider without loaded models must not transcribe")+ } catch {+ XCTAssertEqual((error as NSError).domain, "NemotronProvider")+ XCTAssertEqual(error.localizedDescription, "Nemotron provider is not ready.")+ }+ }+ }++ private func withNemotronFile(+ mode: NemotronProvider.Mode,+ frameCount: Int,+ check: (ASRTranscriptionResult, [[Float]]) throws -> Void+ ) async throws {+ let directory = FileManager.default.temporaryDirectory+ .appendingPathComponent("NemotronFileAudio-\(UUID().uuidString)", isDirectory: true)+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)+ defer { try? FileManager.default.removeItem(at: directory) }+ let fileURL = directory.appendingPathComponent("synthetic.wav")+ try Self.writeAudio(to: fileURL, frameCount: frameCount, sample: 0.25)+ var inputs: [[Float]] = []+ let result = try await NemotronProvider(mode: mode).transcribeFile(at: fileURL) { samples in+ inputs.append(samples)+ return "chunk-\(inputs.count)"+ }+ try check(result, inputs)+ }+ #endif++ private func withTranscription(+ frameCount: Int,+ sample: Float = 0.25,+ check: (TranscriptionResult, RecordingProvider) throws -> Void+ ) async throws {+ let defaults = UserDefaults.standard+ let keys = ["SelectedSpeechModel", "FileTranscriptionSpeakerLabelsEnabled"]+ let savedValues = keys.map { defaults.object(forKey: $0) }+ let settings = SettingsStore.shared+ settings.selectedSpeechModel = .whisperTiny+ settings.fileTranscriptionSpeakerLabelsEnabled = false+ let directory = FileManager.default.temporaryDirectory+ .appendingPathComponent("FileTranscriptionAudio-\(UUID().uuidString)", isDirectory: true)+ defer {+ for (key, value) in zip(keys, savedValues) {+ if let value {+ defaults.set(value, forKey: key)+ } else {+ defaults.removeObject(forKey: key)+ }+ }+ try? FileManager.default.removeItem(at: directory)+ }+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)+ let fileURL = directory.appendingPathComponent("synthetic.wav")+ try Self.writeAudio(to: fileURL, frameCount: frameCount, sample: sample)++ let provider = RecordingProvider()+ let asr = ASRService(meetingModelResidency: MeetingModelResidencyCoordinator())+ asr.modelProvidersForTesting[.whisperTiny] = provider+ asr.isAsrReady = true+ let service = FileTranscriptionService(asrService: asr)+ do {+ let result = try await service.transcribeFile(fileURL)+ XCTAssertEqual(service.currentFileURL, fileURL)+ XCTAssertFalse(service.isTranscribing)+ XCTAssertNil(asr.activeExclusiveActivity)+ try check(result, provider)+ await asr.shutdownForTermination()+ } catch {+ await asr.shutdownForTermination()+ throw error+ }+ }++ private static func writeAudio(to url: URL, frameCount: Int, sample: Float) throws {+ let format = try XCTUnwrap(AVAudioFormat(+ commonFormat: .pcmFormatInt16, sampleRate: 16_000, channels: 1, interleaved: true+ ))+ let file = try AVAudioFile(+ forWriting: url, settings: format.settings, commonFormat: .pcmFormatFloat32, interleaved: false+ )+ let capacity: AVAudioFrameCount = 16_000+ let buffer = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: capacity))+ let channel = try XCTUnwrap(buffer.floatChannelData?[0])+ channel.update(repeating: sample, count: Int(capacity))+ var remaining = frameCount+ while remaining > 0 {+ let count = min(Int(capacity), remaining)+ buffer.frameLength = AVAudioFrameCount(count)+ try file.write(from: buffer)+ remaining -= count+ }+ }+}