Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

fix(transcription): preserve subsecond file audio (#1162)

Alex Efremovc981bc84e3d3
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj
index 6cefe3fb..3164f65c 100644
--- a/Fluid.xcodeproj/project.pbxproj
+++ b/Fluid.xcodeproj/project.pbxproj
@@ -99,6 +99,7 @@
7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */; };
803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 803000000000000000000001 /* MediaPlaybackServiceTests.swift */; };
0B05F11E000000000000A002 /* SupportedFileExtensionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */; };
+ BF1510100000000000000002 /* FileTranscriptionAudioTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1510100000000000000001 /* FileTranscriptionAudioTests.swift */; };
CD0E00000000000000000002 /* MeetingOverlayModelsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD0E00000000000000000001 /* MeetingOverlayModelsTests.swift */; };
CD0F00000000000000000002 /* MeetingProcessingConfigurationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD0F00000000000000000001 /* MeetingProcessingConfigurationTests.swift */; };
CD100000000000000000000002 /* MeetingProviderOptionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD100000000000000000000001 /* MeetingProviderOptionsTests.swift */; };
@@ -294,6 +295,7 @@
7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypingServiceTransientPasteboardTests.swift; sourceTree = "<group>"; };
803000000000000000000001 /* MediaPlaybackServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPlaybackServiceTests.swift; sourceTree = "<group>"; };
0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SupportedFileExtensionsTests.swift; sourceTree = "<group>"; };
+ BF1510100000000000000001 /* FileTranscriptionAudioTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTranscriptionAudioTests.swift; sourceTree = "<group>"; };
CD0E00000000000000000001 /* MeetingOverlayModelsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingOverlayModelsTests.swift; sourceTree = "<group>"; };
CD0F00000000000000000001 /* MeetingProcessingConfigurationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingProcessingConfigurationTests.swift; sourceTree = "<group>"; };
CD100000000000000000000001 /* MeetingProviderOptionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingProviderOptionsTests.swift; sourceTree = "<group>"; };
@@ -553,6 +555,7 @@
C0DE63800000000000000001 /* AppleSpeechLocaleSelectionTests.swift */,
CACEC0010000000000000001 /* KeychainServiceCacheTests.swift */,
0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */,
+ BF1510100000000000000001 /* FileTranscriptionAudioTests.swift */,
FEED67600000000000000001 /* PasteDeliveryCoordinatorTests.swift */,
CD0E00000000000000000001 /* MeetingOverlayModelsTests.swift */,
CD0F00000000000000000001 /* MeetingProcessingConfigurationTests.swift */,
@@ -960,6 +963,7 @@
C0DE63800000000000000002 /* AppleSpeechLocaleSelectionTests.swift in Sources */,
CACEC0010000000000000002 /* KeychainServiceCacheTests.swift in Sources */,
0B05F11E000000000000A002 /* SupportedFileExtensionsTests.swift in Sources */,
+ BF1510100000000000000002 /* FileTranscriptionAudioTests.swift in Sources */,
FEED67600000000000000002 /* PasteDeliveryCoordinatorTests.swift in Sources */,
CD0E00000000000000000002 /* MeetingOverlayModelsTests.swift in Sources */,
CD0F00000000000000000002 /* MeetingProcessingConfigurationTests.swift in Sources */,
diff --git a/Sources/Fluid/Services/FileTranscriptionService.swift b/Sources/Fluid/Services/FileTranscriptionService.swift
index 53252c0a..5d030ede 100644
--- a/Sources/Fluid/Services/FileTranscriptionService.swift
+++ b/Sources/Fluid/Services/FileTranscriptionService.swift
@@ -587,7 +587,7 @@ final class FileTranscriptionService: ObservableObject {
}
// Convert buffer to 16kHz mono Float32 samples
- let samples: [Float]
+ var samples: [Float]
do {
samples = try AudioBufferConverter.monoSamples(
from: buffer,
@@ -597,11 +597,14 @@ final class FileTranscriptionService: ObservableObject {
throw TranscriptionError.audioConversionFailed("Could not resample audio: \(error.localizedDescription)")
}
- // Skip if chunk is too short (< 1 second)
- guard samples.count >= Int(sampleRate) else {
+ guard !samples.isEmpty else {
currentFrame += AVAudioFramePosition(framesToRead)
continue
}
+ // Match dictation: preserve short speech while meeting the ASR minimum.
+ if samples.count < Int(sampleRate) {
+ samples.append(contentsOf: repeatElement(Float(0), count: Int(sampleRate) - samples.count))
+ }
// Transcribe this chunk using the provider (works for both Parakeet and Whisper)
let chunkResult = try await provider.transcribe(samples)
@@ -622,7 +625,7 @@ final class FileTranscriptionService: ObservableObject {
if allTranscriptions.isEmpty {
DebugLogger.shared.warning(
- "No audio chunks were long enough to transcribe (minimum 1 second required)",
+ "No transcription text was produced for this file",
source: "FileTranscriptionService"
)
}
diff --git a/Sources/Fluid/Services/NemotronProvider.swift b/Sources/Fluid/Services/NemotronProvider.swift
index 7c7ab75e..54ec17fa 100644
--- a/Sources/Fluid/Services/NemotronProvider.swift
+++ b/Sources/Fluid/Services/NemotronProvider.swift
@@ -279,6 +279,13 @@ final class NemotronProvider: TranscriptionProvider {
throw Self.makeError("Nemotron provider is not ready.")
}
+ return try await self.transcribeFile(at: fileURL, transcribeChunk: self.transcribeSinglePass)
+ }
+
+ func transcribeFile(
+ at fileURL: URL,
+ transcribeChunk: ([Float]) async throws -> String
+ ) async throws -> ASRTranscriptionResult {
let audioFile = try AVAudioFile(forReading: fileURL)
let sourceFormat = audioFile.processingFormat
let targetSampleRate = 16_000.0
@@ -305,7 +312,7 @@ final class NemotronProvider: TranscriptionProvider {
))
while pendingSamples.count > self.maxTranscriptionSamples {
let end = self.chunkEnd(in: pendingSamples, offset: 0)
- let text = try await self.transcribeSinglePass(Array(pendingSamples[..<end]))
+ let text = try await transcribeChunk(Array(pendingSamples[..<end]))
.trimmingCharacters(in: .whitespacesAndNewlines)
if text.isEmpty == false {
transcriptions.append(text)
@@ -315,9 +322,12 @@ final class NemotronProvider: TranscriptionProvider {
currentFrame += AVAudioFramePosition(framesToRead)
}
- if pendingSamples.count >= Int(targetSampleRate) {
- let result = try await self.transcribeBatched(pendingSamples)
- let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
+ if pendingSamples.isEmpty == false {
+ if pendingSamples.count < self.minimumFinalChunkSamples {
+ pendingSamples.append(contentsOf: repeatElement(Float(0), count: self.minimumFinalChunkSamples - pendingSamples.count))
+ }
+ let text = try await transcribeChunk(pendingSamples)
+ .trimmingCharacters(in: .whitespacesAndNewlines)
if text.isEmpty == false {
transcriptions.append(text)
}
diff --git a/Tests/FluidDictationIntegrationTests/FileTranscriptionAudioTests.swift b/Tests/FluidDictationIntegrationTests/FileTranscriptionAudioTests.swift
new file mode 100644
index 00000000..9b277797
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/FileTranscriptionAudioTests.swift
@@ -0,0 +1,214 @@
+import AVFoundation
+@testable import FluidVoice_Debug
+import XCTest
+
+@MainActor
+final class FileTranscriptionAudioTests: XCTestCase {
+ private final class RecordingProvider: TranscriptionProvider {
+ let name = "Buffered file test provider"
+ let isAvailable = true
+ let isReady = true
+ private(set) var inputs: [[Float]] = []
+
+ func prepare(progressHandler: ((ModelPreparationProgress) -> Void)?) async throws {
+ XCTFail("An already-ready provider must not prepare models")
+ }
+
+ func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ self.inputs.append(samples)
+ // Observe the real file path without adding synthetic text to shared history.
+ return ASRTranscriptionResult(text: "", confidence: 0)
+ }
+ }
+
+ func testSubsecondFilesPreserveSamplesAndAddOnlyTrailingSilence() async throws {
+ for frameCount in [1, 8000, 15_999] {
+ try await self.withTranscription(frameCount: frameCount) { result, provider in
+ XCTAssertEqual(provider.inputs.count, 1, "Source frames: \(frameCount)")
+ let input = try XCTUnwrap(provider.inputs.first)
+ XCTAssertEqual(input.count, 16_000)
+ XCTAssertEqual(Array(input.prefix(frameCount)), Array(repeating: 0.25, count: frameCount))
+ XCTAssertEqual(Array(input.dropFirst(frameCount)), Array(repeating: 0, count: 16_000 - frameCount))
+ XCTAssertEqual(result.duration, Double(frameCount) / 16_000, accuracy: 0.0001)
+ }
+ }
+ }
+
+ func testFilesAtOrAboveOneSecondAreNotPadded() async throws {
+ for frameCount in [16_000, 16_001] {
+ try await self.withTranscription(frameCount: frameCount) { result, provider in
+ XCTAssertEqual(provider.inputs.count, 1, "Source frames: \(frameCount)")
+ let input = try XCTUnwrap(provider.inputs.first)
+ XCTAssertEqual(input, Array(repeating: 0.25, count: frameCount))
+ XCTAssertEqual(result.duration, Double(frameCount) / 16_000, accuracy: 0.0001)
+ }
+ }
+ }
+
+ func testSubsecondTailAfterFullChunkStillReachesProvider() async throws {
+ let fullChunkFrames = 20 * 60 * 16_000
+ try await self.withTranscription(frameCount: fullChunkFrames + 4000) { result, provider in
+ XCTAssertEqual(provider.inputs.count, 2)
+ guard provider.inputs.count == 2 else { return }
+ XCTAssertEqual(provider.inputs[0].count, fullChunkFrames)
+ XCTAssertTrue(provider.inputs[0].allSatisfy { $0 == 0.25 })
+ XCTAssertEqual(
+ provider.inputs[1],
+ Array(repeating: 0.25, count: 4000) + Array(repeating: 0, count: 12_000)
+ )
+ XCTAssertEqual(result.duration, 1200.25, accuracy: 0.0001)
+ }
+ }
+
+ func testEmptyFileDoesNotCreatePaddedInference() async throws {
+ try await self.withTranscription(frameCount: 0) { result, provider in
+ XCTAssertTrue(provider.inputs.isEmpty)
+ XCTAssertTrue(result.text.isEmpty)
+ XCTAssertEqual(result.duration, 0)
+ }
+ }
+
+ func testSubsecondSilenceIsPassedThroughWithoutChangingDuration() async throws {
+ try await self.withTranscription(frameCount: 8000, sample: 0) { result, provider in
+ XCTAssertEqual(provider.inputs.count, 1)
+ let input = try XCTUnwrap(provider.inputs.first)
+ XCTAssertEqual(input, Array(repeating: 0, count: 16_000))
+ XCTAssertEqual(result.duration, 0.5, accuracy: 0.0001)
+ }
+ }
+
+ #if arch(arm64)
+ func testNemotronNativeFilePadsSubsecondInput() async throws {
+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {
+ for frameCount in [1, 8000, 15_999] {
+ try await self.withNemotronFile(mode: mode, frameCount: frameCount) { result, inputs in
+ XCTAssertEqual(inputs.count, 1, "Mode: \(mode), frames: \(frameCount)")
+ let input = try XCTUnwrap(inputs.first)
+ XCTAssertEqual(input, Array(repeating: 0.25, count: frameCount) + Array(repeating: 0, count: 16_000 - frameCount))
+ XCTAssertEqual(result.text, "chunk-1")
+ }
+ }
+ }
+ }
+
+ func testNemotronNativeFileHandlesEmptyAndBoundaryInputs() async throws {
+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {
+ for frameCount in [0, 16_000, 16_001] {
+ try await self.withNemotronFile(mode: mode, frameCount: frameCount) { result, inputs in
+ XCTAssertEqual(inputs.count, frameCount == 0 ? 0 : 1)
+ XCTAssertEqual(inputs.flatMap { $0 }, Array(repeating: 0.25, count: frameCount))
+ XCTAssertEqual(result.text, frameCount == 0 ? "" : "chunk-1")
+ }
+ }
+ }
+ }
+
+ func testNemotronNativeFileKeepsAudioAcrossBatches() async throws {
+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {
+ try await self.withNemotronFile(mode: mode, frameCount: 244_000) { result, inputs in
+ XCTAssertGreaterThan(inputs.count, 1)
+ XCTAssertTrue(inputs.allSatisfy { $0.count >= 16_000 && $0.count <= 240_000 })
+ XCTAssertEqual(inputs.flatMap { $0 }, Array(repeating: 0.25, count: 244_000))
+ XCTAssertEqual(result.text, inputs.indices.map { "chunk-\($0 + 1)" }.joined(separator: " "))
+ }
+ }
+ }
+
+ func testNemotronNativeFileRequiresReadyProvider() async throws {
+ for mode in [NemotronProvider.Mode.offline, .streaming, .streaming320] {
+ let provider = NemotronProvider(mode: mode)
+ do {
+ _ = try await provider.transcribeFile(at: URL(fileURLWithPath: "/unused.wav"))
+ XCTFail("A provider without loaded models must not transcribe")
+ } catch {
+ XCTAssertEqual((error as NSError).domain, "NemotronProvider")
+ XCTAssertEqual(error.localizedDescription, "Nemotron provider is not ready.")
+ }
+ }
+ }
+
+ private func withNemotronFile(
+ mode: NemotronProvider.Mode,
+ frameCount: Int,
+ check: (ASRTranscriptionResult, [[Float]]) throws -> Void
+ ) async throws {
+ let directory = FileManager.default.temporaryDirectory
+ .appendingPathComponent("NemotronFileAudio-\(UUID().uuidString)", isDirectory: true)
+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+ defer { try? FileManager.default.removeItem(at: directory) }
+ let fileURL = directory.appendingPathComponent("synthetic.wav")
+ try Self.writeAudio(to: fileURL, frameCount: frameCount, sample: 0.25)
+ var inputs: [[Float]] = []
+ let result = try await NemotronProvider(mode: mode).transcribeFile(at: fileURL) { samples in
+ inputs.append(samples)
+ return "chunk-\(inputs.count)"
+ }
+ try check(result, inputs)
+ }
+ #endif
+
+ private func withTranscription(
+ frameCount: Int,
+ sample: Float = 0.25,
+ check: (TranscriptionResult, RecordingProvider) throws -> Void
+ ) async throws {
+ let defaults = UserDefaults.standard
+ let keys = ["SelectedSpeechModel", "FileTranscriptionSpeakerLabelsEnabled"]
+ let savedValues = keys.map { defaults.object(forKey: $0) }
+ let settings = SettingsStore.shared
+ settings.selectedSpeechModel = .whisperTiny
+ settings.fileTranscriptionSpeakerLabelsEnabled = false
+ let directory = FileManager.default.temporaryDirectory
+ .appendingPathComponent("FileTranscriptionAudio-\(UUID().uuidString)", isDirectory: true)
+ defer {
+ for (key, value) in zip(keys, savedValues) {
+ if let value {
+ defaults.set(value, forKey: key)
+ } else {
+ defaults.removeObject(forKey: key)
+ }
+ }
+ try? FileManager.default.removeItem(at: directory)
+ }
+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+ let fileURL = directory.appendingPathComponent("synthetic.wav")
+ try Self.writeAudio(to: fileURL, frameCount: frameCount, sample: sample)
+
+ let provider = RecordingProvider()
+ let asr = ASRService(meetingModelResidency: MeetingModelResidencyCoordinator())
+ asr.modelProvidersForTesting[.whisperTiny] = provider
+ asr.isAsrReady = true
+ let service = FileTranscriptionService(asrService: asr)
+ do {
+ let result = try await service.transcribeFile(fileURL)
+ XCTAssertEqual(service.currentFileURL, fileURL)
+ XCTAssertFalse(service.isTranscribing)
+ XCTAssertNil(asr.activeExclusiveActivity)
+ try check(result, provider)
+ await asr.shutdownForTermination()
+ } catch {
+ await asr.shutdownForTermination()
+ throw error
+ }
+ }
+
+ private static func writeAudio(to url: URL, frameCount: Int, sample: Float) throws {
+ let format = try XCTUnwrap(AVAudioFormat(
+ commonFormat: .pcmFormatInt16, sampleRate: 16_000, channels: 1, interleaved: true
+ ))
+ let file = try AVAudioFile(
+ forWriting: url, settings: format.settings, commonFormat: .pcmFormatFloat32, interleaved: false
+ )
+ let capacity: AVAudioFrameCount = 16_000
+ let buffer = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: capacity))
+ let channel = try XCTUnwrap(buffer.floatChannelData?[0])
+ channel.update(repeating: sample, count: Int(capacity))
+ var remaining = frameCount
+ while remaining > 0 {
+ let count = min(Int(capacity), remaining)
+ buffer.frameLength = AVAudioFrameCount(count)
+ try file.write(from: buffer)
+ remaining -= count
+ }
+ }
+}