Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
feat: add experimental incremental parakeet finalization
altic-dev060c45d241fc
diff --git a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolvedindex 0c46ca2f..f9e9ae42 100644--- a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved+++ b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved@@ -34,7 +34,7 @@"location" : "https://github.com/altic-dev/FluidAudio.git","state" : {"branch" : "B/cohere-coreml-asr",- "revision" : "2e885ba247bdbbb3d3e932ba9701a5c165356df7"+ "revision" : "3fd63887eef1dc25edea8263ce4b44aa854d898b"}},{diff --git a/Package.resolved b/Package.resolvedindex 5011ae15..2d1d53c5 100644--- a/Package.resolved+++ b/Package.resolved@@ -33,7 +33,7 @@"location" : "https://github.com/altic-dev/FluidAudio.git","state" : {"branch" : "B/cohere-coreml-asr",- "revision" : "2e885ba247bdbbb3d3e932ba9701a5c165356df7"+ "revision" : "3fd63887eef1dc25edea8263ce4b44aa854d898b"}},{diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swiftindex e4909bbb..706345ed 100644--- a/Sources/Fluid/Persistence/BackupService.swift+++ b/Sources/Fluid/Persistence/BackupService.swift@@ -57,6 +57,8 @@ struct SettingsBackupPayload: Codable, Equatable {let pressAndHoldMode: Boollet hotkeyMode: HotkeyActivationMode?let enableStreamingPreview: Bool+ // Optional so backups created before incremental Parakeet finalization still decode.+ let experimentalParakeetUnifiedFinalEnabled: Bool?// Optional so backups created before the silence filter still decode.let skipSilentRecordingsEnabled: Bool?let enableAIStreaming: Booldiff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swiftindex 91bff4e5..419fe8c2 100644--- a/Sources/Fluid/Persistence/SettingsStore.swift+++ b/Sources/Fluid/Persistence/SettingsStore.swift@@ -1786,6 +1786,16 @@ final class SettingsStore: ObservableObject {}}+ /// Reuses finalized Parakeet windows so long recordings only process their remaining tail at stop.+ /// Experimental and enabled by default; users can fall back to full-buffer finalization.+ var experimentalParakeetUnifiedFinalEnabled: Bool {+ get { self.defaults.object(forKey: Keys.experimentalParakeetUnifiedFinalEnabled) as? Bool ?? true }+ set {+ objectWillChange.send()+ self.defaults.set(newValue, forKey: Keys.experimentalParakeetUnifiedFinalEnabled)+ }+ }+/// Skips clearly silent recordings up to four seconds before invoking ASR./// Opt-in so quiet speech keeps the existing transcription behavior by default.var skipSilentRecordingsEnabled: Bool {@@ -3207,6 +3217,7 @@ final class SettingsStore: ObservableObject {pressAndHoldMode: self.pressAndHoldMode,hotkeyMode: self.hotkeyMode,enableStreamingPreview: self.enableStreamingPreview,+ experimentalParakeetUnifiedFinalEnabled: self.experimentalParakeetUnifiedFinalEnabled,skipSilentRecordingsEnabled: self.skipSilentRecordingsEnabled,enableAIStreaming: self.enableAIStreaming,copyTranscriptionToClipboard: self.copyTranscriptionToClipboard,@@ -3330,6 +3341,9 @@ final class SettingsStore: ObservableObject {self.shareAnonymousAnalytics = payload.shareAnonymousAnalyticsself.hotkeyMode = payload.hotkeyMode ?? (payload.pressAndHoldMode ? .hold : .toggle)self.enableStreamingPreview = payload.enableStreamingPreview+ if let experimentalParakeetUnifiedFinalEnabled = payload.experimentalParakeetUnifiedFinalEnabled {+ self.experimentalParakeetUnifiedFinalEnabled = experimentalParakeetUnifiedFinalEnabled+ }if let skipSilentRecordingsEnabled = payload.skipSilentRecordingsEnabled {self.skipSilentRecordingsEnabled = skipSilentRecordingsEnabled}@@ -5259,6 +5273,7 @@ private extension SettingsStore {static let pressAndHoldMode = "PressAndHoldMode"static let hotkeyMode = "HotkeyMode"static let enableStreamingPreview = "EnableStreamingPreview"+ static let experimentalParakeetUnifiedFinalEnabled = "ExperimentalParakeetUnifiedFinalEnabled"static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled"static let enableAIStreaming = "EnableAIStreaming"static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swiftindex c9f5c0e8..8020acf6 100644--- a/Sources/Fluid/Services/FluidAudioProvider.swift+++ b/Sources/Fluid/Services/FluidAudioProvider.swift@@ -5,6 +5,9 @@ import FluidAudio/// TranscriptionProvider implementation using FluidAudio (optimized for Apple Silicon)/// This wraps the existing FluidAudio-based ASR for use on Apple Silicon Macs.final class FluidAudioProvider: TranscriptionProvider {+ private static let incrementalChunkingThresholdSamples = 240_000+ private static let incrementalAppendBlockSamples = 240_000+struct PronunciationTextReplacement {let wordRange: ClosedRange<Int>let label: String@@ -32,6 +35,8 @@ final class FluidAudioProvider: TranscriptionProvider {private var latestStreamingPreviewText: String = ""private var latestStreamingPreviewSampleCount: Int = 0private var latestStreamingPreviewFinishedAt: TimeInterval?+ private var incrementalSession: ParakeetIncrementalSession?+ private var incrementalAcceptedSampleCount = 0private(set) var isReady: Bool = falseprivate(set) var isWordBoostingActive: Bool = falseprivate(set) var boostedVocabularyTermsCount: Int = 0@@ -166,6 +171,8 @@ final class FluidAudioProvider: TranscriptionProvider {self.latestStreamingPreviewText = ""self.latestStreamingPreviewSampleCount = 0self.latestStreamingPreviewFinishedAt = nil+ self.incrementalSession = nil+ self.incrementalAcceptedSampleCount = 0try Task.checkCancellation()self.isReady = true@@ -184,6 +191,8 @@ final class FluidAudioProvider: TranscriptionProvider {self.latestStreamingPreviewText = ""self.latestStreamingPreviewSampleCount = 0self.latestStreamingPreviewFinishedAt = nil+ self.incrementalSession = nil+ self.incrementalAcceptedSampleCount = 0}func transcribeStreaming(_ samples: [Float]) async throws -> ASRTranscriptionResult {@@ -196,7 +205,35 @@ final class FluidAudioProvider: TranscriptionProvider {}let startedAt = Date().timeIntervalSince1970- let result = try await fullPreviewManager.transcribe(samples, source: AudioSource.microphone)+ let result: ASRResult+ if SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled,+ samples.count > Self.incrementalChunkingThresholdSamples,+ let incrementalManager = self.finalAsrManager ?? self.streamingAsrManager+ {+ do {+ result = try await self.transcribeIncrementalPreview(+ samples,+ manager: incrementalManager+ )+ } catch {+ self.resetIncrementalSession()+ if Task.isCancelled || error is CancellationError {+ throw CancellationError()+ }+ DebugLogger.shared.warning(+ "FluidAudioProvider: Incremental preview failed (\(error.localizedDescription)); using full preview",+ source: "FluidAudioProvider"+ )+ result = try await fullPreviewManager.transcribe(samples, source: AudioSource.microphone)+ }+ } else {+ if !SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled+ || samples.count < self.incrementalAcceptedSampleCount+ {+ self.resetIncrementalSession()+ }+ result = try await fullPreviewManager.transcribe(samples, source: AudioSource.microphone)+ }let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines)self.latestStreamingPreviewText = textself.latestStreamingPreviewSampleCount = samples.count@@ -244,6 +281,7 @@ final class FluidAudioProvider: TranscriptionProvider {}func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {+ defer { self.resetIncrementalSession() }guard let manager = self.finalAsrManager ?? self.streamingAsrManager else {throw NSError(domain: "FluidAudioProvider",@@ -252,6 +290,31 @@ final class FluidAudioProvider: TranscriptionProvider {)}+ if SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled,+ samples.count > Self.incrementalChunkingThresholdSamples,+ self.incrementalSession != nil+ {+ do {+ let startedAt = Date().timeIntervalSince1970+ let result = try await self.transcribeIncrementalFinal(samples)+ self.logFinalBenchmark(+ samples: samples,+ text: result.text,+ startedAt: startedAt,+ usedFallback: false+ )+ return result+ } catch {+ if Task.isCancelled || error is CancellationError {+ throw CancellationError()+ }+ DebugLogger.shared.warning(+ "FluidAudioProvider: Incremental final failed (\(error.localizedDescription)); using full final",+ source: "FluidAudioProvider"+ )+ }+ }+// If the boosted final manager fails, fall back to the unboosted streaming// manager so the user still gets a transcription (just without CTC rescoring).do {@@ -274,6 +337,69 @@ final class FluidAudioProvider: TranscriptionProvider {}}+ private func transcribeIncrementalPreview(+ _ samples: [Float],+ manager: AsrManager+ ) async throws -> ASRResult {+ if samples.count < self.incrementalAcceptedSampleCount {+ self.resetIncrementalSession()+ }+ if self.incrementalSession == nil {+ self.incrementalSession = try await manager.makeIncrementalSession(source: .microphone)+ }+ guard let session = self.incrementalSession else {+ throw NSError(+ domain: "FluidAudioProvider",+ code: -2,+ userInfo: [NSLocalizedDescriptionKey: "Incremental ASR session unavailable"]+ )+ }+ if samples.count > self.incrementalAcceptedSampleCount {+ try await self.appendIncrementalSamples(samples, to: session)+ }+ return try await session.preview()+ }++ private func transcribeIncrementalFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {+ guard let session = self.incrementalSession,+ samples.count >= self.incrementalAcceptedSampleCount+ else {+ throw NSError(+ domain: "FluidAudioProvider",+ code: -3,+ userInfo: [NSLocalizedDescriptionKey: "Incremental ASR session cannot finalize this recording"]+ )+ }+ if samples.count > self.incrementalAcceptedSampleCount {+ try await self.appendIncrementalSamples(samples, to: session)+ }+ let result = try await session.finish(finalAudioSamples: samples)+ let reusedWindows = await session.finalizedWindowCount+ DebugLogger.shared.info(+ "FluidAudioProvider: Incremental final reused \(reusedWindows) stable Parakeet windows",+ source: "FluidAudioProvider"+ )+ return ASRTranscriptionResult(text: result.text, confidence: result.confidence)+ }++ private func appendIncrementalSamples(+ _ samples: [Float],+ to session: ParakeetIncrementalSession+ ) async throws {+ var offset = self.incrementalAcceptedSampleCount+ while offset < samples.count {+ let end = min(offset + Self.incrementalAppendBlockSamples, samples.count)+ try await session.append(Array(samples[offset..<end]))+ offset = end+ self.incrementalAcceptedSampleCount = offset+ }+ }++ private func resetIncrementalSession() {+ self.incrementalSession = nil+ self.incrementalAcceptedSampleCount = 0+ }+private func transcribeFinalResult(_ samples: [Float], manager: AsrManager) async throws -> ASRTranscriptionResult {let matchingEnabled = SettingsStore.shared.pronunciationMatchingEnabled && samples.count <= 16_000 * 15let dictionaryLabels = Self.dictionaryLabels(@@ -472,6 +598,7 @@ final class FluidAudioProvider: TranscriptionProvider {}func clearCache() async throws {+ self.resetIncrementalSession()let baseCacheDir = AsrModels.defaultCacheDirectory().deletingLastPathComponent()let selectedModel = self.modelOverride ?? SettingsStore.shared.selectedSpeechModelDebugLogger.shared.info(diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swiftindex f5d9380d..63fde407 100644--- a/Sources/Fluid/UI/SettingsView.swift+++ b/Sources/Fluid/UI/SettingsView.swift@@ -1480,6 +1480,27 @@ struct SettingsView: View {}.padding(16)}++ ThemedCard(style: .standard) {+ VStack(alignment: .leading, spacing: 14) {+ HStack(spacing: 8) {+ Label("Experimental", systemImage: "flask.fill")+ .font(.headline)+ .foregroundStyle(.primary)+ }++ self.settingsToggleRow(+ title: "Faster Long Dictation",+ description: "For long recordings, reuse completed live windows and process only the remaining tail when you stop.",+ footnote: "Parakeet only. Falls back to normal transcription if reuse is unavailable or fails.",+ isOn: Binding(+ get: { SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled },+ set: { SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled = $0 }+ )+ )+ }+ .padding(16)+ }}.padding(16)}diff --git a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swiftindex 38b83136..a263cd05 100644--- a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift+++ b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift@@ -17,6 +17,7 @@ final class HotkeyShortcutTests: XCTestCase {private let microphoneSelectionMigrationVersionKey = "AppOnlyMicrophoneSelectionMigrationVersion"private let showMicrophoneChangeAlertsKey = "ShowMicrophoneChangeAlerts"private let experimentalDirectAudioCaptureEnabledKey = "ExperimentalDirectAudioCaptureEnabled"+ private let incrementalParakeetEnabledKey = "ExperimentalParakeetUnifiedFinalEnabled"@MainActorfunc testBottomOverlayRapidStopStartStopDoesNotDropFinalHide() async {@@ -140,6 +141,35 @@ final class HotkeyShortcutTests: XCTestCase {XCTAssertNil(decoded.settings.skipSilentRecordingsEnabled)}+ @MainActor+ func testIncrementalParakeetDefaultsOnAndRoundTripsWithoutBreakingLegacyBackups() async throws {+ let defaults = UserDefaults.standard+ let originalValue = defaults.object(forKey: self.incrementalParakeetEnabledKey)+ defer {+ if let originalValue {+ defaults.set(originalValue, forKey: self.incrementalParakeetEnabledKey)+ } else {+ defaults.removeObject(forKey: self.incrementalParakeetEnabledKey)+ }+ }++ defaults.removeObject(forKey: self.incrementalParakeetEnabledKey)+ XCTAssertTrue(SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled)++ SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled = false+ let document = await BackupService.shared.makeBackupDocument()+ XCTAssertEqual(document.settings.experimentalParakeetUnifiedFinalEnabled, false)++ let encoded = try BackupService.shared.encode(document)+ var root = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any])+ var settings = try XCTUnwrap(root["settings"] as? [String: Any])+ settings.removeValue(forKey: "experimentalParakeetUnifiedFinalEnabled")+ root["settings"] = settings++ let legacyBackup = try BackupService.shared.decode(JSONSerialization.data(withJSONObject: root))+ XCTAssertNil(legacyBackup.settings.experimentalParakeetUnifiedFinalEnabled)+ }+@MainActorfunc testLegacySystemModeBackupQueuesMicrophonePriorityMigration() async throws {let document = await BackupService.shared.makeBackupDocument()@@ -976,7 +1006,7 @@ final class HotkeyShortcutTests: XCTestCase {))for _ in 0...AudioCaptureIdlePolicy.SilentPCMRecoveryWatchdog.requiredSilentWindows {XCTAssertFalse(ambientWatchdog.shouldRecover(- isInternalMicrophone: true, isDirectCapture: true, rms: 0.000_1, peak: 0.001+ isInternalMicrophone: true, isDirectCapture: true, rms: 0.0001, peak: 0.001))}}