Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

feat: add experimental incremental parakeet finalization

altic-dev060c45d241fc
diff --git a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
index 0c46ca2f..f9e9ae42 100644
--- a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
+++ b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -34,7 +34,7 @@
"location" : "https://github.com/altic-dev/FluidAudio.git",
"state" : {
"branch" : "B/cohere-coreml-asr",
- "revision" : "2e885ba247bdbbb3d3e932ba9701a5c165356df7"
+ "revision" : "3fd63887eef1dc25edea8263ce4b44aa854d898b"
}
},
{
diff --git a/Package.resolved b/Package.resolved
index 5011ae15..2d1d53c5 100644
--- a/Package.resolved
+++ b/Package.resolved
@@ -33,7 +33,7 @@
"location" : "https://github.com/altic-dev/FluidAudio.git",
"state" : {
"branch" : "B/cohere-coreml-asr",
- "revision" : "2e885ba247bdbbb3d3e932ba9701a5c165356df7"
+ "revision" : "3fd63887eef1dc25edea8263ce4b44aa854d898b"
}
},
{
diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swift
index e4909bbb..706345ed 100644
--- a/Sources/Fluid/Persistence/BackupService.swift
+++ b/Sources/Fluid/Persistence/BackupService.swift
@@ -57,6 +57,8 @@ struct SettingsBackupPayload: Codable, Equatable {
let pressAndHoldMode: Bool
let hotkeyMode: HotkeyActivationMode?
let enableStreamingPreview: Bool
+ // Optional so backups created before incremental Parakeet finalization still decode.
+ let experimentalParakeetUnifiedFinalEnabled: Bool?
// Optional so backups created before the silence filter still decode.
let skipSilentRecordingsEnabled: Bool?
let enableAIStreaming: Bool
diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift
index 91bff4e5..419fe8c2 100644
--- a/Sources/Fluid/Persistence/SettingsStore.swift
+++ b/Sources/Fluid/Persistence/SettingsStore.swift
@@ -1786,6 +1786,16 @@ final class SettingsStore: ObservableObject {
}
}
+ /// Reuses finalized Parakeet windows so long recordings only process their remaining tail at stop.
+ /// Experimental and enabled by default; users can fall back to full-buffer finalization.
+ var experimentalParakeetUnifiedFinalEnabled: Bool {
+ get { self.defaults.object(forKey: Keys.experimentalParakeetUnifiedFinalEnabled) as? Bool ?? true }
+ set {
+ objectWillChange.send()
+ self.defaults.set(newValue, forKey: Keys.experimentalParakeetUnifiedFinalEnabled)
+ }
+ }
+
/// Skips clearly silent recordings up to four seconds before invoking ASR.
/// Opt-in so quiet speech keeps the existing transcription behavior by default.
var skipSilentRecordingsEnabled: Bool {
@@ -3207,6 +3217,7 @@ final class SettingsStore: ObservableObject {
pressAndHoldMode: self.pressAndHoldMode,
hotkeyMode: self.hotkeyMode,
enableStreamingPreview: self.enableStreamingPreview,
+ experimentalParakeetUnifiedFinalEnabled: self.experimentalParakeetUnifiedFinalEnabled,
skipSilentRecordingsEnabled: self.skipSilentRecordingsEnabled,
enableAIStreaming: self.enableAIStreaming,
copyTranscriptionToClipboard: self.copyTranscriptionToClipboard,
@@ -3330,6 +3341,9 @@ final class SettingsStore: ObservableObject {
self.shareAnonymousAnalytics = payload.shareAnonymousAnalytics
self.hotkeyMode = payload.hotkeyMode ?? (payload.pressAndHoldMode ? .hold : .toggle)
self.enableStreamingPreview = payload.enableStreamingPreview
+ if let experimentalParakeetUnifiedFinalEnabled = payload.experimentalParakeetUnifiedFinalEnabled {
+ self.experimentalParakeetUnifiedFinalEnabled = experimentalParakeetUnifiedFinalEnabled
+ }
if let skipSilentRecordingsEnabled = payload.skipSilentRecordingsEnabled {
self.skipSilentRecordingsEnabled = skipSilentRecordingsEnabled
}
@@ -5259,6 +5273,7 @@ private extension SettingsStore {
static let pressAndHoldMode = "PressAndHoldMode"
static let hotkeyMode = "HotkeyMode"
static let enableStreamingPreview = "EnableStreamingPreview"
+ static let experimentalParakeetUnifiedFinalEnabled = "ExperimentalParakeetUnifiedFinalEnabled"
static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled"
static let enableAIStreaming = "EnableAIStreaming"
static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"
diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swift
index c9f5c0e8..8020acf6 100644
--- a/Sources/Fluid/Services/FluidAudioProvider.swift
+++ b/Sources/Fluid/Services/FluidAudioProvider.swift
@@ -5,6 +5,9 @@ import FluidAudio
/// TranscriptionProvider implementation using FluidAudio (optimized for Apple Silicon)
/// This wraps the existing FluidAudio-based ASR for use on Apple Silicon Macs.
final class FluidAudioProvider: TranscriptionProvider {
+ private static let incrementalChunkingThresholdSamples = 240_000
+ private static let incrementalAppendBlockSamples = 240_000
+
struct PronunciationTextReplacement {
let wordRange: ClosedRange<Int>
let label: String
@@ -32,6 +35,8 @@ final class FluidAudioProvider: TranscriptionProvider {
private var latestStreamingPreviewText: String = ""
private var latestStreamingPreviewSampleCount: Int = 0
private var latestStreamingPreviewFinishedAt: TimeInterval?
+ private var incrementalSession: ParakeetIncrementalSession?
+ private var incrementalAcceptedSampleCount = 0
private(set) var isReady: Bool = false
private(set) var isWordBoostingActive: Bool = false
private(set) var boostedVocabularyTermsCount: Int = 0
@@ -166,6 +171,8 @@ final class FluidAudioProvider: TranscriptionProvider {
self.latestStreamingPreviewText = ""
self.latestStreamingPreviewSampleCount = 0
self.latestStreamingPreviewFinishedAt = nil
+ self.incrementalSession = nil
+ self.incrementalAcceptedSampleCount = 0
try Task.checkCancellation()
self.isReady = true
@@ -184,6 +191,8 @@ final class FluidAudioProvider: TranscriptionProvider {
self.latestStreamingPreviewText = ""
self.latestStreamingPreviewSampleCount = 0
self.latestStreamingPreviewFinishedAt = nil
+ self.incrementalSession = nil
+ self.incrementalAcceptedSampleCount = 0
}
func transcribeStreaming(_ samples: [Float]) async throws -> ASRTranscriptionResult {
@@ -196,7 +205,35 @@ final class FluidAudioProvider: TranscriptionProvider {
}
let startedAt = Date().timeIntervalSince1970
- let result = try await fullPreviewManager.transcribe(samples, source: AudioSource.microphone)
+ let result: ASRResult
+ if SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled,
+ samples.count > Self.incrementalChunkingThresholdSamples,
+ let incrementalManager = self.finalAsrManager ?? self.streamingAsrManager
+ {
+ do {
+ result = try await self.transcribeIncrementalPreview(
+ samples,
+ manager: incrementalManager
+ )
+ } catch {
+ self.resetIncrementalSession()
+ if Task.isCancelled || error is CancellationError {
+ throw CancellationError()
+ }
+ DebugLogger.shared.warning(
+ "FluidAudioProvider: Incremental preview failed (\(error.localizedDescription)); using full preview",
+ source: "FluidAudioProvider"
+ )
+ result = try await fullPreviewManager.transcribe(samples, source: AudioSource.microphone)
+ }
+ } else {
+ if !SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled
+ || samples.count < self.incrementalAcceptedSampleCount
+ {
+ self.resetIncrementalSession()
+ }
+ result = try await fullPreviewManager.transcribe(samples, source: AudioSource.microphone)
+ }
let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
self.latestStreamingPreviewText = text
self.latestStreamingPreviewSampleCount = samples.count
@@ -244,6 +281,7 @@ final class FluidAudioProvider: TranscriptionProvider {
}
func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ defer { self.resetIncrementalSession() }
guard let manager = self.finalAsrManager ?? self.streamingAsrManager else {
throw NSError(
domain: "FluidAudioProvider",
@@ -252,6 +290,31 @@ final class FluidAudioProvider: TranscriptionProvider {
)
}
+ if SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled,
+ samples.count > Self.incrementalChunkingThresholdSamples,
+ self.incrementalSession != nil
+ {
+ do {
+ let startedAt = Date().timeIntervalSince1970
+ let result = try await self.transcribeIncrementalFinal(samples)
+ self.logFinalBenchmark(
+ samples: samples,
+ text: result.text,
+ startedAt: startedAt,
+ usedFallback: false
+ )
+ return result
+ } catch {
+ if Task.isCancelled || error is CancellationError {
+ throw CancellationError()
+ }
+ DebugLogger.shared.warning(
+ "FluidAudioProvider: Incremental final failed (\(error.localizedDescription)); using full final",
+ source: "FluidAudioProvider"
+ )
+ }
+ }
+
// If the boosted final manager fails, fall back to the unboosted streaming
// manager so the user still gets a transcription (just without CTC rescoring).
do {
@@ -274,6 +337,69 @@ final class FluidAudioProvider: TranscriptionProvider {
}
}
+ private func transcribeIncrementalPreview(
+ _ samples: [Float],
+ manager: AsrManager
+ ) async throws -> ASRResult {
+ if samples.count < self.incrementalAcceptedSampleCount {
+ self.resetIncrementalSession()
+ }
+ if self.incrementalSession == nil {
+ self.incrementalSession = try await manager.makeIncrementalSession(source: .microphone)
+ }
+ guard let session = self.incrementalSession else {
+ throw NSError(
+ domain: "FluidAudioProvider",
+ code: -2,
+ userInfo: [NSLocalizedDescriptionKey: "Incremental ASR session unavailable"]
+ )
+ }
+ if samples.count > self.incrementalAcceptedSampleCount {
+ try await self.appendIncrementalSamples(samples, to: session)
+ }
+ return try await session.preview()
+ }
+
+ private func transcribeIncrementalFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ guard let session = self.incrementalSession,
+ samples.count >= self.incrementalAcceptedSampleCount
+ else {
+ throw NSError(
+ domain: "FluidAudioProvider",
+ code: -3,
+ userInfo: [NSLocalizedDescriptionKey: "Incremental ASR session cannot finalize this recording"]
+ )
+ }
+ if samples.count > self.incrementalAcceptedSampleCount {
+ try await self.appendIncrementalSamples(samples, to: session)
+ }
+ let result = try await session.finish(finalAudioSamples: samples)
+ let reusedWindows = await session.finalizedWindowCount
+ DebugLogger.shared.info(
+ "FluidAudioProvider: Incremental final reused \(reusedWindows) stable Parakeet windows",
+ source: "FluidAudioProvider"
+ )
+ return ASRTranscriptionResult(text: result.text, confidence: result.confidence)
+ }
+
+ private func appendIncrementalSamples(
+ _ samples: [Float],
+ to session: ParakeetIncrementalSession
+ ) async throws {
+ var offset = self.incrementalAcceptedSampleCount
+ while offset < samples.count {
+ let end = min(offset + Self.incrementalAppendBlockSamples, samples.count)
+ try await session.append(Array(samples[offset..<end]))
+ offset = end
+ self.incrementalAcceptedSampleCount = offset
+ }
+ }
+
+ private func resetIncrementalSession() {
+ self.incrementalSession = nil
+ self.incrementalAcceptedSampleCount = 0
+ }
+
private func transcribeFinalResult(_ samples: [Float], manager: AsrManager) async throws -> ASRTranscriptionResult {
let matchingEnabled = SettingsStore.shared.pronunciationMatchingEnabled && samples.count <= 16_000 * 15
let dictionaryLabels = Self.dictionaryLabels(
@@ -472,6 +598,7 @@ final class FluidAudioProvider: TranscriptionProvider {
}
func clearCache() async throws {
+ self.resetIncrementalSession()
let baseCacheDir = AsrModels.defaultCacheDirectory().deletingLastPathComponent()
let selectedModel = self.modelOverride ?? SettingsStore.shared.selectedSpeechModel
DebugLogger.shared.info(
diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift
index f5d9380d..63fde407 100644
--- a/Sources/Fluid/UI/SettingsView.swift
+++ b/Sources/Fluid/UI/SettingsView.swift
@@ -1480,6 +1480,27 @@ struct SettingsView: View {
}
.padding(16)
}
+
+ ThemedCard(style: .standard) {
+ VStack(alignment: .leading, spacing: 14) {
+ HStack(spacing: 8) {
+ Label("Experimental", systemImage: "flask.fill")
+ .font(.headline)
+ .foregroundStyle(.primary)
+ }
+
+ self.settingsToggleRow(
+ title: "Faster Long Dictation",
+ description: "For long recordings, reuse completed live windows and process only the remaining tail when you stop.",
+ footnote: "Parakeet only. Falls back to normal transcription if reuse is unavailable or fails.",
+ isOn: Binding(
+ get: { SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled },
+ set: { SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled = $0 }
+ )
+ )
+ }
+ .padding(16)
+ }
}
.padding(16)
}
diff --git a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift
index 38b83136..a263cd05 100644
--- a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift
+++ b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift
@@ -17,6 +17,7 @@ final class HotkeyShortcutTests: XCTestCase {
private let microphoneSelectionMigrationVersionKey = "AppOnlyMicrophoneSelectionMigrationVersion"
private let showMicrophoneChangeAlertsKey = "ShowMicrophoneChangeAlerts"
private let experimentalDirectAudioCaptureEnabledKey = "ExperimentalDirectAudioCaptureEnabled"
+ private let incrementalParakeetEnabledKey = "ExperimentalParakeetUnifiedFinalEnabled"
@MainActor
func testBottomOverlayRapidStopStartStopDoesNotDropFinalHide() async {
@@ -140,6 +141,35 @@ final class HotkeyShortcutTests: XCTestCase {
XCTAssertNil(decoded.settings.skipSilentRecordingsEnabled)
}
+ @MainActor
+ func testIncrementalParakeetDefaultsOnAndRoundTripsWithoutBreakingLegacyBackups() async throws {
+ let defaults = UserDefaults.standard
+ let originalValue = defaults.object(forKey: self.incrementalParakeetEnabledKey)
+ defer {
+ if let originalValue {
+ defaults.set(originalValue, forKey: self.incrementalParakeetEnabledKey)
+ } else {
+ defaults.removeObject(forKey: self.incrementalParakeetEnabledKey)
+ }
+ }
+
+ defaults.removeObject(forKey: self.incrementalParakeetEnabledKey)
+ XCTAssertTrue(SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled)
+
+ SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled = false
+ let document = await BackupService.shared.makeBackupDocument()
+ XCTAssertEqual(document.settings.experimentalParakeetUnifiedFinalEnabled, false)
+
+ let encoded = try BackupService.shared.encode(document)
+ var root = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any])
+ var settings = try XCTUnwrap(root["settings"] as? [String: Any])
+ settings.removeValue(forKey: "experimentalParakeetUnifiedFinalEnabled")
+ root["settings"] = settings
+
+ let legacyBackup = try BackupService.shared.decode(JSONSerialization.data(withJSONObject: root))
+ XCTAssertNil(legacyBackup.settings.experimentalParakeetUnifiedFinalEnabled)
+ }
+
@MainActor
func testLegacySystemModeBackupQueuesMicrophonePriorityMigration() async throws {
let document = await BackupService.shared.makeBackupDocument()
@@ -976,7 +1006,7 @@ final class HotkeyShortcutTests: XCTestCase {
))
for _ in 0...AudioCaptureIdlePolicy.SilentPCMRecoveryWatchdog.requiredSilentWindows {
XCTAssertFalse(ambientWatchdog.shouldRecover(
- isInternalMicrophone: true, isDirectCapture: true, rms: 0.000_1, peak: 0.001
+ isInternalMicrophone: true, isDirectCapture: true, rms: 0.0001, peak: 0.001
))
}
}