Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

feat: add Nemotron speech models

BarathwajAnandan3a51da44c68f
diff --git a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
index 5ea70f85..79a2b98b 100644
--- a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
+++ b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -34,7 +34,7 @@
"location" : "https://github.com/altic-dev/FluidAudio.git",
"state" : {
"branch" : "B/cohere-coreml-asr",
- "revision" : "ba6e4359fbb0d00b63e789354acc3f005641cfe4"
+ "revision" : "b71d6a06fbc633b0153d57156fd1fa41beddfd08"
}
},
{
diff --git a/Package.resolved b/Package.resolved
index 1f3d17e9..46e21859 100644
--- a/Package.resolved
+++ b/Package.resolved
@@ -24,7 +24,7 @@
"location" : "https://github.com/altic-dev/FluidAudio.git",
"state" : {
"branch" : "B/cohere-coreml-asr",
- "revision" : "ba6e4359fbb0d00b63e789354acc3f005641cfe4"
+ "revision" : "b71d6a06fbc633b0153d57156fd1fa41beddfd08"
}
},
{
diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swift
index af1ab37a..d3a9ca15 100644
--- a/Sources/Fluid/Persistence/BackupService.swift
+++ b/Sources/Fluid/Persistence/BackupService.swift
@@ -14,6 +14,7 @@ struct SettingsBackupPayload: Codable, Equatable {
let modelReasoningConfigs: [String: SettingsStore.ModelReasoningConfig]
let selectedSpeechModel: SettingsStore.SpeechModel
let selectedCohereLanguage: SettingsStore.CohereLanguage
+ let selectedNemotronLanguage: SettingsStore.NemotronLanguage?
let hotkeyShortcut: HotkeyShortcut
let promptModeHotkeyShortcut: HotkeyShortcut
let promptModeShortcutEnabled: Bool
diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift
index 0f726d1a..8a1c2faf 100644
--- a/Sources/Fluid/Persistence/SettingsStore.swift
+++ b/Sources/Fluid/Persistence/SettingsStore.swift
@@ -2262,6 +2262,7 @@ final class SettingsStore: ObservableObject {
modelReasoningConfigs: self.modelReasoningConfigs,
selectedSpeechModel: self.selectedSpeechModel,
selectedCohereLanguage: self.selectedCohereLanguage,
+ selectedNemotronLanguage: self.selectedNemotronLanguage,
hotkeyShortcut: self.hotkeyShortcut,
promptModeHotkeyShortcut: self.promptModeHotkeyShortcut,
promptModeShortcutEnabled: self.promptModeShortcutEnabled,
@@ -2337,6 +2338,9 @@ final class SettingsStore: ObservableObject {
self.modelReasoningConfigs = payload.modelReasoningConfigs
self.selectedSpeechModel = payload.selectedSpeechModel
self.selectedCohereLanguage = payload.selectedCohereLanguage
+ if let selectedNemotronLanguage = payload.selectedNemotronLanguage {
+ self.selectedNemotronLanguage = selectedNemotronLanguage
+ }
self.hotkeyShortcut = payload.hotkeyShortcut
self.promptModeHotkeyShortcut = payload.promptModeHotkeyShortcut
self.promptModeShortcutEnabled = payload.promptModeShortcutEnabled
@@ -2878,6 +2882,9 @@ final class SettingsStore: ObservableObject {
case parakeetRealtime = "parakeet-realtime"
case qwen3Asr = "qwen3-asr"
case cohereTranscribeSixBit = "cohere-transcribe-6bit"
+ case nemotronOffline = "nemotron-3.5-offline"
+ case nemotronStreaming = "nemotron-3.5-streaming"
+ case nemotronStreaming320 = "nemotron-3.5-streaming-320"
// MARK: - Apple Native
@@ -2906,6 +2913,9 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return "Parakeet Flash (Beta)"
case .qwen3Asr: return "Qwen3 ASR (Beta)"
case .cohereTranscribeSixBit: return "Cohere Transcribe"
+ case .nemotronOffline: return "Nemotron Speech 3.5 - Accurate"
+ case .nemotronStreaming: return "Nemotron Speech 3.5 - Fast"
+ case .nemotronStreaming320: return "Nemotron Speech 3.5 - Fast"
case .appleSpeech: return "Apple ASR Legacy"
case .appleSpeechAnalyzer: return "Apple Speech - macOS 26+"
case .whisperTiny: return "Whisper Tiny"
@@ -2925,6 +2935,9 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return "English Only (Live Streaming)"
case .qwen3Asr: return "30 Languages"
case .cohereTranscribeSixBit: return "14 Languages (Select Manually)"
+ case .nemotronOffline: return "Multilingual (Manual Language)"
+ case .nemotronStreaming: return "Multilingual (320ms Streaming)"
+ case .nemotronStreaming320: return "Multilingual (320ms Streaming)"
case .appleSpeech: return "System Languages"
case .appleSpeechAnalyzer: return "EN, ES, FR, DE, IT, JA, KO, PT, ZH"
case .whisperTiny, .whisperBase, .whisperSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge:
@@ -2939,6 +2952,9 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return "~250 MB"
case .qwen3Asr: return "~2.0 GB"
case .cohereTranscribeSixBit: return "~1.4 GB"
+ case .nemotronOffline: return "~530 MB"
+ case .nemotronStreaming: return "~670 MB"
+ case .nemotronStreaming320: return "~670 MB"
case .appleSpeech: return "Built-in (Zero Download)"
case .appleSpeechAnalyzer: return "Built-in"
case .whisperTiny: return "~75 MB"
@@ -2952,14 +2968,14 @@ final class SettingsStore: ObservableObject {
var requiresAppleSilicon: Bool {
switch self {
- case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit: return true
+ case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return true
default: return false
}
}
var isWhisperModel: Bool {
switch self {
- case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .appleSpeech, .appleSpeechAnalyzer: return false
+ case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeech, .appleSpeechAnalyzer: return false
default: return true
}
}
@@ -3017,6 +3033,9 @@ final class SettingsStore: ObservableObject {
if model == .qwen3Asr, !Self.qwenPreviewEnabled {
return false
}
+ if model == .nemotronStreaming320 {
+ return false
+ }
// Filter by Apple Silicon requirement
if model.requiresAppleSilicon, !CPUArchitecture.isAppleSilicon {
return false
@@ -3052,6 +3071,9 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return "Flash Dictation"
case .qwen3Asr: return "Qwen3 - Multilingual"
case .cohereTranscribeSixBit: return "Cohere - High Accuracy"
+ case .nemotronOffline: return "Nemotron Speech 3.5 - Accurate"
+ case .nemotronStreaming: return "Nemotron Speech 3.5 - Fast"
+ case .nemotronStreaming320: return "Nemotron Speech 3.5 - Fast"
case .appleSpeech: return "Apple ASR Legacy"
case .appleSpeechAnalyzer: return "Apple Speech - macOS 26+"
case .whisperTiny: return "Fast & Light"
@@ -3079,6 +3101,12 @@ final class SettingsStore: ObservableObject {
return "Qwen3 multilingual ASR via FluidAudio. Higher quality, heavier memory footprint."
case .cohereTranscribeSixBit:
return "High-accuracy multilingual transcription. Select the language manually before dictation for best results."
+ case .nemotronOffline:
+ return "NVIDIA Nemotron 3.5 offline transcription with manual language selection."
+ case .nemotronStreaming:
+ return "NVIDIA Nemotron 3.5 low-latency streaming with manual language selection."
+ case .nemotronStreaming320:
+ return "NVIDIA Nemotron 3.5 low-latency streaming with manual language selection."
case .appleSpeech:
return "Built-in macOS speech recognition. No download required."
case .appleSpeechAnalyzer:
@@ -3107,6 +3135,8 @@ final class SettingsStore: ObservableObject {
return 8.0
case .cohereTranscribeSixBit:
return 8.0
+ case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
+ return 8.0
case .appleSpeech, .appleSpeechAnalyzer:
return 2.0 // Built-in, minimal overhead
case .whisperTiny:
@@ -3148,6 +3178,8 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return 5
case .qwen3Asr: return 3
case .cohereTranscribeSixBit: return 3
+ case .nemotronOffline: return 3
+ case .nemotronStreaming, .nemotronStreaming320: return 4
case .appleSpeech: return 4
case .appleSpeechAnalyzer: return 4
case .whisperTiny: return 4
@@ -3167,6 +3199,8 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return 4
case .qwen3Asr: return 4
case .cohereTranscribeSixBit: return 5
+ case .nemotronOffline: return 5
+ case .nemotronStreaming, .nemotronStreaming320: return 4
case .appleSpeech: return 4
case .appleSpeechAnalyzer: return 4
case .whisperTiny: return 2
@@ -3186,6 +3220,8 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return 1.0
case .qwen3Asr: return 0.45
case .cohereTranscribeSixBit: return 0.85
+ case .nemotronOffline: return 0.85
+ case .nemotronStreaming, .nemotronStreaming320: return 0.88
case .appleSpeech: return 0.60
case .appleSpeechAnalyzer: return 0.85
case .whisperTiny: return 0.90
@@ -3205,6 +3241,8 @@ final class SettingsStore: ObservableObject {
case .parakeetRealtime: return 0.75
case .qwen3Asr: return 0.90
case .cohereTranscribeSixBit: return 0.98
+ case .nemotronOffline: return 0.92
+ case .nemotronStreaming, .nemotronStreaming320: return 0.88
case .appleSpeech: return 0.60
case .appleSpeechAnalyzer: return 0.80
case .whisperTiny: return 0.40
@@ -3232,7 +3270,7 @@ final class SettingsStore: ObservableObject {
/// Optimization level for Apple Silicon (for display)
var appleSiliconOptimized: Bool {
switch self {
- case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .appleSpeechAnalyzer:
+ case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeechAnalyzer:
return true
default:
return false
@@ -3243,7 +3281,7 @@ final class SettingsStore: ObservableObject {
/// Large Whisper models are too slow for streaming, so they only do final transcription on stop.
var supportsStreaming: Bool {
switch self {
- case .qwen3Asr, .whisperMedium, .whisperLargeTurbo, .whisperLarge:
+ case .qwen3Asr, .nemotronOffline, .whisperMedium, .whisperLargeTurbo, .whisperLarge:
return false // Too slow for real-time chunk processing
default:
return true // All other models support streaming
@@ -3256,6 +3294,8 @@ final class SettingsStore: ObservableObject {
switch self {
case .parakeetRealtime:
return 0.2
+ case .nemotronStreaming, .nemotronStreaming320:
+ return 0.32
case .cohereTranscribeSixBit:
return 1.0
default:
@@ -3269,6 +3309,8 @@ final class SettingsStore: ObservableObject {
switch self {
case .parakeetRealtime:
return 0.2
+ case .nemotronStreaming, .nemotronStreaming320:
+ return 0.64
case .cohereTranscribeSixBit:
return 1.5
default:
@@ -3288,7 +3330,7 @@ final class SettingsStore: ObservableObject {
/// Which provider this model belongs to
var provider: Provider {
switch self {
- case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime:
+ case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
return .nvidia
case .appleSpeech, .appleSpeechAnalyzer:
return .apple
@@ -3336,6 +3378,30 @@ final class SettingsStore: ObservableObject {
return false
}
return spec.validateArtifacts(at: directory)
+ case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
+ let hint: String
+ switch self {
+ case .nemotronOffline:
+ hint = "nemotron-3.5-asr-offline-6bit-CoreML"
+ default:
+ hint = "nemotron-3.5-asr-streaming320-int8-CoreML"
+ }
+ let directory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?
+ .appendingPathComponent(hint, isDirectory: true)
+ let requiredEntries = [
+ "metadata.json",
+ "preprocessor.mlpackage",
+ "encoder.mlpackage",
+ "decoder.mlpackage",
+ "joint.mlpackage",
+ "joint_decision.mlpackage",
+ "tokenizer.model",
+ ]
+ return directory.map { url in
+ requiredEntries.allSatisfy {
+ FileManager.default.fileExists(atPath: url.appendingPathComponent($0).path)
+ }
+ } ?? false
default:
// Whisper models
guard let whisperFile = self.whisperModelFile else { return false }
@@ -3361,7 +3427,7 @@ final class SettingsStore: ObservableObject {
/// Brand/provider name for the model (NVIDIA, Apple, OpenAI)
var brandName: String {
switch self {
- case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime:
+ case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
return "NVIDIA"
case .qwen3Asr:
return "Qwen"
@@ -3385,7 +3451,7 @@ final class SettingsStore: ObservableObject {
/// Brand color for the provider badge
var brandColorHex: String {
switch self {
- case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime:
+ case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
return "#76B900"
case .qwen3Asr:
return "#E67E22"
@@ -3558,6 +3624,7 @@ private extension SettingsStore {
/// Unified Speech Model (replaces above two)
static let selectedSpeechModel = "SelectedSpeechModel"
static let selectedCohereLanguage = "SelectedCohereLanguage"
+ static let selectedNemotronLanguage = "SelectedNemotronLanguage"
static let externalCoreMLArtifactsDirectories = "ExternalCoreMLArtifactsDirectories"
// Overlay Position
@@ -3757,6 +3824,28 @@ extension SettingsStore {
var tokenString: String { "<|\(self.rawValue)|>" }
}
+ enum NemotronLanguage: String, CaseIterable, Identifiable, Codable {
+ case english = "en"
+ case portuguese = "pt"
+ case spanish = "es"
+ case french = "fr"
+ case german = "de"
+ case italian = "it"
+
+ var id: String { self.rawValue }
+
+ var displayName: String {
+ switch self {
+ case .english: return "English"
+ case .portuguese: return "Portuguese"
+ case .spanish: return "Spanish"
+ case .french: return "French"
+ case .german: return "German"
+ case .italian: return "Italian"
+ }
+ }
+ }
+
// MARK: - Unified Speech Model Selection
/// The selected speech recognition model.
@@ -3771,6 +3860,9 @@ extension SettingsStore {
if model == .qwen3Asr, !SpeechModel.qwenPreviewEnabled {
return SpeechModel.defaultModel
}
+ if model == .nemotronStreaming320 {
+ return .nemotronStreaming
+ }
// Validate model is available on this architecture
if model.requiresAppleSilicon && !CPUArchitecture.isAppleSilicon {
return .whisperBase
@@ -3789,7 +3881,8 @@ extension SettingsStore {
}
set {
objectWillChange.send()
- self.defaults.set(newValue.rawValue, forKey: Keys.selectedSpeechModel)
+ let model = newValue == .nemotronStreaming320 ? SpeechModel.nemotronStreaming : newValue
+ self.defaults.set(model.rawValue, forKey: Keys.selectedSpeechModel)
}
}
@@ -3808,6 +3901,21 @@ extension SettingsStore {
}
}
+ var selectedNemotronLanguage: NemotronLanguage {
+ get {
+ if let rawValue = self.defaults.string(forKey: Keys.selectedNemotronLanguage),
+ let language = NemotronLanguage(rawValue: rawValue)
+ {
+ return language
+ }
+ return .english
+ }
+ set {
+ objectWillChange.send()
+ self.defaults.set(newValue.rawValue, forKey: Keys.selectedNemotronLanguage)
+ }
+ }
+
func externalCoreMLArtifactsDirectory(for model: SpeechModel) -> URL? {
guard let spec = model.externalCoreMLSpec else { return nil }
let paths = self.defaults.dictionary(forKey: Keys.externalCoreMLArtifactsDirectories) as? [String: String] ?? [:]
diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift
index 9062e301..e4490089 100644
--- a/Sources/Fluid/Services/ASRService.swift
+++ b/Sources/Fluid/Services/ASRService.swift
@@ -126,11 +126,10 @@ final class ASRService: ObservableObject {
/// Returns a user-friendly status message for model loading state
var modelStatusMessage: String {
- let usesExternalArtifacts = SettingsStore.shared.selectedSpeechModel.requiresExternalArtifacts
if self.isAsrReady { return "Model ready" }
if self.isDownloadingModel { return "Downloading model..." }
if self.isLoadingModel { return "Loading model into memory..." }
- if self.modelsExistOnDisk { return usesExternalArtifacts ? "Model cached, needs loading" : "Model cached, needs loading" }
+ if self.modelsExistOnDisk { return "Model cached, needs loading" }
return "Model not downloaded"
}
@@ -140,6 +139,7 @@ final class ASRService: ObservableObject {
private var fluidAudioProvider: FluidAudioProvider?
private var parakeetRealtimeProvider: ParakeetRealtimeProvider?
private var externalCoreMLProvider: ExternalCoreMLTranscriptionProvider?
+ private var nemotronProviders: [NemotronProvider.Mode: NemotronProvider] = [:]
private var whisperProvider: WhisperProvider?
private var appleSpeechProvider: AppleSpeechProvider?
/// Stored as Any? because @available cannot be applied to stored properties
@@ -171,11 +171,9 @@ final class ASRService: ObservableObject {
return self.getParakeetRealtimeProvider()
case .cohereTranscribeSixBit:
return self.getExternalCoreMLProvider()
+ case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
+ return self.getNemotronProvider(mode: model.nemotronProviderMode)
case .qwen3Asr:
- DebugLogger.shared.warning(
- "ASRService: Qwen provider removed; falling back to FluidAudio Parakeet path",
- source: "ASRService"
- )
return self.getFluidAudioProvider()
default:
return self.getWhisperProvider()
@@ -217,6 +215,14 @@ final class ASRService: ObservableObject {
return provider
}
+ private func getNemotronProvider(mode: NemotronProvider.Mode) -> NemotronProvider {
+ if let existing = self.nemotronProviders[mode] { return existing }
+ let provider = NemotronProvider(mode: mode)
+ self.nemotronProviders[mode] = provider
+ DebugLogger.shared.info("ASRService: Created \(provider.name) provider", source: "ASRService")
+ return provider
+ }
+
private func getWhisperProvider() -> WhisperProvider {
if let existing = whisperProvider {
return existing
@@ -361,19 +367,19 @@ final class ASRService: ObservableObject {
return AppleSpeechProvider()
case .parakeetTDT, .parakeetTDTv2:
// Create a new provider configured for the specific model
- let provider = FluidAudioProvider(modelOverride: model, configureWordBoosting: false)
- return provider
+ return FluidAudioProvider(modelOverride: model, configureWordBoosting: false)
case .parakeetRealtime:
return ParakeetRealtimeProvider()
case .cohereTranscribeSixBit:
return ExternalCoreMLTranscriptionProvider(modelOverride: model)
+ case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
+ return NemotronProvider(mode: model.nemotronProviderMode)
case .qwen3Asr:
// Qwen support removed; route legacy requests to Parakeet v3.
return FluidAudioProvider(modelOverride: .parakeetTDT, configureWordBoosting: false)
default:
// Whisper models - create provider with specific model override
- let provider = WhisperProvider(modelOverride: model)
- return provider
+ return WhisperProvider(modelOverride: model)
}
}
@@ -2784,6 +2790,16 @@ final class ASRService: ObservableObject {
}
}
+private extension SettingsStore.SpeechModel {
+ var nemotronProviderMode: NemotronProvider.Mode {
+ switch self {
+ case .nemotronStreaming: return .streaming
+ case .nemotronStreaming320: return .streaming320
+ default: return .offline
+ }
+ }
+}
+
private extension ASRService {
/// Stops the streaming timer and waits for the task to complete.
/// This prevents race conditions where the buffer is cleared while
diff --git a/Sources/Fluid/Services/NemotronProvider.swift b/Sources/Fluid/Services/NemotronProvider.swift
new file mode 100644
index 00000000..ecb37a67
--- /dev/null
+++ b/Sources/Fluid/Services/NemotronProvider.swift
@@ -0,0 +1,574 @@
+import Foundation
+
+#if arch(arm64)
+import AVFoundation
+@preconcurrency import CoreML
+import FluidAudio
+
+@available(macOS 14.0, *)
+final class NemotronProvider: TranscriptionProvider {
+ enum Mode: Hashable {
+ case offline
+ case streaming
+ case streaming320
+
+ var displayName: String {
+ switch self {
+ case .offline: return "Nemotron Speech 3.5 - Accurate"
+ case .streaming: return "Nemotron Speech 3.5 - Fast"
+ case .streaming320: return "Nemotron Speech 3.5 - Fast"
+ }
+ }
+
+ var folderHint: String {
+ switch self {
+ case .offline: return "nemotron-3.5-asr-offline-6bit-CoreML"
+ case .streaming: return "nemotron-3.5-asr-streaming320-int8-CoreML"
+ case .streaming320: return "nemotron-3.5-asr-streaming320-int8-CoreML"
+ }
+ }
+
+ var repositoryName: String { self.folderHint }
+ }
+
+ var name: String { self.mode.displayName }
+ var isAvailable: Bool { true }
+ private(set) var isReady: Bool = false
+ var prefersNativeFileTranscription: Bool { true }
+
+ private let repositoryOwner = "BarathwajAnandan"
+ private let repositoryRevision = "main"
+ private let requiredFiles = [
+ "metadata.json",
+ "preprocessor.mlpackage",
+ "encoder.mlpackage",
+ "decoder.mlpackage",
+ "joint.mlpackage",
+ "joint_decision.mlpackage",
+ "tokenizer.model",
+ ]
+
+ private let mode: Mode
+ private var manager: NemotronStreamingAsrManager?
+ private var streamedSampleCount: Int = 0
+ private var activeLanguageCode: String?
+ private var componentProfilingSessionActive = false
+ private var maxTranscriptionSamples: Int = 240_000
+ private let minimumFinalChunkSamples: Int = 16_000
+ private let chunkBoundarySearchRadiusSamples: Int = 32_000
+ private let chunkBoundaryAnalysisWindowSamples: Int = 1280
+ private let chunkBoundaryAnalysisStrideSamples: Int = 320
+ private static var componentProfilingEnabled: Bool {
+ UserDefaults.standard.bool(forKey: "ASRComponentProfilingEnabled")
+ }
+
+ private var folderHint: String { self.mode.folderHint }
+ private var repositoryName: String { self.mode.repositoryName }
+
+ init(mode: Mode = .offline) {
+ self.mode = mode
+ }
+
+ private var cacheDirectory: URL? {
+ FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?
+ .appendingPathComponent(self.folderHint, isDirectory: true)
+ }
+
+ func modelsExistOnDisk() -> Bool {
+ guard let dir = self.cacheDirectory else { return false }
+ return self.requiredFiles.allSatisfy { entry in
+ FileManager.default.fileExists(atPath: dir.appendingPathComponent(entry).path)
+ }
+ }
+
+ func prepare(progressHandler: ((Double) -> Void)? = nil) async throws {
+ guard self.isReady == false else { return }
+ guard let dir = self.cacheDirectory else {
+ throw Self.makeError("Unable to resolve a cache directory for \(self.name).")
+ }
+ try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
+
+ if self.modelsExistOnDisk() {
+ DebugLogger.shared.info(
+ "Nemotron: artifacts present at \(dir.path); skipping download",
+ source: "Nemotron"
+ )
+ progressHandler?(0.8)
+ } else {
+ DebugLogger.shared.info(
+ "Nemotron: artifacts missing; downloading from \(self.repositoryOwner)/\(self.repositoryName)",
+ source: "Nemotron"
+ )
+ let downloader = HuggingFaceModelDownloader(
+ owner: self.repositoryOwner,
+ repo: self.repositoryName,
+ revision: self.repositoryRevision,
+ requiredItems: self.requiredFiles.map {
+ .init(path: $0, isDirectory: $0.hasSuffix(".mlpackage"))
+ }
+ )
+ try await downloader.ensureModelsPresent(at: dir) { progress, _ in
+ progressHandler?(progress * 0.8)
+ }
+ guard self.modelsExistOnDisk() else {
+ throw Self.makeError("Nemotron artifacts incomplete after download at \(dir.path).")
+ }
+ }
+
+ self.maxTranscriptionSamples = Self.loadMaxAudioSamples(from: dir) ?? self.maxTranscriptionSamples
+ let configuration = NemotronStreamingAsrManager.defaultModelConfiguration()
+ configuration.allowLowPrecisionAccumulationOnGPU = true
+ let manager = NemotronStreamingAsrManager(configuration: configuration)
+ try await manager.loadModels(modelDir: dir)
+ try await self.applySelectedLanguage(to: manager)
+ self.manager = manager
+ self.isReady = true
+ progressHandler?(1.0)
+ DebugLogger.shared.info(
+ "Nemotron: provider ready [mode=\(self.mode.displayName), lang=\(SettingsStore.shared.selectedNemotronLanguage.rawValue), maxSamples=\(self.maxTranscriptionSamples)]",
+ source: "Nemotron"
+ )
+ }
+
+ func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ try await self.transcribeFinal(samples)
+ }
+
+ func transcribeStreaming(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ guard self.mode != .offline else {
+ return try await self.transcribeFinal(samples)
+ }
+ guard let manager = self.manager else {
+ throw Self.makeError("Nemotron provider is not ready.")
+ }
+
+ try await self.applySelectedLanguage(to: manager)
+ await self.startComponentProfilingIfNeeded(on: manager)
+ let delta = await self.consumeDelta(from: samples, manager: manager)
+ if delta.isEmpty == false {
+ do {
+ try await manager.appendAudio(Self.makeBuffer(delta))
+ try await manager.processBufferedAudio()
+ } catch {
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ throw error
+ }
+ }
+ let text = await manager.getPartialTranscript()
+ return ASRTranscriptionResult(text: text, confidence: text.isEmpty ? 0 : 1)
+ }
+
+ func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ guard let manager = self.manager else {
+ throw Self.makeError("Nemotron provider is not ready.")
+ }
+ guard samples.isEmpty == false else {
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ await manager.reset()
+ self.streamedSampleCount = 0
+ return ASRTranscriptionResult(text: "", confidence: 0)
+ }
+
+ if self.mode != .offline {
+ try await self.applySelectedLanguage(to: manager)
+ await self.startComponentProfilingIfNeeded(on: manager)
+ let delta = await self.consumeDelta(from: samples, manager: manager)
+ if delta.isEmpty == false {
+ do {
+ try await manager.appendAudio(Self.makeBuffer(delta))
+ try await manager.processBufferedAudio()
+ } catch {
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ await manager.reset()
+ self.streamedSampleCount = 0
+ throw error
+ }
+ }
+
+ let text: String
+ do {
+ text = try await manager.finish()
+ } catch {
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ await manager.reset()
+ self.streamedSampleCount = 0
+ throw error
+ }
+ await self.finishComponentProfilingIfNeeded(on: manager, samples: samples)
+ await manager.reset()
+ self.streamedSampleCount = 0
+ return ASRTranscriptionResult(text: text, confidence: text.isEmpty ? 0 : 1)
+ }
+
+ return try await self.transcribeBatched(samples)
+ }
+
+ func transcribeFile(at fileURL: URL) async throws -> ASRTranscriptionResult {
+ guard self.manager != nil else {
+ throw Self.makeError("Nemotron provider is not ready.")
+ }
+
+ let audioFile = try AVAudioFile(forReading: fileURL)
+ let sourceFormat = audioFile.processingFormat
+ let targetSampleRate = 16_000.0
+ let resampleRatio = targetSampleRate / sourceFormat.sampleRate
+ let sourceFramesPerRead = AVAudioFrameCount(
+ max(1, Double(self.regularChunkSamples) / resampleRatio)
+ )
+ var currentFrame: AVAudioFramePosition = 0
+ var pendingSamples: [Float] = []
+ var transcriptions: [String] = []
+
+ while currentFrame < audioFile.length {
+ let remainingFrames = AVAudioFrameCount(audioFile.length - currentFrame)
+ let framesToRead = min(sourceFramesPerRead, remainingFrames)
+ guard let buffer = AVAudioPCMBuffer(pcmFormat: sourceFormat, frameCapacity: framesToRead) else {
+ throw Self.makeError("Failed to allocate file audio buffer.")
+ }
+
+ audioFile.framePosition = currentFrame
+ try audioFile.read(into: buffer, frameCount: framesToRead)
+ try pendingSamples.append(contentsOf: Self.resampleBuffer(buffer, targetSampleRate: targetSampleRate))
+ while pendingSamples.count > self.maxTranscriptionSamples {
+ let end = self.chunkEnd(in: pendingSamples, offset: 0)
+ let text = try await self.transcribeSinglePass(Array(pendingSamples[..<end]))
+ .trimmingCharacters(in: .whitespacesAndNewlines)
+ if text.isEmpty == false {
+ transcriptions.append(text)
+ }
+ pendingSamples.removeFirst(end)
+ }
+ currentFrame += AVAudioFramePosition(framesToRead)
+ }
+
+ if pendingSamples.count >= Int(targetSampleRate) {
+ let result = try await self.transcribeBatched(pendingSamples)
+ let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
+ if text.isEmpty == false {
+ transcriptions.append(text)
+ }
+ }
+
+ let text = transcriptions.joined(separator: " ")
+ return ASRTranscriptionResult(text: text, confidence: text.isEmpty ? 0 : 1)
+ }
+
+ func clearCache() async throws {
+ if let manager = self.manager {
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ }
+ if let dir = self.cacheDirectory {
+ try? FileManager.default.removeItem(at: dir)
+ }
+ self.manager = nil
+ self.isReady = false
+ self.streamedSampleCount = 0
+ self.activeLanguageCode = nil
+ }
+
+ private func transcribeBatched(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ guard samples.isEmpty == false else { return ASRTranscriptionResult(text: "", confidence: 0) }
+
+ if samples.count <= self.maxTranscriptionSamples {
+ let text = try await self.transcribeSinglePass(samples)
+ return ASRTranscriptionResult(text: text, confidence: text.isEmpty ? 0 : 1)
+ }
+
+ var transcriptions: [String] = []
+ var offset = 0
+ while offset < samples.count {
+ let end = self.chunkEnd(in: samples, offset: offset)
+ let text = try await self.transcribeSinglePass(Array(samples[offset..<end]))
+ .trimmingCharacters(in: .whitespacesAndNewlines)
+ if text.isEmpty == false {
+ transcriptions.append(text)
+ }
+ offset = end
+ }
+
+ let text = transcriptions.joined(separator: " ")
+ return ASRTranscriptionResult(text: text, confidence: text.isEmpty ? 0 : 1)
+ }
+
+ private var regularChunkSamples: Int {
+ max(self.maxTranscriptionSamples - self.minimumFinalChunkSamples, self.minimumFinalChunkSamples)
+ }
+
+ private func chunkEnd(in samples: [Float], offset: Int) -> Int {
+ let maxEnd = min(offset + self.maxTranscriptionSamples, samples.count)
+ guard maxEnd < samples.count else { return samples.count }
+
+ let preferredEnd = min(offset + self.regularChunkSamples, maxEnd)
+ let latestEndWithValidTail = max(offset + self.minimumFinalChunkSamples, samples.count - self.minimumFinalChunkSamples)
+ let upperBound = min(maxEnd, latestEndWithValidTail, preferredEnd + self.chunkBoundarySearchRadiusSamples)
+ let lowerBound = max(offset + self.minimumFinalChunkSamples, preferredEnd - self.chunkBoundarySearchRadiusSamples)
+ guard lowerBound < upperBound else { return preferredEnd }
+
+ return self.quietestBoundary(
+ in: samples,
+ lowerBound: lowerBound,
+ upperBound: upperBound,
+ preferredEnd: preferredEnd
+ ) ?? preferredEnd
+ }
+
+ private func quietestBoundary(
+ in samples: [Float],
+ lowerBound: Int,
+ upperBound: Int,
+ preferredEnd: Int
+ ) -> Int? {
+ let analysisWindow = min(self.chunkBoundaryAnalysisWindowSamples, max(1, upperBound - lowerBound))
+ let halfWindow = max(1, analysisWindow / 2)
+ let stride = max(1, self.chunkBoundaryAnalysisStrideSamples)
+ var bestBoundary: Int?
+ var bestScore = Float.greatestFiniteMagnitude
+ var boundary = lowerBound
+
+ while boundary <= upperBound {
+ let windowStart = max(0, boundary - halfWindow)
+ let windowEnd = min(samples.count, boundary + halfWindow)
+ var energy: Float = 0
+ for index in windowStart..<windowEnd {
+ energy += abs(samples[index])
+ }
+
+ let sampleCount = max(1, windowEnd - windowStart)
+ let distancePenalty = Float(abs(boundary - preferredEnd)) / Float(max(1, self.chunkBoundarySearchRadiusSamples)) * 0.0001
+ let score = energy / Float(sampleCount) + distancePenalty
+ if score < bestScore {
+ bestScore = score
+ bestBoundary = boundary
+ }
+ boundary += stride
+ }
+
+ return bestBoundary
+ }
+
+ private func transcribeSinglePass(_ samples: [Float]) async throws -> String {
+ guard let manager = self.manager else {
+ throw Self.makeError("Nemotron provider is not ready.")
+ }
+ try await self.applySelectedLanguage(to: manager)
+ await self.startComponentProfilingIfNeeded(on: manager)
+ let buffer = try Self.makeBuffer(samples)
+ let text: String
+ do {
+ text = try await manager.transcribe(audioBuffer: buffer)
+ } catch {
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ await manager.reset()
+ self.streamedSampleCount = 0
+ throw error
+ }
+ await self.finishComponentProfilingIfNeeded(on: manager, samples: samples)
+ await manager.reset()
+ self.streamedSampleCount = 0
+ return text
+ }
+
+ private func startComponentProfilingIfNeeded(on manager: NemotronStreamingAsrManager) async {
+ guard Self.componentProfilingEnabled else {
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ return
+ }
+ guard self.componentProfilingSessionActive == false else { return }
+ await manager.setComponentProfilingEnabled(true)
+ self.componentProfilingSessionActive = true
+ }
+
+ private func finishComponentProfilingIfNeeded(on manager: NemotronStreamingAsrManager, samples: [Float]) async {
+ guard self.componentProfilingSessionActive else { return }
+ let profile = await manager.componentProfileSnapshot()
+ self.logComponentProfile(profile, samples: samples)
+ await self.stopComponentProfilingIfNeeded(on: manager)
+ }
+
+ private func stopComponentProfilingIfNeeded(on manager: NemotronStreamingAsrManager) async {
+ guard self.componentProfilingSessionActive else { return }
+ await manager.setComponentProfilingEnabled(false)
+ self.componentProfilingSessionActive = false
+ }
+
+ private func applySelectedLanguage(to manager: NemotronStreamingAsrManager) async throws {
+ let languageCode = SettingsStore.shared.selectedNemotronLanguage.rawValue
+ guard self.activeLanguageCode != languageCode else { return }
+ try await manager.setTargetLanguage(languageCode)
+ self.activeLanguageCode = languageCode
+ }
+
+ private func consumeDelta(from samples: [Float], manager: NemotronStreamingAsrManager) async -> [Float] {
+ if samples.count < self.streamedSampleCount {
+ await manager.reset()
+ self.streamedSampleCount = 0
+ }
+
+ let delta = Array(samples.dropFirst(self.streamedSampleCount))
+ self.streamedSampleCount = samples.count
+ return delta
+ }
+
+ private static func makeBuffer(_ samples: [Float]) throws -> AVAudioPCMBuffer {
+ guard
+ let format = AVAudioFormat(
+ commonFormat: .pcmFormatFloat32, sampleRate: 16_000, channels: 1, interleaved: false
+ ),
+ let buffer = AVAudioPCMBuffer(
+ pcmFormat: format, frameCapacity: AVAudioFrameCount(max(samples.count, 1))
+ )
+ else {
+ throw self.makeError("Failed to allocate audio buffer.")
+ }
+ buffer.frameLength = AVAudioFrameCount(samples.count)
+ if samples.isEmpty == false {
+ guard let channelData = buffer.floatChannelData else {
+ throw self.makeError("Failed to access audio buffer channel data.")
+ }
+ samples.withUnsafeBufferPointer { samplePtr in
+ guard let baseAddress = samplePtr.baseAddress else { return }
+ channelData[0].update(from: baseAddress, count: samples.count)
+ }
+ }
+ return buffer
+ }
+
+ private func logComponentProfile(_ profile: NemotronComponentProfile, samples: [Float]) {
+ let audioMs = Int((Double(samples.count) / 16_000.0 * 1000).rounded())
+ DebugLogger.shared.info(
+ """
+ ASR_COMPONENT provider=nemotron model=\(self.mode.displayName) samples=\(samples.count) audioMs=\(audioMs) \
+ totalMs=\(Self.ms(profile.totalChunkTime)) inputMs=\(Self.ms(profile.audioInputTime)) \
+ preprocessorMs=\(Self.ms(profile.preprocessorTime)) melInputMs=\(Self.ms(profile.melInputTime)) \
+ encoderMs=\(Self.ms(profile.encoderTime)) decodeMs=\(Self.ms(profile.decodeLoopTime)) \
+ decoderMs=\(Self.ms(profile.decoderTime)) jointDecisionMs=\(Self.ms(profile.jointDecisionTime)) \
+ jointMs=\(Self.ms(profile.jointTime)) encoderStepCopyMs=\(Self.ms(profile.encoderStepCopyTime)) \
+ decodeSteps=\(profile.decodeSteps) chunks=\(profile.chunks)
+ """,
+ source: "ASRBenchmark"
+ )
+ }
+
+ private static func ms(_ seconds: Double) -> Int {
+ Int((seconds * 1000).rounded())
+ }
+
+ private static func resampleBuffer(_ buffer: AVAudioPCMBuffer, targetSampleRate: Double) throws -> [Float] {
+ let sourceFormat = buffer.format
+ guard let targetFormat = AVAudioFormat(
+ commonFormat: .pcmFormatFloat32,
+ sampleRate: targetSampleRate,
+ channels: 1,
+ interleaved: false
+ ) else {
+ throw Self.makeError("Failed to create target audio format.")
+ }
+
+ if sourceFormat.sampleRate == targetSampleRate,
+ sourceFormat.channelCount == 1,
+ sourceFormat.commonFormat == .pcmFormatFloat32
+ {
+ guard let channelData = buffer.floatChannelData else {
+ throw Self.makeError("Failed to access audio channel data.")
+ }
+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)))
+ }
+
+ guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
+ throw Self.makeError("Failed to create audio converter.")
+ }
+ let ratio = targetSampleRate / sourceFormat.sampleRate
+ let capacity = AVAudioFrameCount((Double(buffer.frameLength) * ratio).rounded(.up)) + 1024
+ guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
+ throw Self.makeError("Failed to allocate converted audio buffer.")
+ }
+
+ var consumedInput = false
+ var conversionError: NSError?
+ converter.convert(to: outputBuffer, error: &conversionError) { _, status in
+ if consumedInput {
+ status.pointee = .noDataNow
+ return nil
+ }
+ consumedInput = true
+ status.pointee = .haveData
+ return buffer
+ }
+
+ if let conversionError {
+ throw conversionError
+ }
+ guard let channelData = outputBuffer.floatChannelData else {
+ throw Self.makeError("Failed to access converted audio channel data.")
+ }
+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(outputBuffer.frameLength)))
+ }
+
+ private static func loadMaxAudioSamples(from dir: URL) -> Int? {
+ let url = dir.appendingPathComponent("metadata.json")
+ guard
+ let data = try? Data(contentsOf: url),
+ let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
+ else {
+ return nil
+ }
+
+ if let samples = json["max_audio_samples"] as? Int, samples > 0 {
+ return samples
+ }
+ if let shapes = json["shapes"] as? [String: Any],
+ let audioSignal = shapes["audio_signal"] as? [Int],
+ audioSignal.count > 1,
+ audioSignal[1] > 0
+ {
+ return audioSignal[1]
+ }
+ return nil
+ }
+
+ private static func makeError(_ description: String) -> NSError {
+ NSError(domain: "NemotronProvider", code: -1, userInfo: [NSLocalizedDescriptionKey: description])
+ }
+}
+#else
+final class NemotronProvider: TranscriptionProvider {
+ enum Mode {
+ case offline
+ case streaming
+ case streaming320
+
+ var displayName: String {
+ switch self {
+ case .offline: return "Nemotron Speech 3.5 - Accurate"
+ case .streaming: return "Nemotron Speech 3.5 - Fast"
+ case .streaming320: return "Nemotron Speech 3.5 - Fast"
+ }
+ }
+ }
+
+ var name: String { self.mode.displayName }
+ var isAvailable: Bool { false }
+ private(set) var isReady: Bool = false
+ var prefersNativeFileTranscription: Bool { false }
+
+ private let mode: Mode
+
+ init(mode: Mode = .offline) {
+ self.mode = mode
+ }
+
+ func prepare(progressHandler: ((Double) -> Void)? = nil) async throws {
+ throw Self.makeError("Nemotron requires Apple Silicon.")
+ }
+
+ func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ throw Self.makeError("Nemotron requires Apple Silicon.")
+ }
+
+ func modelsExistOnDisk() -> Bool { false }
+
+ private static func makeError(_ description: String) -> NSError {
+ NSError(domain: "NemotronProvider", code: -1, userInfo: [NSLocalizedDescriptionKey: description])
+ }
+}
+#endif
diff --git a/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift b/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift
index a3a7729e..a2c6e75d 100644
--- a/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift
+++ b/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift
@@ -207,6 +207,10 @@ final class VoiceEngineSettingsViewModel: ObservableObject {
return "Qwen3 ASR is a multilingual FluidAudio model with strong quality, but higher memory usage. Requires macOS 15+."
case .cohereTranscribeSixBit:
return "Cohere Transcribe downloads a CoreML pipeline from Hugging Face and caches it locally. Select the language manually before dictation. Best on Apple Silicon with 8GB+ RAM."
+ case .nemotronOffline:
+ return "Nemotron Speech 3.5 Accurate uses NVIDIA CoreML models with manual language selection. Best on Apple Silicon with 8GB+ RAM."
+ case .nemotronStreaming, .nemotronStreaming320:
+ return "Nemotron Speech 3.5 Fast uses NVIDIA's streaming CoreML pipeline with manual language selection. Best for low-latency Apple Silicon dictation."
default:
return "Whisper models support 99 languages and work on any Mac."
}
diff --git a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift
index b339c352..9bd349ad 100644
--- a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift
+++ b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift
@@ -322,6 +322,53 @@ extension VoiceEngineSettingsView {
)
}
+ if model == .nemotronOffline || model == .nemotronStreaming || model == .nemotronStreaming320 {
+ VStack(alignment: .leading, spacing: 8) {
+ HStack(alignment: .center, spacing: 10) {
+ Image(systemName: "globe")
+ .font(.caption)
+ .foregroundStyle(self.theme.palette.accent)
+
+ VStack(alignment: .leading, spacing: 3) {
+ Text("Select Language Manually")
+ .font(.caption)
+ .fontWeight(.semibold)
+ Text("Choose the language prompt injected into Nemotron transcription.")
+ .font(.caption2)
+ .foregroundStyle(.secondary)
+ .lineLimit(2)
+ }
+
+ Spacer(minLength: 8)
+
+ Picker("Nemotron Language", selection: Binding(
+ get: { self.settings.selectedNemotronLanguage },
+ set: { newValue in
+ guard newValue != self.settings.selectedNemotronLanguage else { return }
+ self.settings.selectedNemotronLanguage = newValue
+ }
+ )) {
+ ForEach(SettingsStore.NemotronLanguage.allCases) { language in
+ Text(language.displayName).tag(language)
+ }
+ }
+ .pickerStyle(.menu)
+ .labelsHidden()
+ .disabled(self.viewModel.asr.isRunning)
+ }
+ }
+ .padding(.horizontal, 10)
+ .padding(.vertical, 8)
+ .background(
+ RoundedRectangle(cornerRadius: 8)
+ .fill(self.theme.palette.accent.opacity(0.08))
+ .overlay(
+ RoundedRectangle(cornerRadius: 8)
+ .stroke(self.theme.palette.accent.opacity(0.20), lineWidth: 1)
+ )
+ )
+ }
+
if supportsCustomWords {
HStack(alignment: .center, spacing: 10) {
Image(systemName: "checkmark.seal.fill")