Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
feat(parakeet): add vocabulary boost store and streaming/final boost path
altic-devac86c6df19ff
diff --git a/Sources/Fluid/Resources/parakeet_custom_vocabulary.default.json b/Sources/Fluid/Resources/parakeet_custom_vocabulary.default.jsonnew file mode 100644index 00000000..708f495a--- /dev/null+++ b/Sources/Fluid/Resources/parakeet_custom_vocabulary.default.json@@ -0,0 +1,14 @@+{+ "alpha": 2.5,+ "minCtcScore": -2.0,+ "minSimilarity": 0.72,+ "minCombinedConfidence": 0.65,+ "minTermLength": 3,+ "terms": [+ {+ "text": "FluidVoice",+ "aliases": ["fluid voice", "fluid boys"],+ "weight": 12.0+ }+ ]+}diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swiftindex 14850bd0..e38f2670 100644--- a/Sources/Fluid/Services/FluidAudioProvider.swift+++ b/Sources/Fluid/Services/FluidAudioProvider.swift@@ -13,15 +13,20 @@ final class FluidAudioProvider: TranscriptionProvider {true}- private var asrManager: AsrManager?+ private var streamingAsrManager: AsrManager?+ private var finalAsrManager: AsrManager?private(set) var isReady: Bool = false+ private(set) var isWordBoostingActive: Bool = false+ private(set) var boostedVocabularyTermsCount: Int = 0/// Optional model override - if set, uses this model instead of the global setting./// Used for downloading specific models without changing the active selection.var modelOverride: SettingsStore.SpeechModel?+ private let configureWordBoosting: Bool- init(modelOverride: SettingsStore.SpeechModel? = nil) {+ init(modelOverride: SettingsStore.SpeechModel? = nil, configureWordBoosting: Bool = true) {self.modelOverride = modelOverride+ self.configureWordBoosting = configureWordBoosting}func prepare(progressHandler: ((Double) -> Void)? = nil) async throws {@@ -29,6 +34,7 @@ final class FluidAudioProvider: TranscriptionProvider {let selectedModel = self.modelOverride ?? SettingsStore.shared.selectedSpeechModelDebugLogger.shared.info("FluidAudioProvider: Starting model preparation for \(selectedModel.displayName)", source: "FluidAudioProvider")+ progressHandler?(0.05)// Download and load modelslet models: AsrModels@@ -39,18 +45,64 @@ final class FluidAudioProvider: TranscriptionProvider {// Default to v3 (Multilingual)models = try await AsrModels.downloadAndLoad(version: .v3)}+ progressHandler?(0.70)- // Initialize AsrManager+ // Single manager path minimizes memory on low-resource devices.let manager = AsrManager(config: ASRConfig.default)try await manager.initialize(models: models)- self.asrManager = manager++ self.isWordBoostingActive = false+ self.boostedVocabularyTermsCount = 0++ if self.configureWordBoosting {+ do {+ if let vocabBundle = try await ParakeetVocabularyStore.shared.loadTokenizedVocabularyBundle() {+ try await manager.configureVocabularyBoosting(+ vocabulary: vocabBundle.vocabulary,+ ctcModels: vocabBundle.ctcModels+ )+ self.isWordBoostingActive = true+ self.boostedVocabularyTermsCount = vocabBundle.vocabulary.terms.count+ DebugLogger.shared.info(+ "FluidAudioProvider: Enabled vocabulary boosting with \(self.boostedVocabularyTermsCount) terms",+ source: "FluidAudioProvider"+ )+ } else {+ DebugLogger.shared.debug("FluidAudioProvider: No vocabulary boost terms found; using base ASR manager", source: "FluidAudioProvider")+ }+ } catch {+ DebugLogger.shared.warning("FluidAudioProvider: Failed to configure vocabulary boosting: \(error)", source: "FluidAudioProvider")+ }+ }++ // Boosting now applies in both streaming and final paths.+ self.streamingAsrManager = manager+ self.finalAsrManager = managerself.isReady = true+ progressHandler?(1.0)DebugLogger.shared.info("FluidAudioProvider: Models ready", source: "FluidAudioProvider")}func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult {- guard let manager = asrManager else {+ try await self.transcribeFinal(samples)+ }++ func transcribeStreaming(_ samples: [Float]) async throws -> ASRTranscriptionResult {+ guard let manager = self.streamingAsrManager else {+ throw NSError(+ domain: "FluidAudioProvider",+ code: -1,+ userInfo: [NSLocalizedDescriptionKey: "ASR manager not initialized"]+ )+ }++ let result = try await manager.transcribe(samples, source: AudioSource.microphone)+ return ASRTranscriptionResult(text: result.text, confidence: result.confidence)+ }++ func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {+ guard let manager = self.finalAsrManager ?? self.streamingAsrManager else {throw NSError(domain: "FluidAudioProvider",code: -1,@@ -96,13 +148,16 @@ final class FluidAudioProvider: TranscriptionProvider {}self.isReady = false- self.asrManager = nil+ self.streamingAsrManager = nil+ self.finalAsrManager = nil+ self.isWordBoostingActive = false+ self.boostedVocabularyTermsCount = 0}/// Provides direct access to the underlying AsrManager for advanced use cases/// (e.g., MeetingTranscriptionService sharing)var underlyingManager: AsrManager? {- return self.asrManager+ return self.streamingAsrManager}}#else@@ -112,7 +167,7 @@ final class FluidAudioProvider: TranscriptionProvider {var isAvailable: Bool { false }var isReady: Bool { false }- init(modelOverride: SettingsStore.SpeechModel? = nil) {+ init(modelOverride: SettingsStore.SpeechModel? = nil, configureWordBoosting: Bool = true) {// Intel stub - parameter ignored}diff --git a/Sources/Fluid/Services/ParakeetVocabularyStore.swift b/Sources/Fluid/Services/ParakeetVocabularyStore.swiftnew file mode 100644index 00000000..7a2e486d--- /dev/null+++ b/Sources/Fluid/Services/ParakeetVocabularyStore.swift@@ -0,0 +1,254 @@+import Foundation+#if arch(arm64)+import FluidAudio+#endif++/// JSON-backed store for Parakeet vocabulary boosting terms.+/// Persists to Application Support so it survives app updates and can be user-edited.+final class ParakeetVocabularyStore {+ static let shared = ParakeetVocabularyStore()++ struct VocabularyConfig: Codable, Sendable {+ struct Term: Codable, Hashable, Sendable {+ let text: String+ let weight: Float?+ let aliases: [String]?+ }++ let alpha: Float?+ let minCtcScore: Float?+ let minSimilarity: Float?+ let minCombinedConfidence: Float?+ let minTermLength: Int?+ let terms: [Term]+ }++ struct ResolvedConfig: Sendable {+ let alpha: Float+ let minCtcScore: Float+ let minSimilarity: Float+ let minCombinedConfidence: Float+ let minTermLength: Int+ let terms: [VocabularyConfig.Term]+ }++ enum StoreError: LocalizedError {+ case invalidJSON(String)+ case applicationSupportUnavailable++ var errorDescription: String? {+ switch self {+ case .invalidJSON(let details):+ return "Invalid vocabulary JSON: \(details)"+ case .applicationSupportUnavailable:+ return "Could not access Application Support directory."+ }+ }+ }++ private enum Defaults {+ static let alpha: Float = 2.5+ static let minCtcScore: Float = -2.0+ static let minSimilarity: Float = 0.72+ static let minCombinedConfidence: Float = 0.65+ static let minTermLength: Int = 3+ static let maxTerms: Int = 256+ }++ private let fileName = "parakeet_custom_vocabulary.json"+ private let appSupportFolder = "FluidVoice"++ private init() {}++ func vocabularyFileURL() throws -> URL {+ guard let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else {+ throw StoreError.applicationSupportUnavailable+ }+ let directory = base.appendingPathComponent(self.appSupportFolder, isDirectory: true)+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)+ return directory.appendingPathComponent(self.fileName)+ }++ @discardableResult+ func ensureVocabularyFileExists() throws -> URL {+ let url = try self.vocabularyFileURL()+ guard !FileManager.default.fileExists(atPath: url.path) else { return url }++ if let bundled = Bundle.main.url(forResource: "parakeet_custom_vocabulary.default", withExtension: "json"),+ let bundledText = try? String(contentsOf: bundled, encoding: .utf8)+ {+ try bundledText.write(to: url, atomically: true, encoding: .utf8)+ } else {+ try Self.defaultTemplateJSON().write(to: url, atomically: true, encoding: .utf8)+ }+ return url+ }++ func loadRawJSON() throws -> String {+ let url = try self.ensureVocabularyFileExists()+ return try String(contentsOf: url, encoding: .utf8)+ }++ func validateJSON(_ json: String) throws -> VocabularyConfig {+ let data = Data(json.utf8)+ do {+ return try JSONDecoder().decode(VocabularyConfig.self, from: data)+ } catch {+ throw StoreError.invalidJSON(error.localizedDescription)+ }+ }++ func saveRawJSON(_ json: String) throws {+ _ = try self.validateJSON(json)+ let url = try self.ensureVocabularyFileExists()+ try json.write(to: url, atomically: true, encoding: .utf8)+ NotificationCenter.default.post(name: .parakeetVocabularyDidChange, object: nil)+ }++ func hasAnyBoostTerms() -> Bool {+ (try? self.loadResolvedConfig())?.terms.isEmpty == false+ }++ func loadResolvedConfig() throws -> ResolvedConfig {+ let rawJSON = try self.loadRawJSON()+ let parsed = (try? self.validateJSON(rawJSON)) ?? VocabularyConfig(+ alpha: nil,+ minCtcScore: nil,+ minSimilarity: nil,+ minCombinedConfidence: nil,+ minTermLength: nil,+ terms: []+ )++ let mergedTerms = self.mergeAndNormalizeTerms(jsonTerms: parsed.terms, dictionaryEntries: SettingsStore.shared.customDictionaryEntries)++ return ResolvedConfig(+ alpha: parsed.alpha ?? Defaults.alpha,+ minCtcScore: parsed.minCtcScore ?? Defaults.minCtcScore,+ minSimilarity: parsed.minSimilarity ?? Defaults.minSimilarity,+ minCombinedConfidence: parsed.minCombinedConfidence ?? Defaults.minCombinedConfidence,+ minTermLength: parsed.minTermLength ?? Defaults.minTermLength,+ terms: mergedTerms+ )+ }++ private func mergeAndNormalizeTerms(+ jsonTerms: [VocabularyConfig.Term],+ dictionaryEntries: [SettingsStore.CustomDictionaryEntry]+ ) -> [VocabularyConfig.Term] {+ var mergedByText: [String: VocabularyConfig.Term] = [:]++ func normalizeAliases(_ aliases: [String]?, excluding text: String) -> [String]? {+ let normalized = (aliases ?? [])+ .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }+ .filter { !$0.isEmpty }+ .filter { $0.caseInsensitiveCompare(text) != .orderedSame }+ let deduped = Array(Set(normalized.map { $0.lowercased() })).sorted()+ return deduped.isEmpty ? nil : deduped+ }++ func upsert(_ term: VocabularyConfig.Term) {+ let normalizedText = term.text.trimmingCharacters(in: .whitespacesAndNewlines)+ guard !normalizedText.isEmpty else { return }+ let key = normalizedText.lowercased()++ if let existing = mergedByText[key] {+ let combinedAliases = Array(Set((existing.aliases ?? []) + (term.aliases ?? []))).sorted()+ let combinedWeight = max(existing.weight ?? 0, term.weight ?? 0)+ mergedByText[key] = VocabularyConfig.Term(+ text: existing.text,+ weight: combinedWeight > 0 ? combinedWeight : nil,+ aliases: combinedAliases.isEmpty ? nil : combinedAliases+ )+ return+ }++ mergedByText[key] = VocabularyConfig.Term(+ text: normalizedText,+ weight: term.weight,+ aliases: normalizeAliases(term.aliases, excluding: normalizedText)+ )+ }++ for term in jsonTerms {+ upsert(term)+ }++ for entry in dictionaryEntries {+ let replacement = entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines)+ guard !replacement.isEmpty else { continue }+ let aliases = entry.triggers+ .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }+ .filter { !$0.isEmpty }+ upsert(VocabularyConfig.Term(text: replacement, weight: 10.0, aliases: aliases))+ }++ return mergedByText.values.sorted { lhs, rhs in+ lhs.text.localizedCaseInsensitiveCompare(rhs.text) == .orderedAscending+ }+ }++ static func defaultTemplateJSON() -> String {+ """+ {+ "alpha": 2.5,+ "minCtcScore": -2.0,+ "minSimilarity": 0.72,+ "minCombinedConfidence": 0.65,+ "minTermLength": 3,+ "terms": [+ {+ "text": "FluidVoice",+ "aliases": ["fluid voice", "fluid boys"],+ "weight": 12.0+ }+ ]+ }+ """+ }+}++extension Notification.Name {+ static let parakeetVocabularyDidChange = Notification.Name("ParakeetVocabularyDidChange")+}++#if arch(arm64)+extension ParakeetVocabularyStore {+ /// Creates tokenized vocabulary + CTC models for FluidAudio vocabulary boosting.+ func loadTokenizedVocabularyBundle(+ maxTerms: Int = 256+ ) async throws -> (vocabulary: CustomVocabularyContext, ctcModels: CtcModels)? {+ let resolved = try self.loadResolvedConfig()+ guard !resolved.terms.isEmpty else { return nil }++ let cappedTerms = Array(resolved.terms.prefix(maxTerms))+ let ctcModels = try await CtcModels.downloadAndLoad(variant: .ctc110m)+ let ctcTokenizer = try await CtcTokenizer.load(from: CtcModels.defaultCacheDirectory(for: ctcModels.variant))++ let tokenizedTerms: [CustomVocabularyTerm] = cappedTerms.compactMap { term in+ let tokens = ctcTokenizer.encode(term.text)+ guard !tokens.isEmpty else { return nil }+ return CustomVocabularyTerm(+ text: term.text,+ weight: term.weight,+ aliases: term.aliases,+ tokenIds: nil,+ ctcTokenIds: tokens+ )+ }++ guard !tokenizedTerms.isEmpty else { return nil }++ let vocabulary = CustomVocabularyContext(+ terms: tokenizedTerms,+ alpha: resolved.alpha,+ minCtcScore: resolved.minCtcScore,+ minSimilarity: resolved.minSimilarity,+ minCombinedConfidence: resolved.minCombinedConfidence,+ minTermLength: resolved.minTermLength+ )++ return (vocabulary, ctcModels)+ }+}+#endif