Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

feat(parakeet): add vocabulary boost store and streaming/final boost path

altic-devac86c6df19ff
diff --git a/Sources/Fluid/Resources/parakeet_custom_vocabulary.default.json b/Sources/Fluid/Resources/parakeet_custom_vocabulary.default.json
new file mode 100644
index 00000000..708f495a
--- /dev/null
+++ b/Sources/Fluid/Resources/parakeet_custom_vocabulary.default.json
@@ -0,0 +1,14 @@
+{
+ "alpha": 2.5,
+ "minCtcScore": -2.0,
+ "minSimilarity": 0.72,
+ "minCombinedConfidence": 0.65,
+ "minTermLength": 3,
+ "terms": [
+ {
+ "text": "FluidVoice",
+ "aliases": ["fluid voice", "fluid boys"],
+ "weight": 12.0
+ }
+ ]
+}
diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swift
index 14850bd0..e38f2670 100644
--- a/Sources/Fluid/Services/FluidAudioProvider.swift
+++ b/Sources/Fluid/Services/FluidAudioProvider.swift
@@ -13,15 +13,20 @@ final class FluidAudioProvider: TranscriptionProvider {
true
}
- private var asrManager: AsrManager?
+ private var streamingAsrManager: AsrManager?
+ private var finalAsrManager: AsrManager?
private(set) var isReady: Bool = false
+ private(set) var isWordBoostingActive: Bool = false
+ private(set) var boostedVocabularyTermsCount: Int = 0
/// Optional model override - if set, uses this model instead of the global setting.
/// Used for downloading specific models without changing the active selection.
var modelOverride: SettingsStore.SpeechModel?
+ private let configureWordBoosting: Bool
- init(modelOverride: SettingsStore.SpeechModel? = nil) {
+ init(modelOverride: SettingsStore.SpeechModel? = nil, configureWordBoosting: Bool = true) {
self.modelOverride = modelOverride
+ self.configureWordBoosting = configureWordBoosting
}
func prepare(progressHandler: ((Double) -> Void)? = nil) async throws {
@@ -29,6 +34,7 @@ final class FluidAudioProvider: TranscriptionProvider {
let selectedModel = self.modelOverride ?? SettingsStore.shared.selectedSpeechModel
DebugLogger.shared.info("FluidAudioProvider: Starting model preparation for \(selectedModel.displayName)", source: "FluidAudioProvider")
+ progressHandler?(0.05)
// Download and load models
let models: AsrModels
@@ -39,18 +45,64 @@ final class FluidAudioProvider: TranscriptionProvider {
// Default to v3 (Multilingual)
models = try await AsrModels.downloadAndLoad(version: .v3)
}
+ progressHandler?(0.70)
- // Initialize AsrManager
+ // Single manager path minimizes memory on low-resource devices.
let manager = AsrManager(config: ASRConfig.default)
try await manager.initialize(models: models)
- self.asrManager = manager
+
+ self.isWordBoostingActive = false
+ self.boostedVocabularyTermsCount = 0
+
+ if self.configureWordBoosting {
+ do {
+ if let vocabBundle = try await ParakeetVocabularyStore.shared.loadTokenizedVocabularyBundle() {
+ try await manager.configureVocabularyBoosting(
+ vocabulary: vocabBundle.vocabulary,
+ ctcModels: vocabBundle.ctcModels
+ )
+ self.isWordBoostingActive = true
+ self.boostedVocabularyTermsCount = vocabBundle.vocabulary.terms.count
+ DebugLogger.shared.info(
+ "FluidAudioProvider: Enabled vocabulary boosting with \(self.boostedVocabularyTermsCount) terms",
+ source: "FluidAudioProvider"
+ )
+ } else {
+ DebugLogger.shared.debug("FluidAudioProvider: No vocabulary boost terms found; using base ASR manager", source: "FluidAudioProvider")
+ }
+ } catch {
+ DebugLogger.shared.warning("FluidAudioProvider: Failed to configure vocabulary boosting: \(error)", source: "FluidAudioProvider")
+ }
+ }
+
+ // Boosting now applies in both streaming and final paths.
+ self.streamingAsrManager = manager
+ self.finalAsrManager = manager
self.isReady = true
+ progressHandler?(1.0)
DebugLogger.shared.info("FluidAudioProvider: Models ready", source: "FluidAudioProvider")
}
func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult {
- guard let manager = asrManager else {
+ try await self.transcribeFinal(samples)
+ }
+
+ func transcribeStreaming(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ guard let manager = self.streamingAsrManager else {
+ throw NSError(
+ domain: "FluidAudioProvider",
+ code: -1,
+ userInfo: [NSLocalizedDescriptionKey: "ASR manager not initialized"]
+ )
+ }
+
+ let result = try await manager.transcribe(samples, source: AudioSource.microphone)
+ return ASRTranscriptionResult(text: result.text, confidence: result.confidence)
+ }
+
+ func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {
+ guard let manager = self.finalAsrManager ?? self.streamingAsrManager else {
throw NSError(
domain: "FluidAudioProvider",
code: -1,
@@ -96,13 +148,16 @@ final class FluidAudioProvider: TranscriptionProvider {
}
self.isReady = false
- self.asrManager = nil
+ self.streamingAsrManager = nil
+ self.finalAsrManager = nil
+ self.isWordBoostingActive = false
+ self.boostedVocabularyTermsCount = 0
}
/// Provides direct access to the underlying AsrManager for advanced use cases
/// (e.g., MeetingTranscriptionService sharing)
var underlyingManager: AsrManager? {
- return self.asrManager
+ return self.streamingAsrManager
}
}
#else
@@ -112,7 +167,7 @@ final class FluidAudioProvider: TranscriptionProvider {
var isAvailable: Bool { false }
var isReady: Bool { false }
- init(modelOverride: SettingsStore.SpeechModel? = nil) {
+ init(modelOverride: SettingsStore.SpeechModel? = nil, configureWordBoosting: Bool = true) {
// Intel stub - parameter ignored
}
diff --git a/Sources/Fluid/Services/ParakeetVocabularyStore.swift b/Sources/Fluid/Services/ParakeetVocabularyStore.swift
new file mode 100644
index 00000000..7a2e486d
--- /dev/null
+++ b/Sources/Fluid/Services/ParakeetVocabularyStore.swift
@@ -0,0 +1,254 @@
+import Foundation
+#if arch(arm64)
+import FluidAudio
+#endif
+
+/// JSON-backed store for Parakeet vocabulary boosting terms.
+/// Persists to Application Support so it survives app updates and can be user-edited.
+final class ParakeetVocabularyStore {
+ static let shared = ParakeetVocabularyStore()
+
+ struct VocabularyConfig: Codable, Sendable {
+ struct Term: Codable, Hashable, Sendable {
+ let text: String
+ let weight: Float?
+ let aliases: [String]?
+ }
+
+ let alpha: Float?
+ let minCtcScore: Float?
+ let minSimilarity: Float?
+ let minCombinedConfidence: Float?
+ let minTermLength: Int?
+ let terms: [Term]
+ }
+
+ struct ResolvedConfig: Sendable {
+ let alpha: Float
+ let minCtcScore: Float
+ let minSimilarity: Float
+ let minCombinedConfidence: Float
+ let minTermLength: Int
+ let terms: [VocabularyConfig.Term]
+ }
+
+ enum StoreError: LocalizedError {
+ case invalidJSON(String)
+ case applicationSupportUnavailable
+
+ var errorDescription: String? {
+ switch self {
+ case .invalidJSON(let details):
+ return "Invalid vocabulary JSON: \(details)"
+ case .applicationSupportUnavailable:
+ return "Could not access Application Support directory."
+ }
+ }
+ }
+
+ private enum Defaults {
+ static let alpha: Float = 2.5
+ static let minCtcScore: Float = -2.0
+ static let minSimilarity: Float = 0.72
+ static let minCombinedConfidence: Float = 0.65
+ static let minTermLength: Int = 3
+ static let maxTerms: Int = 256
+ }
+
+ private let fileName = "parakeet_custom_vocabulary.json"
+ private let appSupportFolder = "FluidVoice"
+
+ private init() {}
+
+ func vocabularyFileURL() throws -> URL {
+ guard let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else {
+ throw StoreError.applicationSupportUnavailable
+ }
+ let directory = base.appendingPathComponent(self.appSupportFolder, isDirectory: true)
+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+ return directory.appendingPathComponent(self.fileName)
+ }
+
+ @discardableResult
+ func ensureVocabularyFileExists() throws -> URL {
+ let url = try self.vocabularyFileURL()
+ guard !FileManager.default.fileExists(atPath: url.path) else { return url }
+
+ if let bundled = Bundle.main.url(forResource: "parakeet_custom_vocabulary.default", withExtension: "json"),
+ let bundledText = try? String(contentsOf: bundled, encoding: .utf8)
+ {
+ try bundledText.write(to: url, atomically: true, encoding: .utf8)
+ } else {
+ try Self.defaultTemplateJSON().write(to: url, atomically: true, encoding: .utf8)
+ }
+ return url
+ }
+
+ func loadRawJSON() throws -> String {
+ let url = try self.ensureVocabularyFileExists()
+ return try String(contentsOf: url, encoding: .utf8)
+ }
+
+ func validateJSON(_ json: String) throws -> VocabularyConfig {
+ let data = Data(json.utf8)
+ do {
+ return try JSONDecoder().decode(VocabularyConfig.self, from: data)
+ } catch {
+ throw StoreError.invalidJSON(error.localizedDescription)
+ }
+ }
+
+ func saveRawJSON(_ json: String) throws {
+ _ = try self.validateJSON(json)
+ let url = try self.ensureVocabularyFileExists()
+ try json.write(to: url, atomically: true, encoding: .utf8)
+ NotificationCenter.default.post(name: .parakeetVocabularyDidChange, object: nil)
+ }
+
+ func hasAnyBoostTerms() -> Bool {
+ (try? self.loadResolvedConfig())?.terms.isEmpty == false
+ }
+
+ func loadResolvedConfig() throws -> ResolvedConfig {
+ let rawJSON = try self.loadRawJSON()
+ let parsed = (try? self.validateJSON(rawJSON)) ?? VocabularyConfig(
+ alpha: nil,
+ minCtcScore: nil,
+ minSimilarity: nil,
+ minCombinedConfidence: nil,
+ minTermLength: nil,
+ terms: []
+ )
+
+ let mergedTerms = self.mergeAndNormalizeTerms(jsonTerms: parsed.terms, dictionaryEntries: SettingsStore.shared.customDictionaryEntries)
+
+ return ResolvedConfig(
+ alpha: parsed.alpha ?? Defaults.alpha,
+ minCtcScore: parsed.minCtcScore ?? Defaults.minCtcScore,
+ minSimilarity: parsed.minSimilarity ?? Defaults.minSimilarity,
+ minCombinedConfidence: parsed.minCombinedConfidence ?? Defaults.minCombinedConfidence,
+ minTermLength: parsed.minTermLength ?? Defaults.minTermLength,
+ terms: mergedTerms
+ )
+ }
+
+ private func mergeAndNormalizeTerms(
+ jsonTerms: [VocabularyConfig.Term],
+ dictionaryEntries: [SettingsStore.CustomDictionaryEntry]
+ ) -> [VocabularyConfig.Term] {
+ var mergedByText: [String: VocabularyConfig.Term] = [:]
+
+ func normalizeAliases(_ aliases: [String]?, excluding text: String) -> [String]? {
+ let normalized = (aliases ?? [])
+ .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
+ .filter { !$0.isEmpty }
+ .filter { $0.caseInsensitiveCompare(text) != .orderedSame }
+ let deduped = Array(Set(normalized.map { $0.lowercased() })).sorted()
+ return deduped.isEmpty ? nil : deduped
+ }
+
+ func upsert(_ term: VocabularyConfig.Term) {
+ let normalizedText = term.text.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard !normalizedText.isEmpty else { return }
+ let key = normalizedText.lowercased()
+
+ if let existing = mergedByText[key] {
+ let combinedAliases = Array(Set((existing.aliases ?? []) + (term.aliases ?? []))).sorted()
+ let combinedWeight = max(existing.weight ?? 0, term.weight ?? 0)
+ mergedByText[key] = VocabularyConfig.Term(
+ text: existing.text,
+ weight: combinedWeight > 0 ? combinedWeight : nil,
+ aliases: combinedAliases.isEmpty ? nil : combinedAliases
+ )
+ return
+ }
+
+ mergedByText[key] = VocabularyConfig.Term(
+ text: normalizedText,
+ weight: term.weight,
+ aliases: normalizeAliases(term.aliases, excluding: normalizedText)
+ )
+ }
+
+ for term in jsonTerms {
+ upsert(term)
+ }
+
+ for entry in dictionaryEntries {
+ let replacement = entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard !replacement.isEmpty else { continue }
+ let aliases = entry.triggers
+ .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
+ .filter { !$0.isEmpty }
+ upsert(VocabularyConfig.Term(text: replacement, weight: 10.0, aliases: aliases))
+ }
+
+ return mergedByText.values.sorted { lhs, rhs in
+ lhs.text.localizedCaseInsensitiveCompare(rhs.text) == .orderedAscending
+ }
+ }
+
+ static func defaultTemplateJSON() -> String {
+ """
+ {
+ "alpha": 2.5,
+ "minCtcScore": -2.0,
+ "minSimilarity": 0.72,
+ "minCombinedConfidence": 0.65,
+ "minTermLength": 3,
+ "terms": [
+ {
+ "text": "FluidVoice",
+ "aliases": ["fluid voice", "fluid boys"],
+ "weight": 12.0
+ }
+ ]
+ }
+ """
+ }
+}
+
+extension Notification.Name {
+ static let parakeetVocabularyDidChange = Notification.Name("ParakeetVocabularyDidChange")
+}
+
+#if arch(arm64)
+extension ParakeetVocabularyStore {
+ /// Creates tokenized vocabulary + CTC models for FluidAudio vocabulary boosting.
+ func loadTokenizedVocabularyBundle(
+ maxTerms: Int = 256
+ ) async throws -> (vocabulary: CustomVocabularyContext, ctcModels: CtcModels)? {
+ let resolved = try self.loadResolvedConfig()
+ guard !resolved.terms.isEmpty else { return nil }
+
+ let cappedTerms = Array(resolved.terms.prefix(maxTerms))
+ let ctcModels = try await CtcModels.downloadAndLoad(variant: .ctc110m)
+ let ctcTokenizer = try await CtcTokenizer.load(from: CtcModels.defaultCacheDirectory(for: ctcModels.variant))
+
+ let tokenizedTerms: [CustomVocabularyTerm] = cappedTerms.compactMap { term in
+ let tokens = ctcTokenizer.encode(term.text)
+ guard !tokens.isEmpty else { return nil }
+ return CustomVocabularyTerm(
+ text: term.text,
+ weight: term.weight,
+ aliases: term.aliases,
+ tokenIds: nil,
+ ctcTokenIds: tokens
+ )
+ }
+
+ guard !tokenizedTerms.isEmpty else { return nil }
+
+ let vocabulary = CustomVocabularyContext(
+ terms: tokenizedTerms,
+ alpha: resolved.alpha,
+ minCtcScore: resolved.minCtcScore,
+ minSimilarity: resolved.minSimilarity,
+ minCombinedConfidence: resolved.minCombinedConfidence,
+ minTermLength: resolved.minTermLength
+ )
+
+ return (vocabulary, ctcModels)
+ }
+}
+#endif