Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
fix(dictionary): activate approved pronunciation evidence automatically
altic-dev167eb22112ec
diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swiftindex 55824b9a..268543ba 100644--- a/Sources/Fluid/Services/FluidAudioProvider.swift+++ b/Sources/Fluid/Services/FluidAudioProvider.swift@@ -102,6 +102,8 @@ final class FluidAudioProvider: TranscriptionProvider {private var incrementalAcceptedSampleCount = 0private var recordingGeneration = UUID()private var incrementalPronunciationProfiles: [PronunciationDictionaryProfile] = []+ private var automaticPronunciationProfiles: [PronunciationDictionaryProfile] = []+ private var didLoadAutomaticPronunciationProfiles = falseprivate var usesIncrementalDictation: Bool {self.effectiveExperimentalUnifiedFinalEnabled || self.effectivePronunciationMatchingEnabled@@ -157,12 +159,53 @@ final class FluidAudioProvider: TranscriptionProvider {?? SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled}- private var effectivePronunciationMatchingEnabled: Bool {+ private var explicitPronunciationMatchingEnabled: Bool {self.meetingOptions?.pronunciationMatchingEnabled?? self.enhancementOptions?.pronunciationMatchingEnabled?? SettingsStore.shared.pronunciationMatchingEnabled}+ private var automaticPronunciationMatchingEnabled: Bool {+ self.meetingOptions == nil && self.enhancementOptions == nil+ && SettingsStore.shared.automaticDictionaryLearningEnabled+ }++ private var effectivePronunciationMatchingEnabled: Bool {+ self.explicitPronunciationMatchingEnabled+ || (self.automaticPronunciationMatchingEnabled && !self.automaticPronunciationProfiles.isEmpty)+ }++ /// Read the actor-owned store once after capture begins, never on the microphone startup path.+ private func refreshAutomaticPronunciationProfiles() async throws {+ guard !self.didLoadAutomaticPronunciationProfiles else { return }+ let generation = self.recordingGeneration+ guard self.automaticPronunciationMatchingEnabled, !self.effectiveCustomDictionaryEntries.isEmpty else {+ self.automaticPronunciationProfiles = []+ self.didLoadAutomaticPronunciationProfiles = true+ return+ }+ let stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)+ try self.requireCurrentRecording(generation)+ self.automaticPronunciationProfiles = Self.matchingProfiles(+ stored, entries: self.effectiveCustomDictionaryEntries, includeManual: false+ )+ self.didLoadAutomaticPronunciationProfiles = true+ }++ static func matchingProfiles(+ _ stored: [PronunciationDictionaryProfile],+ entries: [SettingsStore.CustomDictionaryEntry], includeManual: Bool+ ) -> [PronunciationDictionaryProfile] {+ let labels = Self.dictionaryLabels(from: entries)+ return stored.filter { profile in+ guard profile.isEligibleForMatching,+ includeManual || profile.hasOriginalAudio,+ let label = labels[profile.dictionaryEntryID]+ else { return false }+ return !profile.hasOriginalAudio || profile.label.caseInsensitiveCompare(label) == .orderedSame+ }+ }+private var effectiveCustomDictionaryEntries: [SettingsStore.CustomDictionaryEntry] {if let meetingOptions = self.meetingOptions, !meetingOptions.customDictionaryRewritingEnabled {return []@@ -306,6 +349,8 @@ final class FluidAudioProvider: TranscriptionProvider {self.latestStreamingPreviewSampleCount = 0self.latestStreamingPreviewFinishedAt = nilself.recordingGeneration = UUID()+ self.automaticPronunciationProfiles = []+ self.didLoadAutomaticPronunciationProfiles = falseself.resetIncrementalSession()try Task.checkCancellation()@@ -326,6 +371,8 @@ final class FluidAudioProvider: TranscriptionProvider {self.latestStreamingPreviewSampleCount = 0self.latestStreamingPreviewFinishedAt = nilself.recordingGeneration = UUID()+ self.automaticPronunciationProfiles = []+ self.didLoadAutomaticPronunciationProfiles = falseself.resetIncrementalSession()}@@ -339,6 +386,7 @@ final class FluidAudioProvider: TranscriptionProvider {}let generation = self.recordingGeneration+ try await self.refreshAutomaticPronunciationProfiles()let startedAt = Date().timeIntervalSince1970let result: ASRResultif self.usesIncrementalDictation,@@ -485,7 +533,7 @@ final class FluidAudioProvider: TranscriptionProvider {userInfo: [NSLocalizedDescriptionKey: "ASR manager not initialized"])}- let shouldCapture = self.effectivePronunciationMatchingEnabled+ let shouldCapture = self.explicitPronunciationMatchingEnabledawait manager.setPronunciationCustomizationEnabled(shouldCapture)do {let result = try await manager.transcribe(samples, source: AudioSource.microphone)@@ -507,6 +555,7 @@ final class FluidAudioProvider: TranscriptionProvider {func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {let generation = self.recordingGeneration+ try await self.refreshAutomaticPronunciationProfiles()defer {let resetStartedAt = ProcessInfo.processInfo.systemUptimeif self.recordingGeneration == generation { self.resetIncrementalSession() }@@ -675,9 +724,17 @@ final class FluidAudioProvider: TranscriptionProvider {private func pronunciationProfiles() async -> [PronunciationDictionaryProfile] {guard self.effectivePronunciationMatchingEnabled else { return [] }- let labels = Self.dictionaryLabels(from: self.effectiveCustomDictionaryEntries)- let stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)- return stored.filter { $0.isEligibleForMatching && labels[$0.dictionaryEntryID] != nil }+ let stored: [PronunciationDictionaryProfile]+ if self.explicitPronunciationMatchingEnabled {+ stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)+ } else {+ stored = self.automaticPronunciationProfiles+ }+ return Self.matchingProfiles(+ stored,+ entries: self.effectiveCustomDictionaryEntries,+ includeManual: self.explicitPronunciationMatchingEnabled+ )}private static func pronunciationPrototype(_ profile: PronunciationDictionaryProfile) -> PronunciationEmbedding? {@@ -703,6 +760,7 @@ final class FluidAudioProvider: TranscriptionProvider {}func transcribeWithWordTimings(_ samples: [Float]) async throws -> (result: ASRTranscriptionResult, words: [ASRWordTiming]) {+ try await self.refreshAutomaticPronunciationProfiles()guard let manager = self.finalAsrManager ?? self.streamingAsrManager else {throw NSError(domain: "FluidAudioProvider",@@ -753,25 +811,7 @@ final class FluidAudioProvider: TranscriptionProvider {return try (await self.transcribeIncrementalFinal(samples), nil, true)}let matchingEnabled = self.effectivePronunciationMatchingEnabled && samples.count <= 16_000 * 15- let dictionaryLabels = Self.dictionaryLabels(- from: self.effectiveCustomDictionaryEntries- )- let profiles: [PronunciationDictionaryProfile]- if matchingEnabled {- let storedProfiles = await self.pronunciationStore.profiles(- modelKey: self.pronunciationModelKey- )- profiles = storedProfiles.compactMap { storedProfile in- guard storedProfile.isEligibleForMatching,- let currentLabel = dictionaryLabels[storedProfile.dictionaryEntryID]- else { return nil }- var profile = storedProfile- profile.label = currentLabel- return profile- }- } else {- profiles = []- }+ let profiles = matchingEnabled ? await self.pronunciationProfiles() : []// Rewritten text leaves timings on the old tokens; realigning would revert corrections.let textMayBeCorrected = self.isWordBoostingActive || !profiles.isEmptyawait manager.setPronunciationCustomizationEnabled(!profiles.isEmpty)@@ -1029,6 +1069,8 @@ final class FluidAudioProvider: TranscriptionProvider {}func clearCache() async throws {+ self.automaticPronunciationProfiles = []+ self.didLoadAutomaticPronunciationProfiles = falseself.recordingGeneration = UUID()self.resetIncrementalSession()let baseCacheDir = AsrModels.defaultCacheDirectory().deletingLastPathComponent()diff --git a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swiftindex ac1677e8..bc85541c 100644--- a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift+++ b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift@@ -3546,6 +3546,14 @@ extension DictationE2ETests {throw XCTSkip("Set the public speech fixture and Parakeet model paths")}let defaults = UserDefaults.standard+ let previousAutomatic = SettingsStore.shared.automaticDictionaryLearningEnabled+ let previousPreview = SettingsStore.shared.pronunciationMatchingEnabled+ SettingsStore.shared.automaticDictionaryLearningEnabled = true+ SettingsStore.shared.pronunciationMatchingEnabled = false+ defer {+ SettingsStore.shared.automaticDictionaryLearningEnabled = previousAutomatic+ SettingsStore.shared.pronunciationMatchingEnabled = previousPreview+ }let previousEntries = defaults.object(forKey: self.customDictionaryEntriesKey)defer {if let previousEntries { defaults.set(previousEntries, forKey: self.customDictionaryEntriesKey) }@@ -3555,7 +3563,8 @@ extension DictationE2ETests {let folder = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)defer { try? FileManager.default.removeItem(at: folder) }let store = PronunciationDictionaryStore(fileURL: folder.appendingPathComponent("profiles.json"))- let samples = Array(try AudioConverter().resampleAudioFile(path: path).prefix(160000))+ let fullSamples = try AudioConverter().resampleAudioFile(path: path)+ let samples = Array(fullSamples.prefix(160000))let models = try await AsrModels.load(from: URL(fileURLWithPath: modelPath), version: .v3)let manager = AsrManager(config: ASRConfig(tdtConfig: TdtConfig(blankId: AsrModelVersion.v3.blankId), encoderHiddenSize: 1024))try await manager.initialize(models: models)@@ -3574,6 +3583,10 @@ extension DictationE2ETests {let session = AutomaticDictionaryTrainingSession(candidate: candidate, asr: AppServices.shared.asr, audioLearning: worker)let before = await store.allProfiles()XCTAssertTrue(before.isEmpty, "Detection alone cannot activate acoustic learning")+ let provider = FluidAudioProvider(modelOverride: .parakeetTDT, configureWordBoosting: false, pronunciationStore: store)+ try await provider.prepare()+ let beforeApproval = try await provider.transcribeFinal(samples)+ XCTAssertFalse(beforeApproval.text.contains(intended))session.addOnlyCorrection()XCTAssertEqual(session.screen, .success)let entry = try XCTUnwrap(SettingsStore.shared.customDictionaryEntries.first { $0.replacement == intended })@@ -3587,6 +3600,37 @@ extension DictationE2ETests {XCTAssertEqual(learned.first?.enrollments.first?.sourceRecordingID, recording.id)XCTAssertEqual(learned.first?.enrollments.first?.observedText, candidate.heardText)XCTAssertEqual(learned.first?.enrollments.count, 1)+ provider.resetStreamingPreviewCache()+ let corrected = try await provider.transcribeFinal(samples)+ XCTAssertTrue(corrected.text.contains(intended), "Approved audio must work without the Advanced Preview switch")+ provider.resetStreamingPreviewCache()+ let aligned = try await provider.transcribeWithWordTimings(samples)+ XCTAssertTrue(aligned.result.text.contains(intended))+ provider.resetStreamingPreviewCache()+ _ = try await provider.transcribeStreaming(Array(fullSamples.prefix(320000)))+ _ = try await provider.transcribeStreaming(fullSamples)+ let longResult = try await provider.transcribeFinal(fullSamples)+ XCTAssertTrue(longResult.text.contains(intended), "Automatic evidence also participates in incremental long dictation")+ SettingsStore.shared.automaticDictionaryLearningEnabled = false+ provider.resetStreamingPreviewCache()+ let disabled = try await provider.transcribeFinal(samples)+ XCTAssertFalse(disabled.text.contains(intended), "Disabling both modes must preserve baseline ASR")++ }++ func testDictionaryAutomaticProfileSelectionPreservesMeaningAndManualOptIn() {+ let entry = SettingsStore.CustomDictionaryEntry(triggers: ["fluid"], replacement: "Fluid Voice")+ let capture = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3")+ let manual = PronunciationDictionaryProfile(dictionaryEntryID: entry.id, label: entry.replacement, modelKey: capture.modelKey, hiddenSize: 2, enrollments: [capture, capture, capture])+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([manual], entries: [entry], includeManual: false).isEmpty)+ XCTAssertEqual(FluidAudioProvider.matchingProfiles([manual], entries: [entry], includeManual: true).count, 1)+ var original = manual+ original.enrollments[0].originalAudioID = UUID()+ XCTAssertEqual(FluidAudioProvider.matchingProfiles([original], entries: [entry], includeManual: false).count, 1)+ original.label = "Different Meaning"+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([original], entries: [entry], includeManual: false).isEmpty)+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([original], entries: [entry], includeManual: true).isEmpty)+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([manual], entries: [], includeManual: true).isEmpty)}#endif}diff --git a/docs/design/original-audio-learning-implementation.md b/docs/design/original-audio-learning-implementation.mdindex 0b695e90..47de3ada 100644--- a/docs/design/original-audio-learning-implementation.md+++ b/docs/design/original-audio-learning-implementation.md@@ -49,3 +49,14 @@ A separate real 60-second speech experiment enrolled the first occurrence and exThe experiment also found a concrete false edit: Jensen's scored 0.884 against Jensen and could lose its possessive. Original-audio matching now preserves straight or curly apostrophe possessives unless an approved observed variant explicitly taught their removal. Existing possessive labels are not doubled. This is a narrow English possessive safeguard, not a general inflection model. Tests cover both preservation and explicit removal.Raw experiment: scratch/original-audio-learning/evaluation/personal-trials.json and Sources/PersonalProof/main.swift. No thresholds were relaxed from this small sample.+++## Automatic activation follow-up++The final consumer audit found that saved original-audio profiles were still gated by Advanced Preview. Approved original-audio profiles now participate automatically while automatic dictionary learning is enabled, even with Advanced Preview off. Manual-only acoustic profiles remain behind Advanced Preview. Pinned offline options and meeting policy do not inherit automatic matching.++The provider takes a bounded profile snapshot from the store actor on the first transcription call of a recording, after microphone capture has started. Recording reset invalidates that snapshot so a newly approved word is available next time. Empty or ineligible profiles do not switch dictation to the pronunciation path. Short, word-timing and long incremental paths share profile selection; original evidence with a changed intended label is rejected before search, rather than relabelled.++The real-audio approval regression now starts with an empty profile snapshot, approves and learns the correction, resets for the next recording, and verifies actual replacement in short, word-timed and 60-second incremental dictation with Advanced Preview off. It also checks that disabling automatic learning and Advanced Preview restores baseline ASR. This repeated-recording proof is functional validation, not held-out recognition accuracy.++Validation: 76 dictionary/pronunciation and meeting-policy tests passed together, plus the separate production meeting-provider isolation test. Scoped provider lint is clean; full-source formatting and strict lint ran with existing repository violations preserved. The signed private FI build succeeded. Debug pronunciation timings are not treated as optimized latency measurements.