Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

fix(dictionary): activate approved pronunciation evidence automatically

altic-dev167eb22112ec
diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swift
index 55824b9a..268543ba 100644
--- a/Sources/Fluid/Services/FluidAudioProvider.swift
+++ b/Sources/Fluid/Services/FluidAudioProvider.swift
@@ -102,6 +102,8 @@ final class FluidAudioProvider: TranscriptionProvider {
private var incrementalAcceptedSampleCount = 0
private var recordingGeneration = UUID()
private var incrementalPronunciationProfiles: [PronunciationDictionaryProfile] = []
+ private var automaticPronunciationProfiles: [PronunciationDictionaryProfile] = []
+ private var didLoadAutomaticPronunciationProfiles = false
private var usesIncrementalDictation: Bool {
self.effectiveExperimentalUnifiedFinalEnabled || self.effectivePronunciationMatchingEnabled
@@ -157,12 +159,53 @@ final class FluidAudioProvider: TranscriptionProvider {
?? SettingsStore.shared.experimentalParakeetUnifiedFinalEnabled
}
- private var effectivePronunciationMatchingEnabled: Bool {
+ private var explicitPronunciationMatchingEnabled: Bool {
self.meetingOptions?.pronunciationMatchingEnabled
?? self.enhancementOptions?.pronunciationMatchingEnabled
?? SettingsStore.shared.pronunciationMatchingEnabled
}
+ private var automaticPronunciationMatchingEnabled: Bool {
+ self.meetingOptions == nil && self.enhancementOptions == nil
+ && SettingsStore.shared.automaticDictionaryLearningEnabled
+ }
+
+ private var effectivePronunciationMatchingEnabled: Bool {
+ self.explicitPronunciationMatchingEnabled
+ || (self.automaticPronunciationMatchingEnabled && !self.automaticPronunciationProfiles.isEmpty)
+ }
+
+ /// Read the actor-owned store once after capture begins, never on the microphone startup path.
+ private func refreshAutomaticPronunciationProfiles() async throws {
+ guard !self.didLoadAutomaticPronunciationProfiles else { return }
+ let generation = self.recordingGeneration
+ guard self.automaticPronunciationMatchingEnabled, !self.effectiveCustomDictionaryEntries.isEmpty else {
+ self.automaticPronunciationProfiles = []
+ self.didLoadAutomaticPronunciationProfiles = true
+ return
+ }
+ let stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)
+ try self.requireCurrentRecording(generation)
+ self.automaticPronunciationProfiles = Self.matchingProfiles(
+ stored, entries: self.effectiveCustomDictionaryEntries, includeManual: false
+ )
+ self.didLoadAutomaticPronunciationProfiles = true
+ }
+
+ static func matchingProfiles(
+ _ stored: [PronunciationDictionaryProfile],
+ entries: [SettingsStore.CustomDictionaryEntry], includeManual: Bool
+ ) -> [PronunciationDictionaryProfile] {
+ let labels = Self.dictionaryLabels(from: entries)
+ return stored.filter { profile in
+ guard profile.isEligibleForMatching,
+ includeManual || profile.hasOriginalAudio,
+ let label = labels[profile.dictionaryEntryID]
+ else { return false }
+ return !profile.hasOriginalAudio || profile.label.caseInsensitiveCompare(label) == .orderedSame
+ }
+ }
+
private var effectiveCustomDictionaryEntries: [SettingsStore.CustomDictionaryEntry] {
if let meetingOptions = self.meetingOptions, !meetingOptions.customDictionaryRewritingEnabled {
return []
@@ -306,6 +349,8 @@ final class FluidAudioProvider: TranscriptionProvider {
self.latestStreamingPreviewSampleCount = 0
self.latestStreamingPreviewFinishedAt = nil
self.recordingGeneration = UUID()
+ self.automaticPronunciationProfiles = []
+ self.didLoadAutomaticPronunciationProfiles = false
self.resetIncrementalSession()
try Task.checkCancellation()
@@ -326,6 +371,8 @@ final class FluidAudioProvider: TranscriptionProvider {
self.latestStreamingPreviewSampleCount = 0
self.latestStreamingPreviewFinishedAt = nil
self.recordingGeneration = UUID()
+ self.automaticPronunciationProfiles = []
+ self.didLoadAutomaticPronunciationProfiles = false
self.resetIncrementalSession()
}
@@ -339,6 +386,7 @@ final class FluidAudioProvider: TranscriptionProvider {
}
let generation = self.recordingGeneration
+ try await self.refreshAutomaticPronunciationProfiles()
let startedAt = Date().timeIntervalSince1970
let result: ASRResult
if self.usesIncrementalDictation,
@@ -485,7 +533,7 @@ final class FluidAudioProvider: TranscriptionProvider {
userInfo: [NSLocalizedDescriptionKey: "ASR manager not initialized"]
)
}
- let shouldCapture = self.effectivePronunciationMatchingEnabled
+ let shouldCapture = self.explicitPronunciationMatchingEnabled
await manager.setPronunciationCustomizationEnabled(shouldCapture)
do {
let result = try await manager.transcribe(samples, source: AudioSource.microphone)
@@ -507,6 +555,7 @@ final class FluidAudioProvider: TranscriptionProvider {
func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {
let generation = self.recordingGeneration
+ try await self.refreshAutomaticPronunciationProfiles()
defer {
let resetStartedAt = ProcessInfo.processInfo.systemUptime
if self.recordingGeneration == generation { self.resetIncrementalSession() }
@@ -675,9 +724,17 @@ final class FluidAudioProvider: TranscriptionProvider {
private func pronunciationProfiles() async -> [PronunciationDictionaryProfile] {
guard self.effectivePronunciationMatchingEnabled else { return [] }
- let labels = Self.dictionaryLabels(from: self.effectiveCustomDictionaryEntries)
- let stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)
- return stored.filter { $0.isEligibleForMatching && labels[$0.dictionaryEntryID] != nil }
+ let stored: [PronunciationDictionaryProfile]
+ if self.explicitPronunciationMatchingEnabled {
+ stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)
+ } else {
+ stored = self.automaticPronunciationProfiles
+ }
+ return Self.matchingProfiles(
+ stored,
+ entries: self.effectiveCustomDictionaryEntries,
+ includeManual: self.explicitPronunciationMatchingEnabled
+ )
}
private static func pronunciationPrototype(_ profile: PronunciationDictionaryProfile) -> PronunciationEmbedding? {
@@ -703,6 +760,7 @@ final class FluidAudioProvider: TranscriptionProvider {
}
func transcribeWithWordTimings(_ samples: [Float]) async throws -> (result: ASRTranscriptionResult, words: [ASRWordTiming]) {
+ try await self.refreshAutomaticPronunciationProfiles()
guard let manager = self.finalAsrManager ?? self.streamingAsrManager else {
throw NSError(
domain: "FluidAudioProvider",
@@ -753,25 +811,7 @@ final class FluidAudioProvider: TranscriptionProvider {
return try (await self.transcribeIncrementalFinal(samples), nil, true)
}
let matchingEnabled = self.effectivePronunciationMatchingEnabled && samples.count <= 16_000 * 15
- let dictionaryLabels = Self.dictionaryLabels(
- from: self.effectiveCustomDictionaryEntries
- )
- let profiles: [PronunciationDictionaryProfile]
- if matchingEnabled {
- let storedProfiles = await self.pronunciationStore.profiles(
- modelKey: self.pronunciationModelKey
- )
- profiles = storedProfiles.compactMap { storedProfile in
- guard storedProfile.isEligibleForMatching,
- let currentLabel = dictionaryLabels[storedProfile.dictionaryEntryID]
- else { return nil }
- var profile = storedProfile
- profile.label = currentLabel
- return profile
- }
- } else {
- profiles = []
- }
+ let profiles = matchingEnabled ? await self.pronunciationProfiles() : []
// Rewritten text leaves timings on the old tokens; realigning would revert corrections.
let textMayBeCorrected = self.isWordBoostingActive || !profiles.isEmpty
await manager.setPronunciationCustomizationEnabled(!profiles.isEmpty)
@@ -1029,6 +1069,8 @@ final class FluidAudioProvider: TranscriptionProvider {
}
func clearCache() async throws {
+ self.automaticPronunciationProfiles = []
+ self.didLoadAutomaticPronunciationProfiles = false
self.recordingGeneration = UUID()
self.resetIncrementalSession()
let baseCacheDir = AsrModels.defaultCacheDirectory().deletingLastPathComponent()
diff --git a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
index ac1677e8..bc85541c 100644
--- a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
+++ b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
@@ -3546,6 +3546,14 @@ extension DictationE2ETests {
throw XCTSkip("Set the public speech fixture and Parakeet model paths")
}
let defaults = UserDefaults.standard
+ let previousAutomatic = SettingsStore.shared.automaticDictionaryLearningEnabled
+ let previousPreview = SettingsStore.shared.pronunciationMatchingEnabled
+ SettingsStore.shared.automaticDictionaryLearningEnabled = true
+ SettingsStore.shared.pronunciationMatchingEnabled = false
+ defer {
+ SettingsStore.shared.automaticDictionaryLearningEnabled = previousAutomatic
+ SettingsStore.shared.pronunciationMatchingEnabled = previousPreview
+ }
let previousEntries = defaults.object(forKey: self.customDictionaryEntriesKey)
defer {
if let previousEntries { defaults.set(previousEntries, forKey: self.customDictionaryEntriesKey) }
@@ -3555,7 +3563,8 @@ extension DictationE2ETests {
let folder = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
defer { try? FileManager.default.removeItem(at: folder) }
let store = PronunciationDictionaryStore(fileURL: folder.appendingPathComponent("profiles.json"))
- let samples = Array(try AudioConverter().resampleAudioFile(path: path).prefix(160000))
+ let fullSamples = try AudioConverter().resampleAudioFile(path: path)
+ let samples = Array(fullSamples.prefix(160000))
let models = try await AsrModels.load(from: URL(fileURLWithPath: modelPath), version: .v3)
let manager = AsrManager(config: ASRConfig(tdtConfig: TdtConfig(blankId: AsrModelVersion.v3.blankId), encoderHiddenSize: 1024))
try await manager.initialize(models: models)
@@ -3574,6 +3583,10 @@ extension DictationE2ETests {
let session = AutomaticDictionaryTrainingSession(candidate: candidate, asr: AppServices.shared.asr, audioLearning: worker)
let before = await store.allProfiles()
XCTAssertTrue(before.isEmpty, "Detection alone cannot activate acoustic learning")
+ let provider = FluidAudioProvider(modelOverride: .parakeetTDT, configureWordBoosting: false, pronunciationStore: store)
+ try await provider.prepare()
+ let beforeApproval = try await provider.transcribeFinal(samples)
+ XCTAssertFalse(beforeApproval.text.contains(intended))
session.addOnlyCorrection()
XCTAssertEqual(session.screen, .success)
let entry = try XCTUnwrap(SettingsStore.shared.customDictionaryEntries.first { $0.replacement == intended })
@@ -3587,6 +3600,37 @@ extension DictationE2ETests {
XCTAssertEqual(learned.first?.enrollments.first?.sourceRecordingID, recording.id)
XCTAssertEqual(learned.first?.enrollments.first?.observedText, candidate.heardText)
XCTAssertEqual(learned.first?.enrollments.count, 1)
+ provider.resetStreamingPreviewCache()
+ let corrected = try await provider.transcribeFinal(samples)
+ XCTAssertTrue(corrected.text.contains(intended), "Approved audio must work without the Advanced Preview switch")
+ provider.resetStreamingPreviewCache()
+ let aligned = try await provider.transcribeWithWordTimings(samples)
+ XCTAssertTrue(aligned.result.text.contains(intended))
+ provider.resetStreamingPreviewCache()
+ _ = try await provider.transcribeStreaming(Array(fullSamples.prefix(320000)))
+ _ = try await provider.transcribeStreaming(fullSamples)
+ let longResult = try await provider.transcribeFinal(fullSamples)
+ XCTAssertTrue(longResult.text.contains(intended), "Automatic evidence also participates in incremental long dictation")
+ SettingsStore.shared.automaticDictionaryLearningEnabled = false
+ provider.resetStreamingPreviewCache()
+ let disabled = try await provider.transcribeFinal(samples)
+ XCTAssertFalse(disabled.text.contains(intended), "Disabling both modes must preserve baseline ASR")
+
+ }
+
+ func testDictionaryAutomaticProfileSelectionPreservesMeaningAndManualOptIn() {
+ let entry = SettingsStore.CustomDictionaryEntry(triggers: ["fluid"], replacement: "Fluid Voice")
+ let capture = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3")
+ let manual = PronunciationDictionaryProfile(dictionaryEntryID: entry.id, label: entry.replacement, modelKey: capture.modelKey, hiddenSize: 2, enrollments: [capture, capture, capture])
+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([manual], entries: [entry], includeManual: false).isEmpty)
+ XCTAssertEqual(FluidAudioProvider.matchingProfiles([manual], entries: [entry], includeManual: true).count, 1)
+ var original = manual
+ original.enrollments[0].originalAudioID = UUID()
+ XCTAssertEqual(FluidAudioProvider.matchingProfiles([original], entries: [entry], includeManual: false).count, 1)
+ original.label = "Different Meaning"
+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([original], entries: [entry], includeManual: false).isEmpty)
+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([original], entries: [entry], includeManual: true).isEmpty)
+ XCTAssertTrue(FluidAudioProvider.matchingProfiles([manual], entries: [], includeManual: true).isEmpty)
}
#endif
}
diff --git a/docs/design/original-audio-learning-implementation.md b/docs/design/original-audio-learning-implementation.md
index 0b695e90..47de3ada 100644
--- a/docs/design/original-audio-learning-implementation.md
+++ b/docs/design/original-audio-learning-implementation.md
@@ -49,3 +49,14 @@ A separate real 60-second speech experiment enrolled the first occurrence and ex
The experiment also found a concrete false edit: Jensen's scored 0.884 against Jensen and could lose its possessive. Original-audio matching now preserves straight or curly apostrophe possessives unless an approved observed variant explicitly taught their removal. Existing possessive labels are not doubled. This is a narrow English possessive safeguard, not a general inflection model. Tests cover both preservation and explicit removal.
Raw experiment: scratch/original-audio-learning/evaluation/personal-trials.json and Sources/PersonalProof/main.swift. No thresholds were relaxed from this small sample.
+
+
+## Automatic activation follow-up
+
+The final consumer audit found that saved original-audio profiles were still gated by Advanced Preview. Approved original-audio profiles now participate automatically while automatic dictionary learning is enabled, even with Advanced Preview off. Manual-only acoustic profiles remain behind Advanced Preview. Pinned offline options and meeting policy do not inherit automatic matching.
+
+The provider takes a bounded profile snapshot from the store actor on the first transcription call of a recording, after microphone capture has started. Recording reset invalidates that snapshot so a newly approved word is available next time. Empty or ineligible profiles do not switch dictation to the pronunciation path. Short, word-timing and long incremental paths share profile selection; original evidence with a changed intended label is rejected before search, rather than relabelled.
+
+The real-audio approval regression now starts with an empty profile snapshot, approves and learns the correction, resets for the next recording, and verifies actual replacement in short, word-timed and 60-second incremental dictation with Advanced Preview off. It also checks that disabling automatic learning and Advanced Preview restores baseline ASR. This repeated-recording proof is functional validation, not held-out recognition accuracy.
+
+Validation: 76 dictionary/pronunciation and meeting-policy tests passed together, plus the separate production meeting-provider isolation test. Scoped provider lint is clean; full-source formatting and strict lint ran with existing repository violations preserved. The signed private FI build succeeded. Debug pronunciation timings are not treated as optimized latency measurements.