Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

fix(dictionary): prepare bounded pronunciation references in background

altic-dev8a7b99f0187c
diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swift
index 387b37b8..0fe41d3d 100644
--- a/Sources/Fluid/Services/FluidAudioProvider.swift
+++ b/Sources/Fluid/Services/FluidAudioProvider.swift
@@ -67,6 +67,37 @@ final class FluidAudioProvider: TranscriptionProvider {
private static let incrementalChunkingThresholdSamples = 240_000
private static let incrementalAppendBlockSamples = 240_000
+ /// Stable admission keeps a ninth word from evicting references used by the next match.
+ struct TemporalReferenceCache {
+ private var entries: [String: [DictionaryMatchFrames]] = [:]
+ private(set) var byteCount = 0
+ let byteLimit: Int
+
+ init(byteLimit: Int = 32 * 1024 * 1024) { self.byteLimit = byteLimit }
+ // swiftlint:disable:next discouraged_optional_collection
+ subscript(key: String) -> [DictionaryMatchFrames]? { self.entries[key] }
+ func canFit(_ bytes: Int) -> Bool { bytes >= 0 && bytes <= self.byteLimit - self.byteCount }
+
+ @discardableResult mutating func insert(_ frames: [DictionaryMatchFrames], for key: String) -> Bool {
+ if self.entries[key] != nil { return true }
+ guard !frames.isEmpty, frames.allSatisfy(\.isValid) else { return false }
+ let bytes = frames.reduce(0) { $0 + $1.values.count * MemoryLayout<Float>.size }
+ guard self.canFit(bytes) else { return false }
+ self.entries[key] = frames
+ self.byteCount += bytes
+ return true
+ }
+
+ mutating func retain(keys: Set<String>) {
+ self.entries = self.entries.filter { keys.contains($0.key) }
+ self.byteCount = self.entries.values.reduce(0) { total, frames in
+ total + frames.reduce(0) { $0 + $1.values.count * MemoryLayout<Float>.size }
+ }
+ }
+
+ mutating func removeAll() { self.entries.removeAll(); self.byteCount = 0 }
+ }
+
struct PronunciationTextReplacement {
let wordRange: ClosedRange<Int>
let label: String
@@ -123,7 +154,10 @@ final class FluidAudioProvider: TranscriptionProvider {
private var incrementalSharedFeatures = false
private var incrementalSharedEvidence: [Int: [(Range<Int>, DictionaryAcousticEvidence)]] = [:]
private var temporalModels: AsrModels?
- private var temporalReferenceCache: [String: [DictionaryMatchFrames]] = [:]
+ private var temporalReferenceCache = TemporalReferenceCache()
+ private var temporalWarmTask: Task<Void, Never>?
+ private var temporalWarmRequest: (profiles: [PronunciationDictionaryProfile], generation: String)?
+ private var pronunciationProfilesToWarm: [PronunciationDictionaryProfile] = []
/// Optional model override - if set, uses this model instead of the global setting.
/// Used for downloading specific models without changing the active selection.
@@ -187,14 +221,14 @@ final class FluidAudioProvider: TranscriptionProvider {
}
/// Read the actor-owned store once after capture begins, never on the microphone startup path.
- private func refreshAutomaticPronunciationProfiles() async throws {
+ private func refreshAutomaticPronunciationProfiles(warmReferences: Bool = true) async throws {
guard DictionaryMatcherExperiment.sharedFeaturesEnabled else {
self.automaticPronunciationProfiles = []
return
}
guard !self.didLoadAutomaticPronunciationProfiles else { return }
let generation = self.recordingGeneration
- guard self.meetingOptions == nil, self.enhancementOptions == nil, !self.effectiveCustomDictionaryEntries.isEmpty else {
+ guard self.meetingOptions == nil, self.enhancementOptions == nil || self.explicitPronunciationMatchingEnabled, !self.effectiveCustomDictionaryEntries.isEmpty else {
self.automaticPronunciationProfiles = []
self.didLoadAutomaticPronunciationProfiles = true
return
@@ -208,13 +242,13 @@ final class FluidAudioProvider: TranscriptionProvider {
includeOriginal: SettingsStore.shared.automaticDictionaryLearningEnabled
)
self.didLoadAutomaticPronunciationProfiles = true
- if DictionaryMatcherExperiment.sharedFeaturesEnabled, let models = self.temporalModels {
- // Capture has already started. Prepare reference-only features while the user speaks.
- for profile in self.automaticPronunciationProfiles.prefix(8) where !profile.hasOriginalAudio {
- _ = try? await self.temporalReferences(profile, key: DictionaryNegativeEvidenceResolver.profileKey(profile), models: models)
- try self.requireCurrentRecording(generation)
- }
- }
+ let activeProfiles = Self.matchingProfiles(stored, entries: self.effectiveCustomDictionaryEntries, includeManual: self.explicitPronunciationMatchingEnabled)
+ let activeKeys = Set(activeProfiles.map(DictionaryNegativeEvidenceResolver.profileKey))
+ self.temporalReferenceCache.retain(keys: activeKeys)
+ let inspectionIDs = Set(activeProfiles.flatMap(\.enrollments).compactMap(\.inspectionID))
+ self.edgeReferenceCache = self.edgeReferenceCache.filter { inspectionIDs.contains($0.key) }
+ self.pronunciationProfilesToWarm = activeProfiles
+ if warmReferences { self.scheduleTemporalWarmup(activeProfiles) }
}
static func matchingProfiles(
@@ -372,10 +406,7 @@ final class FluidAudioProvider: TranscriptionProvider {
if DictionaryMatcherExperiment.sharedFeaturesEnabled, self.meetingOptions == nil {
let stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)
let eligible = Self.matchingProfiles(stored, entries: self.effectiveCustomDictionaryEntries, includeManual: self.explicitPronunciationMatchingEnabled)
- for profile in eligible.prefix(8) where !profile.hasOriginalAudio {
- _ = try? await self.temporalReferences(profile, key: DictionaryNegativeEvidenceResolver.profileKey(profile), models: models)
- try Task.checkCancellation()
- }
+ self.scheduleTemporalWarmup(eligible)
}
self.streamingAsrManager = streamingManager
self.finalAsrManager = finalManager
@@ -598,7 +629,8 @@ final class FluidAudioProvider: TranscriptionProvider {
func transcribeFinal(_ samples: [Float]) async throws -> ASRTranscriptionResult {
let generation = self.recordingGeneration
- try await self.refreshAutomaticPronunciationProfiles()
+ try await self.refreshAutomaticPronunciationProfiles(warmReferences: false)
+ defer { self.scheduleTemporalWarmup(self.pronunciationProfilesToWarm) }
defer {
let resetStartedAt = ProcessInfo.processInfo.systemUptime
if self.recordingGeneration == generation { self.resetIncrementalSession() }
@@ -829,7 +861,7 @@ final class FluidAudioProvider: TranscriptionProvider {
private func startIncrementalSession(manager: AsrManager) async throws {
let generation = self.recordingGeneration
- let profiles = try await self.prepareEdgeProfiles(await self.pronunciationProfiles(), manager: manager)
+ let profiles = Self.preparedEdgeProfiles(await self.pronunciationProfiles(), cache: self.edgeReferenceCache)
try self.requireCurrentRecording(generation)
let references = DictionaryPronunciationReferences.make(profiles: profiles)
let useSharedFeatures = DictionaryMatcherExperiment.sharedFeaturesEnabled
@@ -894,7 +926,8 @@ final class FluidAudioProvider: TranscriptionProvider {
}
func transcribeWithWordTimings(_ samples: [Float]) async throws -> (result: ASRTranscriptionResult, words: [ASRWordTiming]) {
- try await self.refreshAutomaticPronunciationProfiles()
+ try await self.refreshAutomaticPronunciationProfiles(warmReferences: false)
+ defer { self.scheduleTemporalWarmup(self.pronunciationProfilesToWarm) }
guard let manager = self.finalAsrManager ?? self.streamingAsrManager else {
throw NSError(
domain: "FluidAudioProvider",
@@ -945,7 +978,7 @@ final class FluidAudioProvider: TranscriptionProvider {
return try (await self.transcribeIncrementalFinal(samples), nil, true)
}
let matchingEnabled = self.effectivePronunciationMatchingEnabled && samples.count <= 16_000 * 15
- let profiles = matchingEnabled ? try await self.prepareEdgeProfiles(await self.pronunciationProfiles(), manager: manager) : []
+ let profiles = matchingEnabled ? Self.preparedEdgeProfiles(await self.pronunciationProfiles(), cache: self.edgeReferenceCache) : []
// Rewritten text leaves timings on the old tokens; realigning would revert corrections.
let textMayBeCorrected = self.isWordBoostingActive || !profiles.isEmpty
await manager.setPronunciationCustomizationEnabled(!profiles.isEmpty)
@@ -1025,29 +1058,39 @@ final class FluidAudioProvider: TranscriptionProvider {
return try await manager.pronunciationEmbedding(audioSamples: samples, focalSampleRange: 0..<end)
}
- private func prepareEdgeProfiles(_ profiles: [PronunciationDictionaryProfile], manager: AsrManager) async throws -> [PronunciationDictionaryProfile] {
+ /// Final matching only reads prepared values. Incomplete legacy profiles wait for background warming.
+ static func preparedEdgeProfiles(_ profiles: [PronunciationDictionaryProfile], cache: [UUID: PronunciationEmbedding]) -> [PronunciationDictionaryProfile] {
guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- var prepared = profiles
- for index in prepared.indices where !prepared[index].hasOriginalAudio {
- guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- for sample in prepared[index].enrollments.indices {
- guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- let capture = prepared[index].enrollments[sample]
- guard capture.edgeEmbedding == nil, let id = capture.inspectionID else { continue }
- var embedding = self.edgeReferenceCache[id]
- if embedding == nil, let audio = try? await self.pronunciationStore.inspection(for: id),
- let range = DictionaryPronunciationExperiment.trimmedRange(Array(audio.samples.prefix(audio.recordedSampleCount)))
- {
- guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- embedding = try await self.encodeEdge(Array(audio.samples[range]), manager: manager)
- if self.edgeReferenceCache.count >= 100 { self.edgeReferenceCache.removeAll(keepingCapacity: true) }
- self.edgeReferenceCache[id] = embedding
- }
- prepared[index].enrollments[sample].edgeEmbedding = embedding?.values
- prepared[index].enrollments[sample].edgeFrameCount = embedding?.sourceFrameCount
+ return profiles.compactMap { profile in
+ if profile.hasOriginalAudio { return profile }
+ var prepared = profile
+ for index in prepared.enrollments.indices {
+ if prepared.enrollments[index].edgeEmbedding != nil { continue }
+ guard let id = prepared.enrollments[index].inspectionID, let embedding = cache[id] else { return nil }
+ prepared.enrollments[index].edgeEmbedding = embedding.values
+ prepared.enrollments[index].edgeFrameCount = embedding.sourceFrameCount
}
+ return prepared.edgeCalibration == nil ? nil : prepared
+ }
+ }
+
+ private func warmEdgeProfile(_ profile: PronunciationDictionaryProfile, manager: AsrManager, generation: String) async throws {
+ guard !profile.hasOriginalAudio else { return }
+ for capture in profile.enrollments where capture.edgeEmbedding == nil {
+ try Task.checkCancellation()
+ guard DictionaryMatcherExperiment.sharedFeaturesEnabled,
+ generation == DictionaryMatcherExperiment.generation,
+ let id = capture.inspectionID else { return }
+ if self.edgeReferenceCache[id] != nil { continue }
+ guard self.edgeReferenceCache.count < 512 else { return }
+ let audio = try await self.pronunciationStore.inspection(for: id)
+ guard let range = DictionaryPronunciationExperiment.trimmedRange(Array(audio.samples.prefix(audio.recordedSampleCount))) else { return }
+ let embedding = try await self.encodeEdge(Array(audio.samples[range]), manager: manager)
+ try Task.checkCancellation()
+ guard DictionaryMatcherExperiment.sharedFeaturesEnabled,
+ generation == DictionaryMatcherExperiment.generation else { return }
+ self.edgeReferenceCache[id] = embedding
}
- return prepared
}
// Carry both frame decisions and exact accepted evidence through this bounded refinement pass.
@@ -1105,14 +1148,7 @@ final class FluidAudioProvider: TranscriptionProvider {
// Old isolated-query negatives must not be compared with sentence-context queries.
let evidenceKey = referenceKey + ":shared-features-v1"
guard let query = Self.sharedPronunciationFrames(features, sampleRange: trimmed),
- let models = self.temporalModels else { continue }
- let references: [DictionaryMatchFrames]
- do {
- references = try await self.temporalReferences(profile, key: referenceKey, models: models)
- } catch {
- if error is CancellationError || Task.isCancelled { throw CancellationError() }
- continue
- }
+ let references = self.temporalReferenceCache[referenceKey] else { continue }
guard let decision = await DictionaryExperimentalMatcher.compareSharedFeatures(query: query, references: references, chunked: sharedChunk) else { continue }
guard decision.accepted, DictionaryMatcherExperiment.sharedFeaturesEnabled else { continue }
if compareNegatives {
@@ -1146,22 +1182,70 @@ final class FluidAudioProvider: TranscriptionProvider {
return frames.isValid ? frames : nil
}
+ #if DEBUG
+ /// Test synchronization observes actual preparation; it never starts inference itself.
+ var pronunciationReferencePreparationTask: Task<Void, Never>? { self.temporalWarmTask }
+ var pronunciationReferencesReady: Bool {
+ !self.pronunciationProfilesToWarm.isEmpty
+ && self.pronunciationProfilesToWarm.allSatisfy {
+ self.temporalReferenceCache[DictionaryNegativeEvidenceResolver.profileKey($0)] != nil
+ }
+ && Self.preparedEdgeProfiles(self.pronunciationProfilesToWarm, cache: self.edgeReferenceCache).count == self.pronunciationProfilesToWarm.count
+ }
+ #endif
+
+ /// Preparation runs independently; a cache miss never adds model inference to final matching.
+ private func scheduleTemporalWarmup(_ profiles: [PronunciationDictionaryProfile]) {
+ guard DictionaryMatcherExperiment.sharedFeaturesEnabled, !profiles.isEmpty,
+ let models = self.temporalModels else { return }
+ self.temporalWarmRequest = (profiles, DictionaryMatcherExperiment.generation)
+ guard self.temporalWarmTask == nil else { return }
+ self.temporalWarmTask = Task(priority: .utility) { [weak self] in
+ guard let self else { return }
+ defer { self.temporalWarmTask = nil }
+ while let request = self.temporalWarmRequest, !Task.isCancelled {
+ self.temporalWarmRequest = nil
+ for profile in request.profiles {
+ guard !Task.isCancelled, DictionaryMatcherExperiment.sharedFeaturesEnabled,
+ request.generation == DictionaryMatcherExperiment.generation else { break }
+ let key = DictionaryNegativeEvidenceResolver.profileKey(profile)
+ _ = try? await self.temporalReferences(profile, key: key, models: models)
+ if self.temporalReferenceCache[key] != nil, let manager = self.finalAsrManager ?? self.streamingAsrManager {
+ try? await self.warmEdgeProfile(profile, manager: manager, generation: request.generation)
+ }
+ }
+ }
+ }
+ }
+
private func temporalReferences(_ profile: PronunciationDictionaryProfile, key: String, models: AsrModels) async throws -> [DictionaryMatchFrames] {
guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
if let cached = self.temporalReferenceCache[key] { return cached }
guard profile.enrollments.count >= 3 else { return [] }
+ // Reserve a worst-case three-reference slot before encoding. Stable admission avoids
+ // repeatedly evicting and encoding words when the active dictionary exceeds the budget.
+ let reservation = 3 * 192 * 1024 * MemoryLayout<Float>.size
+ guard self.temporalReferenceCache.canFit(reservation) else { return [] }
+ let pronunciationGeneration = DictionaryMatcherExperiment.generation
var result: [DictionaryMatchFrames] = []
for capture in profile.enrollments.prefix(3) {
- guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- guard let id = capture.inspectionID,
- let audio = try? await self.pronunciationStore.inspection(for: id),
- let range = DictionaryPronunciationExperiment.trimmedRange(Array(audio.samples.prefix(audio.recordedSampleCount))) else { return [] }
- guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- try result.append(await DictionaryTemporalEncoder.encode(Array(audio.samples[range]), models: models))
+ try Task.checkCancellation()
+ guard DictionaryMatcherExperiment.sharedFeaturesEnabled,
+ pronunciationGeneration == DictionaryMatcherExperiment.generation else { return [] }
+ let samples: [Float]
+ if let id = capture.inspectionID {
+ let audio = try await self.pronunciationStore.inspection(for: id)
+ samples = Array(audio.samples.prefix(audio.recordedSampleCount))
+ } else if capture.originalAudioID != nil {
+ samples = try await self.pronunciationStore.originalAudioSamples(for: capture, entryID: profile.dictionaryEntryID)
+ } else { return [] }
+ guard let range = DictionaryPronunciationExperiment.trimmedRange(samples) else { return [] }
+ try result.append(await DictionaryTemporalEncoder.encode(Array(samples[range]), models: models))
}
- guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- if self.temporalReferenceCache.count >= 8 { self.temporalReferenceCache.removeAll(keepingCapacity: true) }
- self.temporalReferenceCache[key] = result
+ try Task.checkCancellation()
+ guard DictionaryMatcherExperiment.sharedFeaturesEnabled,
+ pronunciationGeneration == DictionaryMatcherExperiment.generation else { return [] }
+ guard self.temporalReferenceCache.insert(result, for: key) else { return [] }
return result
}
@@ -1276,7 +1360,6 @@ final class FluidAudioProvider: TranscriptionProvider {
guard !ambiguous else { continue }
accepted.append(candidate)
claimed.formUnion(candidate.wordIndices)
-
}
guard !accepted.isEmpty else { return result.text }
@@ -1420,6 +1503,10 @@ final class FluidAudioProvider: TranscriptionProvider {
self.streamingAsrManager = nil
self.finalAsrManager = nil
self.temporalModels = nil
+ self.temporalWarmTask?.cancel()
+ self.temporalWarmRequest = nil
+ self.pronunciationProfilesToWarm = []
+ self.edgeReferenceCache.removeAll()
self.temporalReferenceCache.removeAll()
self.isWordBoostingActive = false
self.boostedVocabularyTermsCount = 0
diff --git a/Sources/Fluid/Services/PronunciationDictionaryStore.swift b/Sources/Fluid/Services/PronunciationDictionaryStore.swift
index 64d2cd91..f9c53845 100644
--- a/Sources/Fluid/Services/PronunciationDictionaryStore.swift
+++ b/Sources/Fluid/Services/PronunciationDictionaryStore.swift
@@ -169,7 +169,7 @@ actor PronunciationDictionaryStore {
label: label,
modelKey: modelKey,
hiddenSize: first.values.count,
- enrollments: Array(combinedEnrollments.suffix(10)),
+ enrollments: Self.retainedEnrollments(combinedEnrollments),
automaticMatchingEnabled: automaticMatchingEnabled || existingIndex.map { profiles[$0].automaticMatchingEnabled == true } == true,
matchThreshold: existingIndex.flatMap { profiles[$0].matchThreshold }
)
@@ -190,6 +190,42 @@ actor PronunciationDictionaryStore {
}
}
+ /// Keep trained references when automatically learned examples reach the history limit.
+ static func retainedEnrollments(_ captures: [PronunciationEnrollmentCapture]) -> [PronunciationEnrollmentCapture] {
+ let protected = Set(captures.indices.filter { captures[$0].originalAudioID == nil }.suffix(3))
+ let remaining = captures.indices.filter { !protected.contains($0) }.suffix(10 - protected.count)
+ let selected = protected.union(remaining)
+ // Manual references first so the matcher can keep its existing calibration.
+ return selected.sorted { lhs, rhs in
+ let leftManual = captures[lhs].originalAudioID == nil
+ let rightManual = captures[rhs].originalAudioID == nil
+ return leftManual == rightManual ? lhs < rhs : leftManual
+ }.map { captures[$0] }
+ }
+
+ /// Validated focal audio for legacy and new automatically learned references.
+ func originalAudioSamples(for capture: PronunciationEnrollmentCapture, entryID: UUID) throws -> [Float] {
+ guard let id = capture.originalAudioID else { throw PronunciationDictionaryStoreError.inconsistentEnrollment }
+ let url = self.audioURL(id)
+ let size = try url.resourceValues(forKeys: [.fileSizeKey]).fileSize ?? Int.max
+ guard size <= 2_000_000 else { throw PronunciationDictionaryStoreError.inconsistentEnrollment }
+ let record = try JSONDecoder().decode(OriginalAudioRecord.self, from: Data(contentsOf: url))
+ guard record.version == 1, record.sampleRate == 16_000, record.encoding == "float32-little-endian",
+ record.entryID == entryID, record.evidenceID == id, record.modelKey == capture.modelKey,
+ record.pcmFloat32.count % MemoryLayout<Float>.size == 0,
+ record.pcmFloat32.count <= 238_080 * MemoryLayout<Float>.size
+ else {
+ throw PronunciationDictionaryStoreError.inconsistentEnrollment
+ }
+ let samples = record.pcmFloat32.withUnsafeBytes { bytes in
+ stride(from: 0, to: bytes.count, by: MemoryLayout<Float>.size).map { bytes.loadUnaligned(fromByteOffset: $0, as: Float.self) }
+ }
+ let range = record.focalSampleRange
+ guard !range.isEmpty, range.lowerBound >= 0, range.upperBound <= samples.count,
+ samples.allSatisfy(\.isFinite) else { throw PronunciationDictionaryStoreError.inconsistentEnrollment }
+ return Array(samples[range])
+ }
+
/// Reads only the selected example. Legacy profiles explicitly have no evidence.
func inspection(for id: UUID) throws -> DictionaryAudioInspection {
let url = self.inspectionURL(id)
diff --git a/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift b/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift
index b4a677f5..cc7e7f2d 100644
--- a/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift
+++ b/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift
@@ -9,7 +9,6 @@ import FluidAudio
@MainActor
final class CustomDictionaryManualEntryTests: XCTestCase {
-
// Pronunciation features are opt-in in the app; these tests exercise the enabled paths.
private var priorSharedFeaturesFlag: Any?
@@ -23,6 +22,91 @@ final class CustomDictionaryManualEntryTests: XCTestCase {
UserDefaults.standard.set(self.priorSharedFeaturesFlag, forKey: "DictionarySharedFeatureMatcherEnabled")
super.tearDown()
}
+
+ #if arch(arm64)
+ func testLegacyEdgePreparationOnlyUsesReadyCachedReferences() {
+ let ids = (0..<3).map { _ in UUID() }
+ let enrollments = ids.map { id in
+ PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 2, modelKey: "parakeet-v2", inspectionID: id)
+ }
+ let profile = PronunciationDictionaryProfile(dictionaryEntryID: UUID(), label: "Word", modelKey: "parakeet-v2", hiddenSize: 2, enrollments: enrollments)
+ XCTAssertTrue(FluidAudioProvider.preparedEdgeProfiles([profile], cache: [:]).isEmpty)
+ let embedding = PronunciationEmbedding(values: [1, 0], sourceFrameCount: 2)
+ let incomplete = Dictionary(uniqueKeysWithValues: ids.prefix(2).map { ($0, embedding) })
+ XCTAssertTrue(FluidAudioProvider.preparedEdgeProfiles([profile], cache: incomplete).isEmpty)
+ let cache = Dictionary(uniqueKeysWithValues: ids.map { ($0, embedding) })
+ let ready = FluidAudioProvider.preparedEdgeProfiles([profile], cache: cache)
+ XCTAssertEqual(ready.count, 1)
+ XCTAssertTrue(ready[0].enrollments.allSatisfy { $0.edgeEmbedding == [1, 0] })
+ XCTAssertEqual(FluidAudioProvider.preparedEdgeProfiles(ready, cache: [:]), ready, "New persisted edges need no cache or migration")
+ DictionaryMatcherExperiment.setEnabled(false)
+ XCTAssertTrue(FluidAudioProvider.preparedEdgeProfiles(ready, cache: cache).isEmpty)
+ }
+
+ func testTemporalCacheKeepsNinthWordAndRejectsOverflowWithoutEviction() {
+ let frames = [DictionaryMatchFrames(hiddenSize: 2, values: [1, 0])]
+ let entryBytes = 2 * MemoryLayout<Float>.size
+ var cache = FluidAudioProvider.TemporalReferenceCache(byteLimit: 9 * entryBytes)
+ for index in 0..<9 { XCTAssertTrue(cache.insert(frames, for: String(index))) }
+ XCTAssertEqual(cache.byteCount, 9 * entryBytes)
+ XCTAssertEqual(cache["0"], frames, "Adding a ninth word must preserve earlier references")
+ XCTAssertFalse(cache.insert(frames, for: "overflow"))
+ XCTAssertNil(cache["missing"], "An unprepared reference is a pure lookup miss")
+ XCTAssertEqual(cache["0"], frames, "A miss or full cache must not evict existing words")
+ XCTAssertTrue(cache.insert(frames, for: "0"))
+ XCTAssertEqual(cache.byteCount, 9 * entryBytes, "Duplicate warming must not count memory twice")
+ cache.retain(keys: ["0", "8"])
+ XCTAssertEqual(cache.byteCount, 2 * entryBytes)
+ XCTAssertNil(cache["1"])
+ XCTAssertTrue(cache.insert(frames, for: "new"), "Deleting old profiles releases their memory")
+ cache.removeAll()
+ XCTAssertEqual(cache.byteCount, 0)
+ XCTAssertNil(cache["0"])
+ }
+ #endif
+
+ func testOriginalLearningPreservesThreeManualReferences() {
+ let manual = (0..<3).map { index in
+ PronunciationEnrollmentCapture(values: [Float(index)], sourceFrameCount: 1, modelKey: "parakeet-v2", inspectionID: UUID())
+ }
+ let original = (0..<12).map { index in
+ PronunciationEnrollmentCapture(values: [Float(index)], sourceFrameCount: 1, modelKey: "parakeet-v2", originalAudioID: UUID())
+ }
+ let kept = PronunciationDictionaryStore.retainedEnrollments(manual + original)
+ XCTAssertEqual(kept.count, 10)
+ XCTAssertEqual(Array(kept.prefix(3)), manual)
+ XCTAssertEqual(Array(kept.suffix(7)), Array(original.suffix(7)))
+ XCTAssertEqual(PronunciationDictionaryStore.retainedEnrollments(original), Array(original.suffix(10)))
+ XCTAssertEqual(PronunciationDictionaryStore.retainedEnrollments(Array(repeating: manual[0], count: 12)).count, 10)
+ }
+
+ func testOriginalReferenceReadsOnlyFocalPCMAndChecksEntry() async throws {
+ let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: directory) }
+ let store = PronunciationDictionaryStore(fileURL: directory.appendingPathComponent("profiles.json"))
+ let entryID = UUID()
+ let evidence = DictionaryLearningAudioEvidence(
+ recordingID: UUID(),
+ modelKey: "parakeet-v2",
+ observedText: "word",
+ sourceWordRange: 0..<1,
+ sourceSampleRange: 0..<4,
+ focalSampleRange: 1..<3,
+ samples: [0.1, 0.2, 0.3, 0.4]
+ )
+ let capture = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 1, modelKey: evidence.modelKey)
+ let revision = await store.revision(for: entryID)
+ _ = try await store.learnOriginalAudio(entryID: entryID, label: "Word", evidenceID: UUID(), evidence: evidence, capture: capture, expectedRevision: revision)
+ let profiles = await store.allProfiles()
+ let saved = try XCTUnwrap(profiles.first?.enrollments.first)
+ let samples = try await store.originalAudioSamples(for: saved, entryID: entryID)
+ XCTAssertEqual(samples, [0.2, 0.3])
+ do {
+ _ = try await store.originalAudioSamples(for: saved, entryID: UUID())
+ XCTFail("A different entry must not reuse another word's recording")
+ } catch {}
+ }
+
func testVoiceTrainingKeepsPronunciationWithoutEverydayTextAliases() async {
let filtered = await VoiceTrainingAliasFilter.filter(["but", "now", "right now"]) { _ in .checked([]) }
let entries = CustomDictionaryTrainingMerge.mergedEntries(
@@ -340,7 +424,13 @@ extension CustomDictionaryManualEntryTests {
let before = SettingsStore.shared.customDictionaryEntries
let audio = self.inspectionFixture()
let capture = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 1, modelKey: "parakeet-v2", inspectionID: UUID())
- let profile = PronunciationDictionaryProfile(dictionaryEntryID: UUID(), label: "Manimekalai", modelKey: "parakeet-v2", hiddenSize: 2, enrollments: [capture, capture, capture])
+ let profile = PronunciationDictionaryProfile(
+ dictionaryEntryID: UUID(),
+ label: "Manimekalai",
+ modelKey: "parakeet-v2",
+ hiddenSize: 2,
+ enrollments: [capture, capture, capture]
+ )
let candidate = DictionaryMatchReport.Candidate(id: "sample", word: profile.label, sampleNumber: 1, enrollmentCount: 1, eligible: false, score: 0.6, start: 0, end: 0.08)
let report = DictionaryMatchReport(
createdAt: Date(),
diff --git a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
index 6a3e1b27..effbd7a0 100644
--- a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
+++ b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
@@ -1,3 +1,5 @@
+// Existing end-to-end fixture suite is kept together to share its setup and helpers.
+// swiftlint:disable file_length
import AppKit
@testable import FluidVoice_Debug
import Foundation
@@ -9,7 +11,6 @@ import XCTest
@MainActor
final class DictationE2ETests: XCTestCase {
-
// Pronunciation features are opt-in in the app; these tests exercise the enabled paths.
private var priorSharedFeaturesFlag: Any?
@@ -23,6 +24,7 @@ final class DictationE2ETests: XCTestCase {
UserDefaults.standard.set(self.priorSharedFeaturesFlag, forKey: "DictionarySharedFeatureMatcherEnabled")
super.tearDown()
}
+
private let enableTranscriptionSoundsKey = "EnableTranscriptionSounds"
private let transcriptionStartSoundKey = "TranscriptionStartSound"
private let dictationPromptProfilesKey = "DictationPromptProfiles"
@@ -1161,8 +1163,26 @@ extension DictationE2ETests {
XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: hits, profiles: [profile], labels: [id: "FluidVoice"]), "FluidVoice and FluidVoice")
// Deleted dictionary entries and low-confidence hits leave speech untouched.
XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: hits, profiles: [profile], labels: [:]), result.text)
- XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: [PronunciationWindowMatch(prototypeIndex: 0, score: 0.1, frameRange: 162..<170)], profiles: [profile], labels: [id: "FluidVoice"]), result.text)
- XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: [PronunciationWindowMatch(prototypeIndex: 99, score: 1, frameRange: 162..<170)], profiles: [profile], labels: [id: "FluidVoice"]), result.text)
+ XCTAssertEqual(
+ FluidAudioProvider
+ .applyPronunciationMatches(
+ result: result,
+ matches: [PronunciationWindowMatch(prototypeIndex: 0, score: 0.1, frameRange: 162..<170)],
+ profiles: [profile],
+ labels: [id: "FluidVoice"]
+ ),
+ result.text
+ )
+ XCTAssertEqual(
+ FluidAudioProvider
+ .applyPronunciationMatches(
+ result: result,
+ matches: [PronunciationWindowMatch(prototypeIndex: 99, score: 1, frameRange: 162..<170)],
+ profiles: [profile],
+ labels: [id: "FluidVoice"]
+ ),
+ result.text
+ )
}
func testPronunciationStreamingTwoMinuteRealAudio() async throws {
@@ -3340,7 +3360,14 @@ extension DictationE2ETests {
let revision = await store.revision(for: entryID)
var inserted: [Bool] = []
for id in [evidenceID, evidenceID, UUID()] {
- try inserted.append(await store.learnOriginalAudio(entryID: entryID, label: "FluidVoice", evidenceID: id, evidence: evidence, capture: capture, expectedRevision: revision))
+ try inserted.append(await store.learnOriginalAudio(
+ entryID: entryID,
+ label: "FluidVoice",
+ evidenceID: id,
+ evidence: evidence,
+ capture: capture,
+ expectedRevision: revision
+ ))
}
XCTAssertEqual(inserted, [true, false, false], "Rollback must know whether this call actually inserted evidence")
do {
@@ -3350,7 +3377,14 @@ extension DictationE2ETests {
let conflictingEntry = UUID()
let conflictingRevision = await store.revision(for: conflictingEntry)
do {
- try await store.learnOriginalAudio(entryID: conflictingEntry, label: "AnotherWord", evidenceID: evidenceID, evidence: evidence, capture: capture, expectedRevision: conflictingRevision)
+ try await store.learnOriginalAudio(
+ entryID: conflictingEntry,
+ label: "AnotherWord",
+ evidenceID: evidenceID,
+ evidence: evidence,
+ capture: capture,
+ expectedRevision: conflictingRevision
+ )
XCTFail("An event UUID cannot overwrite another word's audio")
} catch { XCTAssertEqual(error as? PronunciationDictionaryStoreError, .inconsistentEnrollment) }
let loaded = await PronunciationDictionaryStore(fileURL: url).allProfiles()
@@ -3379,9 +3413,24 @@ extension DictationE2ETests {
let store = PronunciationDictionaryStore(fileURL: directory.appendingPathComponent("profiles.json"))
let id = UUID()
let revision = await store.revision(for: id)
- let evidence = DictionaryLearningAudioEvidence(recordingID: UUID(), modelKey: "parakeet-v3", observedText: "test", sourceWordRange: 0..<1, sourceSampleRange: 0..<1, focalSampleRange: 0..<1, samples: [0])
+ let evidence = DictionaryLearningAudioEvidence(
+ recordingID: UUID(),
+ modelKey: "parakeet-v3",
+ observedText: "test",
+ sourceWordRange: 0..<1,
+ sourceSampleRange: 0..<1,
+ focalSampleRange: 0..<1,
+ samples: [0]
+ )
do {
- try await store.learnOriginalAudio(entryID: id, label: "Test", evidenceID: UUID(), evidence: evidence, capture: PronunciationEnrollmentCapture(values: [.nan], sourceFrameCount: 1, modelKey: "parakeet-v3"), expectedRevision: revision)
+ try await store.learnOriginalAudio(
+ entryID: id,
+ label: "Test",
+ evidenceID: UUID(),
+ evidence: evidence,
+ capture: PronunciationEnrollmentCapture(values: [.nan], sourceFrameCount: 1, modelKey: "parakeet-v3"),
+ expectedRevision: revision
+ )
XCTFail("Non-finite embeddings must never be activated")
} catch {
XCTAssertEqual(error as? PronunciationDictionaryStoreError, .inconsistentEnrollment)
@@ -3391,7 +3440,14 @@ extension DictationE2ETests {
XCTAssertFalse(FileManager.default.fileExists(atPath: directory.path))
try await store.replaceAllProfiles([])
do {
- try await store.learnOriginalAudio(entryID: id, label: "Test", evidenceID: UUID(), evidence: evidence, capture: PronunciationEnrollmentCapture(values: [1], sourceFrameCount: 1, modelKey: "parakeet-v3"), expectedRevision: revision)
+ try await store.learnOriginalAudio(
+ entryID: id,
+ label: "Test",
+ evidenceID: UUID(),
+ evidence: evidence,
+ capture: PronunciationEnrollmentCapture(values: [1], sourceFrameCount: 1, modelKey: "parakeet-v3"),
+ expectedRevision: revision
+ )
XCTFail("Import must invalidate pending original-audio learning")
} catch { XCTAssertEqual(error as? PronunciationDictionaryStoreError, .staleEvidence) }
}
@@ -3419,7 +3475,10 @@ extension DictationE2ETests {
XCTAssertFalse(incompatible.isEligibleForMatching)
let other = PronunciationDictionaryProfile(dictionaryEntryID: otherID, label: "Other", modelKey: "parakeet-v3", hiddenSize: 2, enrollments: [sample])
let competitor = PronunciationWindowMatch(prototypeIndex: 1, score: 0.88, frameRange: 0..<6)
- XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: [match, competitor], profiles: [profile, other], labels: [id: "FluidVoice", otherID: "Other"]), result.text)
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: result, matches: [match, competitor], profiles: [profile, other], labels: [id: "FluidVoice", otherID: "Other"]),
+ result.text
+ )
}
#endif
}
@@ -3449,7 +3508,15 @@ extension DictationE2ETests {
defer { try? FileManager.default.removeItem(at: directory) }
let store = PronunciationDictionaryStore(fileURL: directory.appendingPathComponent("profiles.json"))
let entry = SettingsStore.CustomDictionaryEntry(triggers: ["flued"], replacement: "Fluid")
- let evidence = DictionaryLearningAudioEvidence(recordingID: UUID(), modelKey: "parakeet-v3", observedText: "flued", sourceWordRange: 0..<1, sourceSampleRange: 0..<1280, focalSampleRange: 0..<1280, samples: Array(repeating: 0, count: 1280))
+ let evidence = DictionaryLearningAudioEvidence(
+ recordingID: UUID(),
+ modelKey: "parakeet-v3",
+ observedText: "flued",
+ sourceWordRange: 0..<1,
+ sourceSampleRange: 0..<1280,
+ focalSampleRange: 0..<1280,
+ samples: Array(repeating: 0, count: 1280)
+ )
let started = self.expectation(description: "Background extraction started")
let cancelled = self.expectation(description: "Dictation cancelled background work")
let extractor = DictionaryLearningTestExtractor(started: started, cancelled: cancelled)
@@ -3482,14 +3549,25 @@ extension DictationE2ETests {
defer { try? FileManager.default.removeItem(at: directory) }
let store = PronunciationDictionaryStore(fileURL: directory.appendingPathComponent("profiles.json"))
let entry = SettingsStore.CustomDictionaryEntry(triggers: ["flued"], replacement: "Fluid")
- let evidence = DictionaryLearningAudioEvidence(recordingID: UUID(), modelKey: "parakeet-v3", observedText: "flued", sourceWordRange: 0..<1, sourceSampleRange: 0..<1, focalSampleRange: 0..<1, samples: [0])
- let service = DictionaryAudioLearningService(store: store,
- lifetime: 0.05,
- extract: { _ in
- XCTFail("Busy dictation must not run the background encoder")
- throw CancellationError()
- }, canProcess: { false },
- isCurrent: { _ in true })
+ let evidence = DictionaryLearningAudioEvidence(
+ recordingID: UUID(),
+ modelKey: "parakeet-v3",
+ observedText: "flued",
+ sourceWordRange: 0..<1,
+ sourceSampleRange: 0..<1,
+ focalSampleRange: 0..<1,
+ samples: [0]
+ )
+ let service = DictionaryAudioLearningService(
+ store: store,
+ lifetime: 0.05,
+ extract: { _ in
+ XCTFail("Busy dictation must not run the background encoder")
+ throw CancellationError()
+ },
+ canProcess: { false },
+ isCurrent: { _ in true }
+ )
for _ in 0..<6 {
service.learn(entry: entry, evidenceID: UUID(), evidence: evidence)
}
@@ -3520,7 +3598,12 @@ extension DictationE2ETests {
let alignment = DictionaryLearningAlignment(modelKey: "parakeet-v3", words: words.map { ASRWordTiming(text: $0.text, start: $0.startTime, end: $0.endTime) })
let recording = try XCTUnwrap(DictionaryLearningRecording(alignment: alignment, samples: samples))
let delivered = words.map(\.text).joined(separator: " ")
- let evidence = try DictionaryLearningAlignmentResolver.resolve(recording: recording, deliveredTextBeforeEdit: delivered, selectedUTF16Range: (delivered as NSString).range(of: selected.text), observedText: selected.text)
+ let evidence = try DictionaryLearningAlignmentResolver.resolve(
+ recording: recording,
+ deliveredTextBeforeEdit: delivered,
+ selectedUTF16Range: (delivered as NSString).range(of: selected.text),
+ observedText: selected.text
+ )
let entry = SettingsStore.CustomDictionaryEntry(triggers: [selected.text], replacement: "LearnedTarget")
let store = PronunciationDictionaryStore(fileURL: directory.appendingPathComponent("profiles.json"))
let service = DictionaryAudioLearningService(store: store, canProcess: { true }, isCurrent: { $0 == entry })
@@ -3583,7 +3666,12 @@ extension DictationE2ETests {
TokenTiming(token: "▁" + parts[0], tokenId: 1, startTime: 0, endTime: 0.24, confidence: 1),
TokenTiming(token: "▁" + parts[1], tokenId: 2, startTime: 0.24, endTime: 0.48, confidence: 1),
])
- let acoustic = FluidAudioProvider.applyPronunciationMatches(result: result, matches: [PronunciationWindowMatch(prototypeIndex: 0, score: score, frameRange: 0..<6)], profiles: [profile], labels: [id: "FluidVoice"])
+ let acoustic = FluidAudioProvider.applyPronunciationMatches(
+ result: result,
+ matches: [PronunciationWindowMatch(prototypeIndex: 0, score: score, frameRange: 0..<6)],
+ profiles: [profile],
+ labels: [id: "FluidVoice"]
+ )
XCTAssertEqual(ASRService.applyCustomDictionary(acoustic), expected)
if text == "fluid boys" { XCTAssertEqual(acoustic, text, "Text fallback must recover an acoustic miss") }
if text == "fluent voice" { XCTAssertEqual(ASRService.applyCustomDictionary(text), text, "Audio must recover a text-rule miss") }
@@ -3593,12 +3681,24 @@ extension DictationE2ETests {
func testDictionaryLearningReconciliationCostAtScale() {
let wordCount = 1200
- let result = ASRResult(text: Array(repeating: "spoken", count: wordCount).joined(separator: " "), confidence: 1, duration: 600, processingTime: 0, tokenTimings: (0..<wordCount).map { index in
- TokenTiming(token: "▁spoken", tokenId: 1, startTime: Double(index) * 0.48, endTime: Double(index + 1) * 0.48, confidence: 1)
- })
+ let result = ASRResult(
+ text: Array(repeating: "spoken", count: wordCount).joined(separator: " "),
+ confidence: 1,
+ duration: 600,
+ processingTime: 0,
+ tokenTimings: (0..<wordCount).map { index in
+ TokenTiming(token: "▁spoken", tokenId: 1, startTime: Double(index) * 0.48, endTime: Double(index + 1) * 0.48, confidence: 1)
+ }
+ )
for count in [1, 10, 1000] {
let profiles = (0..<count).map { index in
- PronunciationDictionaryProfile(dictionaryEntryID: UUID(), label: "Target\(index)", modelKey: "parakeet-v3", hiddenSize: 2, enrollments: [PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3", originalAudioID: UUID(), observedText: "spoken")])
+ PronunciationDictionaryProfile(
+ dictionaryEntryID: UUID(),
+ label: "Target\(index)",
+ modelKey: "parakeet-v3",
+ hiddenSize: 2,
+ enrollments: [PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3", originalAudioID: UUID(), observedText: "spoken")]
+ )
}
let labels = Dictionary(uniqueKeysWithValues: profiles.map { ($0.dictionaryEntryID, $0.label) })
let matches = (0..<count).map { PronunciationWindowMatch(prototypeIndex: $0, score: 0.95, frameRange: ($0 * 6)..<(($0 + 1) * 6)) }
@@ -3666,7 +3766,12 @@ extension DictationE2ETests {
])
let capture = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3", originalAudioID: UUID(), observedText: "fluid")
let profile = PronunciationDictionaryProfile(dictionaryEntryID: entry.id, label: entry.replacement, modelKey: capture.modelKey, hiddenSize: 2, enrollments: [capture])
- let audioOnly = FluidAudioProvider.applyPronunciationMatches(result: result, matches: [PronunciationWindowMatch(prototypeIndex: 0, score: 0.95, frameRange: 0..<6)], profiles: [profile], labels: [entry.id: entry.replacement])
+ let audioOnly = FluidAudioProvider.applyPronunciationMatches(
+ result: result,
+ matches: [PronunciationWindowMatch(prototypeIndex: 0, score: 0.95, frameRange: 0..<6)],
+ profiles: [profile],
+ labels: [entry.id: entry.replacement]
+ )
let textOnly = ASRService.applyCustomDictionary(result.text)
let combined = ASRService.applyCustomDictionary(audioOnly)
XCTAssertEqual(audioOnly, "Fluid Voice plus fluid")
@@ -3688,12 +3793,20 @@ extension DictationE2ETests {
TokenTiming(token: "▁" + spoken, tokenId: 1, startTime: 0, endTime: 0.48, confidence: 1),
TokenTiming(token: "▁project", tokenId: 2, startTime: 0.6, endTime: 0.9, confidence: 1),
])
- XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: [.init(prototypeIndex: 0, score: 0.8835, frameRange: 0..<6)], profiles: [profile], labels: [id: "Jensen"]), result.text)
+ XCTAssertEqual(
+ FluidAudioProvider
+ .applyPronunciationMatches(result: result, matches: [.init(prototypeIndex: 0, score: 0.8835, frameRange: 0..<6)], profiles: [profile], labels: [id: "Jensen"]),
+ result.text
+ )
}
XCTAssertEqual(DictionaryPronunciationDecision.labelPreservingPossessive("Jensen's", heardText: "Jensen's", profile: profile), "Jensen's")
XCTAssertEqual(DictionaryPronunciationDecision.labelPreservingPossessive("Jensen", heardText: "Jensen", profile: profile), "Jensen")
profile.enrollments[0].observedText = "Jensen's"
- XCTAssertEqual(DictionaryPronunciationDecision.labelPreservingPossessive("Jensen", heardText: "Jensen's", profile: profile), "Jensen", "An explicitly taught removal keeps its intended meaning")
+ XCTAssertEqual(
+ DictionaryPronunciationDecision.labelPreservingPossessive("Jensen", heardText: "Jensen's", profile: profile),
+ "Jensen",
+ "An explicitly taught removal keeps its intended meaning"
+ )
}
func testApprovedCorrectionLearnsOriginalRealAudioThroughExistingSession() async throws {
@@ -3731,9 +3844,22 @@ extension DictationE2ETests {
let oldRange = (delivered as NSString).range(of: word.text)
let intended = "PersonalName"
let edited = (delivered as NSString).replacingCharacters(in: oldRange, with: intended)
- var candidate = try XCTUnwrap(AutomaticDictionaryCorrectionDetector.candidate(before: delivered, after: edited, insertedRange: NSRange(location: 0, length: (delivered as NSString).length), allowsInsertionAtEnd: true))
- let recording = try XCTUnwrap(DictionaryLearningRecording(alignment: .init(modelKey: "parakeet-v3", words: words.map { .init(text: $0.text, start: $0.startTime, end: $0.endTime) }), samples: samples))
- candidate.audioEvidence = try DictionaryLearningAlignmentResolver.resolve(recording: recording, deliveredTextBeforeEdit: delivered, selectedUTF16Range: XCTUnwrap(candidate.sourceUTF16Range), observedText: candidate.heardText)
+ var candidate = try XCTUnwrap(AutomaticDictionaryCorrectionDetector.candidate(
+ before: delivered,
+ after: edited,
+ insertedRange: NSRange(location: 0, length: (delivered as NSString).length),
+ allowsInsertionAtEnd: true
+ ))
+ let recording = try XCTUnwrap(DictionaryLearningRecording(
+ alignment: .init(modelKey: "parakeet-v3", words: words.map { .init(text: $0.text, start: $0.startTime, end: $0.endTime) }),
+ samples: samples
+ ))
+ candidate.audioEvidence = try DictionaryLearningAlignmentResolver.resolve(
+ recording: recording,
+ deliveredTextBeforeEdit: delivered,
+ selectedUTF16Range: XCTUnwrap(candidate.sourceUTF16Range),
+ observedText: candidate.heardText
+ )
let worker = DictionaryAudioLearningService(store: store, canProcess: { true })
let session = AutomaticDictionaryTrainingSession(candidate: candidate, asr: AppServices.shared.asr, audioLearning: worker)
let before = await store.allProfiles()
@@ -3777,7 +3903,13 @@ extension DictationE2ETests {
func testWizardPronunciationOptInRequiresThreeSamplesAndPreservesLegacyPolicy() throws {
let entry = SettingsStore.CustomDictionaryEntry(triggers: ["fluid"], replacement: "Fluid Voice")
let capture = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3")
- var profile = PronunciationDictionaryProfile(dictionaryEntryID: entry.id, label: entry.replacement, modelKey: capture.modelKey, hiddenSize: 2, enrollments: [capture, capture])
+ var profile = PronunciationDictionaryProfile(
+ dictionaryEntryID: entry.id,
+ label: entry.replacement,
+ modelKey: capture.modelKey,
+ hiddenSize: 2,
+ enrollments: [capture, capture]
+ )
profile.automaticMatchingEnabled = true
XCTAssertTrue(FluidAudioProvider.matchingProfiles([profile], entries: [entry], includeManual: false, includeOriginal: false).isEmpty)
profile.enrollments.append(capture)
@@ -3792,7 +3924,13 @@ extension DictationE2ETests {
func testDictionaryAutomaticProfileSelectionPreservesMeaningAndManualOptIn() {
let entry = SettingsStore.CustomDictionaryEntry(triggers: ["fluid"], replacement: "Fluid Voice")
let capture = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3")
- let manual = PronunciationDictionaryProfile(dictionaryEntryID: entry.id, label: entry.replacement, modelKey: capture.modelKey, hiddenSize: 2, enrollments: [capture, capture, capture])
+ let manual = PronunciationDictionaryProfile(
+ dictionaryEntryID: entry.id,
+ label: entry.replacement,
+ modelKey: capture.modelKey,
+ hiddenSize: 2,
+ enrollments: [capture, capture, capture]
+ )
XCTAssertTrue(FluidAudioProvider.matchingProfiles([manual], entries: [entry], includeManual: false).isEmpty)
XCTAssertEqual(FluidAudioProvider.matchingProfiles([manual], entries: [entry], includeManual: true).count, 1)
var original = manual
@@ -3865,7 +4003,13 @@ extension DictationE2ETests {
let id = UUID(), otherID = UUID()
let sample = PronunciationEnrollmentCapture(values: [1, 0], sourceFrameCount: 6, modelKey: "parakeet-v3")
var profile = PronunciationDictionaryProfile(dictionaryEntryID: id, label: "Target", modelKey: "parakeet-v3", hiddenSize: 2, enrollments: [sample, sample, sample])
- let result = ASRResult(text: "misheard", confidence: 0.8, duration: 0.48, processingTime: 0.1, tokenTimings: [TokenTiming(token: "misheard", tokenId: 1, startTime: 0, endTime: 0.48, confidence: 0.8)])
+ let result = ASRResult(
+ text: "misheard",
+ confidence: 0.8,
+ duration: 0.48,
+ processingTime: 0.1,
+ tokenTimings: [TokenTiming(token: "misheard", tokenId: 1, startTime: 0, endTime: 0.48, confidence: 0.8)]
+ )
let match = PronunciationWindowMatch(prototypeIndex: 0, score: 0.49, frameRange: 0..<6)
XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: [match], profiles: [profile], labels: [id: "Target"]), "misheard")
profile.matchThreshold = 0.45
@@ -3875,7 +4019,10 @@ extension DictationE2ETests {
let other = PronunciationDictionaryProfile(dictionaryEntryID: otherID, label: "Other", modelKey: "parakeet-v3", hiddenSize: 2, enrollments: [sample, sample, sample])
profile.matchThreshold = 0.45
let otherMatch = PronunciationWindowMatch(prototypeIndex: 1, score: 0.65, frameRange: 0..<6)
- XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: [otherMatch], profiles: [profile, other], labels: [id: "Target", otherID: "Other"]), "misheard")
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: result, matches: [otherMatch], profiles: [profile, other], labels: [id: "Target", otherID: "Other"]),
+ "misheard"
+ )
}
}
#endif
@@ -3899,14 +4046,27 @@ extension DictationE2ETests {
XCTAssertEqual(matches[0].first?.score, 1)
XCTAssertTrue(matches[1].isEmpty && matches[2].isEmpty)
let averaged = try XCTUnwrap(PronunciationEmbeddingMatcher.prototype(from: references.map(\.embedding)))
- XCTAssertTrue(PronunciationEmbeddingMatcher.allMatches(prototypes: [averaged], in: features, windowFrameCounts: [[6]])[0].isEmpty, "The old averaged center misses this valid individual example")
- let result = ASRResult(text: "misheard", confidence: 0.8, duration: 0.48, processingTime: 0.1, tokenTimings: [TokenTiming(token: "misheard", tokenId: 1, startTime: 0, endTime: 0.48, confidence: 0.8)])
+ XCTAssertTrue(
+ PronunciationEmbeddingMatcher.allMatches(prototypes: [averaged], in: features, windowFrameCounts: [[6]])[0].isEmpty,
+ "The old averaged center misses this valid individual example"
+ )
+ let result = ASRResult(
+ text: "misheard",
+ confidence: 0.8,
+ duration: 0.48,
+ processingTime: 0.1,
+ tokenTimings: [TokenTiming(token: "misheard", tokenId: 1, startTime: 0, endTime: 0.48, confidence: 0.8)]
+ )
let hits = matches.enumerated().flatMap { index, hits in
hits.map { PronunciationWindowMatch(prototypeIndex: index, score: $0.score, frameRange: $0.frameRange) }
}
XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: hits, profiles: references.map(\.profile), labels: [id: "Target"]), "Target")
let sameWord = hits + [PronunciationWindowMatch(prototypeIndex: 1, score: 0.99, frameRange: 0..<6)]
- XCTAssertEqual(FluidAudioProvider.applyPronunciationMatches(result: result, matches: sameWord, profiles: references.map(\.profile), labels: [id: "Target"]), "Target", "A second recording of the same word is not a competing word")
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: result, matches: sameWord, profiles: references.map(\.profile), labels: [id: "Target"]),
+ "Target",
+ "A second recording of the same word is not a competing word"
+ )
let otherID = UUID()
let competitor = PronunciationDictionaryProfile(dictionaryEntryID: otherID, label: "Other", modelKey: profile.modelKey, hiddenSize: 2, enrollments: samples)
let competingHits = hits + [PronunciationWindowMatch(prototypeIndex: 3, score: 0.99, frameRange: 0..<6)]
@@ -3928,7 +4088,7 @@ extension DictationE2ETests {
XCTAssertEqual(FluidAudioProvider.sharedPronunciationFrames(features, sampleRange: 1281..<2561)?.values, [0, 1, 1, 1])
XCTAssertNil(FluidAudioProvider.sharedPronunciationFrames(features, sampleRange: -1..<100))
XCTAssertNil(FluidAudioProvider.sharedPronunciationFrames(features, sampleRange: 0..<0))
- XCTAssertNil(FluidAudioProvider.sharedPronunciationFrames(features, sampleRange: 9000..<10000))
+ XCTAssertNil(FluidAudioProvider.sharedPronunciationFrames(features, sampleRange: 9000..<10_000))
}
func testSharedFeatureDictionaryCorpusReplay() async throws {
@@ -3939,10 +4099,21 @@ extension DictationE2ETests {
struct Job: Decodable { let id: String; let word: String; let path: String; let positive: Bool }
let root = URL(fileURLWithPath: path)
let jobs = try JSONDecoder().decode([Job].self, from: Data(contentsOf: root.appendingPathComponent("jobs.json")))
- let keys = ["DictionarySharedFeatureMatcherEnabled", "DictionaryTemporalMatcherEnabled", "DictionaryNegativeLearningEnabled", "DictionaryNegativeComparisonEnabled", "DictionaryPronunciationDebugCapture", "DictionaryEdgeMatchingEnabled"]
+ let keys = [
+ "DictionarySharedFeatureMatcherEnabled",
+ "DictionaryTemporalMatcherEnabled",
+ "DictionaryNegativeLearningEnabled",
+ "DictionaryNegativeComparisonEnabled",
+ "DictionaryPronunciationDebugCapture",
+ "DictionaryEdgeMatchingEnabled",
+ ]
let previous = keys.map { UserDefaults.standard.object(forKey: $0) }
- defer { for (key, value) in zip(keys, previous) { UserDefaults.standard.set(value, forKey: key) } }
- for key in keys { UserDefaults.standard.set(false, forKey: key) }
+ defer { for (key, value) in zip(keys, previous) {
+ UserDefaults.standard.set(value, forKey: key)
+ } }
+ for key in keys {
+ UserDefaults.standard.set(false, forKey: key)
+ }
UserDefaults.standard.set(true, forKey: keys[0])
let folder = FileManager.default.temporaryDirectory.appendingPathComponent("shared-dictionary-\(UUID())")
defer { try? FileManager.default.removeItem(at: folder) }
@@ -3954,7 +4125,10 @@ extension DictationE2ETests {
var enrollments = profile.enrollments
for i in enrollments.indices {
let id = try XCTUnwrap(enrollments[i].inspectionID)
- enrollments[i].pendingInspection = try PropertyListDecoder().decode(DictionaryAudioInspection.self, from: Data(contentsOf: root.appendingPathComponent(id.uuidString + ".plist")))
+ enrollments[i].pendingInspection = try PropertyListDecoder().decode(
+ DictionaryAudioInspection.self,
+ from: Data(contentsOf: root.appendingPathComponent(id.uuidString + ".plist"))
+ )
}
try await store.upsert(dictionaryEntryID: profile.dictionaryEntryID, label: word, modelKey: profile.modelKey, enrollments: enrollments)
entries.append(.init(id: profile.dictionaryEntryID, triggers: [], replacement: word))
@@ -3962,13 +4136,27 @@ extension DictationE2ETests {
let savedProfiles = await store.allProfiles()
let existingProfile = try XCTUnwrap(savedProfiles.first)
do {
- try await store.upsert(dictionaryEntryID: existingProfile.dictionaryEntryID, label: "MustNotSave", modelKey: existingProfile.modelKey, enrollments: existingProfile.enrollments, canPersist: { false })
+ try await store.upsert(
+ dictionaryEntryID: existingProfile.dictionaryEntryID,
+ label: "MustNotSave",
+ modelKey: existingProfile.modelKey,
+ enrollments: existingProfile.enrollments,
+ canPersist: { false }
+ )
XCTFail("Disabled pending save must fail")
- } catch is CancellationError { }
+ } catch is CancellationError {}
let profilesAfterRejectedSave = await store.allProfiles()
XCTAssertEqual(profilesAfterRejectedSave.map(\.label), savedProfiles.map(\.label), "Rejecting a pending save preserves existing voice profiles")
- let provider = FluidAudioProvider(modelOverride: .parakeetTDTv2, configureWordBoosting: false, enhancementOptions: .init(experimentalUnifiedFinalEnabled: false, pronunciationMatchingEnabled: true, customDictionaryEntries: entries), pronunciationStore: store)
+ let provider = FluidAudioProvider(
+ modelOverride: .parakeetTDTv2,
+ configureWordBoosting: false,
+ enhancementOptions: .init(experimentalUnifiedFinalEnabled: false, pronunciationMatchingEnabled: true, customDictionaryEntries: entries),
+ pronunciationStore: store
+ )
+ DictionaryMatcherExperiment.setEnabled(false)
try await provider.prepare()
+ XCTAssertNil(provider.pronunciationReferencePreparationTask, "Off must not schedule reference encoding")
+ XCTAssertFalse(provider.pronunciationReferencesReady)
let savedEntries = SettingsStore.shared.customDictionaryEntries
defer { SettingsStore.shared.customDictionaryEntries = savedEntries }
SettingsStore.shared.customDictionaryEntries = [.init(triggers: ["test replacement key"], replacement: "ReplacementValue")]
@@ -3977,11 +4165,18 @@ extension DictationE2ETests {
XCTAssertEqual(ASRService.applyCustomDictionary("test replacement key"), "ReplacementValue", "Text rules work across pronunciation toggles")
}
// Master off must defeat saved profiles and every legacy audio switch.
- for key in keys { UserDefaults.standard.set(true, forKey: key) }
+ for key in keys {
+ UserDefaults.standard.set(true, forKey: key)
+ }
UserDefaults.standard.set(false, forKey: keys[0])
let probe = try XCTUnwrap(jobs.first { $0.positive })
let pcm = try Data(contentsOf: URL(fileURLWithPath: probe.path)).withUnsafeBytes { Array($0.bindMemory(to: Float.self)) }
- let plain = FluidAudioProvider(modelOverride: .parakeetTDTv2, configureWordBoosting: false, enhancementOptions: .init(experimentalUnifiedFinalEnabled: false, pronunciationMatchingEnabled: false, customDictionaryEntries: []), pronunciationStore: store)
+ let plain = FluidAudioProvider(
+ modelOverride: .parakeetTDTv2,
+ configureWordBoosting: false,
+ enhancementOptions: .init(experimentalUnifiedFinalEnabled: false, pronunciationMatchingEnabled: false, customDictionaryEntries: []),
+ pronunciationStore: store
+ )
try await plain.prepare()
let baseline = try await plain.transcribeFinal(pcm)
let disabled = try await provider.transcribeFinal(pcm)
@@ -3990,9 +4185,27 @@ extension DictationE2ETests {
do {
_ = try await provider.transcribeDictionaryTraining(pcm, capturePronunciation: true)
XCTFail("Off must reject voice enrollment")
- } catch is CancellationError { }
- for key in keys { UserDefaults.standard.set(false, forKey: key) }
- UserDefaults.standard.set(true, forKey: keys[0])
+ } catch is CancellationError {}
+ for key in keys {
+ UserDefaults.standard.set(false, forKey: key)
+ }
+ DictionaryMatcherExperiment.setEnabled(true)
+ provider.resetStreamingPreviewCache()
+ XCTAssertFalse(provider.pronunciationReferencesReady, "Start with an unwarmed provider")
+ let cold = try await provider.transcribeFinal(pcm)
+ XCTAssertEqual(cold.text, baseline.text, "Cold cache misses skip acoustic matching instead of encoding references during final matching")
+ provider.resetStreamingPreviewCache()
+ _ = try await provider.transcribeStreaming(Array(pcm.prefix(32_000)))
+ let warmGeneration = DictionaryMatcherExperiment.generation
+ if let preparation = provider.pronunciationReferencePreparationTask {
+ let finished = expectation(description: "Production pronunciation reference preparation completes")
+ let waiter = Task { await preparation.value; finished.fulfill() }
+ await fulfillment(of: [finished], timeout: 30)
+ waiter.cancel()
+ }
+ XCTAssertEqual(DictionaryMatcherExperiment.generation, warmGeneration, "Readiness must belong to this master-switch generation")
+ XCTAssertTrue(provider.pronunciationReferencesReady, "Accuracy replay requires all production references prepared")
+ provider.resetStreamingPreviewCache()
var falsePositives: [String] = [], falseNegatives: [String] = []
var longFalsePositives: [String] = [], longFalseNegatives: [String] = []
for job in jobs {
@@ -4000,8 +4213,8 @@ extension DictationE2ETests {
provider.resetStreamingPreviewCache()
let result = try await provider.transcribeFinal(samples)
let accepted = result.text.contains(job.word)
- if accepted && !job.positive { falsePositives.append(job.id) }
- if !accepted && job.positive { falseNegatives.append(job.id) }
+ if accepted, !job.positive { falsePositives.append(job.id) }
+ if !accepted, job.positive { falseNegatives.append(job.id) }
print("SHARED_CORPUS id=\(job.id) positive=\(job.positive) accepted=\(accepted)")
if job.id == "page" { XCTAssertFalse(accepted, "Page must remain unchanged") }
do {
@@ -4016,11 +4229,10 @@ extension DictationE2ETests {
let finalized = try await provider.transcribeFinal(long)
print("SHARED_LONG id=\(job.id) stopMs=\((ProcessInfo.processInfo.systemUptime - stop) * 1000)")
let longAccepted = finalized.text.contains(job.word)
- if longAccepted && !job.positive { longFalsePositives.append(job.id) }
- if !longAccepted && job.positive { longFalseNegatives.append(job.id) }
+ if longAccepted, !job.positive { longFalsePositives.append(job.id) }
+ if !longAccepted, job.positive { longFalseNegatives.append(job.id) }
print("SHARED_LONG_DECISION id=\(job.id) positive=\(job.positive) accepted=\(longAccepted)")
if job.id == "page" { XCTAssertFalse(longAccepted, "Page must remain rejected in longer recordings") }
-
}
}
print("SHARED_CORPUS fp=\(falsePositives) fn=\(falseNegatives)")