Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

Improve FluidMeet and custom dictionary workflows (#1206)

ALTIC87d2655c7f85
diff --git a/.gitignore b/.gitignore
index 67fba786..f76353b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -232,3 +232,6 @@ builds/
!README.md
!CONTRIBUTING.md
!.github/pull_request_template.md
+
+# Local Claude Code settings and worktrees
+.claude/
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj
index 38a7a339..d4f88567 100644
--- a/Fluid.xcodeproj/project.pbxproj
+++ b/Fluid.xcodeproj/project.pbxproj
@@ -91,6 +91,7 @@
E5EC00000000000000000092 /* ClockRegressionHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000091 /* ClockRegressionHelper.swift */; };
A6E2246FA3744C7B3E2DB91C /* MeetingLiveTranscriptionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */; };
2952403C87FFDE41C1BC85D5 /* MeetingLiveReplayTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */; };
+ F68E1D074EF74EBAB9C61702 /* MeetingAudioComparisonProbeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F68E1D074EF74EBAB9C61701 /* MeetingAudioComparisonProbeTests.swift */; };
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; };
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; };
DA7200020000000000000002 /* AudioHardwareRecoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */; };
@@ -266,6 +267,7 @@
E5EC00000000000000000091 /* ClockRegressionHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClockRegressionHelper.swift; sourceTree = "<group>"; };
355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveTranscriptionTests.swift; sourceTree = "<group>"; };
7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveReplayTests.swift; sourceTree = "<group>"; };
+ F68E1D074EF74EBAB9C61701 /* MeetingAudioComparisonProbeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingAudioComparisonProbeTests.swift; sourceTree = "<group>"; };
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = "<group>"; };
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = "<group>"; };
DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioHardwareRecoveryTests.swift; sourceTree = "<group>"; };
@@ -530,6 +532,7 @@
E5EC00000000000000000091 /* ClockRegressionHelper.swift */,
355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */,
7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */,
+ F68E1D074EF74EBAB9C61701 /* MeetingAudioComparisonProbeTests.swift */,
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */,
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,
DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */,
@@ -934,6 +937,7 @@
E5EC00000000000000000092 /* ClockRegressionHelper.swift in Sources */,
A6E2246FA3744C7B3E2DB91C /* MeetingLiveTranscriptionTests.swift in Sources */,
2952403C87FFDE41C1BC85D5 /* MeetingLiveReplayTests.swift in Sources */,
+ F68E1D074EF74EBAB9C61702 /* MeetingAudioComparisonProbeTests.swift in Sources */,
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */,
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,
DA7200020000000000000002 /* AudioHardwareRecoveryTests.swift in Sources */,
diff --git a/Sources/Fluid/AppDelegate.swift b/Sources/Fluid/AppDelegate.swift
index 2c769e06..f7bd1f34 100644
--- a/Sources/Fluid/AppDelegate.swift
+++ b/Sources/Fluid/AppDelegate.swift
@@ -367,6 +367,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCenterDele
private func scheduleMeetingAutoDetectorStart() {
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) {
Task { @MainActor in
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
_ = AppServices.shared.meetingAutoDetector
}
}
diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift
index 5b30e39d..9d436c8a 100644
--- a/Sources/Fluid/ContentView.swift
+++ b/Sources/Fluid/ContentView.swift
@@ -161,6 +161,7 @@ enum ShortcutRecordingTarget: Hashable {
case edit
case cancel
case pasteLast
+ case meeting
case dictationPrompt(String)
case newPrompt
@@ -176,6 +177,8 @@ enum ShortcutRecordingTarget: Hashable {
return "Edit Mode"
case .cancel:
return "Cancel Recording"
+ case .meeting:
+ return "Start / Stop Meeting"
case .pasteLast:
return "Paste Last Transcription"
case .dictationPrompt:
@@ -189,7 +192,7 @@ enum ShortcutRecordingTarget: Hashable {
switch self {
case .secondaryDictation, .command, .edit, .pasteLast:
return true
- case .primaryDictation, .cancel, .dictationPrompt, .newPrompt:
+ case .primaryDictation, .cancel, .meeting, .dictationPrompt, .newPrompt:
return false
}
}
@@ -203,7 +206,7 @@ enum ShortcutRecordingTarget: Hashable {
switch self {
case .primaryDictation, .pasteLast:
return true
- case .secondaryDictation, .command, .edit, .cancel, .dictationPrompt, .newPrompt:
+ case .secondaryDictation, .command, .edit, .cancel, .meeting, .dictationPrompt, .newPrompt:
return false
}
}
@@ -434,6 +437,14 @@ struct ContentView: View {
return self.beginDictationRecording(for: .primary, mode: .dictate)
}
+ func beginCommandModeSwitchForTesting() async {
+ await self.beginCommandRecording()
+ }
+
+ func beginRewriteModeSwitchForTesting() async {
+ await self.beginRewriteRecording()
+ }
+
func blockDictationModeSwitchForTesting(savingCancellation: Bool = false, processing: Bool = false) {
self.isSavingCancelledRecording = savingCancellation
self.processingDictationLifecycleID = processing ? self.overlayLifecycleID : nil
@@ -499,6 +510,13 @@ struct ContentView: View {
.onReceive(NotificationCenter.default.publisher(for: .openCustomDictionaryFromVoiceEngine)) { _ in
self.navigateToApp(.customDictionary)
}
+ .onReceive(NotificationCenter.default.publisher(for: .fluidMeetEnabledChanged)) { _ in
+ if self.settings.fluidMeetEnabled, self.appServices.isUIReady {
+ self.menuBarManager.configure(meetingCoordinator: self.appServices.meetingSessionCoordinator)
+ } else if !self.settings.fluidMeetEnabled, self.selectedSidebarItem == .meetingTranscription {
+ self.selectedSidebarItem = .welcome
+ }
+ }
.onReceive(NotificationCenter.default.publisher(for: .appNavigationRequested)) { _ in
self.handlePendingAppNavigation()
}
@@ -771,9 +789,11 @@ struct ContentView: View {
DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) {
DebugLogger.shared.info("🚦 Startup delay complete, signaling UI ready...", source: "ContentView")
self.appServices.signalUIReady()
- self.menuBarManager.configure(
- meetingCoordinator: self.appServices.meetingSessionCoordinator
- )
+ if self.settings.fluidMeetEnabled {
+ self.menuBarManager.configure(
+ meetingCoordinator: self.appServices.meetingSessionCoordinator
+ )
+ }
Task { @MainActor in
await AudioStartupGate.shared.scheduleOpenAfterInitialUISettled()
@@ -1199,6 +1219,10 @@ struct ContentView: View {
}
private func navigateToApp(_ destination: SidebarItem) {
+ guard destination != .meetingTranscription || self.settings.fluidMeetEnabled else {
+ self.openSettings(.fluidMeet)
+ return
+ }
self.clearShortcutRecordingMode()
self.resetSettingsSearch()
self.settingsNavigation.leaveForApp()
@@ -1214,7 +1238,8 @@ struct ContentView: View {
private func closeSettings() {
self.clearShortcutRecordingMode()
self.resetSettingsSearch()
- self.selectedSidebarItem = self.settingsNavigation.dismiss()
+ let destination = self.settingsNavigation.dismiss()
+ self.selectedSidebarItem = destination == .meetingTranscription && !self.settings.fluidMeetEnabled ? .welcome : destination
}
private func resetSettingsSearch() {
@@ -1232,6 +1257,9 @@ struct ContentView: View {
}
private func shortcutConflictMessage(for shortcut: HotkeyShortcut, target: ShortcutRecordingTarget) -> String? {
+ if target == .meeting, !shortcut.isMeetingRecordingShortcut {
+ return "Use a keyboard key with a modifier for meeting recording"
+ }
if shortcut.isMouseShortcut {
guard target.allowsMouseShortcut else {
return "Mouse clicks can only be assigned to Primary Dictation or Paste Last Transcription"
@@ -1247,7 +1275,7 @@ struct ContentView: View {
if configuredShortcut == shortcut {
return "Duplicate with Primary Dictation Shortcut"
}
- if shortcut.conflictsWith(configuredShortcut) {
+ if shortcut.conflictsWith(configuredShortcut, includingKeyboardModifierPrefixes: target == .meeting) {
return "Overlaps Primary Dictation Shortcut — use a different modifier key"
}
}
@@ -1260,6 +1288,7 @@ struct ContentView: View {
configuredShortcuts.append((.secondaryDictation, self.promptModeHotkeyShortcut))
}
let optionalConfiguredShortcuts: [(ShortcutRecordingTarget, HotkeyShortcut?)] = [
+ (.meeting, SettingsStore.shared.meetingRecordingHotkeyShortcut),
(.cancel, self.cancelRecordingHotkeyShortcut),
(.command, self.commandModeHotkeyShortcut),
(.pasteLast, self.pasteLastTranscriptionHotkeyShortcut),
@@ -1269,7 +1298,7 @@ struct ContentView: View {
if configuredShortcut == shortcut {
return "Duplicate with \(otherTarget.title)"
}
- if shortcut.conflictsWith(configuredShortcut) {
+ if shortcut.conflictsWith(configuredShortcut, includingKeyboardModifierPrefixes: target == .meeting || otherTarget == .meeting) {
return "Overlaps \(otherTarget.title) — use a different modifier key"
}
}
@@ -1278,7 +1307,7 @@ struct ContentView: View {
if configuredShortcut == shortcut {
return "Duplicate with \(otherTarget.title)"
}
- if shortcut.conflictsWith(configuredShortcut) {
+ if shortcut.conflictsWith(configuredShortcut, includingKeyboardModifierPrefixes: target == .meeting || otherTarget == .meeting) {
return "Overlaps \(otherTarget.title) — use a different modifier key"
}
}
@@ -1293,7 +1322,7 @@ struct ContentView: View {
if assignment.shortcut == shortcut {
return "Duplicate with Prompt Shortcut"
}
- if shortcut.conflictsWith(assignment.shortcut) {
+ if shortcut.conflictsWith(assignment.shortcut, includingKeyboardModifierPrefixes: target == .meeting) {
return "Overlaps Prompt Shortcut — use a different modifier key"
}
}
@@ -1325,6 +1354,8 @@ struct ContentView: View {
self.rewriteModeHotkeyShortcut = shortcut
SettingsStore.shared.rewriteModeHotkeyShortcut = shortcut
self.hotkeyManager?.updateRewriteModeShortcut(shortcut)
+ case .meeting:
+ SettingsStore.shared.meetingRecordingHotkeyShortcut = shortcut
case .cancel:
self.cancelRecordingHotkeyShortcut = shortcut
SettingsStore.shared.cancelRecordingHotkeyShortcut = shortcut
@@ -1379,7 +1410,7 @@ struct ContentView: View {
case .pasteLast:
self.isPasteLastTranscriptionShortcutEnabled = enabled
SettingsStore.shared.pasteLastTranscriptionShortcutEnabled = enabled
- case .primaryDictation, .cancel, .dictationPrompt, .newPrompt:
+ case .primaryDictation, .cancel, .meeting, .dictationPrompt, .newPrompt:
break
}
}
@@ -1496,11 +1527,13 @@ struct ContentView: View {
Section {
self.sidebarNavigationLink(.commandMode, title: "Command Mode", systemImage: "terminal.fill")
self.sidebarNavigationLink(.fileTranscription, title: "File Transcription", systemImage: "doc.text.fill")
- self.sidebarNavigationLink(
- .meetingTranscription,
- title: "FluidMeet",
- systemImage: "person.2.fill"
- )
+ if self.settings.fluidMeetEnabled {
+ self.sidebarNavigationLink(
+ .meetingTranscription,
+ title: "FluidMeet",
+ systemImage: "person.2.fill"
+ )
+ }
} header: {
self.sidebarSectionHeader("Use")
}
@@ -2099,7 +2132,7 @@ struct ContentView: View {
private var hasPageToolbarActions: Bool {
guard !self.settingsNavigation.isPresented else { return false }
return switch self.selectedSidebarItem ?? .welcome {
- case .stats, .commandMode, .meetingTranscription, .customDictionary, .changelog: true
+ case .stats, .commandMode, .customDictionary, .changelog: true
default: false
}
}
@@ -4861,6 +4894,110 @@ struct ContentView: View {
self.newModelName = ""
}
+ private func beginCommandRecording() async {
+ guard self.canBeginDictationRecording() else { return }
+ DebugLogger.shared.info("Command mode triggered", source: "ContentView")
+ DictationStopSnapshot.captureStartingContext(reusingCapture: self.canSwitchActiveDictationCaptureMode) {
+ self.captureRecordingContext()
+ }
+
+ // Set flag so stopAndProcessTranscription knows to process as command
+ self.setActiveRecordingMode(.command)
+
+ // Set overlay mode to command
+ self.menuBarManager.setOverlayMode(.command)
+
+ guard !self.asr.isRunningOrStarting else { return }
+
+ self.advanceOverlayLifecycle()
+
+ // Start recording immediately for the command
+ DebugLogger.shared.info(
+ "Starting voice recording for command",
+ source: "ContentView"
+ )
+ let captureOverlayLifecycleID = self.overlayLifecycleID
+ let captureTask = Task {
+ guard !Task.isCancelled else {
+ self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .command)
+ return
+ }
+ let startOutcome = await self.asr.start(onCaptureStarted: {
+ TranscriptionSoundPlayer.shared.playStartSound()
+ self.appBench("overlay_phase phase=recording trigger=first_pcm mode=command")
+ })
+ guard !Task.isCancelled else {
+ self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .command)
+ return
+ }
+ if startOutcome == .failed {
+ self.menuBarManager.hideRecordingOverlayImmediately(
+ reason: "command_asr_start_failed"
+ )
+ }
+ }
+ await self.awaitCaptureStart(captureTask)
+ }
+
+ private func beginRewriteRecording() async {
+ guard self.canBeginDictationRecording() else { return }
+
+ DictationStopSnapshot.captureStartingContext(reusingCapture: self.canSwitchActiveDictationCaptureMode) {
+ self.captureRecordingContext()
+ }
+
+ // Try to capture text first while still in the other app
+ let captured = self.rewriteModeService.captureSelectedText()
+ DebugLogger.shared.info("Rewrite mode triggered, text captured: \(captured)", source: "ContentView")
+
+ if !captured {
+ // No text selected - start in "write mode" where user speaks
+ // what to write
+ DebugLogger.shared
+ .info(
+ "No text selected - starting in write/improve mode",
+ source: "ContentView"
+ )
+ self.rewriteModeService.startWithoutSelection()
+ // Set overlay mode to edit
+ self.menuBarManager.setOverlayMode(.edit)
+ } else {
+ // Text was selected - edit mode (with selected context)
+ self.menuBarManager.setOverlayMode(.edit)
+ }
+
+ // Set flag so stopAndProcessTranscription knows to process as rewrite
+ self.setActiveRecordingMode(.edit)
+
+ guard !self.asr.isRunningOrStarting else { return }
+
+ self.advanceOverlayLifecycle()
+
+ // Start recording immediately for the edit instruction
+ DebugLogger.shared.info("Starting voice recording for edit mode", source: "ContentView")
+ let captureOverlayLifecycleID = self.overlayLifecycleID
+ let captureTask = Task {
+ guard !Task.isCancelled else {
+ self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .edit)
+ return
+ }
+ let startOutcome = await self.asr.start(onCaptureStarted: {
+ TranscriptionSoundPlayer.shared.playStartSound()
+ self.appBench("overlay_phase phase=recording trigger=first_pcm mode=edit")
+ })
+ guard !Task.isCancelled else {
+ self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .edit)
+ return
+ }
+ if startOutcome == .failed {
+ self.menuBarManager.hideRecordingOverlayImmediately(
+ reason: "edit_asr_start_failed"
+ )
+ }
+ }
+ await self.awaitCaptureStart(captureTask)
+ }
+
private func initializeHotkeyManagerIfNeeded() {
NotchContentState.shared.onPromptModeSwitchRequested = { mode in
self.handleLivePromptModeSwitch(mode)
@@ -4940,102 +5077,10 @@ struct ContentView: View {
await self.awaitCaptureStart(self.beginDictationRecording(for: selection, mode: .promptMode))
},
commandModeCallback: {
- guard !self.presentExclusiveActivityBlockIfNeeded() else { return }
- DebugLogger.shared.info("Command mode triggered", source: "ContentView")
- self.captureRecordingContext()
-
- // Set flag so stopAndProcessTranscription knows to process as command
- self.setActiveRecordingMode(.command)
-
- // Set overlay mode to command
- self.menuBarManager.setOverlayMode(.command)
-
- guard !self.asr.isRunningOrStarting else { return }
-
- self.advanceOverlayLifecycle()
-
- // Start recording immediately for the command
- DebugLogger.shared.info(
- "Starting voice recording for command",
- source: "ContentView"
- )
- let captureOverlayLifecycleID = self.overlayLifecycleID
- let captureTask = Task {
- guard !Task.isCancelled else {
- self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .command)
- return
- }
- let startOutcome = await self.asr.start(onCaptureStarted: {
- TranscriptionSoundPlayer.shared.playStartSound()
- self.appBench("overlay_phase phase=recording trigger=first_pcm mode=command")
- })
- guard !Task.isCancelled else {
- self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .command)
- return
- }
- if startOutcome == .failed {
- self.menuBarManager.hideRecordingOverlayImmediately(
- reason: "command_asr_start_failed"
- )
- }
- }
- await self.awaitCaptureStart(captureTask)
+ await self.beginCommandRecording()
},
rewriteModeCallback: {
- guard !self.presentExclusiveActivityBlockIfNeeded() else { return }
-
- self.captureRecordingContext()
-
- // Try to capture text first while still in the other app
- let captured = self.rewriteModeService.captureSelectedText()
- DebugLogger.shared.info("Rewrite mode triggered, text captured: \(captured)", source: "ContentView")
-
- if !captured {
- // No text selected - start in "write mode" where user speaks
- // what to write
- DebugLogger.shared
- .info(
- "No text selected - starting in write/improve mode",
- source: "ContentView"
- )
- self.rewriteModeService.startWithoutSelection()
- // Set overlay mode to edit
- self.menuBarManager.setOverlayMode(.edit)
- } else {
- // Text was selected - edit mode (with selected context)
- self.menuBarManager.setOverlayMode(.edit)
- }
-
- // Set flag so stopAndProcessTranscription knows to process as rewrite
- self.setActiveRecordingMode(.edit)
-
- guard !self.asr.isRunningOrStarting else { return }
-
- self.advanceOverlayLifecycle()
-
- // Start recording immediately for the edit instruction
- DebugLogger.shared.info("Starting voice recording for edit mode", source: "ContentView")
- let captureOverlayLifecycleID = self.overlayLifecycleID
- let captureTask = Task {
- guard !Task.isCancelled else {
- self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .edit)
- return
- }
- let startOutcome = await self.asr.start(onCaptureStarted: {
- TranscriptionSoundPlayer.shared.playStartSound()
- self.appBench("overlay_phase phase=recording trigger=first_pcm mode=edit")
- })
- guard !Task.isCancelled else {
- self.clearCancelledCaptureStart(lifecycleID: captureOverlayLifecycleID, mode: .edit)
- return
- }
- if startOutcome == .failed {
- self.menuBarManager.hideRecordingOverlayImmediately(
- reason: "edit_asr_start_failed"
- )
- }
- }
- await self.awaitCaptureStart(captureTask)
+ await self.beginRewriteRecording()
},
isDictateRecordingProvider: {
self.activeRecordingMode == .dictate
@@ -5068,6 +5113,14 @@ struct ContentView: View {
self.handleCancelShortcutResult()
}
+ self.hotkeyManager?.setMeetingRecordingCallback {
+ Task { @MainActor in
+ await self.menuBarManager.toggleMeetingRecordingFromShortcut {
+ try await AppServices.shared.startMeetingRecordingFromMenuBar()
+ }
+ }
+ }
+
// Re-insert the most recent transcription on demand (no clipboard involved).
self.hotkeyManager?.setPasteLastTranscriptionCallback {
self.pasteLastDictationFromHistory()
@@ -5362,9 +5415,18 @@ extension ContentView {
self.activeRecordingMode != .none && self.asr.canSwitchOwnedDictationCaptureMode
}
+ private func clearOwnCaptureBlockErrorIfNeeded() {
+ if self.asr.showError, self.asr.errorTitle == "Dictation Unavailable",
+ self.asr.errorMessage == "Wait for the active dictation to finish."
+ { self.asr.showError = false }
+ }
+
private func canBeginDictationRecording() -> Bool {
guard !self.isSavingCancelledRecording, self.processingDictationLifecycleID == nil else { return false }
- if self.canSwitchActiveDictationCaptureMode { return true }
+ if self.canSwitchActiveDictationCaptureMode {
+ self.clearOwnCaptureBlockErrorIfNeeded()
+ return true
+ }
guard !self.presentExclusiveActivityBlockIfNeeded() else { return false }
return !self.asr.isRunningOrStarting && !self.asr.isDictionaryTrainingCaptureActive
}
@@ -5392,9 +5454,6 @@ extension ContentView {
self.rewriteModeService.clearState()
if switchesOwnedCapture {
- if self.asr.showError, self.asr.errorTitle == "Dictation Unavailable",
- self.asr.errorMessage == "Wait for the active dictation to finish."
- { self.asr.showError = false }
self.menuBarManager.setOverlayMode(.dictation)
DebugLogger.shared.info(
"Dictation capture mode switched from=\(previousMode.rawValue) to=\(mode.rawValue) slot=\(slot.rawValue)",
diff --git a/Sources/Fluid/Models/HotkeyShortcut.swift b/Sources/Fluid/Models/HotkeyShortcut.swift
index 72f2f10e..3e5af74d 100644
--- a/Sources/Fluid/Models/HotkeyShortcut.swift
+++ b/Sources/Fluid/Models/HotkeyShortcut.swift
@@ -238,7 +238,16 @@ extension HotkeyShortcut {
}
/// True when shortcut presses can race because one begins as the other's modifier prefix.
- func conflictsWith(_ other: HotkeyShortcut) -> Bool {
+ func conflictsWith(_ other: HotkeyShortcut, includingKeyboardModifierPrefixes: Bool = false) -> Bool {
+ // Meeting chords cannot wait for a modifier-only dictation shortcut to disambiguate:
+ // that shortcut must keep starting capture immediately.
+ if includingKeyboardModifierPrefixes {
+ let modifierOnly = self.isModifierOnlyShortcut ? self : other
+ let keyed = self.isModifierOnlyShortcut ? other : self
+ if modifierOnly.isModifierOnlyShortcut, !keyed.isModifierOnlyShortcut, !keyed.isMouseShortcut,
+ let modifiers = modifierOnly.expectedModifierFlags, !modifiers.isEmpty,
+ modifiers.isSubset(of: keyed.relevantModifierFlags) { return true }
+ }
if self.isModifierOnlyShortcut, other.isModifierOnlyShortcut {
let lhs = Set(self.normalizedModifierKeyCodes)
let rhs = Set(other.normalizedModifierKeyCodes)
@@ -289,6 +298,11 @@ extension HotkeyShortcut {
}
}
+ /// Meeting recording is a one-shot toggle, independent of dictation activation mode.
+ var isMeetingRecordingShortcut: Bool {
+ !self.isMouseShortcut && !self.isModifierOnlyShortcut && !self.relevantModifierFlags.isEmpty
+ }
+
func matchesRecordingShortcut(keyCode: UInt16, modifiers: NSEvent.ModifierFlags) -> Bool {
!self.requiresModifierForRecording && self.matches(keyCode: keyCode, modifiers: modifiers)
}
diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swift
index 37bc2046..155777a8 100644
--- a/Sources/Fluid/Persistence/BackupService.swift
+++ b/Sources/Fluid/Persistence/BackupService.swift
@@ -42,6 +42,9 @@ struct SettingsBackupPayload: Codable, Equatable {
let rewriteModeSelectedModel: String?
let rewriteModeSelectedProviderID: String
let rewriteModeLinkedToGlobal: Bool
+ // Optional so older backups preserve the user's current meeting shortcut.
+ let meetingRecordingHotkeyShortcut: HotkeyShortcut?
+ let meetingRecordingShortcutEnabled: Bool?
let cancelRecordingHotkeyShortcut: HotkeyShortcut?
// Optional so older backup files (which predate this setting) still decode.
let pasteLastTranscriptionHotkeyShortcut: HotkeyShortcut?
diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift
index ad179bf3..54ea7850 100644
--- a/Sources/Fluid/Persistence/SettingsStore.swift
+++ b/Sources/Fluid/Persistence/SettingsStore.swift
@@ -2113,6 +2113,17 @@ final class SettingsStore: ObservableObject {
}
}
+ /// Feature availability is separate from the user's meeting detection preferences.
+ @MainActor var fluidMeetEnabled: Bool {
+ get { self.defaults.object(forKey: Keys.fluidMeetEnabled) as? Bool ?? true }
+ set {
+ guard newValue != self.fluidMeetEnabled else { return }
+ objectWillChange.send()
+ self.defaults.set(newValue, forKey: Keys.fluidMeetEnabled)
+ NotificationCenter.default.post(name: .fluidMeetEnabledChanged, object: nil)
+ }
+ }
+
/// Tier 1 native (Zoom/Teams/Webex) meeting auto-detection (default: ON).
var meetingAutoDetectEnabled: Bool {
get {
@@ -3206,6 +3217,26 @@ final class SettingsStore: ObservableObject {
}
}
+ /// Unassigned by default. Meeting recording never claims a shortcut until explicitly configured.
+ var meetingRecordingHotkeyShortcut: HotkeyShortcut? {
+ get {
+ guard let data = self.defaults.data(forKey: Keys.meetingRecordingHotkeyShortcut),
+ let shortcut = try? JSONDecoder().decode(HotkeyShortcut.self, from: data),
+ shortcut.isMeetingRecordingShortcut else { return nil }
+ return shortcut
+ }
+ set {
+ objectWillChange.send()
+ guard let newValue, newValue.isMeetingRecordingShortcut,
+ let data = try? JSONEncoder().encode(newValue)
+ else {
+ self.defaults.removeObject(forKey: Keys.meetingRecordingHotkeyShortcut)
+ return
+ }
+ self.defaults.set(data, forKey: Keys.meetingRecordingHotkeyShortcut)
+ }
+ }
+
// MARK: - Paste Last Transcription Settings
/// Whether the "Paste Last Transcription" global hotkey is active. Opt-in and off by default.
@@ -3656,6 +3687,8 @@ final class SettingsStore: ObservableObject {
rewriteModeSelectedModel: self.rewriteModeSelectedModel,
rewriteModeSelectedProviderID: self.rewriteModeSelectedProviderID,
rewriteModeLinkedToGlobal: self.rewriteModeLinkedToGlobal,
+ meetingRecordingHotkeyShortcut: self.meetingRecordingHotkeyShortcut,
+ meetingRecordingShortcutEnabled: self.meetingRecordingHotkeyShortcut != nil,
cancelRecordingHotkeyShortcut: self.cancelRecordingHotkeyShortcut,
pasteLastTranscriptionHotkeyShortcut: self.pasteLastTranscriptionHotkeyShortcut,
pasteLastTranscriptionShortcutEnabled: self.pasteLastTranscriptionShortcutEnabled,
@@ -3808,6 +3841,9 @@ final class SettingsStore: ObservableObject {
self.rewriteModeSelectedModel = payload.rewriteModeSelectedModel
self.rewriteModeSelectedProviderID = payload.rewriteModeSelectedProviderID
self.rewriteModeLinkedToGlobal = payload.rewriteModeLinkedToGlobal
+ if let enabled = payload.meetingRecordingShortcutEnabled {
+ self.meetingRecordingHotkeyShortcut = enabled ? payload.meetingRecordingHotkeyShortcut : nil
+ }
self.cancelRecordingHotkeyShortcut = payload.cancelRecordingHotkeyShortcut
// Both guarded so restoring an older backup (which predates these fields) doesn't wipe a
// currently-configured shortcut or leave the feature enabled with no shortcut bound.
@@ -5887,6 +5923,7 @@ private extension SettingsStore {
static let meetingRecordingDefaults = "MeetingRecordingDefaults"
static let meetingAudioRetentionPolicy = "MeetingAudioRetentionPolicy"
static let meetingTranscriptionBackendID = "MeetingTranscriptionBackendID"
+ static let fluidMeetEnabled = "FluidMeetEnabled"
static let meetingAutoDetectEnabled = "MeetingAutoDetectEnabled"
static let meetingAutoDetectBrowserEnabled = "MeetingAutoDetectBrowserEnabled"
static let microphoneSelectionMode = "MicrophoneSelectionMode"
@@ -5940,6 +5977,7 @@ private extension SettingsStore {
static let commandModeSelectedProviderID = "CommandModeSelectedProviderID"
static let commandModeHotkeyShortcut = "CommandModeHotkeyShortcut"
static let commandModeConfirmBeforeExecute = "CommandModeConfirmBeforeExecute"
+ static let meetingRecordingHotkeyShortcut = "MeetingRecordingHotkeyShortcut"
static let cancelRecordingHotkeyShortcut = "CancelRecordingHotkeyShortcut"
static let pasteLastTranscriptionHotkeyShortcut = "PasteLastTranscriptionHotkeyShortcut"
static let pasteLastTranscriptionShortcutEnabled = "PasteLastTranscriptionShortcutEnabled"
@@ -6473,3 +6511,7 @@ extension SettingsStore {
return newModel
}
}
+
+extension Notification.Name {
+ static let fluidMeetEnabledChanged = Notification.Name("FluidMeetEnabledChanged")
+}
diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift
index e9a50245..3e1b5e85 100644
--- a/Sources/Fluid/Services/ASRService.swift
+++ b/Sources/Fluid/Services/ASRService.swift
@@ -7472,12 +7472,28 @@ final class ASRService: ObservableObject {
}
private static var cachedDictionaryPatterns: [CachedDictionaryPattern] = []
+ private static var cachedBuiltInDictionaryPatterns: [CachedDictionaryPattern] = []
+ private static var cachedUserDictionaryProtections: [NSRegularExpression] = []
private static var dictionaryCacheNeedsRebuild: Bool = true
/// Rebuilds the regex cache if dictionary has changed.
/// Called lazily on first apply after settings change.
private static func rebuildDictionaryCache() {
- let entries = SettingsStore.shared.customDictionaryEntries
+ let userEntries = SettingsStore.shared.customDictionaryEntries
+ self.cachedDictionaryPatterns = self.dictionaryPatterns(for: userEntries)
+ self.cachedBuiltInDictionaryPatterns = self.dictionaryPatterns(for: BuiltInDictionary.entries(excludingSpellingsIn: userEntries))
+ self.cachedUserDictionaryProtections = self.cachedDictionaryPatterns.map(\.regex)
+ var protectedPatterns = Set(self.cachedUserDictionaryProtections.map(\.pattern))
+ for entry in userEntries where !entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+ let pattern = self.dictionaryPattern(for: entry.replacement)
+ guard protectedPatterns.insert(pattern).inserted,
+ let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) else { continue }
+ self.cachedUserDictionaryProtections.append(regex)
+ }
+ self.dictionaryCacheNeedsRebuild = false
+ }
+
+ private static func dictionaryPatterns(for entries: [SettingsStore.CustomDictionaryEntry]) -> [CachedDictionaryPattern] {
var patterns: [CachedDictionaryPattern] = []
for entry in entries {
@@ -7513,10 +7529,51 @@ final class ASRService: ObservableObject {
}
}
- self.cachedDictionaryPatterns = patterns.sorted {
+ return patterns.sorted {
$0.regex.pattern.utf16.count > $1.regex.pattern.utf16.count
}
- self.dictionaryCacheNeedsRebuild = false
+ }
+
+ /// Apply defaults only outside user-defined phrases and completed user replacements.
+ /// Match against one immutable string so built-in edits cannot move protected ranges.
+ private static func applyBuiltInDictionary(
+ to text: String,
+ onReplacement: (CachedDictionaryPattern, NSRange, String, String) -> Void
+ ) -> String {
+ let fullRange = NSRange(text.startIndex..., in: text)
+ var candidates: [(pattern: CachedDictionaryPattern, match: NSTextCheckingResult)] = []
+ for pattern in self.cachedBuiltInDictionaryPatterns {
+ candidates += pattern.regex.matches(in: text, range: fullRange).map { (pattern, $0) }
+ }
+ guard !candidates.isEmpty else { return text }
+ candidates.sort {
+ if $0.match.range.location != $1.match.range.location { return $0.match.range.location < $1.match.range.location }
+ return $0.match.range.length > $1.match.range.length
+ }
+ let protected = self.cachedUserDictionaryProtections.flatMap {
+ $0.matches(in: text, range: fullRange).map(\.range)
+ }.sorted { $0.location < $1.location }
+ var protectedIndex = 0
+ var previousEnd = 0
+ var accepted: [(pattern: CachedDictionaryPattern, match: NSTextCheckingResult)] = []
+ for candidate in candidates {
+ let range = candidate.match.range
+ while protectedIndex < protected.count, NSMaxRange(protected[protectedIndex]) <= range.location {
+ protectedIndex += 1
+ }
+ guard range.location >= previousEnd,
+ protectedIndex == protected.count || protected[protectedIndex].location >= NSMaxRange(range) else { continue }
+ accepted.append(candidate)
+ previousEnd = NSMaxRange(range)
+ }
+ let output = NSMutableString(string: text)
+ for (pattern, match) in accepted.reversed() {
+ let replacement = pattern.regex.replacementString(for: match, in: text, offset: 0, template: pattern.template)
+ let heard = (text as NSString).substring(with: match.range)
+ if heard != replacement { onReplacement(pattern, match.range, heard, replacement) }
+ output.replaceCharacters(in: match.range, with: replacement)
+ }
+ return output as String
}
private static func dictionaryPattern(
@@ -7554,16 +7611,14 @@ final class ASRService: ObservableObject {
/// Uses case-insensitive matching with word boundaries.
/// Optimized: caches compiled regexes to avoid per-call compilation overhead.
static func applyCustomDictionary(_ text: String) -> String {
- // Fast path: no entries configured
- let entries = SettingsStore.shared.customDictionaryEntries
- guard !entries.isEmpty else { return text }
+ // Built-in spellings always exist, so the compiled patterns decide whether there is work.
// Rebuild cache if needed (lazy initialization)
if self.dictionaryCacheNeedsRebuild {
self.rebuildDictionaryCache()
}
- guard !self.cachedDictionaryPatterns.isEmpty else {
+ guard !self.cachedDictionaryPatterns.isEmpty || !self.cachedBuiltInDictionaryPatterns.isEmpty else {
return text
}
@@ -7571,6 +7626,21 @@ final class ASRService: ObservableObject {
// Apply cached regexes - O(n) where n = number of patterns
var replacementCount = 0
+ func recordReplacement(_ pattern: CachedDictionaryPattern, _ range: NSRange, _ heard: String, _ replacement: String) {
+ replacementCount += 1
+ guard replacementCount <= DictionaryReplacementDiagnostics.maximumEvents else { return }
+ DebugLogger.shared.debug(
+ "DICTIONARY_REPLACEMENT stage=exact_rule entryID=\(pattern.entryID) " +
+ "trigger=\(DictionaryReplacementDiagnostics.quoted(pattern.trigger)) " +
+ "heard=\(DictionaryReplacementDiagnostics.quoted(heard)) " +
+ "replacement=\(DictionaryReplacementDiagnostics.quoted(replacement)) " +
+ "utf16Location=\(range.location) utf16Length=\(range.length)",
+ source: "CustomDictionary"
+ )
+ }
+ // Defaults cannot consume an explicit user phrase. Apply user rules last so a
+ // default's corrected spelling can also follow the user's chosen replacement.
+ result = self.applyBuiltInDictionary(to: result, onReplacement: recordReplacement)
for pattern in self.cachedDictionaryPatterns {
result = DictionaryReplacementProtection.replacingMatches(
in: result,
@@ -7578,16 +7648,7 @@ final class ASRService: ObservableObject {
template: pattern.template,
canonical: pattern.canonical,
onReplacement: { range, heard, replacement in
- replacementCount += 1
- guard replacementCount <= DictionaryReplacementDiagnostics.maximumEvents else { return }
- DebugLogger.shared.debug(
- "DICTIONARY_REPLACEMENT stage=exact_rule entryID=\(pattern.entryID) " +
- "trigger=\(DictionaryReplacementDiagnostics.quoted(pattern.trigger)) " +
- "heard=\(DictionaryReplacementDiagnostics.quoted(heard)) " +
- "replacement=\(DictionaryReplacementDiagnostics.quoted(replacement)) " +
- "utf16Location=\(range.location) utf16Length=\(range.length)",
- source: "CustomDictionary"
- )
+ recordReplacement(pattern, range, heard, replacement)
}
)
}
diff --git a/Sources/Fluid/Services/AppServices+MeetingMenuBar.swift b/Sources/Fluid/Services/AppServices+MeetingMenuBar.swift
index 83cda5cc..3d1458ca 100644
--- a/Sources/Fluid/Services/AppServices+MeetingMenuBar.swift
+++ b/Sources/Fluid/Services/AppServices+MeetingMenuBar.swift
@@ -3,11 +3,13 @@ import Foundation
extension AppServices {
/// Starts the same coordinator used by FluidMeet without posting a navigation request.
func startMeetingRecordingFromMenuBar() async throws {
+ guard SettingsStore.shared.fluidMeetEnabled else { throw MeetingCoordinatorError.featureDisabled }
let defaults = SettingsStore.shared.meetingRecordingDefaults
let target = defaults.mode == .onlineCall ? self.meetingAutomaticTarget : nil
let preferredInputUID = SettingsStore.shared.preferredInputDeviceUID
let microphones = await MeetingCaptureSourceCatalog.microphoneSnapshot()
- let applications = try await target == nil ? [] : MeetingCaptureSourceCatalog.availableApplications()
+ let needsApplications = defaults.mode == .onlineCall && (target != nil || (defaults.isConfigured && defaults.applicationBundleIdentifier != nil))
+ let applications = try await needsApplications ? MeetingCaptureSourceCatalog.availableApplications() : []
try Task.checkCancellation()
// Never capture a replacement app when detection changed during source discovery.
if let target, target != self.meetingAutomaticTarget {
@@ -20,6 +22,7 @@ extension AppServices {
applications: applications,
preferredInputUID: preferredInputUID
)
+ guard SettingsStore.shared.fluidMeetEnabled else { throw MeetingCoordinatorError.featureDisabled }
_ = try await self.meetingSessionCoordinator.startRecording(configuration: configuration)
}
@@ -52,6 +55,11 @@ extension AppServices {
}
match.windowID = target.windowID
application = match
+ } else if defaults.mode == .onlineCall, defaults.isConfigured, let bundleIdentifier = defaults.applicationBundleIdentifier {
+ guard let saved = defaults.savedApplication(in: applications), let pid = saved.processID, pid > 0 else {
+ throw MeetingCaptureError.applicationUnavailable(bundleIdentifier)
+ }
+ application = saved
}
let mode: MeetingCaptureMode = application == nil ? .inRoom : .onlineCall
return MeetingCaptureConfiguration(
diff --git a/Sources/Fluid/Services/AppServices.swift b/Sources/Fluid/Services/AppServices.swift
index 2018c4a9..23cccf10 100644
--- a/Sources/Fluid/Services/AppServices.swift
+++ b/Sources/Fluid/Services/AppServices.swift
@@ -41,7 +41,7 @@ final class AppServices: ObservableObject {
DebugLogger.shared.info("🚦 UI Ready signal received - services can now initialize", source: "AppServices")
self.isUIReady = true
}
- _ = self.meetingAutoDetector
+ self.refreshFluidMeetAvailability()
}
// MARK: - Lazy Services
@@ -132,24 +132,26 @@ final class AppServices: ObservableObject {
)
self._meetingSessionCoordinator = coordinator
self.setupMeetingCoordinatorForwarding()
- // The auto-detected episode itself stays consumed until the meeting's own evidence ends
- // (window/URL gone ≥60s) — not merely because our recording stopped — so only the nudge
- // and the end-of-meeting watch are dropped here.
+ // Attach end monitoring to every captured call, including manual and menu-bar starts.
+ // Start-prompt episodes remain consumed until the meeting's own evidence ends.
coordinator.$state
.receive(on: DispatchQueue.main)
.sink { [weak self, weak coordinator] state in
// Deliveries are queued, so read the state as it is now before calling a recording over.
- switch coordinator?.state {
- case .preparing, .recording, .recordingDegraded: break
- default: self?._meetingAutoDetector?.recordingStopped()
- }
+ Self.synchronizeMeetingEndWatch(
+ state: coordinator?.state,
+ session: coordinator?.activeSession,
+ detector: self?._meetingAutoDetector
+ )
guard !MeetingOverlayVisibility.isVisible(for: state) else { return }
MeetingStillRecordingNudgeController.shared.hide()
}
.store(in: &self.cancellables)
Task { @MainActor in
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
MeetingRecordingPillController.shared.activate(coordinator: coordinator)
await coordinator.ensureRestored()
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
await coordinator.sweepExpiredAudio()
}
return coordinator
@@ -169,20 +171,26 @@ final class AppServices: ObservableObject {
browserTabReader: AXBrowserTabReader(),
activityGate: LiveDetectionActivityGate(),
clock: SystemClock(),
- isNativeDetectionEnabled: { SettingsStore.shared.meetingAutoDetectEnabled },
+ isNativeDetectionEnabled: { SettingsStore.shared.fluidMeetEnabled && SettingsStore.shared.meetingAutoDetectEnabled },
isBrowserDetectionEnabled: {
- SettingsStore.shared.meetingAutoDetectEnabled && SettingsStore.shared.meetingAutoDetectBrowserEnabled
+ SettingsStore.shared.fluidMeetEnabled && SettingsStore.shared.meetingAutoDetectEnabled && SettingsStore.shared.meetingAutoDetectBrowserEnabled
},
lastSpeechAt: { [weak self] in self?._meetingSessionCoordinator?.lastLiveSpeechAt }
)
self._meetingAutoDetector = detector
self.wireMeetingAutoDetector(detector)
- detector.start()
+ if SettingsStore.shared.fluidMeetEnabled { detector.start() }
DebugLogger.shared.info("detector-started", source: "MeetingAutoDetector")
return detector
}
private func wireMeetingAutoDetector(_ detector: MeetingAutoDetector) {
+ // The detector is lazy; a manual recording may already be running when it is created.
+ Self.synchronizeMeetingEndWatch(
+ state: self._meetingSessionCoordinator?.state,
+ session: self._meetingSessionCoordinator?.activeSession,
+ detector: detector
+ )
detector.onAutomaticTargetChanged = { [weak self] target in
self?.meetingAutomaticTarget = target
}
@@ -228,8 +236,33 @@ final class AppServices: ObservableObject {
}
}
+ /// Inputs must be the coordinator's current snapshot, not a queued state notification.
+ static func synchronizeMeetingEndWatch(
+ state: MeetingCoordinatorState?,
+ session: MeetingSession?,
+ detector: MeetingAutoDetector?
+ ) {
+ guard let detector else { return }
+ switch state {
+ case .preparing:
+ break
+ case let .recording(id), let .recordingDegraded(id):
+ guard let session, session.id == id else {
+ detector.recordingStopped()
+ return
+ }
+ detector.recordingStarted(
+ sessionID: id,
+ application: session.mode == .onlineCall ? session.capturedApplication : nil
+ )
+ default:
+ detector.recordingStopped()
+ }
+ }
+
private func startAutoDetectedRecording(episodeID: UUID) async throws {
- guard let detector = self._meetingAutoDetector,
+ guard SettingsStore.shared.fluidMeetEnabled,
+ let detector = self._meetingAutoDetector,
let target = detector.resolvedTarget(for: episodeID),
detector.canStart(episodeID: episodeID)
else {
@@ -280,6 +313,11 @@ final class AppServices: ObservableObject {
private var cancellables = Set<AnyCancellable>()
private init() {
+ NotificationCenter.default.publisher(for: .fluidMeetEnabledChanged)
+ .sink { [weak self] _ in
+ MainActor.assumeIsolated { self?.refreshFluidMeetAvailability() }
+ }
+ .store(in: &self.cancellables)
// CRITICAL: Services are NOT created here.
// They are created lazily on first access, which happens AFTER the UI is ready.
// This ensures SwiftUI's AttributeGraph has finished processing before
@@ -287,6 +325,30 @@ final class AppServices: ObservableObject {
DebugLogger.shared.info("📦 AppServices singleton created (services not yet initialized)", source: "AppServices")
}
+ /// Do not abandon an active recording or processing job when changing availability.
+ func setFluidMeetEnabled(_ enabled: Bool) {
+ guard enabled || !self.hasActiveMeetingSessionActivity else { return }
+ SettingsStore.shared.fluidMeetEnabled = enabled
+ }
+
+ private func refreshFluidMeetAvailability() {
+ guard SettingsStore.shared.fluidMeetEnabled else {
+ self._meetingAutoDetector?.stop()
+ self._meetingAutoDetector = nil
+ self.meetingAutomaticTarget = nil
+ self.meetingAutoDetectHealth = nil
+ MeetingDetectionPromptController.shared.cancelForFeatureDisable()
+ MeetingStillRecordingNudgeController.shared.hide()
+ MeetingRecordingPillController.shared.deactivate()
+ return
+ }
+ guard self.isUIReady else { return }
+ if let coordinator = self._meetingSessionCoordinator {
+ MeetingRecordingPillController.shared.activate(coordinator: coordinator)
+ }
+ _ = self.meetingAutoDetector
+ }
+
// MARK: - Change Forwarding
/// Forward AudioHardwareObserver changes to trigger UI updates
@@ -329,7 +391,7 @@ final class AppServices: ObservableObject {
// Access the properties to trigger lazy initialization
_ = self.audioObserver
_ = self.asr
- _ = self.meetingAutoDetector
+ self.refreshFluidMeetAvailability()
DebugLogger.shared.info("✅ All services initialized", source: "AppServices")
}
diff --git a/Sources/Fluid/Services/AutomaticDictionaryCorrectionTracker.swift b/Sources/Fluid/Services/AutomaticDictionaryCorrectionTracker.swift
index 2d11ceef..5edb9daa 100644
--- a/Sources/Fluid/Services/AutomaticDictionaryCorrectionTracker.swift
+++ b/Sources/Fluid/Services/AutomaticDictionaryCorrectionTracker.swift
@@ -183,7 +183,11 @@ final class AutomaticDictionaryCorrectionTracker {
private static let maximumFieldLength = 100_000
private static let verificationAttempts = 20
private static let verificationDelayNanoseconds: UInt64 = 50_000_000
- private static let observationDurationNanoseconds: UInt64 = 30_000_000_000
+ nonisolated static func observationDuration(hasVoiceSpellings: Bool) -> Duration {
+ // Keep observing through confirmation and the edit debounce. Ordinary dictations stay short.
+ hasVoiceSpellings ? DictionaryVoiceSpellingLearner.confirmationWindow + .seconds(5) : .seconds(30)
+ }
+
private static let completionSignalDelayNanoseconds: UInt64 = 1_000_000_000
private static let inactivityFallbackNanoseconds: UInt64 = 3_000_000_000
@@ -231,14 +235,7 @@ final class AutomaticDictionaryCorrectionTracker {
self.suggestionGeneration = UUID()
self.evidencePreparationTask?.cancel()
self.evidencePreparationTask = nil
- self.verificationTask?.cancel()
- self.verificationTask = nil
- self.timeoutTask?.cancel()
- self.timeoutTask = nil
- self.debounceTask?.cancel()
- self.debounceTask = nil
- self.removeObserver()
- self.session = nil
+ self.stopObservation()
DictionaryCorrectionOverlayController.shared.hide()
}
@@ -266,6 +263,11 @@ final class AutomaticDictionaryCorrectionTracker {
allowsInsertionAtEnd: allowsInsertionAtEnd
)
+ // Reject on the edit event, before the debounce can race the confirmation deadline.
+ if isInside, let recording = session.learningRecording {
+ DictionaryVoiceSpellingLearner.shared.cancel(recordingID: recording.id)
+ }
+
if var pending = session.pendingCorrection {
let continuesCandidate = pending.correctedRange.map {
AutomaticDictionaryCorrectionDetector.changeContinuesCandidate(
@@ -453,8 +455,12 @@ final class AutomaticDictionaryCorrectionTracker {
)
CFRunLoopAddSource(CFRunLoopGetMain(), AXObserverGetRunLoopSource(createdObserver), .commonModes)
+ if let learningRecording {
+ DictionaryVoiceSpellingLearner.shared.propose(learningRecording.alignment.voiceSpellings, recordingID: learningRecording.id)
+ }
+ let duration = Self.observationDuration(hasVoiceSpellings: learningRecording?.alignment.voiceSpellings.isEmpty == false)
self.timeoutTask = Task { @MainActor [weak self] in
- try? await Task.sleep(nanoseconds: Self.observationDurationNanoseconds)
+ try? await Task.sleep(for: duration)
guard !Task.isCancelled else { return }
self?.cancel()
}
@@ -483,6 +489,9 @@ final class AutomaticDictionaryCorrectionTracker {
insertedRange: pending.insertedRange,
allowsInsertionAtEnd: true
)
+ if let candidate {
+ DictionaryVoiceSpellingLearner.shared.reject(editedText: candidate.heardText)
+ }
var context: DictionaryLearningCorrectionContext?
if DictionaryMatcherExperiment.sharedFeaturesEnabled, let recording = self.session?.learningRecording, let range = candidate?.sourceUTF16Range {
let before = pending.beforeValue as NSString
@@ -618,6 +627,9 @@ final class AutomaticDictionaryCorrectionTracker {
}
private func stopObservation() {
+ if let recording = self.session?.learningRecording {
+ DictionaryVoiceSpellingLearner.shared.cancel(recordingID: recording.id)
+ }
self.verificationTask?.cancel()
self.verificationTask = nil
self.timeoutTask?.cancel()
diff --git a/Sources/Fluid/Services/AutomaticDictionaryTrainingSession.swift b/Sources/Fluid/Services/AutomaticDictionaryTrainingSession.swift
index 94520e2c..6775456c 100644
--- a/Sources/Fluid/Services/AutomaticDictionaryTrainingSession.swift
+++ b/Sources/Fluid/Services/AutomaticDictionaryTrainingSession.swift
@@ -244,9 +244,7 @@ final class AutomaticDictionaryTrainingSession: ObservableObject {
pronunciationGeneration == DictionaryMatcherExperiment.generation, !self.isCancelled else { return }
guard !filtered.accepted.isEmpty else {
self.hasError = true
- self.statusMessage = filtered.lookupAvailable
- ? "These recordings contain everyday words. Try again, or add the correction manually."
- : "Couldn't check these words. Try saving again, or add the correction manually."
+ self.statusMessage = filtered.rejectionMessage
return
}
self.persist(triggers: filtered.accepted)
diff --git a/Sources/Fluid/Services/BuiltInDictionary.swift b/Sources/Fluid/Services/BuiltInDictionary.swift
new file mode 100644
index 00000000..209139f3
--- /dev/null
+++ b/Sources/Fluid/Services/BuiltInDictionary.swift
@@ -0,0 +1,22 @@
+import Foundation
+
+/// Spellings shipped with the app and applied by the text dictionary layer. They do not appear in
+/// the Custom Dictionary list and cannot be edited or removed. A user entry with the same spelling
+/// always takes precedence.
+nonisolated enum BuiltInDictionary {
+ static let entries: [SettingsStore.CustomDictionaryEntry] = [
+ .init(triggers: ["claud", "clawed"], replacement: "Claude"),
+ .init(triggers: ["post hoc"], replacement: "PostHog"),
+ .init(triggers: ["huggingface", "hugging face"], replacement: "Hugging Face"),
+ .init(triggers: ["fluid voice"], replacement: "FluidVoice"),
+ ]
+
+ /// Built-in entries without any spelling the user already maps, so the user's choice wins.
+ static func entries(excludingSpellingsIn userEntries: [SettingsStore.CustomDictionaryEntry]) -> [SettingsStore.CustomDictionaryEntry] {
+ let taken = Set(userEntries.flatMap(\.triggers).map { $0.lowercased() })
+ return self.entries.compactMap { entry in
+ let triggers = entry.triggers.filter { !taken.contains($0) }
+ return triggers.isEmpty ? nil : SettingsStore.CustomDictionaryEntry(id: entry.id, triggers: triggers, replacement: entry.replacement)
+ }
+ }
+}
diff --git a/Sources/Fluid/Services/DictationStopSnapshot.swift b/Sources/Fluid/Services/DictationStopSnapshot.swift
index d3b41da2..c18f063a 100644
--- a/Sources/Fluid/Services/DictationStopSnapshot.swift
+++ b/Sources/Fluid/Services/DictationStopSnapshot.swift
@@ -2,6 +2,12 @@ import AppKit
/// One immutable decision for a normal dictation, made before awaiting ASR finalization.
struct DictationStopSnapshot {
+ /// Command/Edit can reuse the running capture without changing its original field or formatting context.
+ static func captureStartingContext(reusingCapture: Bool, capture: () -> Void) {
+ guard !reusingCapture else { return }
+ capture()
+ }
+
let target: TypingService.RecordingTargetContext?
private(set) var appInfo: (name: String, bundleId: String, windowTitle: String)
let route: DictationProviderRoute
diff --git a/Sources/Fluid/Services/DictionaryAudioSpan.swift b/Sources/Fluid/Services/DictionaryAudioSpan.swift
new file mode 100644
index 00000000..14f1cfec
--- /dev/null
+++ b/Sources/Fluid/Services/DictionaryAudioSpan.swift
@@ -0,0 +1,110 @@
+import Foundation
+
+/// Text rules go first: words covered by a saved spelling, or already spelled correctly, belong to
+/// the text layer and an audio match never touches them. An audio match replaces only the other
+/// words, and always a whole run of words that looks like the target, never half of a mishearing.
+nonisolated enum DictionaryAudioSpan {
+ private static let maximumRunLength = 5
+ /// In the grey zone the written words must look at least this much like the word or a saved
+ /// spelling (1 = identical). Real grey-zone uses scored 0.57-0.86, e.g. "boob tea" for Booptie.
+ static let greyZoneMinimumResemblance = 0.5
+ /// A neighbouring word joins only when it makes the run clearly closer, so a stray letter
+ /// shared with the target ("is" + "Nickdom" against "Snigdham") does not pull it in.
+ private static let joinMargin = 0.1
+
+ /// Lowercased letters and digits only, so "Nickdom?" and "nickdom" compare equal.
+ static func normalized(_ text: String) -> String {
+ String(text.lowercased().unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) }.map(Character.init))
+ }
+
+ /// Indices of transcript words that a saved spelling, or an already correct word, covers.
+ static func textClaimed(words: [String], entries: [SettingsStore.CustomDictionaryEntry]) -> Set<Int> {
+ let tokens = words.map(self.normalized)
+ var claimed = Set<Int>()
+ for entry in entries {
+ for phrase in entry.triggers + [entry.replacement] {
+ let parts = phrase.split(whereSeparator: \.isWhitespace).map { self.normalized(String($0)) }.filter { !$0.isEmpty }
+ guard !parts.isEmpty, parts.count <= tokens.count else { continue }
+ for start in 0...(tokens.count - parts.count) where Array(tokens[start..<(start + parts.count)]) == parts {
+ claimed.formUnion(start..<(start + parts.count))
+ }
+ }
+ }
+ return claimed
+ }
+
+ /// The contiguous run of unclaimed words, starting from the words the match touched, that looks
+ /// most like one of `targets`. Neighbouring words join only while they make the run look closer.
+ static func span(words: [String], touched: [Int], claimed: Set<Int>, targets: [String]) -> ClosedRange<Int>? {
+ let tokens = words.map(self.normalized)
+ let goals = targets.map(self.normalized).filter { !$0.isEmpty }
+ guard !goals.isEmpty else { return nil }
+ func score(_ run: ClosedRange<Int>) -> Double {
+ let text = run.map { tokens[$0] }.joined()
+ return goals.map { self.similarity(text, $0) }.max() ?? 0
+ }
+ func free(_ index: Int) -> Bool { tokens.indices.contains(index) && !claimed.contains(index) }
+
+ var best: (run: ClosedRange<Int>, score: Double)?
+ for group in self.groups(touched.filter(free).sorted()) {
+ var run = group
+ var current = score(run)
+ while true {
+ var options: [(run: ClosedRange<Int>, gain: Double)] = []
+ if free(run.lowerBound - 1), run.count < self.maximumRunLength {
+ let grown = (run.lowerBound - 1)...run.upperBound
+ options.append((grown, score(grown) - current - self.joinMargin))
+ }
+ if free(run.upperBound + 1), run.count < self.maximumRunLength {
+ let grown = run.lowerBound...(run.upperBound + 1)
+ options.append((grown, score(grown) - current - self.joinMargin))
+ }
+ if run.count > 1 {
+ for shrunk in [(run.lowerBound + 1)...run.upperBound, run.lowerBound...(run.upperBound - 1)] {
+ options.append((shrunk, score(shrunk) - current))
+ }
+ }
+ guard let next = options.max(by: { $0.gain < $1.gain }), next.gain > 1e-9 else { break }
+ run = next.run
+ current = score(next.run)
+ }
+ if best.map({ current > $0.score }) ?? true { best = (run, current) }
+ }
+ return best?.run
+ }
+
+ /// How much a run of words looks like the closest target, 0-1.
+ static func resemblance(words: [String], run: ClosedRange<Int>, targets: [String]) -> Double {
+ guard run.lowerBound >= 0, run.upperBound < words.count else { return 0 }
+ let text = run.map { self.normalized(words[$0]) }.joined()
+ return targets.map { self.similarity(text, self.normalized($0)) }.max() ?? 0
+ }
+
+ /// 1 for identical text, falling toward 0 as more letters must change.
+ static func similarity(_ a: String, _ b: String) -> Double {
+ let x = Array(a), y = Array(b)
+ guard !x.isEmpty || !y.isEmpty else { return 1 }
+ var previous = Array(0...y.count)
+ for i in 1...max(1, x.count) where !x.isEmpty {
+ var row = [i] + [Int](repeating: 0, count: y.count)
+ for j in stride(from: 1, through: y.count, by: 1) {
+ row[j] = min(previous[j] + 1, row[j - 1] + 1, previous[j - 1] + (x[i - 1] == y[j - 1] ? 0 : 1))
+ }
+ previous = row
+ }
+ let distance = x.isEmpty ? y.count : previous[y.count]
+ return 1 - Double(distance) / Double(max(x.count, y.count))
+ }
+
+ private static func groups(_ sorted: [Int]) -> [ClosedRange<Int>] {
+ var result: [ClosedRange<Int>] = []
+ for index in sorted {
+ if let last = result.last, last.upperBound + 1 == index {
+ result[result.count - 1] = last.lowerBound...index
+ } else {
+ result.append(index...index)
+ }
+ }
+ return result
+ }
+}
diff --git a/Sources/Fluid/Services/DictionaryLearningEvidence.swift b/Sources/Fluid/Services/DictionaryLearningEvidence.swift
index fc7c8811..1fa3d440 100644
--- a/Sources/Fluid/Services/DictionaryLearningEvidence.swift
+++ b/Sources/Fluid/Services/DictionaryLearningEvidence.swift
@@ -6,6 +6,14 @@ nonisolated struct DictionaryLearningAlignment: Sendable {
let words: [ASRWordTiming]
var acousticOutput: String? = nil
var acousticEvidence: [DictionaryAcousticEvidence] = []
+ /// New spellings that voice matching replaced in this dictation, learned unless the user undoes them.
+ var voiceSpellings: [DictionaryVoiceSpelling] = []
+}
+
+nonisolated struct DictionaryVoiceSpelling: Equatable, Sendable {
+ let entryID: UUID
+ let label: String
+ let spelling: String
}
/// Short-lived ownership of original PCM, independent from the consume-once history snapshot.
diff --git a/Sources/Fluid/Services/DictionaryNegativeExamples.swift b/Sources/Fluid/Services/DictionaryNegativeExamples.swift
index 6f84b335..911fc877 100644
--- a/Sources/Fluid/Services/DictionaryNegativeExamples.swift
+++ b/Sources/Fluid/Services/DictionaryNegativeExamples.swift
@@ -9,6 +9,11 @@ nonisolated struct DictionaryAcousticEvidence: Sendable {
let modelKey: String
let sourceWordRange: Range<Int>
let frames: DictionaryMatchFrames
+ var matchFrameRange: Range<Int>? = nil
+
+ func aligned(to wordRange: Range<Int>, label: String) -> Self {
+ Self(id: self.id, entryID: self.entryID, label: label, profileKey: self.profileKey, modelKey: self.modelKey, sourceWordRange: wordRange, frames: self.frames, matchFrameRange: self.matchFrameRange)
+ }
}
nonisolated struct DictionaryNegativeCorrection: Sendable {
diff --git a/Sources/Fluid/Services/DictionaryPronunciationExperiment.swift b/Sources/Fluid/Services/DictionaryPronunciationExperiment.swift
index 7a307610..39310c79 100644
--- a/Sources/Fluid/Services/DictionaryPronunciationExperiment.swift
+++ b/Sources/Fluid/Services/DictionaryPronunciationExperiment.swift
@@ -23,6 +23,52 @@ nonisolated enum DictionaryPronunciationExperiment {
return max(range.lowerBound, first - 1920)..<min(range.upperBound, last + 1920)
}
+ /// Cuts a single-word recording to the word itself. The recognizer's word times locate it;
+ /// the word's own loudness tightens the edges, and a final p/t/k pop after a short gap is kept.
+ /// Absolute loudness alone keeps room noise and silence, so it is only the fallback.
+ static func wordRange(_ samples: [Float], words: [DictionaryAudioInspection.Word]) -> Range<Int>? {
+ guard let first = words.first, let last = words.last, !samples.isEmpty else { return nil }
+ let rate = 16_000.0, block = 160
+ func rms(_ range: Range<Int>) -> Float {
+ let slice = samples[range]
+ return sqrt(slice.reduce(Float(0)) { $0 + $1 * $1 } / Float(max(1, slice.count)))
+ }
+ let lower = max(0, Int((first.start - 0.16) * rate))
+ let upper = min(samples.count, Int((last.end + 0.16) * rate))
+ guard upper - lower >= block * 3 else { return nil }
+ let levels = stride(from: lower, to: upper - block + 1, by: block).map { rms($0..<($0 + block)) }
+ let leadingNoise = stride(from: 0, to: min(samples.count, 4800) - block + 1, by: block)
+ .map { rms($0..<($0 + block)) }.sorted()
+ let noise = leadingNoise.isEmpty ? 0 : leadingNoise[leadingNoise.count / 2]
+ let gate = max((levels.max() ?? 0) * 0.08, noise * 3)
+ guard let firstLoud = levels.firstIndex(where: { $0 > gate }),
+ let lastLoud = levels.lastIndex(where: { $0 > gate }) else { return nil }
+ var end = lower + (lastLoud + 1) * block
+ let burstLimit = min(samples.count, end + 4000)
+ var position = end
+ while position + block <= burstLimit {
+ if rms(position..<(position + block)) > max(noise * 4, 0.003) { end = position + block }
+ position += block
+ }
+ let pad = 480
+ let start = max(0, lower + firstLoud * block - pad)
+ end = min(samples.count, end + pad)
+ return start < end ? start..<end : nil
+ }
+
+ /// Encoder frames (1280 samples each) that contain speech. Words recorded slowly, such as
+ /// "Slides… A… I", carry pauses that vanish when the same word is said quickly in a sentence.
+ static func spokenFrames(_ samples: [Float], frames: Range<Int>) -> [Int] {
+ let levels = frames.map { frame -> Float in
+ let lower = min(samples.count, frame * 1280), upper = min(samples.count, lower + 1280)
+ guard upper > lower else { return 0 }
+ return sqrt(samples[lower..<upper].reduce(Float(0)) { $0 + $1 * $1 } / Float(upper - lower))
+ }
+ let gate = max((levels.max() ?? 0) * 0.08, 1e-4)
+ let spoken = zip(frames, levels).filter { $0.1 > gate }.map(\.0)
+ return spoken.count >= 3 ? spoken : Array(frames)
+ }
+
// swiftlint:disable:next discouraged_optional_collection
static func normalized(_ values: [Float]) -> [Float]? {
guard !values.isEmpty, values.allSatisfy(\.isFinite) else { return nil }
diff --git a/Sources/Fluid/Services/DictionaryPronunciationReferences.swift b/Sources/Fluid/Services/DictionaryPronunciationReferences.swift
index c92395c9..2714b9c4 100644
--- a/Sources/Fluid/Services/DictionaryPronunciationReferences.swift
+++ b/Sources/Fluid/Services/DictionaryPronunciationReferences.swift
@@ -12,7 +12,9 @@ enum DictionaryPronunciationReferences {
static func make(profiles: [PronunciationDictionaryProfile], hiddenSize: Int? = nil, compatibleModelKey: String? = nil) -> [Reference] {
profiles.flatMap { profile -> [Reference] in
let compatible = compatibleModelKey.map {
- profile.modelKey == $0 && ParakeetSpeechModelCatalog.descriptor(forInstalledPronunciationModelKey: $0) != nil
+ (profile.modelKey == $0 || ParakeetSpeechModelCatalog.sharesPronunciationFamily(profile.modelKey, with: $0)) &&
+ (ParakeetSpeechModelCatalog.descriptor(forInstalledPronunciationModelKey: $0) != nil ||
+ ParakeetSpeechModelCatalog.sharesPronunciationFamily($0, with: $0))
} ?? !ParakeetSpeechModelCatalog.isOutdatedCompactPronunciationModelKey(profile.modelKey)
guard compatible, profile.enrollments.allSatisfy({ $0.modelKey == profile.modelKey })
else { return [] }
diff --git a/Sources/Fluid/Services/DictionaryStampMatcher.swift b/Sources/Fluid/Services/DictionaryStampMatcher.swift
new file mode 100644
index 00000000..b8669c70
--- /dev/null
+++ b/Sources/Fluid/Services/DictionaryStampMatcher.swift
@@ -0,0 +1,190 @@
+import Accelerate
+import Foundation
+
+/// Finds a recorded word inside a dictation by sliding each recording's encoder frames along
+/// the dictation (subsequence dynamic time warping). The best stretch may start and end anywhere,
+/// and its length may differ from the recording, so the word can be faster or slower in a sentence.
+nonisolated enum DictionaryStampMatcher {
+ // Distance = mean per-frame cosine distance along the matched path (0 identical, 1 unrelated).
+ // Tuned 2026-10-20 on 2,007 checks (5 words x the user's last 500 dictations + test sentences):
+ // real uses 0.03-0.20, deliberate sound-alikes 0.180 and 0.198, ordinary speech never below 0.268.
+ // At these values: 40 of 43 real uses caught, 0 of 1,964 others swapped (a fixed 0.14 caught 32).
+
+ /// Below this, the sound alone is enough to replace.
+ static let sureDistance: Float = 0.16
+ /// Between `sureDistance` and this, replace only if the words also look like the word
+ /// (see `DictionaryAudioSpan.greyZoneMinimumResemblance`). At or above it, never replace.
+ static let greyZoneLimit: Float = 0.175
+ static let maximumQueryFrames = 1500
+
+ struct Hit: Equatable, Sendable {
+ let frames: Range<Int>
+ let distance: Float
+ }
+
+ /// A dictation's frames, normalized once and shared by every dictionary word's search.
+ struct Query: Sendable {
+ let values: [Float]
+ let count: Int
+ let hiddenSize: Int
+ }
+
+ static func prepare(_ query: DictionaryMatchFrames) -> Query? {
+ let m = query.count
+ guard m > 0, m <= self.maximumQueryFrames, query.values.count == m * query.hiddenSize,
+ query.values.allSatisfy(\.isFinite) else { return nil }
+ return Query(values: self.normalizedRows(query), count: m, hiddenSize: query.hiddenSize)
+ }
+
+ /// Up to `limit` non-overlapping stretches of `query` closer than `greyZoneLimit` to any reference.
+ static func hits(references: [DictionaryMatchFrames], in query: DictionaryMatchFrames, limit: Int = 3) -> [Hit] {
+ guard let prepared = self.prepare(query) else { return [] }
+ return self.hits(references: references, in: prepared, limit: limit)
+ }
+
+ static func hits(references: [DictionaryMatchFrames], in query: Query, limit: Int = 3) -> [Hit] {
+ self.hits(words: [references], in: query, limit: limit).first ?? []
+ }
+
+ /// Every dictionary word's recordings against one dictation. All recording frames are compared
+ /// with the dictation in a single matrix multiply, about 7x faster than one multiply per recording
+ /// (100 words: 0.8 ms instead of 5.5 ms on an M-series Mac).
+ static func hits(words: [[DictionaryMatchFrames]], in query: Query, limit: Int = 3) -> [[Hit]] {
+ let m = query.count, hidden = query.hiddenSize
+ let usable = words.map { references in
+ references.filter {
+ $0.hiddenSize == hidden && !$0.values.isEmpty && $0.values.count == $0.count * hidden && $0.values.allSatisfy(\.isFinite)
+ }
+ }
+ let totalRows = usable.reduce(0) { $0 + $1.reduce(0) { $0 + $1.count } }
+ guard m > 0, totalRows > 0 else { return words.map { _ in [] } }
+ var stacked = [Float]()
+ stacked.reserveCapacity(totalRows * hidden)
+ for references in usable {
+ for reference in references {
+ stacked += self.normalizedRows(reference)
+ }
+ }
+ var similarity = [Float](repeating: 0, count: totalRows * m)
+ stacked.withUnsafeBufferPointer { r in
+ query.values.withUnsafeBufferPointer { q in
+ similarity.withUnsafeMutableBufferPointer { s in
+ cblas_sgemm(
+ CblasRowMajor,
+ CblasNoTrans,
+ CblasTrans,
+ Int32(totalRows),
+ Int32(m),
+ Int32(hidden),
+ 1,
+ r.baseAddress,
+ Int32(hidden),
+ q.baseAddress,
+ Int32(hidden),
+ 0,
+ s.baseAddress,
+ Int32(m)
+ )
+ }
+ }
+ }
+ var row = 0
+ return similarity.withUnsafeBufferPointer { sim in
+ usable.map { references in
+ var bestDistance = [Float](repeating: .infinity, count: m)
+ var bestStart = [Int](repeating: 0, count: m)
+ for reference in references {
+ let n = reference.count
+ let (distances, starts) = self.endScores(similarity: UnsafeBufferPointer(rebasing: sim[(row * m)..<((row + n) * m)]), rows: n, columns: m)
+ row += n
+ for j in 0..<m where distances[j] < bestDistance[j] {
+ bestDistance[j] = distances[j]
+ bestStart[j] = starts[j]
+ }
+ }
+ return self.pick(bestDistance: bestDistance, bestStart: bestStart, limit: limit)
+ }
+ }
+ }
+
+ private static func pick(bestDistance: [Float], bestStart: [Int], limit: Int) -> [Hit] {
+ let m = bestDistance.count
+ var hits: [Hit] = []
+ var available = [Bool](repeating: true, count: m)
+ while hits.count < limit {
+ var candidate: Int?
+ for j in 0..<m where available[j] && bestDistance[j] < self.greyZoneLimit {
+ if candidate.map({ bestDistance[j] < bestDistance[$0] }) ?? true { candidate = j }
+ }
+ guard let end = candidate else { break }
+ let range = bestStart[end]..<(end + 1)
+ hits.append(Hit(frames: range, distance: bestDistance[end]))
+ for j in 0..<m where available[j] && (bestStart[j]..<(j + 1)).overlaps(range) {
+ available[j] = false
+ }
+ }
+ return hits
+ }
+
+ private static func normalizedRows(_ frames: DictionaryMatchFrames) -> [Float] {
+ var values = frames.values
+ let hidden = frames.hiddenSize
+ for row in 0..<frames.count {
+ let start = row * hidden
+ var norm: Float = 0
+ values.withUnsafeBufferPointer { buffer in
+ guard let base = buffer.baseAddress else { return }
+ vDSP_svesq(base + start, 1, &norm, vDSP_Length(hidden))
+ }
+ let scale = 1 / max(sqrt(norm), 1e-8)
+ for d in start..<(start + hidden) {
+ values[d] *= scale
+ }
+ }
+ return values
+ }
+
+ /// For every possible end frame in the query: the length-normalized path cost of the best
+ /// alignment of the whole reference ending there, and where that alignment starts.
+ private static func endScores(similarity: UnsafeBufferPointer<Float>, rows n: Int, columns m: Int) -> ([Float], [Int]) {
+ // Two rows swapped in place: no per-row copies, no bounds checks in the hot loop.
+ var previousCost = [Float](repeating: 0, count: m), cost = [Float](repeating: 0, count: m)
+ var previousStart = Array(0..<m), start = Array(0..<m)
+ for j in 0..<m {
+ previousCost[j] = 1 - similarity[j]
+ }
+ let sim = similarity
+ for i in 1..<max(1, n) {
+ previousCost.withUnsafeBufferPointer { pc in
+ previousStart.withUnsafeBufferPointer { ps in
+ cost.withUnsafeMutableBufferPointer { c in
+ start.withUnsafeMutableBufferPointer { st in
+ let row = i * m
+ c[0] = pc[0] + 1 - sim[row]
+ st[0] = ps[0]
+ for j in 1..<max(1, m) {
+ var best = pc[j], from = ps[j]
+ if pc[j - 1] < best { best = pc[j - 1]; from = ps[j - 1] }
+ if c[j - 1] < best { best = c[j - 1]; from = st[j - 1] }
+ c[j] = best + 1 - sim[row + j]
+ st[j] = from
+ }
+ }
+ }
+ }
+ }
+ swap(&previousCost, &cost)
+ swap(&previousStart, &start)
+ }
+ cost = previousCost
+ start = previousStart
+ var distances = [Float](repeating: .infinity, count: m)
+ for j in 0..<m {
+ let matched = j + 1 - start[j]
+ // A word in a sentence may be up to twice as fast or slow as the recording, no more.
+ guard matched * 2 >= n, matched <= n * 2 else { continue }
+ distances[j] = cost[j] / Float(matched + n)
+ }
+ return (distances, start)
+ }
+}
diff --git a/Sources/Fluid/Services/DictionaryTrainingEndpointMonitor.swift b/Sources/Fluid/Services/DictionaryTrainingEndpointMonitor.swift
index 1bc4e5ca..2516f4b7 100644
--- a/Sources/Fluid/Services/DictionaryTrainingEndpointMonitor.swift
+++ b/Sources/Fluid/Services/DictionaryTrainingEndpointMonitor.swift
@@ -1,5 +1,41 @@
+import Combine
import Foundation
+/// Gives the user a visible break between examples. Initial capture is never delayed.
+@MainActor
+final class DictionaryTrainingRestart: ObservableObject {
+ @Published private(set) var isWaiting = false
+ private var task: Task<Void, Never>?
+ private var requestID: UUID?
+
+ func schedule(
+ delay: Duration = .seconds(1),
+ canContinue: @escaping () -> Bool,
+ restart: @escaping () async -> Void
+ ) {
+ self.cancel()
+ let requestID = UUID()
+ self.requestID = requestID
+ self.isWaiting = true
+ self.task = Task { @MainActor [weak self] in
+ do { try await Task.sleep(for: delay) } catch { return }
+ guard let self, self.requestID == requestID, !Task.isCancelled else { return }
+ self.task = nil
+ self.requestID = nil
+ self.isWaiting = false
+ guard canContinue() else { return }
+ await restart()
+ }
+ }
+
+ func cancel() {
+ self.task?.cancel()
+ self.task = nil
+ self.requestID = nil
+ self.isWaiting = false
+ }
+}
+
struct DictionaryTrainingAudioCursor {
private(set) var sampleOffset = 0
private var generation: Int
diff --git a/Sources/Fluid/Services/DictionaryVoiceSpellingLearner.swift b/Sources/Fluid/Services/DictionaryVoiceSpellingLearner.swift
new file mode 100644
index 00000000..8db0180f
--- /dev/null
+++ b/Sources/Fluid/Services/DictionaryVoiceSpellingLearner.swift
@@ -0,0 +1,134 @@
+import Foundation
+
+/// Voice matching caught a word under a new spelling ("Nickdom" for Snigdham). If the user keeps
+/// the replacement, the spelling becomes a saved text rule, so next time the text layer replaces it
+/// directly and voice matching is not needed. Editing the replaced word within the window drops it.
+@MainActor
+final class DictionaryVoiceSpellingLearner {
+ static let shared = DictionaryVoiceSpellingLearner()
+
+ nonisolated static let confirmationWindow: Duration = .seconds(120)
+ private static let maximumPending = 20
+ private static let maximumSpellingsPerWord = 20
+
+ private struct Pending {
+ let id: UUID
+ let spelling: DictionaryVoiceSpelling
+ let recordingID: UUID?
+ let task: Task<Void, Never>
+ }
+
+ private var pending: [Pending] = []
+ private let window: Duration
+ private let isEnabled: @MainActor () -> Bool
+ private let canWrite: @MainActor () -> Bool
+ private let load: @MainActor () -> [SettingsStore.CustomDictionaryEntry]
+ private let save: @MainActor ([SettingsStore.CustomDictionaryEntry]) -> Void
+
+ init(
+ window: Duration = DictionaryVoiceSpellingLearner.confirmationWindow,
+ isEnabled: @escaping @MainActor () -> Bool = { SettingsStore.shared.automaticDictionaryLearningEnabled },
+ canWrite: @escaping @MainActor () -> Bool = { !AppServices.shared.asr.isRunningOrStarting },
+ load: @escaping @MainActor () -> [SettingsStore.CustomDictionaryEntry] = { SettingsStore.shared.customDictionaryEntries },
+ save: @escaping @MainActor ([SettingsStore.CustomDictionaryEntry]) -> Void = {
+ SettingsStore.shared.customDictionaryEntries = $0
+ // The text layer caches compiled rules; without this the learned spelling would not apply yet.
+ // No vocabulary notification: that would restart the speech engine for a text-only change.
+ ASRService.invalidateDictionaryCache()
+ }
+ ) {
+ self.window = window
+ self.isEnabled = isEnabled
+ self.canWrite = canWrite
+ self.load = load
+ self.save = save
+ }
+
+ var pendingCount: Int { self.pending.count }
+
+ /// Called once the dictation has been delivered where its edits can be observed.
+ func propose(_ spellings: [DictionaryVoiceSpelling], recordingID: UUID? = nil) {
+ guard self.isEnabled() else { return }
+ for spelling in spellings where Self.isLearnable(spelling) {
+ guard self.pending.count < Self.maximumPending,
+ !self.pending.contains(where: { $0.spelling == spelling }) else { continue }
+ let id = UUID()
+ let window = self.window
+ let task = Task { @MainActor [weak self] in
+ do { try await Task.sleep(for: window) } catch { return }
+ // Keep learned-rule writes outside an active capture.
+ for _ in 0..<30 {
+ guard let self, !Task.isCancelled else { return }
+ if self.canWrite() { break }
+ do { try await Task.sleep(for: .seconds(2)) } catch { return }
+ }
+ self?.commit(id)
+ }
+ self.pending.append(Pending(id: id, spelling: spelling, recordingID: recordingID, task: task))
+ }
+ }
+
+ /// The user edited a delivered word. Any pending spelling for that word was a wrong match.
+ func reject(editedText: String) {
+ let edited = DictionaryAudioSpan.normalized(editedText)
+ guard !edited.isEmpty else { return }
+ for item in self.pending where DictionaryAudioSpan.normalized(item.spelling.label) == edited {
+ item.task.cancel()
+ DebugLogger.shared.info("DICTIONARY_VOICE_SPELLING dropped after user edit", source: "CustomDictionary")
+ }
+ self.pending.removeAll { DictionaryAudioSpan.normalized($0.spelling.label) == edited }
+ }
+
+ func cancelAll() {
+ self.pending.forEach { $0.task.cancel() }
+ self.pending.removeAll()
+ }
+
+ /// An unobserved field cannot confirm that its replacements were kept.
+ func cancel(recordingID: UUID) {
+ for item in self.pending where item.recordingID == recordingID {
+ item.task.cancel()
+ }
+ self.pending.removeAll { $0.recordingID == recordingID }
+ }
+
+ private func commit(_ id: UUID) {
+ guard let index = self.pending.firstIndex(where: { $0.id == id }) else { return }
+ let spelling = self.pending.remove(at: index).spelling
+ guard self.isEnabled(), self.canWrite() else { return }
+ var entries = self.load()
+ // The word may have been deleted or renamed meanwhile; only add to the same word.
+ guard let entryIndex = entries.firstIndex(where: {
+ $0.id == spelling.entryID && $0.replacement.caseInsensitiveCompare(spelling.label) == .orderedSame
+ }) else { return }
+ let trigger = spelling.spelling.trimmingCharacters(in: .whitespacesAndNewlines.union(.punctuationCharacters)).lowercased()
+ let existing = entries[entryIndex].triggers.map(DictionaryAudioSpan.normalized)
+ guard !trigger.isEmpty, !existing.contains(DictionaryAudioSpan.normalized(trigger)),
+ entries[entryIndex].triggers.count < Self.maximumSpellingsPerWord else { return }
+ // A manual rule may have been added while this proposal was waiting. It owns its
+ // spellings and canonical words, including phrases contained in this new spelling.
+ let otherEntries = entries.filter { $0.id != spelling.entryID }
+ let words = trigger.split(whereSeparator: \.isWhitespace).map(String.init)
+ guard DictionaryAudioSpan.textClaimed(words: words, entries: otherEntries).isEmpty else { return }
+ entries[entryIndex] = SettingsStore.CustomDictionaryEntry(
+ id: entries[entryIndex].id,
+ triggers: entries[entryIndex].triggers + [trigger],
+ replacement: entries[entryIndex].replacement
+ )
+ self.save(entries)
+ DebugLogger.shared.info("DICTIONARY_VOICE_SPELLING saved new spelling for a dictionary word", source: "CustomDictionary")
+ }
+
+ /// Never learn a spelling that is the word itself, or a bare common word that would
+ /// replace ordinary speech everywhere.
+ static func isLearnable(_ spelling: DictionaryVoiceSpelling) -> Bool {
+ let heard = DictionaryAudioSpan.normalized(spelling.spelling)
+ guard !heard.isEmpty, heard.count <= 60, heard != DictionaryAudioSpan.normalized(spelling.label) else { return false }
+ return !Self.commonWords.contains(heard)
+ }
+
+ private static let commonWords: Set<String> = [
+ "a", "an", "the", "and", "or", "but", "to", "of", "in", "on", "at", "it", "is", "i", "you", "he", "she",
+ "we", "they", "me", "my", "be", "do", "so", "no", "yes", "this", "that", "there", "then", "them", "with",
+ ]
+}
diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swift
index 39eb8965..5be15d2b 100644
--- a/Sources/Fluid/Services/FluidAudioProvider.swift
+++ b/Sources/Fluid/Services/FluidAudioProvider.swift
@@ -274,7 +274,15 @@ final class FluidAudioProvider: TranscriptionProvider {
private var incrementalSharedFeatures = false
private var incrementalSharedEvidence: [Int: [(Range<Int>, DictionaryAcousticEvidence)]] = [:]
private var temporalModels: AsrModels?
- private var temporalReferenceCache = TemporalReferenceCache()
+ /// Prepared word recordings, per speech model. Saving a dictionary word replaces this provider, so the
+ /// cache lives outside it: otherwise every voice word would be re-encoded after each save, and the next
+ /// dictation would wait for all of them instead of only the new word.
+ private static var sharedTemporalReferenceCaches: [String: TemporalReferenceCache] = [:]
+ private var temporalReferenceCache: TemporalReferenceCache {
+ get { Self.sharedTemporalReferenceCaches[self.pronunciationModelKey] ?? TemporalReferenceCache() }
+ set { Self.sharedTemporalReferenceCaches[self.pronunciationModelKey] = newValue }
+ }
+
private var temporalWarmTask: Task<Void, Never>?
private var temporalWarmRequest: (profiles: [PronunciationDictionaryProfile], generation: String)?
private var pronunciationProfilesToWarm: [PronunciationDictionaryProfile] = []
@@ -363,7 +371,7 @@ final class FluidAudioProvider: TranscriptionProvider {
self.didLoadAutomaticPronunciationProfiles = true
return
}
- let stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)
+ let stored = await self.pronunciationStore.sharedProfiles(modelKey: self.pronunciationModelKey, hiddenSize: self.temporalModels?.version.encoderHiddenSize)
try self.requireCurrentRecording(generation)
self.automaticPronunciationProfiles = Self.matchingProfiles(
stored,
@@ -611,7 +619,7 @@ final class FluidAudioProvider: TranscriptionProvider {
}
if DictionaryMatcherExperiment.sharedFeaturesEnabled, self.meetingOptions == nil {
- let stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)
+ let stored = await self.pronunciationStore.sharedProfiles(modelKey: self.pronunciationModelKey, hiddenSize: self.temporalModels?.version.encoderHiddenSize)
let eligible = Self.matchingProfiles(stored, entries: self.effectiveCustomDictionaryEntries, includeManual: self.explicitPronunciationMatchingEnabled)
self.scheduleTemporalWarmup(eligible)
}
@@ -981,13 +989,23 @@ final class FluidAudioProvider: TranscriptionProvider {
for (frames, item) in items {
let indices = wordIndex.substantiallyOverlappingWordIndices(startTime: Double(frames.lowerBound) * 0.08, endTime: Double(frames.upperBound) * 0.08, minimumOverlapRatio: 0)
guard let first = indices.first, let last = indices.last else { continue }
- acousticEvidence.append(DictionaryAcousticEvidence(id: item.id, entryID: item.entryID, label: item.label, profileKey: item.profileKey, modelKey: item.modelKey, sourceWordRange: first..<(last + 1), frames: item.frames))
+ acousticEvidence.append(DictionaryAcousticEvidence(
+ id: item.id,
+ entryID: item.entryID,
+ label: item.label,
+ profileKey: item.profileKey,
+ modelKey: item.modelKey,
+ sourceWordRange: first..<(last + 1),
+ frames: item.frames,
+ matchFrameRange: frames
+ ))
}
}
} else if let manager = self.finalAsrManager ?? self.streamingAsrManager, originalText == result.text {
matches = try await self.refineEdgeMatches(matches, profiles: profiles, samples: samples, manager: manager, transcript: result.text, timings: result.tokenTimings ?? [], evidence: &acousticEvidence, temporalMatches: &temporalMatches)
}
try self.requireCurrentRecording(generation)
+ var voiceSpellings: [DictionaryVoiceSpelling] = []
let text = self.effectivePronunciationMatchingEnabled && originalText == result.text
? Self.applyPronunciationMatches(
result: result,
@@ -995,9 +1013,11 @@ final class FluidAudioProvider: TranscriptionProvider {
profiles: profiles,
labels: Self.dictionaryLabels(from: self.effectiveCustomDictionaryEntries).filter { eligibleIDs.contains($0.key) },
temporalMatches: temporalMatches,
- onAccepted: { id, label, range in
- acceptedEvidence += acousticEvidence.filter { $0.entryID == id && $0.label == label && $0.sourceWordRange == range }
- }
+ entries: self.effectiveCustomDictionaryEntries + BuiltInDictionary.entries,
+ onAccepted: { id, label, range, frames in
+ acceptedEvidence += Self.acceptedAcousticEvidence(acousticEvidence, entryID: id, label: label, wordRange: range, frameRange: frames)
+ },
+ onReplaced: { voiceSpellings.append($0) }
) : result.text
let finishFinishedAt = ProcessInfo.processInfo.systemUptime
DebugLogger.shared.debug(
@@ -1010,6 +1030,7 @@ final class FluidAudioProvider: TranscriptionProvider {
var alignment = self.learningAlignment(for: result)
alignment?.acousticOutput = text
alignment?.acousticEvidence = acceptedEvidence
+ alignment?.voiceSpellings = voiceSpellings
return ASRTranscriptionResult(
text: text,
confidence: result.confidence,
@@ -1058,7 +1079,7 @@ final class FluidAudioProvider: TranscriptionProvider {
guard self.effectivePronunciationMatchingEnabled else { return [] }
let stored: [PronunciationDictionaryProfile]
if self.explicitPronunciationMatchingEnabled {
- stored = await self.pronunciationStore.profiles(modelKey: self.pronunciationModelKey)
+ stored = await self.pronunciationStore.sharedProfiles(modelKey: self.pronunciationModelKey, hiddenSize: self.temporalModels?.version.encoderHiddenSize)
} else {
stored = self.automaticPronunciationProfiles
}
@@ -1234,19 +1255,23 @@ final class FluidAudioProvider: TranscriptionProvider {
temporalMatches: &temporalMatches,
features: features
)
+ var voiceSpellings: [DictionaryVoiceSpelling] = []
let corrected = Self.applyPronunciationMatches(
result: result,
matches: DictionaryMatcherExperiment.sharedFeaturesEnabled ? refined : [],
profiles: references.map(\.profile),
labels: Self.dictionaryLabels(from: self.effectiveCustomDictionaryEntries),
temporalMatches: temporalMatches,
- onAccepted: { id, label, range in
- acceptedEvidence += acousticEvidence.filter { $0.entryID == id && $0.label == label && $0.sourceWordRange == range }
- }
+ entries: self.effectiveCustomDictionaryEntries + BuiltInDictionary.entries,
+ onAccepted: { id, label, range, frames in
+ acceptedEvidence += Self.acceptedAcousticEvidence(acousticEvidence, entryID: id, label: label, wordRange: range, frameRange: frames)
+ },
+ onReplaced: { voiceSpellings.append($0) }
)
var alignment = self.learningAlignment(for: result)
alignment?.acousticOutput = corrected
alignment?.acousticEvidence = acceptedEvidence
+ alignment?.voiceSpellings = voiceSpellings
return (
ASRTranscriptionResult(
text: corrected,
@@ -1273,7 +1298,7 @@ final class FluidAudioProvider: TranscriptionProvider {
static func preparedEdgeProfiles(_ profiles: [PronunciationDictionaryProfile], cache: [UUID: PronunciationEmbedding]) -> [PronunciationDictionaryProfile] {
guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
return profiles.compactMap { profile in
- if profile.hasOriginalAudio { return profile }
+ if profile.hasOriginalAudio || !profile.hasTemporalRecordings { return profile }
var prepared = profile
for index in prepared.enrollments.indices {
if prepared.enrollments[index].edgeEmbedding != nil { continue }
@@ -1326,56 +1351,94 @@ final class FluidAudioProvider: TranscriptionProvider {
var used: [Int: [Range<Int>]] = [:]
let compareNegatives = DictionaryMatcherExperiment.compareNegatives
let collectNegatives = DictionaryMatcherExperiment.collectNegatives
- var sharedComparisons = 0
let negativeRevision = compareNegatives ? await DictionaryNegativeExampleStore.shared.revision() : nil
+ // Words recorded in the Add Word screen: slide each recording along the whole dictation.
+ // This does not depend on the averaged-window shortlist, which misses words said differently in a sentence.
+ let query = DictionaryStampMatcher.prepare(DictionaryMatchFrames(hiddenSize: features.hiddenSize, values: features.values))
+ // Right after a word is saved, its recordings are still being prepared. A dictation that
+ // finishes first would be checked against nothing, so wait briefly for that one preparation.
+ let deadline = ContinuousClock.now + .seconds(2)
+ while self.temporalWarmTask != nil, ContinuousClock.now < deadline,
+ profiles.contains(where: { $0.hasTemporalRecordings && self.temporalReferenceCache[DictionaryNegativeEvidenceResolver.profileKey($0)] == nil })
+ {
+ try await Task.sleep(for: .milliseconds(40))
+ }
+ var stampedProfiles = Set<String>()
+ var stamped: [(index: Int, profile: PronunciationDictionaryProfile, referenceKey: String, references: [DictionaryMatchFrames])] = []
+ for (index, profile) in profiles.enumerated() where profile.hasTemporalRecordings {
+ let referenceKey = DictionaryNegativeEvidenceResolver.profileKey(profile)
+ guard stampedProfiles.insert(referenceKey).inserted,
+ let references = self.temporalReferenceCache[referenceKey] else { continue }
+ stamped.append((index, profile, referenceKey, references))
+ }
+ // One comparison for every recorded word at once; per word it costs a fraction of a millisecond.
+ let stampHits = query.map { DictionaryStampMatcher.hits(words: stamped.map(\.references), in: $0) } ?? []
+ for (word, hits) in zip(stamped, stampHits) {
+ try Task.checkCancellation()
+ let (index, profile, referenceKey, references) = word
+ // Old isolated-query negatives must not be compared with sentence-context queries.
+ let targetSuffix = profile.modelKey == self.pronunciationModelKey ? "" : ":" + self.pronunciationModelKey
+ let evidenceKey = referenceKey + ":shared-features-v1" + targetSuffix
+ for hit in hits {
+ let indices = wordIndex.substantiallyOverlappingWordIndices(
+ startTime: Double(hit.frames.lowerBound) * 0.08,
+ endTime: Double(hit.frames.upperBound) * 0.08,
+ minimumOverlapRatio: 0
+ )
+ guard let first = indices.first, let last = indices.last else { continue }
+ let hitFrames = DictionaryMatchFrames(
+ hiddenSize: features.hiddenSize,
+ values: Array(features.values[(hit.frames.lowerBound * features.hiddenSize)..<(hit.frames.upperBound * features.hiddenSize)])
+ )
+ if compareNegatives {
+ let negatives = await DictionaryNegativeExampleStore.shared.frames(entryID: profile.dictionaryEntryID, profileKey: evidenceKey, modelKey: self.pronunciationModelKey)
+ let allowed = await DictionaryExperimentalMatcher.compareNegative(query: hitFrames, references: references, negatives: negatives)
+ if !allowed, DictionaryMatcherExperiment.compareNegatives,
+ await DictionaryNegativeExampleStore.shared.revision() == negativeRevision { continue }
+ }
+ try Task.checkCancellation()
+ guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { continue }
+ temporalMatches.insert(TemporalMatchKey(prototypeIndex: index, frameRange: hit.frames))
+ output.append(PronunciationWindowMatch(prototypeIndex: index, score: 1 - hit.distance, frameRange: hit.frames))
+ if collectNegatives, DictionaryMatcherExperiment.collectNegatives {
+ evidence.append(DictionaryAcousticEvidence(
+ id: UUID(),
+ entryID: profile.dictionaryEntryID,
+ label: profile.label,
+ profileKey: evidenceKey,
+ modelKey: self.pronunciationModelKey,
+ sourceWordRange: first..<(last + 1),
+ frames: hitFrames,
+ matchFrameRange: hit.frames
+ ))
+ }
+ DebugLogger.shared.info(
+ "DICTIONARY_STAMP_MATCH distance=\(String(format: "%.3f", hit.distance)) frames=\(hit.frames.lowerBound)-\(hit.frames.upperBound)",
+ source: "PronunciationMatching"
+ )
+ }
+ }
// Bounded experimental reranking: up to three non-overlapping candidates per word, 24 comparisons total.
for match in matches.sorted(by: { $0.score > $1.score }) {
try Task.checkCancellation()
guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { break }
- guard profiles.indices.contains(match.prototypeIndex) else { continue }
+ guard profiles.indices.contains(match.prototypeIndex),
+ !profiles[match.prototypeIndex].hasTemporalRecordings else { continue }
// Score the complete overlapping transcript span, never just the prefix of a split word.
let indices = wordIndex.substantiallyOverlappingWordIndices(
startTime: Double(match.frameRange.lowerBound) * 0.08,
endTime: Double(match.frameRange.upperBound) * 0.08,
minimumOverlapRatio: 0
)
- guard let first = indices.first, let last = indices.last else { continue }
- let firstFrame = max(0, Int(floor(words[first].startTime / 0.08)))
- let lastFrame = max(firstFrame + 1, Int(ceil(words[last].endTime / 0.08)))
- let alignedFrames = firstFrame..<lastFrame
- let lower = min(samples.count, firstFrame * 1280)
- let upper = min(samples.count, max(lower, lastFrame * 1280))
- let range = lower..<upper
- guard !range.isEmpty, range.count <= 16_000 * 15,
- (used[match.prototypeIndex]?.count ?? 0) < 3,
- !(used[match.prototypeIndex] ?? []).contains(where: { $0.overlaps(range) }),
- let trimmed = DictionaryPronunciationExperiment.trimmedRange(samples, within: range) else { continue }
- do {
- guard sharedComparisons < 24 else { break }
- used[match.prototypeIndex, default: []].append(range)
- sharedComparisons += 1
- let profile = profiles[match.prototypeIndex]
- let referenceKey = DictionaryNegativeEvidenceResolver.profileKey(profile)
- // Old isolated-query negatives must not be compared with sentence-context queries.
- let evidenceKey = referenceKey + ":shared-features-v1"
- guard let query = Self.sharedPronunciationFrames(features, sampleRange: trimmed),
- let references = self.temporalReferenceCache[referenceKey] else { continue }
- guard let decision = await DictionaryExperimentalMatcher.compareSharedFeatures(query: query, references: references, chunked: sharedChunk) else { continue }
- guard decision.accepted, DictionaryMatcherExperiment.sharedFeaturesEnabled else { continue }
- if compareNegatives {
- let negatives = await DictionaryNegativeExampleStore.shared.frames(entryID: profile.dictionaryEntryID, profileKey: evidenceKey, modelKey: profile.modelKey)
- let allowed = await DictionaryExperimentalMatcher.compareNegative(query: query, references: references, negatives: negatives)
- if !allowed, DictionaryMatcherExperiment.compareNegatives,
- await DictionaryNegativeExampleStore.shared.revision() == negativeRevision { continue }
- }
- try Task.checkCancellation()
- guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { continue }
- temporalMatches.insert(TemporalMatchKey(prototypeIndex: match.prototypeIndex, frameRange: alignedFrames))
- output.append(PronunciationWindowMatch(prototypeIndex: match.prototypeIndex, score: decision.meanRelative, frameRange: alignedFrames))
- if collectNegatives, DictionaryMatcherExperiment.collectNegatives {
- evidence.append(DictionaryAcousticEvidence(id: UUID(), entryID: profile.dictionaryEntryID, label: profile.label, profileKey: evidenceKey, modelKey: profile.modelKey, sourceWordRange: first..<(last + 1), frames: query))
- }
- continue
+ guard !indices.isEmpty else { continue }
+ let range = match.frameRange.lowerBound..<match.frameRange.upperBound
+ guard (used[match.prototypeIndex]?.count ?? 0) < 3,
+ !(used[match.prototypeIndex] ?? []).contains(where: { $0.overlaps(range) }) else { continue }
+ used[match.prototypeIndex, default: []].append(range)
+ // Words learned from corrections have no recordings; the averaged score decides.
+ let heard = indices.map { words[$0].text }.joined(separator: " ")
+ if DictionaryPronunciationDecision.accepts(score: match.score, heardText: heard, profile: profiles[match.prototypeIndex]) {
+ output.append(match)
}
}
return DictionaryMatcherExperiment.sharedFeaturesEnabled && DictionaryMatcherExperiment.generation == pronunciationGeneration ? output : []
@@ -1431,8 +1494,9 @@ final class FluidAudioProvider: TranscriptionProvider {
private func temporalReferences(_ profile: PronunciationDictionaryProfile, key: String, models: AsrModels) async throws -> [DictionaryMatchFrames] {
guard DictionaryMatcherExperiment.sharedFeaturesEnabled else { return [] }
- guard profile.modelKey == self.pronunciationModelKey,
- profile.enrollments.allSatisfy({ $0.modelKey == profile.modelKey })
+ guard profile.modelKey == self.pronunciationModelKey ||
+ ParakeetSpeechModelCatalog.sharesPronunciationFamily(profile.modelKey, with: self.pronunciationModelKey),
+ profile.enrollments.allSatisfy({ $0.modelKey == profile.modelKey })
else { throw PronunciationDictionaryStoreError.outdatedModelRevision }
if let cached = self.temporalReferenceCache[key] { return cached }
guard profile.enrollments.count >= 3 else { return [] }
@@ -1449,7 +1513,27 @@ final class FluidAudioProvider: TranscriptionProvider {
let samples: [Float]
if let id = capture.inspectionID {
let audio = try await self.pronunciationStore.inspection(for: id)
- samples = Array(audio.samples.prefix(audio.recordedSampleCount))
+ let recorded = Array(audio.samples.prefix(audio.recordedSampleCount))
+ // Encode the whole recording, then keep only the word's frames. The word keeps its
+ // natural surroundings for the encoder, and the reference excludes silence and noise.
+ if let word = DictionaryPronunciationExperiment.wordRange(recorded, words: audio.words),
+ recorded.count <= 240_000
+ {
+ let encoded = try await DictionaryTemporalEncoder.encode(recorded, models: models)
+ guard !encoded.values.isEmpty else { return [] }
+ let first = min(encoded.count - 1, word.lowerBound / 1280)
+ let last = min(encoded.count, max(first + 1, (word.upperBound + 1279) / 1280))
+ let hidden = encoded.hiddenSize
+ let spoken = DictionaryPronunciationExperiment.spokenFrames(recorded, frames: first..<last)
+ let frames = DictionaryMatchFrames(
+ hiddenSize: hidden,
+ values: spoken.flatMap { encoded.values[($0 * hidden)..<(($0 + 1) * hidden)] }
+ )
+ guard frames.isValid else { return [] }
+ result.append(frames)
+ continue
+ }
+ samples = recorded
} else if capture.originalAudioID != nil {
samples = try await self.pronunciationStore.originalAudioSamples(for: capture, entryID: profile.dictionaryEntryID)
} else { return [] }
@@ -1498,25 +1582,38 @@ final class FluidAudioProvider: TranscriptionProvider {
return capture
}
+ /// Keep the selected acoustic hit's identity while aligning its rough word times to the final replacement.
+ static func acceptedAcousticEvidence(
+ _ evidence: [DictionaryAcousticEvidence], entryID: UUID, label: String, wordRange: Range<Int>, frameRange: Range<Int>
+ ) -> [DictionaryAcousticEvidence] {
+ evidence.filter { $0.entryID == entryID && $0.matchFrameRange == frameRange }
+ .map { $0.aligned(to: wordRange, label: label) }
+ }
+
static func applyPronunciationMatches(
result: ASRResult,
matches: [PronunciationWindowMatch],
profiles: [PronunciationDictionaryProfile],
labels: [UUID: String],
temporalMatches: Set<TemporalMatchKey> = [],
- onAccepted: ((UUID, String, Range<Int>) -> Void)? = nil
+ entries: [SettingsStore.CustomDictionaryEntry] = [],
+ onAccepted: ((UUID, String, Range<Int>, Range<Int>) -> Void)? = nil,
+ onReplaced: ((DictionaryVoiceSpelling) -> Void)? = nil
) -> String {
guard let timings = result.tokenTimings else { return result.text }
let words = WordAudioChunkExtractor.words(from: timings)
guard !words.isEmpty else { return result.text }
let wordIndex = WordAudioOverlapIndex(words: words)
+ // The text layer runs after this and owns every word a saved spelling already covers.
+ let textClaimed = DictionaryAudioSpan.textClaimed(words: words.map(\.text), entries: entries)
struct Candidate {
let entryID: UUID
let label: String
let score: Float
let wordIndices: [Int]
+ let frameRange: Range<Int>
}
var candidates: [Candidate] = []
for match in matches {
@@ -1527,14 +1624,29 @@ final class FluidAudioProvider: TranscriptionProvider {
else { continue }
let startTime = Double(match.frameRange.lowerBound) * 0.08
let endTime = Double(match.frameRange.upperBound) * 0.08
- let indices = wordIndex.substantiallyOverlappingWordIndices(
- startTime: startTime,
- endTime: endTime
- )
- guard !indices.isEmpty else { continue }
let profile = profiles[match.prototypeIndex]
- let heard = indices.map { words[$0].text }.joined(separator: " ")
let temporalAccepted = temporalMatches.contains(TemporalMatchKey(prototypeIndex: match.prototypeIndex, frameRange: match.frameRange))
+ let indices: [Int]
+ if temporalAccepted {
+ // Recognizer word times are rough, so start from every word the match touches and
+ // settle on the whole run of free words that looks most like this word.
+ let touched = wordIndex.substantiallyOverlappingWordIndices(startTime: startTime, endTime: endTime, minimumOverlapRatio: 0)
+ let spellings = entries.first { $0.id == profile.dictionaryEntryID }?.triggers ?? []
+ let targets = [label] + spellings
+ guard let run = DictionaryAudioSpan.span(words: words.map(\.text), touched: touched, claimed: textClaimed, targets: targets)
+ else { continue }
+ // Grey zone: the sound is close but not certain, so the words must also look like the word.
+ if 1 - match.score >= DictionaryStampMatcher.sureDistance,
+ DictionaryAudioSpan.resemblance(words: words.map(\.text), run: run, targets: targets)
+ < DictionaryAudioSpan.greyZoneMinimumResemblance
+ { continue }
+ indices = Array(run)
+ } else {
+ indices = wordIndex.substantiallyOverlappingWordIndices(startTime: startTime, endTime: endTime)
+ guard indices.allSatisfy({ !textClaimed.contains($0) }) else { continue }
+ }
+ guard !indices.isEmpty else { continue }
+ let heard = indices.map { words[$0].text }.joined(separator: " ")
guard temporalAccepted || DictionaryPronunciationDecision.accepts(score: match.score, heardText: heard, profile: profile) else { continue }
var correctedLabel = label
if profile.hasOriginalAudio {
@@ -1543,7 +1655,7 @@ final class FluidAudioProvider: TranscriptionProvider {
label, heardText: heard, profile: profile
)
}
- candidates.append(Candidate(entryID: profile.dictionaryEntryID, label: correctedLabel, score: match.score, wordIndices: indices))
+ candidates.append(Candidate(entryID: profile.dictionaryEntryID, label: correctedLabel, score: match.score, wordIndices: indices, frameRange: match.frameRange))
}
var leaders: [Int: [(label: String, score: Float)]] = [:]
@@ -1581,7 +1693,8 @@ final class FluidAudioProvider: TranscriptionProvider {
guard let first = candidate.wordIndices.first, let last = candidate.wordIndices.last else { return nil }
let heard = candidate.wordIndices.map { words[$0].text }.joined(separator: " ")
if DictionaryNegativeEvidenceResolver.normalized(heard) != DictionaryNegativeEvidenceResolver.normalized(candidate.label) {
- onAccepted?(candidate.entryID, candidate.label, first..<(last + 1))
+ onAccepted?(candidate.entryID, candidate.label, first..<(last + 1), candidate.frameRange)
+ onReplaced?(DictionaryVoiceSpelling(entryID: candidate.entryID, label: candidate.label, spelling: heard))
}
return PronunciationTextReplacement(wordRange: first...last, label: candidate.label)
}
diff --git a/Sources/Fluid/Services/GlobalHotkeyManager.swift b/Sources/Fluid/Services/GlobalHotkeyManager.swift
index a735c648..3416407f 100644
--- a/Sources/Fluid/Services/GlobalHotkeyManager.swift
+++ b/Sources/Fluid/Services/GlobalHotkeyManager.swift
@@ -308,6 +308,8 @@ final class GlobalHotkeyManager: NSObject {
private var shortcutCaptureHandler: ((NSEvent) -> NSEvent?)?
private let isSessionLockedProvider: () -> Bool
private var cancelCallback: (() -> CancelHandlingResult)?
+ private var meetingRecordingCallback: (() -> Void)?
+ private var meetingShortcutPressedKey: UInt16?
private var pasteLastTranscriptionCallback: (() -> Void)?
private var hotkeyMode: HotkeyActivationMode = SettingsStore.shared.hotkeyMode
private let automaticTapThresholdSeconds: TimeInterval = 0.4
@@ -699,14 +701,6 @@ final class GlobalHotkeyManager: NSObject {
self.scheduleActiveShortcutLog(reason: "shortcuts updated")
}
- func setCancelCallback(_ callback: @escaping () -> CancelHandlingResult) {
- self.cancelCallback = callback
- }
-
- func setPasteLastTranscriptionCallback(_ callback: @escaping () -> Void) {
- self.pasteLastTranscriptionCallback = callback
- }
-
private func setupGlobalHotkeyWithRetry() {
for attempt in 1...self.maxRetryAttempts {
DebugLogger.shared.debug("Setup attempt \(attempt)/\(self.maxRetryAttempts)", source: "GlobalHotkeyManager")
@@ -1287,6 +1281,10 @@ final class GlobalHotkeyManager: NSObject {
let eventModifiers = Self.modifierFlags(from: event.flags)
let isAutorepeat = event.getIntegerValueField(.keyboardEventAutorepeat) != 0
+ if self.handleMeetingRecordingShortcut(type: type, keyCode: keyCode, modifiers: eventModifiers, isAutorepeat: isAutorepeat) {
+ return nil
+ }
+
switch type {
case .keyDown:
// A fresh down proves the previous same-key release was missed. Do not let its
@@ -1961,6 +1959,7 @@ final class GlobalHotkeyManager: NSObject {
}
func resetModifierOnlyShortcutTracking(reason: ModifierTrackingResetReason = .shortcutCapture) {
+ self.meetingShortcutPressedKey = nil
self.invalidateAllRecordingActions()
self.modifierPressReceivedAt = nil
self.currentStopPressReceivedAt = nil
@@ -2772,6 +2771,9 @@ extension GlobalHotkeyManager {
if self.commandModeShortcutEnabled, let shortcut = self.commandModeShortcut {
shortcuts.append(shortcut)
}
+ if SettingsStore.shared.fluidMeetEnabled, let shortcut = SettingsStore.shared.meetingRecordingHotkeyShortcut {
+ shortcuts.append(shortcut)
+ }
if let shortcut = SettingsStore.shared.cancelRecordingHotkeyShortcut {
shortcuts.append(shortcut)
}
@@ -2873,3 +2875,34 @@ extension GlobalHotkeyManager {
}
}
}
+
+extension GlobalHotkeyManager {
+ func setCancelCallback(_ callback: @escaping () -> CancelHandlingResult) {
+ self.cancelCallback = callback
+ }
+
+ func setPasteLastTranscriptionCallback(_ callback: @escaping () -> Void) {
+ self.pasteLastTranscriptionCallback = callback
+ }
+
+ func setMeetingRecordingCallback(_ callback: @escaping () -> Void) {
+ self.meetingRecordingCallback = callback
+ }
+
+ private func handleMeetingRecordingShortcut(type: CGEventType, keyCode: UInt16, modifiers: NSEvent.ModifierFlags, isAutorepeat: Bool) -> Bool {
+ if type == .keyUp, self.meetingShortcutPressedKey == keyCode {
+ self.meetingShortcutPressedKey = nil
+ return true
+ }
+ guard type == .keyDown, SettingsStore.shared.fluidMeetEnabled,
+ let shortcut = SettingsStore.shared.meetingRecordingHotkeyShortcut,
+ shortcut.matches(keyCode: keyCode, modifiers: modifiers),
+ !self.isSessionLockedProvider() else { return false }
+ self.markOtherInputDuringModifierOnly()
+ if !isAutorepeat {
+ self.meetingShortcutPressedKey = keyCode
+ self.meetingRecordingCallback?()
+ }
+ return true
+ }
+}
diff --git a/Sources/Fluid/Services/Meeting/AEC/MeetingAECActivationController.swift b/Sources/Fluid/Services/Meeting/AEC/MeetingAECActivationController.swift
index f967dcc8..0d48fb93 100644
--- a/Sources/Fluid/Services/Meeting/AEC/MeetingAECActivationController.swift
+++ b/Sources/Fluid/Services/Meeting/AEC/MeetingAECActivationController.swift
@@ -12,13 +12,16 @@ nonisolated enum MeetingAECActivationState: Equatable, Sendable {
}
nonisolated struct MeetingAECActivationController: Equatable, Sendable {
+ static let formatValidationTimeoutSeconds: TimeInterval = 2
private(set) var generation: UInt64 = 0
private(set) var state: MeetingAECActivationState = .raw
+ private var armedAtUptime: TimeInterval?
/// Invalidates construction, a ready/armed candidate, or an active engine synchronously.
mutating func invalidate() {
self.generation &+= 1
self.state = .raw
+ self.armedAtUptime = nil
}
/// Reserves the next generation. A second constructor can never overlap the first one.
@@ -57,12 +60,20 @@ nonisolated struct MeetingAECActivationController: Equatable, Sendable {
/// Called after `startCapture` (or immediately after a route-time candidate commit when the
/// stream is already running). Actual render and capture callbacks still remain raw.
- mutating func armReadyCandidate() -> Bool {
+ mutating func armReadyCandidate(now: TimeInterval = ProcessInfo.processInfo.systemUptime) -> Bool {
guard case let .ready(token) = self.state, token == self.generation else { return false }
self.state = .armed(token, renderValidated: false, captureValidated: false)
+ self.armedAtUptime = now
return true
}
+ /// Checked by incoming callbacks, without another timer. A missing reference stream must not
+ /// keep otherwise valid microphone speech blocked for the entire meeting.
+ func formatValidationTimedOut(now: TimeInterval = ProcessInfo.processInfo.systemUptime) -> Bool {
+ guard case .armed = self.state, let armedAtUptime else { return false }
+ return now - armedAtUptime >= Self.formatValidationTimeoutSeconds
+ }
+
/// Returns true only on the callback that completes both real-format observations. That
/// callback itself stays raw; a subsequent callback is the first one allowed to see `.active`.
@discardableResult
diff --git a/Sources/Fluid/Services/Meeting/AEC/MeetingAECModels.swift b/Sources/Fluid/Services/Meeting/AEC/MeetingAECModels.swift
index 173617c7..c673351c 100644
--- a/Sources/Fluid/Services/Meeting/AEC/MeetingAECModels.swift
+++ b/Sources/Fluid/Services/Meeting/AEC/MeetingAECModels.swift
@@ -20,6 +20,7 @@ nonisolated enum MeetingAECFailure: String, Error, Equatable, Sendable {
case invalidDuration
case unsupportedFormat
case unsupportedSampleRate
+ case formatValidationTimedOut
case formatChanged
case nonFiniteSamples
case streamChanged
@@ -147,7 +148,7 @@ nonisolated enum MeetingAECConstants {
static let provenance = MeetingAECProvenance(
upstreamRevision: "d9bd07ba5f614156021df666ba4052ac73cf7953",
- bridgeConfigurationID: "webrtc-aec3-48k-mono-cxx20-no-protobuf-no-log-v1",
+ bridgeConfigurationID: "webrtc-aec3-48k-mono-speech-protection-v2",
sampleRateHz: sampleRateHz,
frameDurationMilliseconds: 10
)
diff --git a/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputCommitter.swift b/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputCommitter.swift
index fda6e8c9..49e19ef6 100644
--- a/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputCommitter.swift
+++ b/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputCommitter.swift
@@ -14,8 +14,8 @@ final nonisolated class MeetingAECOutputCommitter: @unchecked Sendable {
private enum State: Equatable {
case unprotected
- case promotionPending(UInt64)
- case protected
+ case promotionPending(UInt64, MeetingMicrophoneEchoProtection)
+ case admitted(MeetingMicrophoneEchoProtection)
}
private let lock = NSLock()
@@ -24,6 +24,7 @@ final nonisolated class MeetingAECOutputCommitter: @unchecked Sendable {
private let transcriptGate: MeetingMicrophoneTranscriptGate
private let liveAudioHandler: (@Sendable (MeetingAudioTrackKind, CMSampleBuffer) -> Void)?
private let terminalFailure: @Sendable (MeetingAECFailure) -> Void
+ private let protectionChanged: @Sendable (MeetingMicrophoneEchoProtection) -> Void
private let pending = DispatchGroup()
private var state: State = .unprotected
private var generation: UInt64 = 0
@@ -41,6 +42,7 @@ final nonisolated class MeetingAECOutputCommitter: @unchecked Sendable {
producerEpoch: UInt64 = 0,
transcriptGate: MeetingMicrophoneTranscriptGate,
liveAudioHandler: (@Sendable (MeetingAudioTrackKind, CMSampleBuffer) -> Void)?,
+ protectionChanged: @escaping @Sendable (MeetingMicrophoneEchoProtection) -> Void = { _ in },
terminalFailure: @escaping @Sendable (MeetingAECFailure) -> Void
) {
self.writer = writer
@@ -48,104 +50,109 @@ final nonisolated class MeetingAECOutputCommitter: @unchecked Sendable {
self.transcriptGate = transcriptGate
self.liveAudioHandler = liveAudioHandler
self.terminalFailure = terminalFailure
+ self.protectionChanged = protectionChanged
}
/// Warm-up frames are processed and recorded but remain in the unprotected era. The first
/// frame *after* the 200-frame attestation performs the atomic durable promotion.
func commitProcessed(_ sampleBuffer: CMSampleBuffer, mayPromote: Bool) {
+ let hasPromotion = self.lock.withLock { self.state != .unprotected }
+ guard mayPromote || hasPromotion else {
+ guard self.enqueueOrFail(sampleBuffer, startSeconds: Self.seconds(of: sampleBuffer)) else { return }
+ _ = self.transcriptGate.observeAndAdmit(sampleBuffer)
+ return
+ }
+ self.commitAdmitted(sampleBuffer, protection: .softwareEchoCancelled)
+ }
+
+ /// A filtering failure changes processing provenance, not whether captured speech exists.
+ /// Persist the honest fallback state before offering any of its audio to live transcription.
+ func commitUnfiltered(_ sampleBuffer: CMSampleBuffer) {
+ self.commitAdmitted(sampleBuffer, protection: .unfiltered)
+ }
+
+ private func commitAdmitted(_ sampleBuffer: CMSampleBuffer, protection: MeetingMicrophoneEchoProtection) {
let sampleBox = SampleBox(sampleBuffer)
let startSeconds = Self.seconds(of: sampleBuffer)
- let action = self.lock.withLock { () -> (State, UInt64?) in
+ let action = self.lock.withLock { () -> (token: UInt64?, gateEpoch: UInt64) in
switch self.state {
- case .protected:
- return (.protected, nil)
- case .promotionPending:
- return (self.state, nil)
- case .unprotected where mayPromote:
+ case let .admitted(current) where current == protection:
+ return (nil, self.promotionGateEpoch)
+ case let .promotionPending(_, current) where current == protection:
+ return (nil, self.promotionGateEpoch)
+ default:
self.generation &+= 1
let token = self.generation
self.promotionGateEpoch = self.transcriptGate.invalidationEpoch
- self.state = .promotionPending(token)
- return (self.state, token)
- case .unprotected:
- return (.unprotected, nil)
+ self.state = .promotionPending(token, protection)
+ return (token, self.promotionGateEpoch)
}
}
- switch action.0 {
- case .protected:
- guard self.enqueueOrFail(sampleBox.sampleBuffer, startSeconds: startSeconds) else { return }
- if self.transcriptGate.observeAndAdmit(sampleBox.sampleBuffer) {
- self.liveAudioHandler?(.microphone, sampleBox.sampleBuffer)
+ guard let token = action.token else {
+ // The existing bounded writer queue orders live offers after the era commit and all
+ // earlier samples, including a rapid filtered-to-unfiltered transition.
+ _ = self.enqueueOrFail(sampleBox.sampleBuffer, startSeconds: startSeconds) { [self] in
+ self.offerLive(sampleBox, gateEpoch: action.gateEpoch)
}
- case .unprotected:
- guard self.enqueueOrFail(sampleBox.sampleBuffer, startSeconds: startSeconds) else { return }
- _ = self.transcriptGate.observeAndAdmit(sampleBox.sampleBuffer)
- case .promotionPending where action.1 == nil:
- guard self.enqueueOrFail(sampleBox.sampleBuffer, startSeconds: startSeconds) else { return }
- _ = self.transcriptGate.observeAndAdmit(sampleBox.sampleBuffer)
- case let .promotionPending(token):
- self.beginPendingCommit()
- self.writer.enqueue(
- sampleBox.sampleBuffer,
- producerEpoch: self.producerEpoch,
- afterPersistingMetadata: { track in
- Self.applyEra(
- to: &track,
- protection: .softwareEchoCancelled,
- provenance: MeetingAECConstants.provenance,
- startSeconds: startSeconds
- )
- },
- // Promotion failure must preserve the older conservative in-memory track.
- failClosed: false
- ) { [self] result in
- defer { self.endPendingCommit() }
- switch result {
- case .success:
- let openAttempt = self.lock.withLock { () -> (Bool, UInt64) in
- guard self.generation == token, self.state == .promotionPending(token) else {
- return (false, 0)
- }
- return (true, self.promotionGateEpoch)
- }
- guard openAttempt.0 else { return }
- // A route/lifecycle close racing this completion holds the gate closed; land a
- // compensating conservative write after the optimistic era just persisted.
- guard self.transcriptGate.updateIfNotInvalidated(
- .softwareEchoCancelled,
- since: openAttempt.1
- ) else {
- self.lock.withLock {
- self.generation &+= 1
- self.state = .unprotected
- }
- self.persistSafetyEra(startSeconds: startSeconds)
- return
- }
- if self.transcriptGate.observeAndAdmit(sampleBox.sampleBuffer) {
- self.liveAudioHandler?(.microphone, sampleBox.sampleBuffer)
- }
- // Keep the state pending until the first admitted live offer has completed.
- // A later callback can therefore never overtake the promotion frame on a
- // different queue. An external invalidation wins by changing this state first.
- self.lock.withLock {
- guard self.generation == token, self.state == .promotionPending(token) else {
- return
- }
- self.state = .protected
- }
- case let .failure(error):
- self.lock.withLock {
- if self.generation == token { self.state = .unprotected }
+ return
+ }
+ self.beginPendingCommit()
+ self.writer.enqueue(
+ sampleBox.sampleBuffer,
+ producerEpoch: self.producerEpoch,
+ afterPersistingMetadata: { track in
+ Self.applyEra(
+ to: &track,
+ protection: protection,
+ provenance: protection == .softwareEchoCancelled ? MeetingAECConstants.provenance : nil,
+ startSeconds: startSeconds
+ )
+ },
+ // Promotion failure must preserve the older conservative in-memory track.
+ failClosed: false
+ ) { [self] result in
+ defer { self.endPendingCommit() }
+ switch result {
+ case .success:
+ // A route/lifecycle close racing this completion holds the gate closed; land a
+ // compensating conservative write after the optimistic era just persisted.
+ guard self.transcriptGate.updateIfNotInvalidated(
+ protection,
+ since: action.gateEpoch
+ ) else {
+ let needsSafetyEra = self.lock.withLock { () -> Bool in
+ guard self.generation == token else { return false }
+ self.generation &+= 1
+ self.state = .unprotected
+ return true
}
- _ = self.transcriptGate.invalidate()
- self.reportTerminalFailure(Self.terminalFailure(for: error))
+ if needsSafetyEra { self.persistSafetyEra(startSeconds: startSeconds) }
+ return
}
+ self.offerLive(sampleBox, gateEpoch: action.gateEpoch)
+ if self.transcriptGate.invalidationEpoch == action.gateEpoch {
+ self.protectionChanged(protection)
+ }
+ self.lock.withLock {
+ if self.generation == token { self.state = .admitted(protection) }
+ }
+ case let .failure(error):
+ self.lock.withLock {
+ if self.generation == token { self.state = .unprotected }
+ }
+ _ = self.transcriptGate.invalidate()
+ self.reportTerminalFailure(Self.terminalFailure(for: error))
}
}
}
+ private func offerLive(_ sample: SampleBox, gateEpoch: UInt64) {
+ if self.transcriptGate.observeAndAdmit(sample.sampleBuffer, since: gateEpoch) {
+ self.liveAudioHandler?(.microphone, sample.sampleBuffer)
+ }
+ }
+
/// Records a raw/bypass sample. If the prior state could have been protected, its demotion and
/// this sample are one writer command, so raw bytes cannot land beneath optimistic metadata.
func commitRaw(_ sampleBuffer: CMSampleBuffer, failure: MeetingAECFailure) {
@@ -227,6 +234,17 @@ final nonisolated class MeetingAECOutputCommitter: @unchecked Sendable {
continuation.resume()
}
}
+ // Live offers share the bounded PCM queue, so also drain samples behind the era commit.
+ _ = await self.writer.snapshot()
+ }
+
+ /// Close live delivery without relabelling successfully filtered audio at Stop.
+ func finish() {
+ self.lock.withLock {
+ self.generation &+= 1
+ self.state = .unprotected
+ }
+ _ = self.transcriptGate.invalidate()
}
func failTerminal(_ failure: MeetingAECFailure) {
@@ -263,8 +281,12 @@ final nonisolated class MeetingAECOutputCommitter: @unchecked Sendable {
/// The plain enqueue carries no era command, so an overflow must still leave the in-memory
/// and persisted track unprotected before the terminal stop completes.
- private func enqueueOrFail(_ sampleBuffer: CMSampleBuffer, startSeconds: Double?) -> Bool {
- guard self.writer.enqueue(sampleBuffer, producerEpoch: self.producerEpoch) else {
+ private func enqueueOrFail(
+ _ sampleBuffer: CMSampleBuffer,
+ startSeconds: Double?,
+ onConsumed: (@Sendable () -> Void)? = nil
+ ) -> Bool {
+ guard self.writer.enqueue(sampleBuffer, producerEpoch: self.producerEpoch, onConsumed: onConsumed) else {
self.lock.withLock {
self.generation &+= 1
self.state = .unprotected
diff --git a/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputRouteClassifier.swift b/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputRouteClassifier.swift
index f2479241..80f4c567 100644
--- a/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputRouteClassifier.swift
+++ b/Sources/Fluid/Services/Meeting/AEC/MeetingAECOutputRouteClassifier.swift
@@ -7,8 +7,8 @@ nonisolated enum MeetingAECOutputRouteDisposition: Equatable, Sendable {
case ambiguous
}
-/// Positive route classifier for the direct SCK microphone path. Absence of headphone evidence is
-/// never sufficient to infer a speaker; the initial deployment supports only built-in speakers.
+/// Stable output routes may attempt AEC regardless of transport. Admission still depends on real
+/// PCM format/clock checks and warm-up; this classification alone never blesses processed audio.
nonisolated enum MeetingAECOutputRouteClassifier {
static func classify(
first: MeetingOutputRouteSnapshot,
@@ -27,12 +27,7 @@ nonisolated enum MeetingAECOutputRouteClassifier {
{
return .physicallyClosed
}
- guard !route.isBluetooth else { return .ambiguous }
- guard !debugDisabled,
- route.isBuiltIn,
- !route.isHeadphonesDataSource,
- !route.terminalTypes.contains(kAudioStreamTerminalTypeHeadphones)
- else { return .ambiguous }
+ guard !debugDisabled else { return .ambiguous }
return .supportedSpeaker
}
}
diff --git a/Sources/Fluid/Services/Meeting/AEC/MeetingAECPipeline.swift b/Sources/Fluid/Services/Meeting/AEC/MeetingAECPipeline.swift
index 6f850fb2..1b8aaaf5 100644
--- a/Sources/Fluid/Services/Meeting/AEC/MeetingAECPipeline.swift
+++ b/Sources/Fluid/Services/Meeting/AEC/MeetingAECPipeline.swift
@@ -8,18 +8,23 @@ final nonisolated class MeetingAECPipeline: @unchecked Sendable {
private var joiner = MeetingAECStreamJoiner()
private let processor: any MeetingAECProcessing
let committer: MeetingAECOutputCommitter
+ private let diagnosticObserver: (@Sendable (MeetingAECJoinedFrame, [Float], MeetingAECBridgeStatistics) -> Void)?
private(set) var stopped = false
+ /// Retry AEC on a new route, not on every callback while the current route is failing.
+ private var fallbackFailure: MeetingAECFailure?
init(
processor: any MeetingAECProcessing,
- committer: MeetingAECOutputCommitter
+ committer: MeetingAECOutputCommitter,
+ diagnosticObserver: (@Sendable (MeetingAECJoinedFrame, [Float], MeetingAECBridgeStatistics) -> Void)? = nil
) {
self.processor = processor
self.committer = committer
+ self.diagnosticObserver = diagnosticObserver
}
func consumeRender(_ sampleBuffer: CMSampleBuffer, stream: MeetingAECStreamToken) {
- guard !self.stopped else { return }
+ guard !self.stopped, self.fallbackFailure == nil else { return }
switch MeetingAECPCMAdapter.extract(sampleBuffer, kind: .render, stream: stream) {
case let .success(block):
self.handle(self.joiner.append(block))
@@ -33,6 +38,10 @@ final nonisolated class MeetingAECPipeline: @unchecked Sendable {
self.committer.commitRaw(sampleBuffer, failure: .stopped)
return
}
+ if self.fallbackFailure != nil {
+ self.committer.commitUnfiltered(sampleBuffer)
+ return
+ }
switch MeetingAECPCMAdapter.extract(sampleBuffer, kind: .capture, stream: stream) {
case let .success(block):
self.handle(self.joiner.append(block))
@@ -60,14 +69,14 @@ final nonisolated class MeetingAECPipeline: @unchecked Sendable {
boundary: MeetingMicrophoneTranscriptGate.InvalidationBoundary?
) {
guard !self.stopped else { return }
- self.handle(self.joiner.flush(reason: reason))
+ self.handle(self.joiner.flush(reason: reason), allowFallback: false)
self.committer.invalidateForExternalBoundary(boundary)
}
func stop(boundary: MeetingMicrophoneTranscriptGate.InvalidationBoundary?) {
guard !self.stopped else { return }
self.handle(self.joiner.flush(reason: .stopped))
- self.committer.invalidateForExternalBoundary(boundary)
+ self.committer.finish()
self.stopped = true
}
@@ -76,20 +85,21 @@ final nonisolated class MeetingAECPipeline: @unchecked Sendable {
currentRawCapture: CMSampleBuffer?
) {
self.handle(self.joiner.flush(reason: failure))
+ self.fallbackFailure = failure
if let currentRawCapture {
- self.committer.commitRaw(currentRawCapture, failure: failure)
+ self.committer.commitUnfiltered(currentRawCapture)
}
}
- private func handle(_ emissions: [MeetingAECJoinerEmission]) {
- var mayContinueProcessing = true
+ private func handle(_ emissions: [MeetingAECJoinerEmission], allowFallback: Bool = true) {
+ var flushRemainingCapture = false
for emission in emissions {
switch emission {
case let .paired(joined):
- if mayContinueProcessing {
- mayContinueProcessing = self.process(joined)
+ if self.fallbackFailure == nil {
+ if !self.process(joined, allowFallback: allowFallback) { flushRemainingCapture = true }
} else {
- self.commitRaw(joined.capture, failure: .bridgeProcessing)
+ self.commitRaw(joined.capture, failure: self.fallbackFailure ?? .bridgeProcessing, allowFallback: allowFallback)
}
case let .bypass(slice, failure):
guard let raw = MeetingAECPCMAdapter.synthesize(
@@ -99,26 +109,34 @@ final nonisolated class MeetingAECPipeline: @unchecked Sendable {
self.committer.invalidateForExternalBoundary(nil)
continue
}
- self.committer.commitRaw(raw, failure: failure)
- case .reset:
- self.resetProcessor()
- self.committer.invalidateForExternalBoundary(nil)
+ if allowFallback {
+ self.fallbackFailure = failure
+ self.committer.commitUnfiltered(raw)
+ } else {
+ self.committer.commitRaw(raw, failure: failure)
+ }
+ case let .reset(failure):
+ if allowFallback, failure != .stopped { self.fallbackFailure = failure }
}
}
+ if flushRemainingCapture {
+ self.handle(self.joiner.flush(reason: self.fallbackFailure ?? .bridgeProcessing), allowFallback: allowFallback)
+ }
}
@discardableResult
- private func process(_ joined: MeetingAECJoinedFrame) -> Bool {
+ private func process(_ joined: MeetingAECJoinedFrame, allowFallback: Bool) -> Bool {
do {
let result = try self.processor.process(
render: joined.render.samples,
capture: joined.capture.samples
)
+ self.diagnosticObserver?(joined, result.samples, result.statistics)
guard let output = MeetingAECPCMAdapter.synthesize(
samples: result.samples,
presentationTime: joined.capture.presentationTime
) else {
- self.commitRawAndReset(joined.capture, failure: .outputSynthesis)
+ self.beginFallback(joined.capture, failure: .outputSynthesis, allowFallback: allowFallback)
return false
}
let mayPromote = joined.clockAttested
@@ -126,37 +144,31 @@ final nonisolated class MeetingAECPipeline: @unchecked Sendable {
self.committer.commitProcessed(output, mayPromote: mayPromote)
return true
} catch let failure as MeetingAECFailure {
- self.commitRawAndReset(joined.capture, failure: failure)
+ self.beginFallback(joined.capture, failure: failure, allowFallback: allowFallback)
return false
} catch {
- self.commitRawAndReset(joined.capture, failure: .bridgeProcessing)
+ self.beginFallback(joined.capture, failure: .bridgeProcessing, allowFallback: allowFallback)
return false
}
}
- private func commitRawAndReset(_ capture: MeetingAECFrame, failure: MeetingAECFailure) {
- self.commitRaw(capture, failure: failure)
- self.joiner = MeetingAECStreamJoiner()
- self.resetProcessor()
+ private func beginFallback(_ capture: MeetingAECFrame, failure: MeetingAECFailure, allowFallback: Bool) {
+ self.fallbackFailure = failure
+ self.commitRaw(capture, failure: failure, allowFallback: allowFallback)
}
- private func commitRaw(_ capture: MeetingAECFrame, failure: MeetingAECFailure) {
+ private func commitRaw(_ capture: MeetingAECFrame, failure: MeetingAECFailure, allowFallback: Bool = true) {
if let raw = MeetingAECPCMAdapter.synthesize(
samples: capture.samples,
presentationTime: capture.presentationTime
) {
- self.committer.commitRaw(raw, failure: failure)
+ if allowFallback {
+ self.committer.commitUnfiltered(raw)
+ } else {
+ self.committer.commitRaw(raw, failure: failure)
+ }
} else {
self.committer.invalidateForExternalBoundary(nil)
}
}
-
- private func resetProcessor() {
- do {
- try self.processor.reset()
- } catch {
- self.stopped = true
- self.committer.failTerminal(.bridgeInitialization)
- }
- }
}
diff --git a/Sources/Fluid/Services/Meeting/Backend/MeetingAnalysisManifest.swift b/Sources/Fluid/Services/Meeting/Backend/MeetingAnalysisManifest.swift
index 0f6dee71..53115b36 100644
--- a/Sources/Fluid/Services/Meeting/Backend/MeetingAnalysisManifest.swift
+++ b/Sources/Fluid/Services/Meeting/Backend/MeetingAnalysisManifest.swift
@@ -344,6 +344,7 @@ nonisolated enum MeetingCanonicalAdmissionRationale: String, Codable, CaseIterab
case onlineMicrophoneVoiceProcessed
case onlineMicrophoneAcousticallyClosed
case onlineMicrophoneSoftwareEchoCancelled
+ case onlineMicrophoneUnfiltered
case onlineMicrophoneUnprotected
/// Legacy-era audio stays usable only through the existing explicit legacy compatibility
/// policy. The canonical backend has no such policy, so it is inadmissible here.
@@ -357,7 +358,8 @@ nonisolated enum MeetingCanonicalAdmissionRationale: String, Codable, CaseIterab
.inRoomMicrophone,
.onlineMicrophoneVoiceProcessed,
.onlineMicrophoneAcousticallyClosed,
- .onlineMicrophoneSoftwareEchoCancelled:
+ .onlineMicrophoneSoftwareEchoCancelled,
+ .onlineMicrophoneUnfiltered:
return .admissible
case .onlineMicrophoneUnprotected,
.onlineMicrophoneLegacyUnclassified,
@@ -383,6 +385,7 @@ nonisolated enum MeetingCanonicalAdmission {
case .voiceProcessed: return .onlineMicrophoneVoiceProcessed
case .acousticallyClosed: return .onlineMicrophoneAcousticallyClosed
case .softwareEchoCancelled: return .onlineMicrophoneSoftwareEchoCancelled
+ case .unfiltered: return .onlineMicrophoneUnfiltered
case .unprotected: return .onlineMicrophoneUnprotected
case .legacyUnclassified: return .onlineMicrophoneLegacyUnclassified
case .none: return .onlineMicrophoneProtectionMissing
diff --git a/Sources/Fluid/Services/Meeting/Backend/MeetingParakeetNemotronBackend.swift b/Sources/Fluid/Services/Meeting/Backend/MeetingParakeetNemotronBackend.swift
index b8954267..e335b808 100644
--- a/Sources/Fluid/Services/Meeting/Backend/MeetingParakeetNemotronBackend.swift
+++ b/Sources/Fluid/Services/Meeting/Backend/MeetingParakeetNemotronBackend.swift
@@ -13,7 +13,7 @@ import Foundation
final class MeetingParakeetNemotronBackend: MeetingTranscriptionBackend {
static let descriptor = MeetingBackendDescriptor(
id: .parakeetNemotron,
- version: "5",
+ version: "8",
execution: .local,
supportedLanguageCodes: VoiceEngineLanguageCatalog.parakeetV3LanguageIDs,
supportedTrackKinds: Set(MeetingAudioTrackKind.allCases),
@@ -465,7 +465,6 @@ final class MeetingParakeetNemotronBackend: MeetingTranscriptionBackend {
guard physicalEnd > epochStart else { return [] }
var wordUnits: [MeetingFinalTextUnit] = []
- var previousAssignedWord: (token: MeetingBackendSpeakerToken, end: TimeInterval)?
let sortedWords = output.words.enumerated()
.sorted { ($0.element.start, $0.offset) < ($1.element.start, $1.offset) }
for (index, word) in sortedWords {
@@ -491,19 +490,12 @@ final class MeetingParakeetNemotronBackend: MeetingTranscriptionBackend {
guard end > start else { continue }
guard let spanIDs = Self.intersectingSpanIDs(start: start, end: end, spans: work.spans)
else { continue }
- let coverageAssignment = Self.assignment(
+ let speaker = Self.assignment(
start: start,
end: end,
activity: activity,
allowDominance: true
)
- let speaker = Self.handoffAssignment(
- coverageAssignment,
- start: start,
- end: end,
- activity: activity,
- previousAssignedWord: previousAssignedWord
- )
wordUnits.append(MeetingFinalTextUnit(
id: "unit:\(attemptID.uuidString):\(work.epoch.id):\(index)",
trackID: work.track.id,
@@ -516,14 +508,28 @@ final class MeetingParakeetNemotronBackend: MeetingTranscriptionBackend {
analysisSpanIDs: spanIDs,
confidence: nil
))
- if case let .assigned(token) = speaker {
- previousAssignedWord = (token, end)
- } else {
- // An uncertain word is a hard context break; never chain a later guess through it.
- previousAssignedWord = nil
+ }
+ if !wordUnits.isEmpty {
+ let recovered = MeetingWordBoundaryRecovery.assignments(
+ units: wordUnits,
+ activityByEpoch: [work.epoch.id: activity]
+ )
+ return wordUnits.map { unit in
+ guard case .unassigned = unit.speaker, let speaker = recovered[unit.id] else { return unit }
+ return MeetingFinalTextUnit(
+ id: unit.id,
+ trackID: unit.trackID,
+ analysisEpochID: unit.analysisEpochID,
+ precision: unit.precision,
+ text: unit.text,
+ analysisStart: unit.analysisStart,
+ analysisEnd: unit.analysisEnd,
+ speaker: speaker,
+ analysisSpanIDs: unit.analysisSpanIDs,
+ confidence: unit.confidence
+ )
}
}
- if !wordUnits.isEmpty { return wordUnits }
let text = output.text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !text.isEmpty else { return [] }
@@ -619,8 +625,6 @@ final class MeetingParakeetNemotronBackend: MeetingTranscriptionBackend {
/// Nemotron emits diarization decisions on this cadence (plan §4); a boundary sliver shorter
/// than one output hop is model noise, not evidence of a second speaker.
private nonisolated static let nemotronCadenceMaximumFloorSeconds: TimeInterval = 0.08
- private nonisolated static let handoffMaximumWordDurationSeconds: TimeInterval = 0.8
- private nonisolated static let handoffMaximumPreviousWordGapSeconds: TimeInterval = 0.8
/// Conservative initial product policy for multi-candidate words, isolated here so the ratios
/// are named and covered at exact boundaries instead of living as unexplained literals.
@@ -728,61 +732,6 @@ final class MeetingParakeetNemotronBackend: MeetingTranscriptionBackend {
}
}
- /// Deliberately simple second pass for short turn-boundary words. The user chose the more
- /// readable handoff behavior despite its known risk for genuine interruption/backchannel
- /// speech. Coverage remains the first-pass authority; this only resolves an otherwise
- /// ambiguous two-slot result with immediate assigned-word context.
- nonisolated static func handoffAssignment(
- _ coverageAssignment: MeetingBackendSpeakerAssignment,
- start: TimeInterval,
- end: TimeInterval,
- activity: [MeetingBackendSpeakerActivity],
- previousAssignedWord: (token: MeetingBackendSpeakerToken, end: TimeInterval)?
- ) -> MeetingBackendSpeakerAssignment {
- guard case let .ambiguous(candidates) = coverageAssignment,
- candidates.count == 2,
- let previousAssignedWord,
- candidates.contains(previousAssignedWord.token)
- else { return coverageAssignment }
-
- let duration = end - start
- let gap = start - previousAssignedWord.end
- guard duration.isFinite,
- duration > 0,
- duration <= Self.handoffMaximumWordDurationSeconds,
- gap.isFinite,
- gap >= -Self.nemotronCadenceMaximumFloorSeconds,
- gap <= Self.handoffMaximumPreviousWordGapSeconds
- else { return coverageAssignment }
-
- guard let other = candidates.first(where: { $0 != previousAssignedWord.token }) else {
- return coverageAssignment
- }
- let otherOnsets = activity.compactMap { entry -> TimeInterval? in
- guard entry.token == other,
- min(entry.end, end) - max(entry.start, start)
- > MeetingAnalysisManifestSchema.mappingToleranceSeconds
- else { return nil }
- return entry.start
- }
- guard let otherOnset = otherOnsets.min() else { return coverageAssignment }
- let onsetDelta = otherOnset - start
- let resolved: MeetingBackendSpeakerAssignment = abs(onsetDelta)
- <= Self.nemotronCadenceMaximumFloorSeconds
- ? .assigned(other)
- : .assigned(previousAssignedWord.token)
-
- #if DEBUG
- Self.logHandoffDecision(
- duration: duration,
- previousWordGap: gap,
- otherOnsetDelta: onsetDelta,
- assignedEntrant: abs(onsetDelta) <= Self.nemotronCadenceMaximumFloorSeconds
- )
- #endif
- return resolved
- }
-
#if DEBUG
/// Structured, DEBUG-only measurement of a multi-candidate word decision. Deliberately carries
/// only durations, coverage seconds and the resulting decision kind — never transcript text,
@@ -813,22 +762,5 @@ final class MeetingParakeetNemotronBackend: MeetingTranscriptionBackend {
)
}
- private nonisolated static func logHandoffDecision(
- duration: TimeInterval,
- previousWordGap: TimeInterval,
- otherOnsetDelta: TimeInterval,
- assignedEntrant: Bool
- ) {
- DebugLogger.shared.info(
- String(
- format: "[speakerHandoff] duration=%.3fs previousGap=%.3fs otherOnsetDelta=%.3fs decision=%@",
- duration,
- previousWordGap,
- otherOnsetDelta,
- assignedEntrant ? "entrant" : "incumbent"
- ),
- source: "MeetingParakeetNemotronBackend"
- )
- }
#endif
}
diff --git a/Sources/Fluid/Services/Meeting/Backend/MeetingTranscriptAssembler.swift b/Sources/Fluid/Services/Meeting/Backend/MeetingTranscriptAssembler.swift
index e0d6d6b1..8d9a72e7 100644
--- a/Sources/Fluid/Services/Meeting/Backend/MeetingTranscriptAssembler.swift
+++ b/Sources/Fluid/Services/Meeting/Backend/MeetingTranscriptAssembler.swift
@@ -5,8 +5,10 @@ nonisolated enum MeetingSpeakerActivityCoverage {
static func isOutside(
_ unit: MeetingFinalTextUnit,
activityByEpoch: [MeetingAnalysisEpochID: [MeetingBackendSpeakerActivity]],
- tolerance: TimeInterval = MeetingAnalysisManifestSchema.mappingToleranceSeconds
+ tolerance: TimeInterval = MeetingAnalysisManifestSchema.mappingToleranceSeconds,
+ boundaryAssignments: [String: MeetingBackendSpeakerAssignment] = [:]
) -> Bool {
+ if boundaryAssignments[unit.id] != nil { return false }
guard let activity = activityByEpoch[unit.analysisEpochID], !activity.isEmpty else {
return false
}
@@ -16,6 +18,86 @@ nonisolated enum MeetingSpeakerActivityCoverage {
}
}
+/// Word times from TDT and 80ms diarizer frames are estimates, not sample-aligned bounds.
+/// Recover one adjacent boundary word, never an arbitrary run of text in silence. This shared
+/// policy keeps backend attribution, assembly visibility and echo-target visibility consistent.
+nonisolated enum MeetingWordBoundaryRecovery {
+ private static let hop: TimeInterval = 0.08
+ private static let maximumWordDuration: TimeInterval = 1.2
+ private static let epsilon = MeetingAnalysisManifestSchema.mappingToleranceSeconds
+
+ static func assignments(
+ units: [MeetingFinalTextUnit],
+ activityByEpoch: [MeetingAnalysisEpochID: [MeetingBackendSpeakerActivity]]
+ ) -> [String: MeetingBackendSpeakerAssignment] {
+ var result: [String: MeetingBackendSpeakerAssignment] = [:]
+ for (epoch, epochUnits) in Dictionary(grouping: units, by: \.analysisEpochID) {
+ let activity = activityByEpoch[epoch] ?? []
+ guard !activity.isEmpty else { continue }
+ let ordered = epochUnits.enumerated().sorted {
+ ($0.element.analysisStart, $0.offset) < ($1.element.analysisStart, $1.offset)
+ }.map(\.element)
+ for index in ordered.indices.dropFirst() {
+ let word = ordered[index]
+ let previous = ordered[index - 1]
+ if result[previous.id] == nil,
+ let assignment = Self.assignment(for: word, previous: previous, activity: activity)
+ {
+ result[word.id] = assignment
+ }
+ }
+ }
+ return result
+ }
+
+ private static func assignment(
+ for word: MeetingFinalTextUnit,
+ previous: MeetingFinalTextUnit,
+ activity: [MeetingBackendSpeakerActivity]
+ ) -> MeetingBackendSpeakerAssignment? {
+ guard word.precision == .word, previous.precision == .word,
+ word.trackID == previous.trackID,
+ word.analysisEpochID == previous.analysisEpochID,
+ MeetingEvidenceInterval.isValid(start: word.analysisStart, end: word.analysisEnd),
+ MeetingEvidenceInterval.isValid(start: previous.analysisStart, end: previous.analysisEnd),
+ word.analysisEnd - word.analysisStart <= self.maximumWordDuration + self.epsilon,
+ abs(word.analysisStart - previous.analysisEnd) <= self.hop + self.epsilon,
+ !Set(word.analysisSpanIDs).isDisjoint(with: previous.analysisSpanIDs),
+ case let .assigned(token) = previous.speaker
+ else { return nil }
+ switch word.speaker {
+ case .unassigned: break
+ case let .assigned(current) where current == token:
+ // A directly supported word needs no repair and can still anchor its own ending.
+ guard !activity.contains(where: {
+ min($0.end, word.analysisEnd) - max($0.start, word.analysisStart) > Self.epsilon
+ }) else { return nil }
+ default: return nil // Never override another speaker or genuine ambiguity.
+ }
+ // Nemotron endpoints are Float32. One fixed microsecond is too small even at 19s;
+ // allow two ULPs, bounded to 2ms, rather than extending the 80ms speech-boundary policy.
+ let roundingAllowance = min(0.002, max(Self.epsilon, 2 * Double(Float(word.analysisStart).ulp)))
+ let previousDuration = previous.analysisEnd - previous.analysisStart
+ // The anchor itself needs substantial measured coverage. Recovered tails cannot become
+ // anchors and recursively admit an ASR hallucination continuing into silence.
+ let anchored = activity.contains {
+ $0.token == token
+ && min($0.end, previous.analysisEnd) - max($0.start, previous.analysisStart)
+ >= 0.5 * previousDuration - min(roundingAllowance, 0.01 * previousDuration)
+ && abs($0.end - word.analysisStart) <= Self.hop + roundingAllowance
+ }
+ guard anchored else { return nil }
+ // Even a short competing onset vetoes inherited identity. Keep the word, but keep its
+ // speaker unknown rather than pretending timing proximity proves who said it.
+ let competing = activity.contains {
+ $0.token != token
+ && min($0.end, word.analysisEnd + Self.hop) - max($0.start, previous.analysisStart)
+ > Self.epsilon
+ }
+ return competing ? .unassigned : .assigned(token)
+ }
+}
+
// Stage C2b1 of `MEETING_TRANSCRIPTION_IMPLEMENTATION_PLAN.md` (§4) and
// `MEETING_TRANSCRIPTION_BACKEND_ARCHITECTURE_PLAN.md` §4.6: the pure assembler every backend
// shares. It reads a frozen plan, a validated analysis manifest, final evidence, per-span
@@ -177,6 +259,10 @@ nonisolated struct MeetingTranscriptAssembler {
)
let receiptsBySpanID = Dictionary(grouping: receipts, by: \.spanID)
let activityByEpoch = Dictionary(grouping: evidence.speakerActivity) { $0.token.analysisEpochID }
+ let boundaryAssignments = MeetingWordBoundaryRecovery.assignments(
+ units: evidence.units.filter { $0.quarantineReason(in: manifest) == nil },
+ activityByEpoch: activityByEpoch
+ )
let inadmissibleGapsByTrack = Dictionary(grouping: manifest.allGaps.filter {
$0.reason == .inadmissibleCaptureEra && $0.recordedInterval != nil
}) { $0.trackID }
@@ -254,7 +340,8 @@ nonisolated struct MeetingTranscriptAssembler {
guard duplicate.quarantineReason(in: manifest) == nil,
!MeetingSpeakerActivityCoverage.isOutside(
duplicate,
- activityByEpoch: activityByEpoch
+ activityByEpoch: activityByEpoch,
+ boundaryAssignments: boundaryAssignments
)
else {
throw MeetingAssemblyError.echoDuplicateTargetNotVisible(
@@ -272,7 +359,9 @@ nonisolated struct MeetingTranscriptAssembler {
}
}
- if MeetingSpeakerActivityCoverage.isOutside(unit, activityByEpoch: activityByEpoch) {
+ if MeetingSpeakerActivityCoverage.isOutside(
+ unit, activityByEpoch: activityByEpoch, boundaryAssignments: boundaryAssignments
+ ) {
dispositions.append(MeetingTextUnitDispositionRecord(
unitID: unit.id,
disposition: .outsideActivity,
diff --git a/Sources/Fluid/Services/Meeting/Backend/MeetingUnitEchoVerdictProvider.swift b/Sources/Fluid/Services/Meeting/Backend/MeetingUnitEchoVerdictProvider.swift
index dfebd96d..9d1d70bf 100644
--- a/Sources/Fluid/Services/Meeting/Backend/MeetingUnitEchoVerdictProvider.swift
+++ b/Sources/Fluid/Services/Meeting/Backend/MeetingUnitEchoVerdictProvider.swift
@@ -63,6 +63,10 @@ nonisolated struct MeetingTextOverlapEchoVerdictProvider: MeetingUnitEchoVerdict
var microphoneUnits: [PositionedUnit] = []
var applicationUnits: [PositionedUnit] = []
let activityByEpoch = Dictionary(grouping: evidence.speakerActivity) { $0.token.analysisEpochID }
+ let boundaryAssignments = MeetingWordBoundaryRecovery.assignments(
+ units: evidence.units.filter { $0.quarantineReason(in: manifest) == nil },
+ activityByEpoch: activityByEpoch
+ )
for unit in evidence.units {
try Task.checkCancellation()
@@ -85,7 +89,8 @@ nonisolated struct MeetingTextOverlapEchoVerdictProvider: MeetingUnitEchoVerdict
// duplicate target, so it is not echo evidence either (the assembler proves this).
guard !MeetingSpeakerActivityCoverage.isOutside(
unit,
- activityByEpoch: activityByEpoch
+ activityByEpoch: activityByEpoch,
+ boundaryAssignments: boundaryAssignments
) else { continue }
applicationUnits.append(positioned)
case .microphone:
diff --git a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTrackEngine.swift b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTrackEngine.swift
index 8fc65a89..2f049dd1 100644
--- a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTrackEngine.swift
+++ b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTrackEngine.swift
@@ -12,6 +12,7 @@ nonisolated protocol MeetingLiveRecognizer: Sendable {
func prepareModels() async throws
func appendAudio(_ buffer: AVAudioPCMBuffer) async throws
func processBufferedAudio() async throws
+ func finish() async throws -> String
func reset() async
func setEouCallback(_ callback: @escaping @Sendable (String) -> Void) async
func setPartialTranscriptCallback(_ callback: @escaping @Sendable (String) -> Void) async
@@ -39,6 +40,9 @@ nonisolated struct MeetingLiveDiagnosticSnapshot: Sendable {
var maxProcessSeconds: Double = 0
var firstPartialAfterResetSeconds: Double?
var feedSeconds: Double = 0
+ var textPauses = 0
+ var textResumptions = 0
+ var lastResetReason: String?
}
/// Drives one `StreamingEouAsrManager` for one capture track. Two instances are required — the
@@ -55,17 +59,30 @@ actor MeetingLiveTrackEngine {
typealias ReadyHandler = @Sendable (MeetingAudioTrackKind) -> Void
private let kind: MeetingAudioTrackKind
+ private var comparisonProbe: MeetingAudioComparisonProbe?
+
+ func attachComparisonProbe(_ probe: MeetingAudioComparisonProbe) {
+ self.comparisonProbe = probe
+ probe.recordEvent("recognizer-probe-attached", track: self.kind)
+ }
+
/// Sized to ride out first-inference CoreML warmup without dropping audio: at the ~10.7ms
/// microphone buffers observed in practice this holds ~5.5s, and app-audio buffers twice that.
/// A 24-slot queue held only ~0.25s and shed 5s of meeting-opening audio during warmup.
nonisolated let queue = MeetingLiveBoundedQueue<MeetingLiveSampleCopy.Sample>(capacity: 512)
private static let chunkSize: StreamingChunkSize = .ms160
- private static let maxOpenUtteranceSeconds: Double = 20
- private static let partialStallSeconds: Double = 2.5
+ private static let partialStallSeconds: Double = 120
+ private static let pauseObservationSeconds: Double = 5
/// The recognizer buffers a full chunk before it can decode it, so the speech behind the first
/// token started at least one chunk before the audio we were feeding when that token arrived.
private static let decoderLookback = CMTime(value: CMTimeValue(chunkSize.durationMs), timescale: 1000)
private let manager: any MeetingLiveRecognizer
+ private enum RecognizerCallback: Sendable {
+ case partial(String)
+ case end(String)
+ }
+
+ private let recognizerCallbacks = MeetingLiveBoundedQueue<RecognizerCallback>(capacity: 64)
private let uptime: @Sendable () -> Double
private let diagnosticsEnabled: Bool
private var diagnosticSnapshot = MeetingLiveDiagnosticSnapshot()
@@ -100,6 +117,19 @@ actor MeetingLiveTrackEngine {
private var diagUtterances = 0
private var diagLastFlowLog: Double = 0
private var diagLastEouUptime: Double?
+ private var firstFeedUptime: Double?
+ private var lastTextUpdateUptime: Double?
+ private var textPauseReported = false
+ private var lastInputRMS: Double = 0
+ private var inputEnergySinceFlow: Double = 0
+ private var inputSamplesSinceFlow = 0
+ private var peakInputRMSSinceFlow: Double = 0
+ private var modelProcessCompletions = 0
+
+ private enum ResetReason: String {
+ case startupWarmup, stopped, modelEndOfUtterance, noTextTimeout
+ case processingError, audioQueueOverflow, callbackQueueOverflow
+ }
private var diagLabel: String { self.kind == .microphone ? "MIC" : "APP" }
@@ -173,6 +203,7 @@ actor MeetingLiveTrackEngine {
}
func stop() async {
+ self.comparisonProbe?.recordEvent("recognizer-stop", track: self.kind)
self.isStopped = true
self.onPartial = nil
self.onUtterance = nil
@@ -185,7 +216,7 @@ actor MeetingLiveTrackEngine {
self.isModelReady = false
await self.drainTask?.value
self.drainTask = nil
- await self.resetRecognizer()
+ await self.resetRecognizer(reason: .stopped)
}
/// Fast, synchronous, non-blocking hand-off from the capture tee. On saturation the queue drops
@@ -198,11 +229,33 @@ actor MeetingLiveTrackEngine {
}
private func installCallbacks() async {
- await self.manager.setEouCallback { [weak self] transcript in
- Task { await self?.handleEOU(transcript) }
+ let callbacks = self.recognizerCallbacks
+ await self.manager.setEouCallback { transcript in
+ callbacks.enqueue(.end(transcript))
}
- await self.manager.setPartialTranscriptCallback { [weak self] transcript in
- Task { await self?.handlePartial(transcript) }
+ await self.manager.setPartialTranscriptCallback { transcript in
+ callbacks.enqueue(.partial(transcript))
+ }
+ }
+
+ /// Model callbacks can arrive while inference is suspended. Apply them only after the
+ /// decode call returns, so an endpoint cannot reset the decoder during that same inference.
+ private func applyRecognizerCallbacks() async {
+ let dropped = self.recognizerCallbacks.takeDroppedCount()
+ if dropped > 0 {
+ self.comparisonProbe?.recordEvent("recognizer-callbacks-dropped", track: self.kind, detail: String(dropped))
+ await self.resetUtterance(reason: .callbackQueueOverflow)
+ self.onDegraded?(self.kind, "Live captions missed model updates and resynced.")
+ return
+ }
+ for callback in self.recognizerCallbacks.drainAll() {
+ guard !self.isStopped else { return }
+ switch callback {
+ case let .partial(text): self.handlePartial(text)
+ case let .end(text):
+ await self.handleEOU(text)
+ return
+ }
}
}
@@ -223,7 +276,7 @@ actor MeetingLiveTrackEngine {
} catch {
DebugLogger.shared.warning("Live captions warmup failed (continuing): \(error)", source: "MeetingLive")
}
- await self.resetRecognizer()
+ await self.resetRecognizer(reason: .startupWarmup)
self.diag(String(format: "warmup complete in %.2fs", self.uptime() - started))
}
@@ -260,11 +313,28 @@ actor MeetingLiveTrackEngine {
private func consume(_ sample: MeetingLiveSampleCopy.Sample) async {
guard let converted = self.convert(sample.buffer) else {
+ self.comparisonProbe?.recordEvent("recognizer-conversion-failed", track: self.kind, pts: sample.pts.seconds)
if self.diagnosticsEnabled { self.diagnosticSnapshot.conversionFailures += 1 }
self.diag("convert FAILED (sr=\(sample.buffer.format.sampleRate) ch=\(sample.buffer.format.channelCount))")
return
}
+ if let comparisonProbe, comparisonProbe.isRecordingDiagnostics,
+ let input = meetingMicrophoneSynthesizeSampleBuffer(from: converted, presentationTime: sample.pts)
+ {
+ comparisonProbe.offer(input, stage: self.kind == .microphone ? .microphoneRecognizerInput : .browserRecognizerInput)
+ }
self.currentFeedPTS = sample.pts
+ if self.firstFeedUptime == nil { self.firstFeedUptime = self.uptime() }
+ if self.comparisonProbe?.isRecordingDiagnostics == true || self.diagnosticsEnabled,
+ let channel = converted.floatChannelData?[0], converted.frameLength > 0
+ {
+ let samples = UnsafeBufferPointer(start: channel, count: Int(converted.frameLength))
+ let energy = samples.reduce(0.0) { $0 + Double($1) * Double($1) }
+ self.lastInputRMS = sqrt(energy / Double(samples.count))
+ self.inputEnergySinceFlow += energy
+ self.inputSamplesSinceFlow += samples.count
+ self.peakInputRMSSinceFlow = max(self.peakInputRMSSinceFlow, self.lastInputRMS)
+ }
self.lastConsumedPTS = sample.pts + sample.duration
self.diagSamplesConsumed += 1
self.diagSecondsConsumed += sample.duration.seconds
@@ -272,16 +342,22 @@ actor MeetingLiveTrackEngine {
self.diagnosticSnapshot.convertedBuffers += 1
self.diagnosticSnapshot.feedSeconds += sample.duration.seconds
}
- self.logFlowIfDue()
do {
if self.diagnosticsEnabled { self.diagnosticSnapshot.appendStarts += 1 }
+ self.comparisonProbe?.recordEvent("recognizer-append-start", track: self.kind, pts: sample.pts.seconds)
try await self.manager.appendAudio(converted)
+ self.comparisonProbe?.recordEvent("recognizer-append-complete", track: self.kind, pts: sample.pts.seconds)
if self.diagnosticsEnabled {
self.diagnosticSnapshot.appendCompletions += 1
self.diagnosticSnapshot.processStarts += 1
}
let processStarted = self.diagnosticsEnabled ? self.uptime() : 0
+ self.comparisonProbe?.recordEvent("recognizer-decode-start", track: self.kind, pts: sample.pts.seconds)
try await self.manager.processBufferedAudio()
+ self.modelProcessCompletions += 1
+ self.comparisonProbe?.recordEvent("recognizer-decode-complete", track: self.kind, pts: sample.pts.seconds)
+ await self.applyRecognizerCallbacks()
+ self.logFlowIfDue()
if self.diagnosticsEnabled {
self.diagnosticSnapshot.processCompletions += 1
self.diagnosticSnapshot.maxProcessSeconds = max(
@@ -291,39 +367,44 @@ actor MeetingLiveTrackEngine {
await self.rotateLongUtteranceIfNeeded()
} catch {
if self.diagnosticsEnabled { self.diagnosticSnapshot.appendOrProcessErrors += 1 }
+ self.comparisonProbe?.recordEvent("recognizer-processing-failed", track: self.kind, pts: sample.pts.seconds, detail: error.localizedDescription)
DebugLogger.shared.warning("Live captions audio processing failed: \(error)", source: "MeetingLive")
self.onDegraded?(self.kind, "Live captions hit an error and resynced.")
- await self.resetUtterance()
+ await self.resetUtterance(reason: .processingError)
}
}
- /// The decoder's own endpointing is unreliable in the field: it can stop emitting partials
- /// mid-utterance (observed after as little as 10s / ~120 chars) and never fire EOU, freezing
- /// captions on stale text. Two fallbacks close the turn: the decoder has gone quiet for a few
- /// seconds (a pause or a stall — either way the turn is over), or the utterance hit the hard
- /// duration ceiling. Both finalize the current partial and reset for a fresh turn.
+ /// A gap in text is not proof the model is stuck. Let the model finish phrases naturally;
+ /// recover only after two minutes without a partial, never merely because a phrase is long.
private func rotateLongUtteranceIfNeeded() async {
guard let start = self.utteranceStartPTS,
let end = self.lastConsumedPTS,
let turnID = self.utteranceTurnID
else { return }
- let openSeconds = end.seconds - start.seconds
let quietSeconds = self.lastPartialUptime.map { self.uptime() - $0 }
- let hitCeiling = openSeconds > Self.maxOpenUtteranceSeconds
- let decoderQuiet = (quietSeconds ?? 0) > Self.partialStallSeconds && !self.lastPartialText.isEmpty
- guard hitCeiling || decoderQuiet else { return }
- let text = self.lastPartialText.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard (quietSeconds ?? 0) >= Self.partialStallSeconds, !self.lastPartialText.isEmpty else { return }
+ self.comparisonProbe?.recordEvent("forced-recognizer-endpoint", track: self.kind, pts: end.seconds, detail: "no partial update for \(Self.partialStallSeconds) seconds")
+ var text = self.lastPartialText.trimmingCharacters(in: .whitespacesAndNewlines)
+ self.comparisonProbe?.recordEvent("recognizer-flush-start", track: self.kind, pts: end.seconds)
+ do {
+ let finished = try await self.manager.finish().trimmingCharacters(in: .whitespacesAndNewlines)
+ if !finished.isEmpty { text = finished }
+ self.comparisonProbe?.recordEvent("recognizer-flush-complete", track: self.kind, pts: end.seconds, text: text)
+ } catch {
+ self.comparisonProbe?.recordEvent("recognizer-flush-failed", track: self.kind, detail: error.localizedDescription)
+ self.onDegraded?(self.kind, "Live captions could not finish a phrase and resynced.")
+ }
self.utteranceStartPTS = nil
self.utteranceTurnID = nil
self.lastPartialText = ""
self.lastPartialUptime = nil
- await self.resetRecognizer()
+ await self.resetRecognizer(reason: .noTextTimeout)
guard !self.isStopped, !text.isEmpty else { return }
self.lastUtteranceEndPTS = end
self.diagUtterances += 1
self.diag(String(
format: "forced EOU (%@) span=[%.2f–%.2f] chars=%d",
- hitCeiling ? "open>\(Int(Self.maxOpenUtteranceSeconds))s" : String(format: "quiet %.1fs", quietSeconds ?? 0),
+ String(format: "quiet %.1fs", quietSeconds ?? 0),
start.seconds,
end.seconds,
text.count
@@ -334,10 +415,29 @@ actor MeetingLiveTrackEngine {
/// Once every 5s of wall clock: proves audio is still reaching the recognizer, and how long the
/// current utterance has been open without an endpoint.
private func logFlowIfDue() {
- guard self.diagnosticsEnabled else { return }
+ guard self.diagnosticsEnabled || self.comparisonProbe?.isRecordingDiagnostics == true else { return }
let now = self.uptime()
guard now - self.diagLastFlowLog >= 5 else { return }
self.diagLastFlowLog = now
+ let textGap = now - (self.lastTextUpdateUptime ?? self.firstFeedUptime ?? now)
+ let windowRMS = sqrt(self.inputEnergySinceFlow / Double(max(1, self.inputSamplesSinceFlow)))
+ let detail = "feedBuffers=\(self.diagSamplesConsumed) audioSeconds=\(self.diagSecondsConsumed) "
+ + "decodeCompletions=\(self.modelProcessCompletions) textGapSeconds=\(textGap) "
+ + "partialUpdates=\(self.diagPartials) queueDepth=\(self.queue.count) "
+ + "inputRMS=\(self.lastInputRMS) windowRMS=\(windowRMS) peakBufferRMS=\(self.peakInputRMSSinceFlow) "
+ + "everProducedText=\(self.lastTextUpdateUptime != nil)"
+ self.inputEnergySinceFlow = 0
+ self.inputSamplesSinceFlow = 0
+ self.peakInputRMSSinceFlow = 0
+ self.comparisonProbe?.recordEvent("model-flow", track: self.kind, pts: self.lastConsumedPTS?.seconds, detail: detail)
+ if textGap >= Self.pauseObservationSeconds, !self.textPauseReported {
+ self.textPauseReported = true
+ let producedText = self.lastTextUpdateUptime != nil
+ if self.diagnosticsEnabled, producedText { self.diagnosticSnapshot.textPauses += 1 }
+ let step = producedText ? "model-text-pause" : "model-waiting-for-first-text"
+ self.comparisonProbe?.recordEvent(step, track: self.kind, pts: self.lastConsumedPTS?.seconds, detail: detail)
+ self.diag("\(step); not a stuck-model verdict: \(detail)")
+ }
let openFor = self.utteranceStartPTS.map { start in
(self.lastConsumedPTS ?? start).seconds - start.seconds
} ?? 0
@@ -367,6 +467,17 @@ actor MeetingLiveTrackEngine {
/// swallowed every second of preceding silence into the utterance's span.
private func handlePartial(_ text: String) {
guard !self.isStopped else { return }
+ let now = self.uptime()
+ let firstText = self.lastTextUpdateUptime == nil
+ if self.textPauseReported || firstText {
+ let gap = now - (self.lastTextUpdateUptime ?? self.firstFeedUptime ?? now)
+ let step = firstText ? "model-first-text" : "model-text-resumed"
+ self.comparisonProbe?.recordEvent(step, track: self.kind, pts: self.currentFeedPTS?.seconds, detail: "textGapSeconds=\(gap) decodeCompletions=\(self.modelProcessCompletions)")
+ self.diag("\(step) after \(gap)s; decodeCompletions=\(self.modelProcessCompletions)")
+ if self.diagnosticsEnabled, !firstText { self.diagnosticSnapshot.textResumptions += 1 }
+ }
+ self.textPauseReported = false
+ self.lastTextUpdateUptime = now
if self.diagnosticsEnabled {
self.diagnosticSnapshot.partialCallbacks += 1
if self.diagnosticSnapshot.firstPartialAfterResetSeconds == nil,
@@ -407,7 +518,7 @@ actor MeetingLiveTrackEngine {
self.utteranceTurnID = nil
self.lastPartialText = ""
self.lastPartialUptime = nil
- await self.resetRecognizer()
+ await self.resetRecognizer(reason: .modelEndOfUtterance)
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !self.isStopped, !trimmed.isEmpty else { return }
guard let start, let end, let turnID else {
@@ -432,6 +543,7 @@ actor MeetingLiveTrackEngine {
/// resetting per sample keeps a 250-sample burst from issuing 250 decoder resets and 250
/// degraded notifications; the drain loop performs exactly one resync before it next consumes.
private func noteDrop() {
+ self.comparisonProbe?.recordEvent("recognizer-queue-dropped-buffer", track: self.kind)
self.diagDrops += 1
self.pendingResync = true
}
@@ -440,27 +552,31 @@ actor MeetingLiveTrackEngine {
guard self.pendingResync else { return }
self.pendingResync = false
self.diag("queue saturated — resyncing after \(self.diagDrops) dropped samples")
- await self.resetUtterance()
+ await self.resetUtterance(reason: .audioQueueOverflow)
self.onDegraded?(self.kind, "Live captions dropped audio and resynced.")
}
- private func resetUtterance() async {
+ private func resetUtterance(reason: ResetReason) async {
self.utteranceStartPTS = nil
self.utteranceTurnID = nil
self.lastPartialText = ""
self.lastPartialUptime = nil
self.lastConsumedPTS = nil
- await self.resetRecognizer()
+ await self.resetRecognizer(reason: reason)
}
- private func resetRecognizer() async {
+ private func resetRecognizer(reason: ResetReason) async {
+ self.comparisonProbe?.recordEvent("recognizer-reset", track: self.kind, detail: reason.rawValue)
+ self.diag("reset reason=\(reason.rawValue)")
if self.diagnosticsEnabled {
self.diagnosticSnapshot.resetInvocations += 1
+ self.diagnosticSnapshot.lastResetReason = reason.rawValue
self.diagnosticSnapshot.wrapperGeneration += 1
self.diagnosticSnapshot.firstPartialAfterResetSeconds = nil
self.diagnosticResetUptime = nil
}
await self.manager.reset()
+ _ = self.recognizerCallbacks.drainAll()
if self.diagnosticsEnabled {
self.diagnosticSnapshot.recognizerResetReturned += 1
self.diagnosticResetUptime = self.uptime()
@@ -471,6 +587,7 @@ actor MeetingLiveTrackEngine {
/// Deterministic P0 seam: bypasses capture/model loading, but exercises production conversion,
/// append/process and existing rotation policy. It deliberately does not add recovery policy.
func diagnosticConsume(_ sample: MeetingLiveSampleCopy.Sample) async { await self.consume(sample) }
+ func diagnosticInstallCallbacks() async { await self.installCallbacks() }
func diagnosticPartial(_ text: String) { self.handlePartial(text) }
func diagnosticSnapshotValue() -> MeetingLiveDiagnosticSnapshot { self.diagnosticSnapshot }
#endif
diff --git a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift
index dfb2f07d..daafc3d0 100644
--- a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift
+++ b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift
@@ -32,6 +32,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
DebugLogger.shared.info(message(), source: "MeetingLive")
}
+ private var comparisonProbe: MeetingAudioComparisonProbe?
private var snapshot: MeetingLiveTranscriptSnapshot = .empty
private var recentThemUtterances: [MeetingLiveUtterance] = []
private var offerCounts: [MeetingAudioTrackKind: Int] = [:]
@@ -49,6 +50,26 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
self.onUpdate = onUpdate
}
+ func attachComparisonProbe(_ probe: MeetingAudioComparisonProbe) {
+ self.stateLock.withLock { self.comparisonProbe = probe }
+ #if arch(arm64)
+ let engines = self.stateLock.withLock { [self.microphoneEngine, self.applicationEngine].compactMap { $0 } }
+ for engine in engines {
+ Task { await engine.attachComparisonProbe(probe) }
+ }
+ #endif
+ probe.recordEvent("live-probe-attached")
+ }
+
+ func recordComparisonCaptureEvent(_ event: MeetingCaptureEvent) {
+ guard let probe = self.stateLock.withLock({ self.comparisonProbe }), probe.isRecordingDiagnostics else { return }
+ probe.recordEvent("capture-event", detail: String(describing: event))
+ }
+
+ private func recordProbe(_ step: String, kind: MeetingAudioTrackKind? = nil, pts: Double? = nil, text: String? = nil, detail: String? = nil) {
+ self.stateLock.withLock { self.comparisonProbe }?.recordEvent(step, track: kind, pts: pts, text: text, detail: detail)
+ }
+
/// Clears any in-progress microphone partial at a capture-side splice, so it can't straddle eras.
func resetMicrophoneUtterance() {
self.publish { $0.settingPartial(nil, for: .you) }
@@ -56,6 +77,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
func setMicrophoneCaptureMethod(_ method: MeetingAudioTrackCaptureMethod?) {
self.stateLock.withLock { self.microphoneCaptureMethod = method }
+ self.recordProbe("microphone-capture-method", detail: method?.rawValue ?? "unknown")
self.diag(
"[live/ECHO] setter-armed captureMethod=\(method.map(String.init(describing:)) ?? "nil")"
)
@@ -98,11 +120,13 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
func offer(kind: MeetingAudioTrackKind, sampleBuffer: CMSampleBuffer) {
#if arch(arm64)
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
+ self.recordProbe("live-audio-offered", kind: kind, pts: pts.seconds)
guard pts.isNumeric else { return }
// Keep the shared capture clock even when this track has no caption engine.
self.originBox.establish(pts)
guard self.stateLock.withLock({ kind == .microphone ? self.microphoneEngine != nil : self.applicationEngine != nil }) else { return }
guard let sample = MeetingLiveSampleCopy.copy(sampleBuffer) else {
+ self.recordProbe("live-sample-copy-failed", kind: kind, pts: pts.seconds)
self.diag("[live/tee] sample copy FAILED kind=\(kind)")
return
}
@@ -170,6 +194,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
guard let origin = self.originBox.current else { return }
let startSeconds = MeetingLiveTimeConversion.sessionSeconds(pts: start, origin: origin)
let speaker = Self.speaker(for: kind)
+ self.recordProbe("partial-before-display", kind: kind, pts: startSeconds, text: text)
let partial = MeetingLivePartial(id: id, text: text, start: startSeconds)
self.publish { $0.settingPartial(partial, for: speaker) }
}
@@ -181,6 +206,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
let endSeconds = MeetingLiveTimeConversion.sessionSeconds(pts: end, origin: origin)
let speaker = Self.speaker(for: kind)
+ self.recordProbe("utterance-before-filter", kind: kind, pts: startSeconds, text: text)
let updated: MeetingLiveTranscriptSnapshot = self.stateLock.withLock {
self.snapshot.revision &+= 1
if speaker == .you {
@@ -191,6 +217,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
windowSeconds: Self.echoWindowSeconds
)
if MeetingLiveEchoFilter.shouldSuppress(micText: text, recentThemText: recentThem) {
+ self.comparisonProbe?.recordEvent("utterance-echo-suppressed", track: kind, pts: startSeconds, text: text, detail: recentThem)
self.snapshot = self.snapshot.settingPartial(nil, for: .you).markingFinalized(id)
return self.snapshot
}
@@ -200,6 +227,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
self.recentThemUtterances.append(record)
self.recentThemUtterances.removeAll { $0.end < startSeconds - Self.echoWindowSeconds }
}
+ self.comparisonProbe?.recordEvent("utterance-displayed", track: kind, pts: startSeconds, text: text)
let utterance = MeetingLiveUtterance(id: id, speaker: speaker, text: text, start: startSeconds, end: endSeconds)
self.snapshot = self.snapshot.inserting(utterance).settingPartial(nil, for: speaker)
return self.snapshot
@@ -208,6 +236,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
}
private func handleDegraded(kind: MeetingAudioTrackKind, reason: String) {
+ self.recordProbe("live-degraded", kind: kind, detail: reason)
DebugLogger.shared.warning("Live captions degraded: \(reason)", source: "MeetingLive")
self.publish { $0.settingAvailability(.degraded(reason: reason)) }
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift b/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift
index 77eaaa13..14c8d431 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift
@@ -61,6 +61,7 @@ final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable {
private let manifestURL: URL
private let chunkDuration: TimeInterval
private let eventHandler: EventHandler
+ private let diagnosticSampleObserver: (@Sendable (CMSampleBuffer, UInt64) -> Void)?
private let encoder: JSONEncoder
private let ledger: MeetingAudioChunkLedgerStore
@@ -108,7 +109,8 @@ final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable {
// PCM persistence is intentionally synchronous on the serialized writer queue. Keep the
// queue bounded, but allow roughly 1.3 seconds of capture jitter at a 48 kHz/10 ms tap
// cadence while the filesystem drains a burst.
- pendingSlotLimit: Int = 128
+ pendingSlotLimit: Int = 128,
+ diagnosticSampleObserver: (@Sendable (CMSampleBuffer, UInt64) -> Void)? = nil
) throws {
self.trackID = track.id
self.track = track
@@ -119,6 +121,7 @@ final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable {
self.manifestURL = self.trackDirectory.appendingPathComponent("track.json")
self.chunkDuration = max(60, chunkDuration)
self.eventHandler = eventHandler
+ self.diagnosticSampleObserver = diagnosticSampleObserver
self.ledger = MeetingAudioChunkLedgerStore(sessionDirectory: sessionDirectory)
self.queue = DispatchQueue(
label: "com.fluidvoice.meeting.writer.\(track.kind.rawValue)",
@@ -144,7 +147,11 @@ final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable {
}
@discardableResult
- func enqueue(_ sampleBuffer: CMSampleBuffer, producerEpoch: UInt64 = 0) -> Bool {
+ func enqueue(
+ _ sampleBuffer: CMSampleBuffer,
+ producerEpoch: UInt64 = 0,
+ onConsumed: (@Sendable () -> Void)? = nil
+ ) -> Bool {
self.stateLock.lock()
guard self.lifecycle == .accepting else {
self.stateLock.unlock()
@@ -168,6 +175,7 @@ final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable {
self.queue.async { [weak self, sampleBox] in
defer { self?.pendingSlots.signal() }
self?.consume(sampleBox.sampleBuffer, producerEpoch: sampleBox.producerEpoch)
+ onConsumed?()
}
self.stateLock.unlock()
return true
@@ -375,6 +383,7 @@ final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable {
}
private func consume(_ sourceSampleBuffer: CMSampleBuffer, producerEpoch: UInt64) {
+ self.diagnosticSampleObserver?(sourceSampleBuffer, producerEpoch)
guard CMSampleBufferIsValid(sourceSampleBuffer), CMSampleBufferDataIsReady(sourceSampleBuffer) else {
self.recordDroppedSample(detail: "Capture delivered an invalid audio sample.")
return
diff --git a/Sources/Fluid/Services/Meeting/MeetingAudioComparisonProbe.swift b/Sources/Fluid/Services/Meeting/MeetingAudioComparisonProbe.swift
new file mode 100644
index 00000000..14328417
--- /dev/null
+++ b/Sources/Fluid/Services/Meeting/MeetingAudioComparisonProbe.swift
@@ -0,0 +1,284 @@
+@preconcurrency import AVFoundation
+import CoreMedia
+import Foundation
+
+/// Local investigation tool. Diagnostic PCM never enters session history,
+/// playback, captions, echo decisions or final transcription. Disabled unless explicitly armed.
+final nonisolated class MeetingAudioComparisonProbe: @unchecked Sendable {
+ static let nextRecordingKey = "MeetingAudioComparisonProbeNextRecording"
+ static let requestExpirationKey = "MeetingAudioComparisonProbeExpiresAt"
+ static let requestLifetime: TimeInterval = 30 * 60
+ static let maximumSeconds: TimeInterval = 300
+
+ enum Stage: String, CaseIterable, Codable, Sendable {
+ case browser
+ case microphoneInput = "microphone-before-aec"
+ case microphoneOutput = "microphone-after-aec"
+ case microphoneRecognizerInput = "microphone-recognizer-input"
+ case browserRecognizerInput = "browser-recognizer-input"
+ }
+
+ private struct Context: Encodable {
+ let sessionID: UUID
+ let mode: MeetingCaptureMode
+ let microphone: MeetingMicrophoneIdentity
+ let application: MeetingApplicationIdentity?
+ let operatingSystem: String
+ let appVersion: String
+ let appBuild: String
+ }
+
+ private struct Report: Encodable {
+ let context: Context
+ let schemaVersion = 1
+ let purpose = "Compare capture input with the microphone audio saved by FluidMeet. Input is already subject to any macOS or device processing."
+ let maximumSeconds: TimeInterval
+ let completed: Bool
+ let hasAllStages: Bool
+ let tracks: [String: MeetingAudioTrack]
+ let droppedDiagnosticEvents: Int
+ }
+
+ private struct Event: Encodable {
+ let uptime: TimeInterval
+ let step: String
+ let track: String?
+ let pts: Double?
+ let text: String?
+ let detail: String?
+ }
+
+ let directory: URL
+ private let context: Context
+ private let eventQueue = DispatchQueue(label: "com.fluidvoice.meeting.audio-comparison.events", qos: .utility)
+ private let eventSlots = DispatchSemaphore(value: 128)
+ private let eventFile: FileHandle
+ private var droppedDiagnosticEvents = 0
+ private var deadlineTimer: DispatchSourceTimer?
+ private let writers: [Stage: MeetingAudioChunkWriter]
+ private let lock = NSLock()
+ private let uptime: @Sendable () -> TimeInterval
+ private let deadline: TimeInterval
+ private let durationLimit: TimeInterval
+ private var accepting = true
+ private var finalization: Task<Void, Never>?
+ private let onUsefulComparison: @Sendable () -> Void
+
+ var isRecordingDiagnostics: Bool {
+ self.lock.withLock { self.accepting && self.uptime() < self.deadline }
+ }
+
+ /// Short attempts and in-room recordings must not consume an online audio comparison.
+ /// Expiry bounds the request even if no usable recording is made.
+ static func requestExpiration(mode: MeetingCaptureMode, defaults: UserDefaults = .standard, now: TimeInterval = Date.timeIntervalSinceReferenceDate) -> TimeInterval? {
+ guard defaults.bool(forKey: self.nextRecordingKey) else { return nil }
+ var expiration = defaults.double(forKey: self.requestExpirationKey)
+ if expiration == 0 {
+ expiration = now + self.requestLifetime
+ defaults.set(expiration, forKey: self.requestExpirationKey)
+ }
+ guard expiration.isFinite, expiration > now else {
+ defaults.removeObject(forKey: self.nextRecordingKey)
+ defaults.removeObject(forKey: self.requestExpirationKey)
+ return nil
+ }
+ return mode == .onlineCall ? expiration : nil
+ }
+
+ static func completeRequest(expiration: TimeInterval, defaults: UserDefaults = .standard) {
+ // A previous probe finishing must not clear a newly armed request.
+ guard defaults.double(forKey: self.requestExpirationKey) == expiration else { return }
+ defaults.removeObject(forKey: self.nextRecordingKey)
+ defaults.removeObject(forKey: self.requestExpirationKey)
+ }
+
+ init(
+ session: MeetingSession,
+ directory: URL,
+ maximumSeconds: TimeInterval = MeetingAudioComparisonProbe.maximumSeconds,
+ uptime: @escaping @Sendable () -> TimeInterval = { ProcessInfo.processInfo.systemUptime },
+ onUsefulComparison: @escaping @Sendable () -> Void = {}
+ ) throws {
+ self.directory = directory
+ self.onUsefulComparison = onUsefulComparison
+ self.context = Context(
+ sessionID: session.id,
+ mode: session.mode,
+ microphone: session.selectedMicrophone,
+ application: session.capturedApplication,
+ operatingSystem: ProcessInfo.processInfo.operatingSystemVersionString,
+ appVersion: Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown",
+ appBuild: Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "unknown"
+ )
+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700])
+ try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path)
+ self.uptime = uptime
+ self.durationLimit = min(Self.maximumSeconds, max(0, maximumSeconds))
+ self.deadline = uptime() + self.durationLimit
+ var writers: [Stage: MeetingAudioChunkWriter] = [:]
+ for stage in Stage.allCases {
+ let track = MeetingAudioTrack(
+ id: UUID(),
+ kind: stage == .browser || stage == .browserRecognizerInput ? .applicationAudio : .microphone,
+ sourceIdentifier: stage.rawValue,
+ sourceDisplayName: stage.rawValue,
+ format: nil,
+ timebase: session.timebase,
+ health: .waiting,
+ chunks: []
+ )
+ writers[stage] = try MeetingAudioChunkWriter(
+ track: track,
+ sessionDirectory: directory.appendingPathComponent(stage.rawValue, isDirectory: true),
+ chunkDuration: 60,
+ eventHandler: { _ in },
+ pendingSlotLimit: 32
+ )
+ }
+ self.writers = writers
+ let eventURL = directory.appendingPathComponent("events.jsonl")
+ try Data().write(to: eventURL)
+ self.eventFile = try FileHandle(forWritingTo: eventURL)
+ try Self.write(Report(context: self.context, maximumSeconds: min(Self.maximumSeconds, max(0, maximumSeconds)), completed: false, hasAllStages: false, tracks: [:], droppedDiagnosticEvents: 0), to: directory)
+ let timer = DispatchSource.makeTimerSource(queue: .global(qos: .utility))
+ timer.schedule(deadline: .now() + self.durationLimit)
+ timer.setEventHandler { [weak self] in
+ guard let self else { return }
+ Task { await self.stop() }
+ }
+ self.deadlineTimer = timer
+ timer.resume()
+ }
+
+ func recordAECFrame(_ frame: MeetingAECJoinedFrame, output: [Float], statistics: MeetingAECBridgeStatistics) {
+ guard self.isRecordingDiagnostics else { return }
+ func rms(_ samples: [Float]) -> Double {
+ guard !samples.isEmpty else { return 0 }
+ return sqrt(samples.reduce(0.0) { $0 + Double($1) * Double($1) } / Double(samples.count))
+ }
+ let delay = statistics.estimatedDelayMilliseconds.map { String($0) } ?? "unknown"
+ let residual = statistics.residualEchoLikelihood.map { String($0) } ?? "unknown"
+ let detail = "renderRMS=\(rms(frame.render.samples)) captureRMS=\(rms(frame.capture.samples)) outputRMS=\(rms(output)) "
+ + "clockAttested=\(frame.clockAttested) pairedFrames=\(frame.pairedFrameCount) "
+ + "estimatedDelayMs=\(delay) residualEcho=\(residual) resets=\(statistics.resets)"
+ self.recordEvent("aec-frame", track: .microphone, pts: frame.capture.presentationTime.seconds, detail: detail)
+ }
+
+ /// Small, bounded asynchronous event log, including pre-filter text only for this explicit test.
+ func recordEvent(_ step: String, track: MeetingAudioTrackKind? = nil, pts: Double? = nil, text: String? = nil, detail: String? = nil) {
+ self.lock.lock()
+ guard self.accepting, self.uptime() < self.deadline else { self.lock.unlock(); return }
+ guard self.eventSlots.wait(timeout: .now()) == .success else {
+ self.droppedDiagnosticEvents += 1
+ self.lock.unlock()
+ return
+ }
+ let event = Event(
+ uptime: self.uptime(),
+ step: step,
+ track: track?.rawValue,
+ pts: pts?.isFinite == true ? pts : nil,
+ text: text.map { String($0.prefix(8192)) },
+ detail: detail.map { String($0.prefix(2048)) }
+ )
+ self.eventQueue.async { [self] in
+ defer { self.eventSlots.signal() }
+ do {
+ var data = try JSONEncoder().encode(event)
+ data.append(0x0a)
+ try self.eventFile.write(contentsOf: data)
+ } catch {
+ self.lock.withLock { self.droppedDiagnosticEvents += 1 }
+ }
+ }
+ self.lock.unlock()
+ }
+
+ /// Capture is already stopped. Allow pending caption callbacks to be recorded without delaying
+ /// normal Stop or holding capture hardware, then close all diagnostic files.
+ func finishAfterCaptionTeardown() {
+ self.recordEvent("capture-stopped")
+ Task { [self] in
+ try? await Task.sleep(nanoseconds: 3_000_000_000)
+ await self.stop()
+ }
+ }
+
+ /// A bounded enqueue only. Disk work stays on independent writer queues; saturation is
+ /// reported in the probe's track health rather than blocking or changing normal capture.
+ func offer(_ sample: CMSampleBuffer, stage: Stage, producerEpoch: UInt64 = 0) {
+ let expired = self.lock.withLock { () -> Bool? in
+ guard self.accepting else { return nil }
+ if self.uptime() >= self.deadline {
+ self.accepting = false
+ return true
+ }
+ self.writers[stage]?.enqueue(sample, producerEpoch: producerEpoch)
+ return false
+ }
+ if expired == true { Task { await self.stop() } }
+ }
+
+ func stop() async {
+ let task = self.lock.withLock { () -> Task<Void, Never> in
+ if let finalization = self.finalization { return finalization }
+ self.accepting = false
+ self.deadlineTimer?.cancel()
+ self.deadlineTimer = nil
+ let task = Task { [self] in
+ var tracks: [String: MeetingAudioTrack] = [:]
+ for stage in Stage.allCases {
+ if let writer = self.writers[stage] { tracks[stage.rawValue] = await writer.stop() }
+ }
+ await withCheckedContinuation { continuation in
+ self.eventQueue.async { [self] in
+ try? self.eventFile.synchronize()
+ try? self.eventFile.close()
+ continuation.resume()
+ }
+ }
+ do {
+ let hasAllStages = Stage.allCases.allSatisfy { stage in
+ tracks[stage.rawValue]?.chunks.contains(where: { $0.finalizationState == .finalized }) == true
+ }
+ let report = Report(
+ context: self.context,
+ maximumSeconds: self.durationLimit,
+ completed: true,
+ hasAllStages: hasAllStages,
+ tracks: tracks,
+ droppedDiagnosticEvents: self.lock.withLock { self.droppedDiagnosticEvents }
+ )
+ try Self.write(report, to: self.directory)
+ if hasAllStages, Self.hasUsefulComparison(tracks) { self.onUsefulComparison() }
+ } catch {
+ DebugLogger.shared.warning("Audio comparison probe report could not be saved: \(error)", source: "MeetingAudioComparisonProbe")
+ }
+ }
+ self.finalization = task
+ return task
+ }
+ await task.value
+ }
+
+ static func hasUsefulComparison(_ tracks: [String: MeetingAudioTrack]) -> Bool {
+ Stage.allCases.allSatisfy { stage in
+ guard let track = tracks[stage.rawValue], track.health.droppedSampleCount == 0 else { return false }
+ let seconds = track.chunks.reduce(0.0) { total, chunk in
+ guard chunk.finalizationState == .finalized,
+ let asset = chunk.captureAnalysisAsset,
+ let frames = asset.frameCount,
+ let rate = asset.sampleRate, rate > 0 else { return total }
+ return total + Double(frames) / rate
+ }
+ return seconds >= 30
+ }
+ }
+
+ private static func write(_ report: Report, to directory: URL) throws {
+ let encoder = JSONEncoder()
+ encoder.dateEncodingStrategy = .iso8601
+ encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+ try encoder.encode(report).write(to: directory.appendingPathComponent("comparison.json"), options: .atomic)
+ }
+}
diff --git a/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift b/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
index 292bd42b..dd608541 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
@@ -158,6 +158,7 @@ protocol WindowSnapshotProviding: AnyObject {
nonisolated struct BrowserTabURL: Sendable, Equatable {
var host: String
var path: String
+ var callState: MeetingBrowserCallState = .unknown
}
@MainActor
@@ -967,7 +968,8 @@ final class AXBrowserTabReader: BrowserTabReading {
let windowElement = window as! AXUIElement
let document = self.stringAttribute(windowElement, attribute: "AXDocument")
- if let document, let url = self.parse(document) {
+ let documentURL = document.flatMap(self.parse)
+ if let url = documentURL, !MeetingBrowserCallEvidence.supports(url) {
return .found(url)
}
@@ -976,6 +978,7 @@ final class AXBrowserTabReader: BrowserTabReading {
if let found = self.findWebAreaURL(root: windowElement, depth: 0, budget: &budget, timedOut: &timedOut) {
return .found(found)
}
+ if let documentURL { return .found(documentURL) }
return timedOut ? .timedOut : .notFound("document=\(document == nil ? "nil" : "set") webarea=miss")
}
@@ -993,7 +996,8 @@ final class AXBrowserTabReader: BrowserTabReading {
guard budget > 0 else { return nil }
let role = self.stringAttribute(child, attribute: kAXRoleAttribute as String) ?? ""
if role == "AXWebArea" {
- if let urlString = self.stringAttribute(child, attribute: "AXURL"), let url = self.parse(urlString) {
+ if let urlString = self.stringAttribute(child, attribute: "AXURL"), var url = self.parse(urlString) {
+ url.callState = MeetingBrowserCallAXReader.state(webArea: child, url: url)
return url
}
continue
diff --git a/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift b/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
index e5db4312..05492a03 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
@@ -18,6 +18,8 @@ final class MeetingAutoDetector {
/// How long the meeting app must have let go of its audio, or been gone, before we ask whether
/// the meeting ended. Short blips (a device switch, a crash and relaunch) stay under it.
static let endSignalGraceSeconds: TimeInterval = 20
+ /// A positive post-call page does not need the reconnect grace used for ambiguous audio loss.
+ static let browserPostCallGraceSeconds: TimeInterval = 2
/// People still talking means the meeting is not over for them, whatever the app did. The
/// question waits until nobody has said anything for this long.
static let speechQuietSeconds: TimeInterval = 45
@@ -109,6 +111,7 @@ final class MeetingAutoDetector {
/// The meeting app behind our own live recording. Kept apart from the episode, which is
/// retired by rules written for the next start prompt and can go while we still record.
private struct WatchedRecording {
+ var sessionID: MeetingSessionID? = nil
var pid: Int32
var bundleIdentifier: String
var tier: MeetingDetectionTier
@@ -119,6 +122,8 @@ final class MeetingAutoDetector {
/// Browser meetings only: when the browser was first seen without the microphone.
var browserMicReleasedAt: Date?
var browserMicUnknown = false
+ var browserCallEvidenceKey: String?
+ var browserCallEndedAt: Date?
var asked = false
/// The user waved the question away. It stays away for the rest of this recording.
var silenced = false
@@ -222,21 +227,22 @@ final class MeetingAutoDetector {
self.lastLoggedRejectionByPID.removeAll()
}
- private func pollTick(runGeneration: UInt64) async {
+ func pollTick(runGeneration: UInt64? = nil) async {
+ let runGeneration = runGeneration ?? self.runGeneration
guard self.isCurrentRun(runGeneration) else { return }
let now = self.clock.now()
- if self.isNativeDetectionEnabled() {
+ if self.shouldPollAudio {
await self.pollAudioProcessActivity(at: now, expectedRunGeneration: runGeneration)
guard self.isCurrentRun(runGeneration) else { return }
}
if self.shouldCollectEvidence(at: now) {
- if self.isNativeDetectionEnabled() {
- let nativePIDs = Set(self.records.filter { $0.value.tier == .nativeTier1 }.keys)
+ if self.isNativeDetectionEnabled() || self.watchedRecording?.tier == .nativeTier1 {
+ let nativePIDs = Set(self.records.filter { $0.value.tier == .nativeTier1 && self.shouldObserve($0.value) }.keys)
if !nativePIDs.isEmpty {
self.handleWindowSnapshot(self.windowSnapshotProvider.snapshot(interestPIDs: nativePIDs), at: now)
}
}
- if self.isBrowserDetectionEnabled() {
+ if self.isBrowserDetectionEnabled() || self.watchedRecording?.tier == .browserTier2 {
await self.pollBrowserTabsIfDue(at: now, runGeneration: runGeneration)
guard self.isCurrentRun(runGeneration) else { return }
}
@@ -249,11 +255,17 @@ final class MeetingAutoDetector {
!Task.isCancelled && self.runGeneration == generation && self.pollTask != nil
}
+ private var shouldPollAudio: Bool { self.isNativeDetectionEnabled() || self.watchedRecording != nil }
+
+ private func shouldObserve(_ record: CandidateRecord) -> Bool {
+ if record.tier == .nativeTier1 ? self.isNativeDetectionEnabled() : self.isBrowserDetectionEnabled() { return true }
+ guard let watched = self.watchedRecording else { return false }
+ return watched.pid == record.pid && watched.bundleIdentifier == record.bundleIdentifier && watched.incarnation == record.incarnation
+ }
+
private func shouldCollectEvidence(at now: Date) -> Bool {
+ if self.watchedRecording != nil { return true }
if !self.episodesByKey.isEmpty { return true }
- // A browser we were recording is gone. Keep reading tabs so a relaunched one that is back
- // on the meeting can be recognised.
- if let watched = self.watchedRecording, watched.tier == .browserTier2, !self.isStillRunning(watched) { return true }
return self.records.values.contains { record in
record.hasLiveWindow || record.windowLostAt != nil
|| record.audioEvidenceAt.map { now.timeIntervalSince($0) <= Self.confirmDeadlineSeconds } == true
@@ -262,11 +274,12 @@ final class MeetingAutoDetector {
private func pollBrowserTabsIfDue(at now: Date, runGeneration: UInt64) async {
for (pid, record) in self.records where record.tier == .browserTier2 {
- guard self.isCurrentRun(runGeneration), self.isBrowserDetectionEnabled() else { return }
+ guard self.isCurrentRun(runGeneration) else { return }
+ guard self.shouldObserve(record) else { continue }
let lastPoll = self.lastBrowserPollAt[pid]
guard lastPoll.map { now.timeIntervalSince($0) >= 2 } ?? true else { continue }
let url = await self.browserTabReader.frontmostTabURL(bundleIdentifier: record.bundleIdentifier, processID: pid)
- guard self.isCurrentRun(runGeneration), self.isBrowserDetectionEnabled(),
+ guard self.isCurrentRun(runGeneration), self.shouldObserve(record),
self.records[pid]?.incarnation == record.incarnation
else { continue }
self.lastBrowserPollAt[pid] = now
@@ -278,7 +291,8 @@ final class MeetingAutoDetector {
func handleWorkspaceEvent(_ event: WorkspaceEvent, at now: Date) {
guard let tier = MeetingAppRegistry.tier(forBundleIdentifier: event.bundleIdentifier) else { return }
- guard tier == .nativeTier1 ? self.isNativeDetectionEnabled() : self.isBrowserDetectionEnabled() else { return }
+ let watchesProcess = self.watchedRecording.map { $0.pid == event.processID && $0.bundleIdentifier == event.bundleIdentifier } == true
+ guard watchesProcess || (tier == .nativeTier1 ? self.isNativeDetectionEnabled() : self.isBrowserDetectionEnabled()) else { return }
switch event.kind {
case .launched:
@@ -357,12 +371,12 @@ final class MeetingAutoDetector {
private func pollAudioProcessActivity(at now: Date, expectedRunGeneration: UInt64?) async {
guard !Task.isCancelled,
- expectedRunGeneration.map({ self.runGeneration == $0 && self.pollTask != nil && self.isNativeDetectionEnabled() }) ?? true
+ expectedRunGeneration.map({ self.isCurrentRun($0) && self.shouldPollAudio }) ?? true
else { return }
- let candidates = self.records.filter { $0.value.tier == .nativeTier1 }
+ let candidates = self.records.filter { $0.value.tier == .nativeTier1 && self.shouldObserve($0.value) }
let snapshot = await self.audioProcessActivity.snapshot()
guard !Task.isCancelled,
- expectedRunGeneration.map({ self.runGeneration == $0 && self.pollTask != nil && self.isNativeDetectionEnabled() }) ?? true
+ expectedRunGeneration.map({ self.isCurrentRun($0) && self.shouldPollAudio }) ?? true
else { return }
self.noteWatchedBrowserMicrophone(snapshot, at: now)
guard let activeInputByPID = MeetingAudioProcessResolver.activeOwnerInputByPID(snapshot: snapshot) else {
@@ -381,7 +395,7 @@ final class MeetingAutoDetector {
self.audioQueryUnknown = false
for (pid, record) in candidates {
guard !Task.isCancelled,
- expectedRunGeneration.map({ self.runGeneration == $0 && self.pollTask != nil && self.isNativeDetectionEnabled() }) ?? true,
+ expectedRunGeneration.map({ self.isCurrentRun($0) && self.shouldObserve(record) }) ?? true,
self.records[pid]?.incarnation == record.incarnation
else { continue }
let isActive = activeInputByPID[pid] != nil
@@ -606,7 +620,16 @@ final class MeetingAutoDetector {
// MARK: - Browser evidence (Tier 2)
func handleBrowserTabURL(_ url: BrowserTabURL?, pid: Int32, bundleIdentifier: String, at now: Date) {
- guard let url, MeetingInCallURLMatcher.isInCallURL(host: url.host, path: url.path) else {
+ if let url { self.noteBrowserCallState(url, pid: pid, bundleIdentifier: bundleIdentifier, at: now) }
+ if let url, url.callState == .ended, MeetingBrowserCallEvidence.supports(url) {
+ self.records[pid]?.windowEvidenceAt = nil
+ let key = "pid:\(pid)|url:\(url.host)\(url.path)"
+ if let episode = self.episodesByKey[key], !episode.consumed {
+ self.episodesByKey[key] = nil
+ self.onEpisodeInvalidated?(episode.id)
+ }
+ }
+ guard let url, url.callState != .ended, MeetingInCallURLMatcher.isInCallURL(host: url.host, path: url.path) else {
// Unreadable AXURL (nil) or the tab navigated off an in-call URL: fail closed and,
// if this record had live evidence, start the loss clock so the episode can end.
if self.records[pid]?.hasLiveWindow == true {
@@ -629,6 +652,27 @@ final class MeetingAutoDetector {
self.attemptConfirm(pid: pid, at: now)
}
+ private func noteBrowserCallState(_ url: BrowserTabURL, pid: Int32, bundleIdentifier: String, at now: Date) {
+ guard MeetingBrowserCallEvidence.supports(url),
+ var watched = self.watchedRecording, watched.tier == .browserTier2,
+ watched.pid == pid, watched.bundleIdentifier == bundleIdentifier, self.isStillRunning(watched)
+ else { return }
+ let key = "url:\(url.host)\(url.path)"
+ if watched.browserCallEvidenceKey == nil, url.callState == .inCall { watched.browserCallEvidenceKey = key }
+ guard watched.browserCallEvidenceKey == key else { return }
+ switch url.callState {
+ case .ended:
+ if watched.browserCallEndedAt == nil {
+ watched.browserCallEndedAt = now
+ }
+ case .inCall:
+ watched.browserCallEndedAt = nil
+ case .unknown:
+ break // A failed read or a tab switch cannot undo a positive post-call observation.
+ }
+ self.watchedRecording = watched
+ }
+
// MARK: - Confirm + episode lifecycle
private func attemptConfirm(pid: Int32, at now: Date) {
@@ -750,15 +794,56 @@ final class MeetingAutoDetector {
self.activeConsumedEpisodeKey = key
self.dismissalTimestamps.removeAll()
let record = self.records[episode.pid]
+ // The shared recording-state observer may already have attached this session.
+ // Consuming its start prompt must not reset a pending or dismissed end question.
+ if let watched = self.watchedRecording,
+ watched.pid == episode.pid, watched.bundleIdentifier == episode.bundleIdentifier,
+ watched.incarnation == record?.incarnation
+ { return }
self.watchedRecording = WatchedRecording(
pid: episode.pid,
bundleIdentifier: episode.bundleIdentifier,
tier: episode.tier,
incarnation: record?.incarnation,
- sawProcessAudio: record?.processAudioWasActive == true
+ sawProcessAudio: record?.processAudioWasActive == true,
+ browserCallEvidenceKey: record?.windowEvidenceKey
)
}
+ /// Called for every real recording, including manual and menu-bar starts. This observes
+ /// the explicitly captured app; it never confirms an episode or changes capture/source state.
+ func recordingStarted(sessionID: MeetingSessionID, application: MeetingApplicationIdentity?) {
+ guard self.watchedRecording?.sessionID != sessionID else { return }
+ guard let application, let pid = application.processID, pid > 0,
+ let tier = MeetingAppRegistry.tier(forBundleIdentifier: application.bundleIdentifier)
+ else {
+ self.recordingStopped()
+ return
+ }
+ self.armIfNeeded(pid: pid, bundleIdentifier: application.bundleIdentifier, tier: tier)
+ guard let record = self.records[pid], record.bundleIdentifier == application.bundleIdentifier else {
+ self.recordingStopped()
+ return
+ }
+ if var watched = self.watchedRecording,
+ watched.sessionID == nil, watched.pid == pid,
+ watched.bundleIdentifier == application.bundleIdentifier, watched.incarnation == record.incarnation
+ {
+ watched.sessionID = sessionID
+ self.watchedRecording = watched
+ } else {
+ self.watchedRecording = WatchedRecording(
+ sessionID: sessionID,
+ pid: pid,
+ bundleIdentifier: application.bundleIdentifier,
+ tier: tier,
+ incarnation: record.incarnation,
+ sawProcessAudio: record.processAudioWasActive,
+ browserCallEvidenceKey: record.windowEvidenceKey
+ )
+ }
+ }
+
/// Our recording is over, however it ended. Nothing is left to ask about.
func recordingStopped() {
self.watchedRecording = nil
@@ -898,6 +983,9 @@ final class MeetingAutoDetector {
if let releasedAt = watched.browserMicReleasedAt {
moments.append(releasedAt.addingTimeInterval(Self.endSignalGraceSeconds))
}
+ if let endedAt = watched.browserCallEndedAt {
+ moments.append(endedAt.addingTimeInterval(Self.browserPostCallGraceSeconds))
+ }
} else {
audioUnknown = self.audioQueryUnknown
if record.windowEvidenceAt == nil, let windowLostAt = record.windowLostAt {
@@ -916,8 +1004,10 @@ final class MeetingAutoDetector {
watched.appGoneAt = goneAt
askAt = goneAt.addingTimeInterval(Self.endSignalGraceSeconds)
}
- // Without live captions there is no speech to wait for, and the app's own signal decides.
- let stillTalking = self.lastSpeechAt().map { now.timeIntervalSince($0) < Self.speechQuietSeconds } == true
+ // Speech can explain an ambiguous disconnect. A positive post-call page is already an
+ // explicit end; its question still leaves capture running until the user chooses Stop.
+ let hasExplicitBrowserEnd = watched.tier == .browserTier2 && watched.browserCallEndedAt != nil
+ let stillTalking = !hasExplicitBrowserEnd && self.lastSpeechAt().map { now.timeIntervalSince($0) < Self.speechQuietSeconds } == true
let shouldAsk = askAt.map { now >= $0 } == true && !stillTalking && !watched.asked
// Every sign of the end is gone again: the call resumed. An audio query we cannot read is
// not that, so it changes nothing. The question can then come back at the next real end.
diff --git a/Sources/Fluid/Services/Meeting/MeetingBrowserCallEvidence.swift b/Sources/Fluid/Services/Meeting/MeetingBrowserCallEvidence.swift
new file mode 100644
index 00000000..9b864c83
--- /dev/null
+++ b/Sources/Fluid/Services/Meeting/MeetingBrowserCallEvidence.swift
@@ -0,0 +1,80 @@
+import ApplicationServices
+import Foundation
+
+/// URL presence alone cannot distinguish a connected call from its post-call page.
+nonisolated enum MeetingBrowserCallState: Sendable, Equatable {
+ case unknown
+ case inCall
+ case ended
+}
+
+/// Positive controls observed on Google Meet's English post-call and connected-call pages.
+/// Unrecognised/localised copy stays unknown; it must never become an inferred end.
+nonisolated struct MeetingBrowserCallEvidence {
+ private var leftHeading = false
+ private var rejoinButton = false
+ private var leaveButton = false
+
+ static func supports(_ url: BrowserTabURL) -> Bool {
+ url.host.lowercased() == "meet.google.com" && MeetingInCallURLMatcher.isInCallURL(host: url.host, path: url.path)
+ }
+
+ mutating func observe(role: String, label: String) {
+ let label = label.trimmingCharacters(in: .whitespacesAndNewlines)
+ if role == "AXHeading", label == "You left the meeting" { self.leftHeading = true }
+ if role == "AXButton", label == "Rejoin" { self.rejoinButton = true }
+ if role == "AXButton", label == "Leave call" { self.leaveButton = true }
+ }
+
+ var state: MeetingBrowserCallState {
+ if self.leftHeading, self.rejoinButton, !self.leaveButton { return .ended }
+ if self.leaveButton, !self.leftHeading, !self.rejoinButton { return .inCall }
+ return .unknown
+ }
+}
+
+/// Runs on the browser reader's existing utility queue with the same AX messaging timeout.
+/// Reads only labels of buttons/headings; never records page text, meeting IDs or participants.
+nonisolated enum MeetingBrowserCallAXReader {
+ private static let maxElements = 160
+ private static let maxDepth = 10
+ private static let maxChildren = 32
+ private static let leafRoles: Set<String> = ["AXStaticText", "AXImage", "AXLink", "AXTextField", "AXTextArea", "AXHeading", "AXButton"]
+
+ static func state(webArea: AXUIElement, url: BrowserTabURL) -> MeetingBrowserCallState {
+ guard MeetingBrowserCallEvidence.supports(url) else { return .unknown }
+ let deadline = ContinuousClock.now.advanced(by: .milliseconds(150))
+ var evidence = MeetingBrowserCallEvidence()
+ var pending: [(AXUIElement, Int)] = [(webArea, 0)]
+ var index = 0
+ while index < pending.count, index < Self.maxElements {
+ guard ContinuousClock.now < deadline else { return .unknown }
+ let (element, depth) = pending[index]
+ index += 1
+ guard let role = self.string(element, attribute: kAXRoleAttribute as String) else { return .unknown }
+ if role == "AXHeading" || role == "AXButton" {
+ let label = [kAXTitleAttribute, kAXDescriptionAttribute, kAXValueAttribute]
+ .lazy.compactMap { self.string(element, attribute: $0 as String) }
+ .first { !$0.isEmpty } ?? ""
+ evidence.observe(role: role, label: label)
+ }
+ guard depth < Self.maxDepth, !Self.leafRoles.contains(role) else { continue }
+ var childrenValue: CFTypeRef?
+ let status = AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &childrenValue)
+ if status == .cannotComplete { return .unknown }
+ guard status == .success, let children = childrenValue as? [AXUIElement] else { continue }
+ let remaining = Self.maxElements - pending.count
+ guard remaining > 0 else { break }
+ for child in children.prefix(min(Self.maxChildren, remaining)) {
+ pending.append((child, depth + 1))
+ }
+ }
+ return evidence.state
+ }
+
+ private static func string(_ element: AXUIElement, attribute: String) -> String? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else { return nil }
+ return value as? String
+ }
+}
diff --git a/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
index f04b8ee6..2e4f420f 100644
--- a/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
@@ -30,6 +30,7 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
var sessionID: MeetingSessionID
var runtime: any MeetingCaptureRuntime
var writers: [MeetingAudioChunkWriter]
+ var comparisonProbe: MeetingAudioComparisonProbe?
}
private var activeCapture: ActiveCapture?
@@ -61,13 +62,35 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
)
try Self.verifyMicrophonePermission()
+ let comparisonProbe: MeetingAudioComparisonProbe?
+ if let expiration = MeetingAudioComparisonProbe.requestExpiration(mode: configuration.mode) {
+ let directory = FileManager.default.temporaryDirectory
+ .appendingPathComponent("FluidVoice-AudioComparison", isDirectory: true)
+ .appendingPathComponent(session.id.uuidString, isDirectory: true)
+ comparisonProbe = try? MeetingAudioComparisonProbe(session: session, directory: directory, onUsefulComparison: {
+ MeetingAudioComparisonProbe.completeRequest(expiration: expiration)
+ })
+ DebugLogger.shared.info("Audio comparison probe \(comparisonProbe == nil ? "failed to start" : "armed") at \(directory.path)", source: "MeetingCaptureEngine")
+ } else {
+ comparisonProbe = nil
+ }
+ if let comparisonProbe { eventHandler(.audioComparisonProbeStarted(comparisonProbe)) }
+ // If setup exits before an active capture owns the probe, still finalize its files.
+ var probeOwnedByCapture = false
+ defer {
+ if !probeOwnedByCapture, let comparisonProbe { Task { await comparisonProbe.stop() } }
+ }
let tracks = Self.makeTracks(session: session, configuration: configuration)
let writers = try tracks.map { track in
try MeetingAudioChunkWriter(
track: track,
sessionDirectory: sessionDirectory,
chunkDuration: configuration.chunkDuration,
- eventHandler: eventHandler
+ eventHandler: eventHandler,
+ diagnosticSampleObserver: comparisonProbe.map { probe in
+ let stage: MeetingAudioComparisonProbe.Stage = track.kind == .microphone ? .microphoneOutput : .browser
+ return { sample, epoch in probe.offer(sample, stage: stage, producerEpoch: epoch) }
+ }
)
}
let writersByKind = Dictionary(uniqueKeysWithValues: zip(tracks.map(\.kind), writers))
@@ -127,6 +150,7 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
application: application,
microphone: configuration.microphone,
includeMicrophone: true,
+ comparisonProbe: comparisonProbe,
microphoneEchoProtection: .unprotected,
applicationWriter: applicationWriter,
microphoneWriter: microphoneWriter,
@@ -173,8 +197,10 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
self.activeCapture = ActiveCapture(
sessionID: session.id,
runtime: runtime,
- writers: writers
+ writers: writers,
+ comparisonProbe: comparisonProbe
)
+ probeOwnedByCapture = true
// Re-snapshot: a VPIO commit during start() promotes provenance on the writer's own track.
let settledTracks = await withTaskGroup(of: MeetingAudioTrack.self, returning: [MeetingAudioTrack].self) { group in
for writer in writers {
@@ -225,6 +251,7 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .writerStop)
#endif
let tracks = await Self.stopWriters(activeCapture.writers)
+ activeCapture.comparisonProbe?.finishAfterCaptionTeardown()
#if DEBUG
AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .writerStop)
#endif
@@ -567,6 +594,8 @@ nonisolated enum MeetingDirectAEC3Availability {
private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCaptureRuntime, SCStreamOutput, SCStreamDelegate,
@unchecked Sendable
{
+ private let comparisonProbe: MeetingAudioComparisonProbe?
+ private let microphonePCMNormalizer = MeetingMicrophonePCMNormalizer()
/// HAL notifications can arrive while the default input/output pair is still converging.
/// Admission stays closed until the latest route revision has settled.
private static let rawRouteSettleSeconds: Double = 0.75
@@ -647,12 +676,14 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
requestedMicrophone: MeetingMicrophoneIdentity,
activeMicrophone: MeetingMicrophoneIdentity,
includeMicrophone: Bool,
+ comparisonProbe: MeetingAudioComparisonProbe?,
microphoneEchoProtection: MeetingMicrophoneEchoProtection,
applicationWriter: MeetingAudioChunkWriter,
microphoneWriter: MeetingAudioChunkWriter,
eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void,
liveAudioHandler: (@Sendable (MeetingAudioTrackKind, CMSampleBuffer) -> Void)?
) {
+ self.comparisonProbe = comparisonProbe
self.stream = stream
self.scope = scope
self.application = application
@@ -677,6 +708,7 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
application: MeetingApplicationIdentity,
microphone: MeetingMicrophoneIdentity,
includeMicrophone: Bool = true,
+ comparisonProbe: MeetingAudioComparisonProbe? = nil,
microphoneEchoProtection: MeetingMicrophoneEchoProtection = .unprotected,
applicationWriter: MeetingAudioChunkWriter,
microphoneWriter: MeetingAudioChunkWriter,
@@ -710,6 +742,7 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
requestedMicrophone: microphone,
activeMicrophone: resolvedMicrophone,
includeMicrophone: includeMicrophone,
+ comparisonProbe: comparisonProbe,
microphoneEchoProtection: microphoneEchoProtection,
applicationWriter: applicationWriter,
microphoneWriter: microphoneWriter,
@@ -824,9 +857,12 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
var firstSnapshot = MeetingCaptureEngine.currentOutputRouteSnapshot()
try? await Task.sleep(nanoseconds: 100_000_000)
var secondSnapshot = MeetingCaptureEngine.currentOutputRouteSnapshot()
+ var settleAttempts = 1
while firstSnapshot != secondSnapshot,
+ settleAttempts < 3,
self.stateLock.withLock({ self.outputRouteRevision == baselineRevision && !self.stopping })
{
+ settleAttempts += 1
try? await Task.sleep(nanoseconds: 250_000_000)
firstSnapshot = MeetingCaptureEngine.currentOutputRouteSnapshot()
try? await Task.sleep(nanoseconds: 100_000_000)
@@ -1015,6 +1051,8 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
#if DEBUG
self.recordPairedDiagnostic(sampleBuffer, output: .microphone)
#endif
+ let sampleBuffer = self.microphonePCMNormalizer.normalize(sampleBuffer)
+ self.comparisonProbe?.offer(sampleBuffer, stage: .microphoneInput, producerEpoch: streamToken.generation)
switch aecState {
case .active:
pipeline?.consumeCapture(sampleBuffer, stream: streamToken)
@@ -1057,6 +1095,10 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
stream: SCStream,
token: MeetingAECStreamToken
) {
+ if self.stateLock.withLock({ self.aecActivation.formatValidationTimedOut() }) {
+ self.deactivateAECOnCallbackQueue(reason: .formatValidationTimedOut)
+ return
+ }
if let failure = MeetingAECPipeline.firstFormatFailure(sampleBuffer, kind: kind, stream: token) {
self.deactivateAECOnCallbackQueue(reason: failure)
return
@@ -1081,6 +1123,15 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
}
detached?.invalidate(reason: reason, boundary: boundary)
if let detached { self.retainRetiredCommitterIfPending(detached.committer) }
+ let revision = self.stateLock.withLock { self.outputRouteRevision }
+ self.routeProtectionTasks.enqueue { [weak self] in
+ guard let self, self.isCurrentRouteRevision(revision) else { return }
+ do {
+ try await self.resumeDirectMicrophone(.unfiltered, revision: revision)
+ } catch {
+ self.handleAECTerminalFailure(.metadataPersistence)
+ }
+ }
}
private func applySettledRouteDisposition(
@@ -1096,36 +1147,7 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
switch disposition {
case .physicallyClosed:
- // The gate epoch is captured before persistence so a route/lifecycle invalidation
- // racing the write below can never be overwritten by this stale open.
- let gateEpoch = self.microphoneTranscriptGate.invalidationEpoch
- let start = self.microphoneTranscriptGate.latestObservedSeconds()
- try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
- MeetingAECOutputCommitter.applyEra(
- to: &track,
- protection: .acousticallyClosed,
- provenance: nil,
- startSeconds: start
- )
- }
- let mayOpen = self.stateLock.withLock {
- self.outputRouteRevision == revision && !self.stopping
- }
- if mayOpen,
- self.microphoneTranscriptGate.updateIfNotInvalidated(.acousticallyClosed, since: gateEpoch)
- {} else {
- // The route invalidation may have queued its conservative command before the
- // stale optimistic write above. Append a compensating fail-closed write after it.
- let unsafeStart = self.microphoneTranscriptGate.invalidate()?.unsafeStartSeconds
- try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
- MeetingAECOutputCommitter.applyEra(
- to: &track,
- protection: .unprotected,
- provenance: nil,
- startSeconds: unsafeStart
- )
- }
- }
+ try await self.resumeDirectMicrophone(.acousticallyClosed, revision: revision)
case .supportedSpeaker:
let start = self.microphoneTranscriptGate.latestObservedSeconds()
try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
@@ -1140,23 +1162,34 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
do {
try await self.installAECForSupportedSpeaker(revision: revision)
} catch {
- self.eventHandler(.interrupted(
- kind: .microphoneChanged,
- trackID: self.microphoneWriter.trackID,
- detail: "Speaker-route AEC initialization failed; microphone remains raw and unprotected."
- ))
+ try await self.resumeDirectMicrophone(.unfiltered, revision: revision)
}
case .ambiguous:
- let start = self.microphoneTranscriptGate.latestObservedSeconds()
+ try await self.resumeDirectMicrophone(.unfiltered, revision: revision)
+ }
+ }
+
+ /// Echo provenance and live delivery commit together, without changing capture method or
+ /// restarting either device. A stale route cannot reopen the gate after a newer invalidation.
+ private func resumeDirectMicrophone(_ protection: MeetingMicrophoneEchoProtection, revision: UInt64) async throws {
+ guard self.stateLock.withLock({
+ self.outputRouteRevision == revision && !self.stopping && !self.rebuilding && !self.currentStreamStopped
+ }) else { return }
+ let gateEpoch = self.microphoneTranscriptGate.invalidationEpoch
+ let start = self.microphoneTranscriptGate.latestObservedSeconds()
+ try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
+ MeetingAECOutputCommitter.applyEra(to: &track, protection: protection, provenance: nil, startSeconds: start)
+ }
+ let mayOpen = self.stateLock.withLock {
+ self.outputRouteRevision == revision && !self.stopping && !self.rebuilding && !self.currentStreamStopped
+ }
+ if mayOpen, self.microphoneTranscriptGate.updateIfNotInvalidated(protection, since: gateEpoch) {
+ self.eventHandler(.microphoneEchoProtectionChanged(protection))
+ } else {
+ let unsafeStart = self.microphoneTranscriptGate.invalidate()?.unsafeStartSeconds
try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
- MeetingAECOutputCommitter.applyEra(
- to: &track,
- protection: .unprotected,
- provenance: nil,
- startSeconds: start
- )
+ MeetingAECOutputCommitter.applyEra(to: &track, protection: .unprotected, provenance: nil, startSeconds: unsafeStart)
}
- _ = self.microphoneTranscriptGate.invalidate()
}
}
@@ -1192,11 +1225,22 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
writer: self.microphoneWriter,
producerEpoch: reserved.ownership.streamGeneration,
transcriptGate: self.microphoneTranscriptGate,
- liveAudioHandler: self.liveAudioHandler
+ liveAudioHandler: self.liveAudioHandler,
+ protectionChanged: { [eventHandler = self.eventHandler] protection in
+ eventHandler(.microphoneEchoProtectionChanged(protection))
+ }
) { [weak self] failure in
self?.handleAECTerminalFailure(failure)
}
- let candidate = MeetingAECPipeline(processor: processor, committer: committer)
+ let candidate = MeetingAECPipeline(
+ processor: processor,
+ committer: committer,
+ diagnosticObserver: self.comparisonProbe.map { probe in
+ { frame, output, statistics in
+ probe.recordAECFrame(frame, output: output, statistics: statistics)
+ }
+ }
+ )
let committed = self.stateLock.withLock { () -> Bool in
let mayCommit = MeetingCaptureOwnershipGate.mayCommitAEC(
@@ -1396,6 +1440,14 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
}
self.routeProtectionTasks.enqueue { [weak self] in
guard let self else { return }
+ if self.stateLock.withLock({ self.routeListenerDegraded }) {
+ do {
+ try await self.resumeDirectMicrophone(.unfiltered, revision: revision)
+ } catch {
+ self.handleAECTerminalFailure(.metadataPersistence)
+ }
+ return
+ }
guard let settled = await self.settleRouteAndInput(revision: revision) else { return }
var disposition = settled.disposition
var inputDetail: String?
@@ -1490,9 +1542,9 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
try? await Task.sleep(nanoseconds: 100_000_000)
let second = MeetingCaptureEngine.currentOutputRouteSnapshot()
guard self.stateLock.withLock({ self.outputRouteRevision == revision && !self.stopping }) else { return nil }
- guard first == second else {
+ if first != second {
attempt += 1
- continue
+ if attempt < 3 { continue }
}
let disposition = MeetingAECOutputRouteClassifier.classify(
first: first,
@@ -1904,6 +1956,7 @@ private final nonisolated class ScreenCaptureRuntimeDelegateProxy: NSObject, SCS
private final nonisolated class InRoomMicrophoneCaptureRuntime: NSObject, MeetingCaptureRuntime,
AVCaptureAudioDataOutputSampleBufferDelegate, @unchecked Sendable
{
+ private let microphonePCMNormalizer = MeetingMicrophonePCMNormalizer()
var captureScope: MeetingCaptureScope? { nil }
private let session = AVCaptureSession()
@@ -1997,6 +2050,7 @@ private final nonisolated class InRoomMicrophoneCaptureRuntime: NSObject, Meetin
let shouldAccept = !self.stopping
self.stateLock.unlock()
guard shouldAccept else { return }
+ let sampleBuffer = self.microphonePCMNormalizer.normalize(sampleBuffer)
self.writer.enqueue(sampleBuffer)
self.liveAudioHandler?(.microphone, sampleBuffer)
}
@@ -2122,6 +2176,7 @@ final nonisolated class MeetingSerializedCaptureLifecycle: @unchecked Sendable {
final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
AVCaptureAudioDataOutputSampleBufferDelegate, @unchecked Sendable
{
+ private let microphonePCMNormalizer = MeetingMicrophonePCMNormalizer()
private static let interruptionEndedWaitSeconds: Double = 3
let deviceUID: String?
@@ -2268,7 +2323,7 @@ final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
if shouldAccept { self.deliveredSampleCount &+= 1 }
self.stateLock.unlock()
guard shouldAccept else { return }
- self.onSample(sampleBuffer)
+ self.onSample(self.microphonePCMNormalizer.normalize(sampleBuffer))
}
func sampleCount() -> UInt64 {
@@ -2715,10 +2770,8 @@ nonisolated enum MeetingCapturePathDecision: Sendable, Equatable {
case screenCaptureKit(reason: String)
}
-/// Pre-production builds exercise direct AEC3 by default on a positively classified built-in-
-/// speaker route. The escape hatch is intentionally fail-closed and also prevents
-/// `ScreenCaptureMeetingRuntime` from constructing AEC3 if selection reaches it through another
-/// path.
+/// Stable speaker routes can attempt direct AEC3 after format and clock validation. The escape
+/// hatch disables filtering; it does not disable recording or unfiltered transcription.
nonisolated enum MeetingDirectAEC3Gate {
// TODO: Remove this escape hatch once direct AEC3 has completed real-meeting testing and is stable.
static let disableEnvironmentKey = "FLUIDVOICE_DISABLE_AEC3"
@@ -2958,16 +3011,16 @@ nonisolated struct MeetingOutputRouteSnapshot: Sendable, Equatable {
}
nonisolated enum MeetingRawMicrophoneProtection {
- /// Closed must be positively identified. Unknown, aggregate, HDMI, virtual, and speaker
- /// routes remain unprotected even when they are not the built-in speakers.
+ /// Headphones need positive evidence. Other raw routes keep speech with an explicit echo
+ /// warning; this classification never pretends the audio was filtered or acoustically closed.
static func classify(_ route: MeetingOutputRouteSnapshot) -> MeetingMicrophoneEchoProtection {
- guard route.deviceExists else { return .unprotected }
+ guard route.deviceExists else { return .unfiltered }
if (route.isBuiltIn && route.isHeadphonesDataSource)
|| route.terminalTypes.contains(kAudioStreamTerminalTypeHeadphones)
{
return .acousticallyClosed
}
- return .unprotected
+ return .unfiltered
}
}
@@ -3011,7 +3064,7 @@ final nonisolated class MeetingMicrophoneTranscriptGate: @unchecked Sendable {
/// Atomically observes the sample boundary and decides live admission. A route callback taking
/// the same lock therefore has an exact conservative cut: samples ordered after invalidation
/// cannot leak into live transcription under the prior route's protection.
- func observeAndAdmit(_ sampleBuffer: CMSampleBuffer) -> Bool {
+ func observeAndAdmit(_ sampleBuffer: CMSampleBuffer, since epoch: UInt64? = nil) -> Bool {
let time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
return self.lock.withLock {
if time.isValid, time.isNumeric {
@@ -3020,7 +3073,7 @@ final nonisolated class MeetingMicrophoneTranscriptGate: @unchecked Sendable {
self.latestPresentationSeconds = max(self.latestPresentationSeconds ?? seconds, seconds)
}
}
- return self.protection.admitsTranscript
+ return (epoch == nil || epoch == self.invalidationCounter) && self.protection.admitsTranscript
}
}
@@ -4509,6 +4562,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
private func publishTranscriptProtection(_ protection: MeetingMicrophoneEchoProtection) {
let mayOpen = self.stateLock.withLock { !self.routeChangeDirty && !self.stopRequested }
self.transcriptGate.update(mayOpen ? protection : .unprotected)
+ if mayOpen { self.eventHandler(.microphoneEchoProtectionChanged(protection)) }
}
private func handleMicEvent(_ event: MeetingMicrophoneCaptureEvent, owner: MeetingMicrophoneCapture) {
diff --git a/Sources/Fluid/Services/Meeting/MeetingMicrophonePCMNormalizer.swift b/Sources/Fluid/Services/Meeting/MeetingMicrophonePCMNormalizer.swift
new file mode 100644
index 00000000..f99665a0
--- /dev/null
+++ b/Sources/Fluid/Services/Meeting/MeetingMicrophonePCMNormalizer.swift
@@ -0,0 +1,110 @@
+@preconcurrency import AVFoundation
+import CoreMedia
+import Foundation
+
+/// Adapts native microphone LPCM before the writer, echo processing and live captions fan out.
+/// One converter is retained per producer, rebuilt only when its format changes. No resampling,
+/// channel mixing, capture delay or opening-frame discard is involved.
+final nonisolated class MeetingMicrophonePCMNormalizer: @unchecked Sendable {
+ private let lock = NSLock()
+ private var sourceFormat: AVAudioFormat?
+ private var converter: AVAudioConverter?
+
+ /// Invalid/unsupported input is returned intact so the existing writer still reports its
+ /// failure and degraded health. Conversion must never silently discard a capture callback.
+ func normalize(_ sample: CMSampleBuffer) -> CMSampleBuffer {
+ guard CMSampleBufferIsValid(sample), CMSampleBufferDataIsReady(sample),
+ let description = CMSampleBufferGetFormatDescription(sample)
+ else { return sample }
+ if (try? MeetingPCMFormatContract(formatDescription: description)) != nil { return sample }
+ return self.lock.withLock {
+ (try? self.convert(sample, description: description)) ?? sample
+ }
+ }
+
+ private func convert(_ sample: CMSampleBuffer, description: CMFormatDescription) throws -> CMSampleBuffer {
+ let format = try MeetingPCMFormatResolver.resolve(description)
+ let asbd = format.streamDescription.pointee
+ let frames = CMSampleBufferGetNumSamples(sample)
+ let channels = Int(format.channelCount)
+ guard asbd.mFormatID == kAudioFormatLinearPCM,
+ asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger != 0,
+ [16, 24, 32].contains(asbd.mBitsPerChannel),
+ format.sampleRate.isFinite, format.sampleRate > 0,
+ channels > 0, frames > 0,
+ // Bound malformed callback metadata to at most 4 MiB of Float32 output.
+ frames <= 1_048_576 / channels,
+ let input = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames))
+ else { throw MeetingPCMFormatContractError.unsupported("invalid native integer microphone buffer") }
+ input.frameLength = AVAudioFrameCount(frames)
+ guard CMSampleBufferCopyPCMDataIntoAudioBufferList(
+ sample, at: 0, frameCount: Int32(frames), into: input.mutableAudioBufferList
+ ) == noErr else { throw MeetingPCMFormatContractError.unsupported("invalid microphone PCM payload") }
+
+ if self.sourceFormat != format || self.converter == nil {
+ let target: AVAudioFormat?
+ if let layout = format.channelLayout {
+ target = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: format.sampleRate, interleaved: true, channelLayout: layout)
+ } else {
+ target = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: format.sampleRate, channels: format.channelCount, interleaved: true)
+ }
+ guard let target, let converter = AVAudioConverter(from: format, to: target) else {
+ throw MeetingPCMFormatContractError.unsupported("cannot convert microphone PCM")
+ }
+ self.sourceFormat = format
+ self.converter = converter
+ }
+ guard let converter = self.converter,
+ let output = AVAudioPCMBuffer(pcmFormat: converter.outputFormat, frameCapacity: input.frameLength)
+ else { throw MeetingPCMFormatContractError.unsupported("cannot allocate microphone PCM") }
+ try converter.convert(to: output, from: input)
+ guard output.frameLength == input.frameLength else {
+ throw MeetingPCMFormatContractError.unsupported("incomplete microphone PCM conversion")
+ }
+ return try Self.makeSample(output, timingFrom: sample)
+ }
+
+ private static func makeSample(_ buffer: AVAudioPCMBuffer, timingFrom source: CMSampleBuffer) throws -> CMSampleBuffer {
+ let audio = buffer.audioBufferList.pointee.mBuffers
+ let byteCount = Int(audio.mDataByteSize)
+ var block: CMBlockBuffer?
+ guard let data = audio.mData,
+ CMBlockBufferCreateWithMemoryBlock(
+ allocator: kCFAllocatorDefault,
+ memoryBlock: nil,
+ blockLength: byteCount,
+ blockAllocator: nil,
+ customBlockSource: nil,
+ offsetToData: 0,
+ dataLength: byteCount,
+ flags: kCMBlockBufferAssureMemoryNowFlag,
+ blockBufferOut: &block
+ ) == noErr, let block,
+ CMBlockBufferReplaceDataBytes(with: data, blockBuffer: block, offsetIntoDestination: 0, dataLength: byteCount) == noErr
+ else { throw MeetingPCMFormatContractError.unsupported("cannot own converted microphone PCM") }
+ var count = 0
+ guard CMSampleBufferGetSampleTimingInfoArray(source, entryCount: 0, arrayToFill: nil, entriesNeededOut: &count) == noErr,
+ count > 0, count <= CMSampleBufferGetNumSamples(source)
+ else { throw MeetingPCMFormatContractError.unsupported("invalid microphone timing") }
+ var timing = [CMSampleTimingInfo](repeating: CMSampleTimingInfo(), count: count)
+ guard CMSampleBufferGetSampleTimingInfoArray(source, entryCount: count, arrayToFill: &timing, entriesNeededOut: nil) == noErr else {
+ throw MeetingPCMFormatContractError.unsupported("cannot preserve microphone timing")
+ }
+ var result: CMSampleBuffer?
+ guard CMSampleBufferCreateReady(
+ allocator: kCFAllocatorDefault,
+ dataBuffer: block,
+ formatDescription: buffer.format.formatDescription,
+ sampleCount: Int(buffer.frameLength),
+ sampleTimingEntryCount: count,
+ sampleTimingArray: &timing,
+ sampleSizeEntryCount: 0,
+ sampleSizeArray: nil,
+ sampleBufferOut: &result
+ ) == noErr, let result else { throw MeetingPCMFormatContractError.unsupported("cannot create converted microphone sample") }
+ if let attachments = CMCopyDictionaryOfAttachments(allocator: kCFAllocatorDefault, target: source, attachmentMode: kCMAttachmentMode_ShouldPropagate) {
+ CMSetAttachments(result, attachments: attachments, attachmentMode: kCMAttachmentMode_ShouldPropagate)
+ }
+ return result
+ }
+}
diff --git a/Sources/Fluid/Services/Meeting/MeetingModels.swift b/Sources/Fluid/Services/Meeting/MeetingModels.swift
index 2ac38ba5..fe702dfe 100644
--- a/Sources/Fluid/Services/Meeting/MeetingModels.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingModels.swift
@@ -56,6 +56,8 @@ nonisolated enum MeetingInterruptionKind: String, Codable, Sendable {
case voiceProcessingOverload
/// Final processing completed with typed gaps or quarantined evidence.
case transcriptCoverageIncomplete
+ case echoFilteringUnavailable
+ case echoFilteringRestored
}
nonisolated enum MeetingFailureDomain: String, Codable, Sendable {
@@ -481,6 +483,9 @@ nonisolated enum MeetingMicrophoneEchoProtection: String, Codable, Sendable {
/// Microphone PCM processed by the pinned WebRTC AEC3 bridge after clock attestation and
/// warm-up. This is software provenance, not a claim that the physical route is closed.
case softwareEchoCancelled
+ /// Valid captured microphone audio retained for transcription when echo filtering cannot run.
+ /// This explicitly reports possible playback leakage; it never claims echo was removed.
+ case unfiltered
/// Pre-v9 audio that was historically transcribed without route evidence. It remains eligible
/// for backward compatibility, but must never be rewritten as positively verified protection.
case legacyUnclassified
@@ -489,6 +494,10 @@ nonisolated enum MeetingMicrophoneEchoProtection: String, Codable, Sendable {
var admitsTranscript: Bool {
self != .unprotected
}
+
+ var transcriptWarning: String? {
+ self == .unfiltered ? "Echo filtering is unavailable. Your microphone is still transcribed, but may include speaker audio." : nil
+ }
}
nonisolated struct MeetingAECProvenance: Codable, Equatable, Sendable {
@@ -863,6 +872,35 @@ nonisolated enum MeetingAudioRetentionPolicy: String, Codable, CaseIterable, Sen
}
}
+/// A speaker editor draft. Nothing is persisted until names and merges are saved together.
+nonisolated struct MeetingSpeakerEdits: Equatable, Sendable {
+ var names: [SessionSpeakerID: String]
+ var merges: [SessionSpeakerID: SessionSpeakerID] = [:]
+
+ mutating func setMerge(source: SessionSpeakerID, into target: SessionSpeakerID?) {
+ self.merges[source] = target
+ guard let target else { return }
+ // Keep selections direct when the destination itself is subsequently merged.
+ for id in self.merges.keys where self.merges[id] == source {
+ self.merges[id] = target
+ }
+ }
+
+ func resolvedMerges() throws -> [SessionSpeakerID: SessionSpeakerID] {
+ var resolved: [SessionSpeakerID: SessionSpeakerID] = [:]
+ for source in self.merges.keys {
+ var visited: Set<SessionSpeakerID> = [source]
+ var target = source
+ while let next = self.merges[target] {
+ guard visited.insert(next).inserted else { throw MeetingDomainError.cannotMergeSpeakers }
+ target = next
+ }
+ resolved[source] = target
+ }
+ return resolved
+ }
+}
+
nonisolated struct MeetingSession: Codable, Identifiable, Equatable, Sendable {
static let currentSchemaVersion = 1
@@ -947,6 +985,19 @@ nonisolated struct MeetingSession: Codable, Identifiable, Equatable, Sendable {
(self.endedAt ?? Date()).timeIntervalSince(self.startedAt)
}
+ var microphoneEchoWarning: String? {
+ guard self.mode == .onlineCall else { return nil }
+ switch self.state {
+ case .preparing, .recording, .recordingDegraded, .stopping:
+ guard self.events.last(where: { $0.kind == .echoFilteringUnavailable || $0.kind == .echoFilteringRestored })?.kind == .echoFilteringUnavailable else { return nil }
+ default:
+ // A short raw tail at Stop is kept too, but is not a failed AEC attempt. The explicit
+ // status event distinguishes an actual filtering fallback from that final partial frame.
+ guard self.events.contains(where: { $0.kind == .echoFilteringUnavailable }) else { return nil }
+ }
+ return MeetingMicrophoneEchoProtection.unfiltered.transcriptWarning
+ }
+
/// Speakers not merged away into another speaker — the set the UI should render.
var activeSpeakers: [MeetingSessionSpeaker] {
self.speakers.filter { $0.mergedIntoSpeakerID == nil }
@@ -982,6 +1033,47 @@ nonisolated struct MeetingSession: Codable, Identifiable, Equatable, Sendable {
self.updatedAt = Date()
}
+ /// Merge only the immediate matching neighbors of a manually tagged Unknown. Keep the
+ /// selected ID so correction/undo targets remain stable, and never combine audio tracks.
+ mutating func mergeTaggedSegmentWithNeighbors(id: MeetingTranscriptSegmentID) -> Set<MeetingTranscriptSegmentID> {
+ let ordered = self.transcriptSegments.enumerated().sorted {
+ ($0.element.start, $0.element.end, $0.offset) < ($1.element.start, $1.element.end, $1.offset)
+ }.map(\.element)
+ guard let position = ordered.firstIndex(where: { $0.id == id }),
+ let speakerID = ordered[position].speakerID,
+ !ordered[position].isEcho
+ else { return [id] }
+ let target = ordered[position]
+ func matches(_ candidate: MeetingTranscriptSegment) -> Bool {
+ candidate.speakerID == speakerID && candidate.sourceTrackID == target.sourceTrackID
+ && !candidate.isEcho && candidate.overlap == .none
+ && (candidate.attributionState == nil || candidate.attributionState == .assigned)
+ && candidate.status == target.status && candidate.completeness == target.completeness
+ }
+ var pieces = [target]
+ if position > 0, matches(ordered[position - 1]) { pieces.insert(ordered[position - 1], at: 0) }
+ if position + 1 < ordered.count, matches(ordered[position + 1]) { pieces.append(ordered[position + 1]) }
+ let ids = Set(pieces.map(\.id))
+ guard pieces.count > 1, let index = self.transcriptSegments.firstIndex(where: { $0.id == id }) else { return ids }
+ func joined(_ values: [String]) -> String {
+ values.reduce("") { left, right in
+ guard !left.isEmpty else { return right }
+ guard let first = right.unicodeScalars.first else { return left }
+ return left + (CharacterSet.punctuationCharacters.contains(first) ? "" : " ") + right
+ }
+ }
+ var merged = target
+ merged.start = pieces.map(\.start).min() ?? target.start
+ merged.end = pieces.map(\.end).max() ?? target.end
+ merged.text = joined(pieces.map(\.text))
+ if pieces.contains(where: { $0.originalText != nil }) {
+ merged.originalText = joined(pieces.map { $0.originalText ?? $0.text })
+ }
+ self.transcriptSegments[index] = merged
+ self.transcriptSegments.removeAll { $0.id != id && ids.contains($0.id) }
+ return ids
+ }
+
/// Gives one unresolved segment its own manual speaker identity. The identity is intentionally
/// scoped to this segment: naming one unknown must never rename or relabel another unknown.
@discardableResult
@@ -1302,7 +1394,11 @@ final nonisolated class MeetingCaptureMethodChangeAck: @unchecked Sendable {
}
nonisolated enum MeetingCaptureEvent: Sendable {
+ /// Explicit local investigation only; never persisted as a session interruption.
+ case audioComparisonProbeStarted(MeetingAudioComparisonProbe)
case trackHealth(trackID: MeetingAudioTrackID, health: MeetingTrackHealth)
+ /// Presentation-only status. The writer owns persisted capture eras; this cannot restart capture.
+ case microphoneEchoProtectionChanged(MeetingMicrophoneEchoProtection)
case chunkFinalized(trackID: MeetingAudioTrackID, chunk: MeetingAudioChunk, format: MeetingAudioFormat)
case interrupted(kind: MeetingInterruptionKind, trackID: MeetingAudioTrackID?, detail: String?)
/// `ack` is `nil` for downgrades — the mic is already dead, no race to gate on.
diff --git a/Sources/Fluid/Services/Meeting/MeetingProcessingConfiguration.swift b/Sources/Fluid/Services/Meeting/MeetingProcessingConfiguration.swift
index fded7aa1..fa782015 100644
--- a/Sources/Fluid/Services/Meeting/MeetingProcessingConfiguration.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingProcessingConfiguration.swift
@@ -23,9 +23,8 @@ nonisolated enum MeetingConfigFingerprintEncoding {
nonisolated struct MeetingFinalProcessingConfiguration: Equatable, Sendable {
static let defaultASRModel = "parakeet-tdt-v2"
static let defaultLanguageCode = "en"
- /// Mirrors `MeetingProcessingPipeline.pipelineVersion`; that class is `@MainActor`, so a
- /// MainActor test guards this literal against drift instead of referencing it here.
- static let defaultPipelineVersion = 13
+ /// One shared immutable version keeps frozen requests and saved attempts consistent.
+ static let defaultPipelineVersion = MeetingProcessingPipeline.pipelineVersion
let asrModel: String
let languageCode: String
diff --git a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
index 4a28623b..9c93fc15 100644
--- a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
@@ -644,8 +644,8 @@ private final class MeetingProviderLanguagePin {
final class MeetingProcessingPipeline: MeetingProcessingControlling {
/// Bump whenever classification rules change so a resumed run can't mix rules mid-session.
/// Era-awareness did NOT bump this: the mic pass re-runs every time, and multi-era tracks cannot predate this build.
- /// Version 13: short ambiguous words receive the opted-in turn-handoff second pass.
- static let pipelineVersion = 13
+ /// Version 15: retain unfiltered microphone speech when echo filtering is unavailable.
+ nonisolated static let pipelineVersion = 15
/// Per-turn engines starve on short turns, and an all-empty chunk collapses to unlabeled.
static let perTurnTurnMergeGapSeconds = 5.0
@@ -1699,10 +1699,41 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
status: segment.status.rawValue,
completeness: segment.completeness.rawValue,
isLikelyEcho: segment.isLikelyEcho,
- attributionState: segment.attributionState?.rawValue
+ attributionState: (segment.attributionState ?? (segment.speakerID != nil ? .assigned : .unassigned)).rawValue
)
}
+ /// Product policy: unresolved text belongs to the preceding visible speaker. Keep the
+ /// original word evidence in the sidecar; only the published transcript inherits a label.
+ /// An opening Unknown stays unresolved until there is a preceding speaker to inherit.
+ nonisolated static func inheritPreviousSpeaker(
+ _ segments: [MeetingTranscriptSegment]
+ ) -> [MeetingTranscriptSegment] {
+ let ordered = segments.enumerated().sorted {
+ ($0.element.start, $0.element.end, $0.element.sourceTrackID.uuidString, $0.offset)
+ < ($1.element.start, $1.element.end, $1.element.sourceTrackID.uuidString, $1.offset)
+ }.map(\.element)
+ var previousSpeakerID: SessionSpeakerID?
+ return ordered.map { segment in
+ guard !segment.isEcho else { return segment }
+ var updated = segment
+ let unresolved = segment.speakerID == nil
+ || segment.attributionState == .unassigned
+ || segment.attributionState == .overlappingSpeakers
+ || segment.attributionState == .timingUncertain
+ if unresolved {
+ if let previousSpeakerID {
+ updated.speakerID = previousSpeakerID
+ updated.attributionState = .assigned
+ updated.overlap = .none
+ }
+ } else {
+ previousSpeakerID = segment.speakerID
+ }
+ return updated
+ }
+ }
+
/// Canonical evidence remains word-granular in the sidecar, but product session JSON and UI
/// rows stay turn-granular. Folds each track chronologically and merges only a segment with
/// the immediately preceding turn on that same track — never across an intervening
@@ -1714,7 +1745,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
maximumGapSeconds: TimeInterval = MeetingProcessingPipeline.canonicalTurnMergeGapSeconds,
maximumDurationSeconds: TimeInterval = MeetingProcessingPipeline.canonicalTurnMaximumDurationSeconds
) -> [MeetingTranscriptSegment] {
- let byTrack = Dictionary(grouping: segments, by: \.sourceTrackID)
+ let byTrack = Dictionary(grouping: Self.inheritPreviousSpeaker(segments), by: \.sourceTrackID)
var merged: [MeetingTranscriptSegment] = []
for (_, members) in byTrack {
let ordered = members.enumerated().sorted {
diff --git a/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
index 149e4ab4..1c8da95a 100644
--- a/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
@@ -173,10 +173,7 @@ final class MeetingSessionCoordinator: ObservableObject {
case rename(speakerID: SessionSpeakerID, previousName: String)
case reassign(
segmentID: MeetingTranscriptSegmentID,
- previousSpeakerID: SessionSpeakerID?,
- previousOverlap: MeetingTranscriptOverlap,
- previousAttributionState: MeetingTranscriptAttributionState?,
- previousRevision: Int
+ previousSegments: [(index: Int, segment: MeetingTranscriptSegment)]
)
case nameUnknown(
segmentID: MeetingTranscriptSegmentID,
@@ -191,6 +188,10 @@ final class MeetingSessionCoordinator: ObservableObject {
movedSegments: [(id: MeetingTranscriptSegmentID, previousRevision: Int)]
)
case renameBatch([(speakerID: SessionSpeakerID, previousName: String)])
+ case speakerEdits(
+ previousSpeakers: [MeetingSessionSpeaker],
+ movedSegments: [(id: MeetingTranscriptSegmentID, speakerID: SessionSpeakerID?, revision: Int)]
+ )
}
private var correctionUndoStacks: [MeetingSessionID: [TranscriptCorrection]] = [:]
@@ -348,6 +349,7 @@ final class MeetingSessionCoordinator: ObservableObject {
passiveOffer: MeetingSession?,
resolvingSourceID: MeetingSessionID?
) async throws -> MeetingSession {
+ guard SettingsStore.shared.fluidMeetEnabled else { throw MeetingCoordinatorError.featureDisabled }
guard self.startReservation == nil else { throw MeetingCoordinatorError.recordingAlreadyActive }
guard !self.hasBlockingActivityTask,
self.canBeginNewRecording, !self.isDeleting
@@ -390,8 +392,10 @@ final class MeetingSessionCoordinator: ObservableObject {
// open indefinitely; reserving meeting audio first would unnecessarily block dictation.
try await self.validateRecordingModels()
try Task.checkCancellation()
+ guard SettingsStore.shared.fluidMeetEnabled else { throw MeetingCoordinatorError.featureDisabled }
try await self.capture.preflightPermissions()
try Task.checkCancellation()
+ guard SettingsStore.shared.fluidMeetEnabled else { throw MeetingCoordinatorError.featureDisabled }
guard self.startReservation == reservation,
self.activeSession?.id == passiveOffer?.id,
!self.hasBlockingActivityTaskExceptStart,
@@ -406,6 +410,7 @@ final class MeetingSessionCoordinator: ObservableObject {
let lease = try await self.audioArbiter.acquireMeetingCapture()
do {
try Task.checkCancellation()
+ guard SettingsStore.shared.fluidMeetEnabled else { throw MeetingCoordinatorError.featureDisabled }
guard self.startReservation == reservation,
self.activeSession?.id == passiveOffer?.id,
!self.hasBlockingActivityTaskExceptStart,
@@ -873,7 +878,7 @@ final class MeetingSessionCoordinator: ObservableObject {
private func resolveSession(_ id: MeetingSessionID) async throws -> MeetingSession {
if let active = self.activeSession, active.id == id { return active }
if let completed = self.latestCompletedSession, completed.id == id { return completed }
- if let loaded = try await self.store.load(id: id) { return loaded }
+ if let loaded = try await self.store.loadTranscript(id: id) { return loaded }
throw MeetingCoordinatorError.noActiveMeeting
}
@@ -932,6 +937,28 @@ final class MeetingSessionCoordinator: ObservableObject {
}
}
+ @discardableResult
+ func editSpeakers(sessionID: MeetingSessionID, edits: MeetingSpeakerEdits) async throws -> MeetingSession {
+ try await self.performCorrection(sessionID: sessionID) { session in
+ let merges = try edits.resolvedMerges()
+ let previousSpeakers = session.speakers
+ let movedSegments: [(id: MeetingTranscriptSegmentID, speakerID: SessionSpeakerID?, revision: Int)] = session.transcriptSegments.compactMap { segment in
+ guard let speakerID = segment.speakerID, merges[speakerID] != nil else { return nil }
+ return (id: segment.id, speakerID: segment.speakerID, revision: segment.revision)
+ }
+ for source in merges.keys.sorted(by: { $0.uuidString < $1.uuidString }) {
+ if let target = merges[source] { try session.mergeSpeakers(source, into: target) }
+ }
+ for (speakerID, name) in edits.names {
+ guard session.speakers.contains(where: { $0.id == speakerID && $0.mergedIntoSpeakerID == nil }) else { continue }
+ let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines)
+ guard !trimmed.isEmpty else { continue }
+ try session.renameSpeaker(id: speakerID, to: trimmed)
+ }
+ return .speakerEdits(previousSpeakers: previousSpeakers, movedSegments: movedSegments)
+ }
+ }
+
@discardableResult
func reassignSegment(
sessionID: MeetingSessionID,
@@ -942,17 +969,16 @@ final class MeetingSessionCoordinator: ObservableObject {
guard let segment = session.transcriptSegments.first(where: { $0.id == segmentID }) else {
throw MeetingDomainError.segmentNotFound
}
- let previousSpeakerID = segment.speakerID
- let previousOverlap = segment.overlap
- let previousAttributionState = segment.attributionState
- let previousRevision = segment.revision
+ let before = session.transcriptSegments
try session.reassignSegment(id: segmentID, to: speakerID)
+ let affectedIDs = segment.speakerID == nil
+ ? session.mergeTaggedSegmentWithNeighbors(id: segmentID)
+ : Set([segmentID])
return .reassign(
segmentID: segmentID,
- previousSpeakerID: previousSpeakerID,
- previousOverlap: previousOverlap,
- previousAttributionState: previousAttributionState,
- previousRevision: previousRevision
+ previousSegments: before.enumerated().compactMap {
+ affectedIDs.contains($0.element.id) ? (index: $0.offset, segment: $0.element) : nil
+ }
)
}
}
@@ -1126,7 +1152,7 @@ final class MeetingSessionCoordinator: ObservableObject {
let sessions: [MeetingSession]
do {
- sessions = try await self.store.loadAll()
+ sessions = try await self.store.loadHistory()
} catch {
DebugLogger.shared.warning(
"Retention sweep could not load sessions; leaving the current timer as-is: \(error)",
@@ -1231,18 +1257,12 @@ final class MeetingSessionCoordinator: ObservableObject {
guard let index = session.speakers.firstIndex(where: { $0.id == speakerID }) else { return false }
session.speakers[index].displayName = previousName
return true
- case let .reassign(
- segmentID,
- previousSpeakerID,
- previousOverlap,
- previousAttributionState,
- previousRevision
- ):
- guard let index = session.transcriptSegments.firstIndex(where: { $0.id == segmentID }) else { return false }
- session.transcriptSegments[index].speakerID = previousSpeakerID
- session.transcriptSegments[index].overlap = previousOverlap
- session.transcriptSegments[index].attributionState = previousAttributionState
- session.transcriptSegments[index].revision = previousRevision
+ case let .reassign(segmentID, previousSegments):
+ guard session.transcriptSegments.contains(where: { $0.id == segmentID }) else { return false }
+ session.transcriptSegments.removeAll { $0.id == segmentID }
+ for original in previousSegments.sorted(by: { $0.index < $1.index }) {
+ session.transcriptSegments.insert(original.segment, at: min(original.index, session.transcriptSegments.count))
+ }
return true
case let .nameUnknown(
segmentID,
@@ -1279,6 +1299,16 @@ final class MeetingSessionCoordinator: ObservableObject {
restored = true
}
return restored
+ case let .speakerEdits(previousSpeakers, movedSegments):
+ guard Set(session.speakers.map(\.id)) == Set(previousSpeakers.map(\.id)) else { return false }
+ let previous = Dictionary(uniqueKeysWithValues: movedSegments.map { ($0.id, $0) })
+ session.speakers = previousSpeakers
+ for index in session.transcriptSegments.indices {
+ guard let original = previous[session.transcriptSegments[index].id] else { continue }
+ session.transcriptSegments[index].speakerID = original.speakerID
+ session.transcriptSegments[index].revision = original.revision
+ }
+ return true
}
}
@@ -1767,7 +1797,23 @@ final class MeetingSessionCoordinator: ObservableObject {
private func handleCaptureEvent(_ event: MeetingCaptureEvent, generation: UUID) {
guard self.captureGeneration == generation, var session = self.activeSession else { return }
+ self.liveTranscriptionCoordinator?.recordComparisonCaptureEvent(event)
switch event {
+ case let .audioComparisonProbeStarted(probe):
+ self.liveTranscriptionCoordinator?.attachComparisonProbe(probe)
+ return // Diagnostic-only: no recording, persistence, navigation or transcript writes.
+ case let .microphoneEchoProtectionChanged(protection):
+ let kind: MeetingInterruptionKind = protection == .unfiltered ? .echoFilteringUnavailable : .echoFilteringRestored
+ if session.events.last(where: { $0.kind == .echoFilteringUnavailable || $0.kind == .echoFilteringRestored })?.kind == kind {
+ return
+ }
+ session.events.append(MeetingSessionEvent(
+ id: UUID(),
+ occurredAt: Date(),
+ kind: kind,
+ trackID: nil,
+ detail: protection.transcriptWarning
+ ))
case let .trackHealth(trackID, health):
if health.status == .healthy {
self.startupWriterFailures.remove(trackID)
@@ -2170,6 +2216,7 @@ final class MeetingSessionCoordinator: ObservableObject {
}
enum MeetingCoordinatorError: LocalizedError {
+ case featureDisabled
case recordingAlreadyActive
case noActiveMeeting
case noRecoverableAudio
@@ -2180,6 +2227,8 @@ enum MeetingCoordinatorError: LocalizedError {
var errorDescription: String? {
switch self {
+ case .featureDisabled:
+ return "Enable FluidMeet in Settings to record a meeting."
case .recordingAlreadyActive:
return "Another meeting recording is already active."
case .noActiveMeeting:
diff --git a/Sources/Fluid/Services/Meeting/MeetingSessionStore.swift b/Sources/Fluid/Services/Meeting/MeetingSessionStore.swift
index 03802a36..2e70b5f1 100644
--- a/Sources/Fluid/Services/Meeting/MeetingSessionStore.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingSessionStore.swift
@@ -7,6 +7,8 @@ nonisolated protocol MeetingSessionStoring: Sendable {
func save(_ session: MeetingSession) async throws
func load(id: MeetingSessionID) async throws -> MeetingSession?
func loadAll() async throws -> [MeetingSession]
+ func loadHistory() async throws -> [MeetingSession]
+ func loadTranscript(id: MeetingSessionID) async throws -> MeetingSession?
func loadRecoverable() async throws -> [MeetingSession]
func sessionDirectory(for id: MeetingSessionID) async throws -> URL
func existingSessionDirectory(for id: MeetingSessionID) async throws -> URL?
@@ -16,6 +18,12 @@ nonisolated protocol MeetingSessionStoring: Sendable {
func deleteAudioFiles(for id: MeetingSessionID) async throws
}
+// Test/custom stores retain their existing behavior unless they provide the lighter reads.
+nonisolated extension MeetingSessionStoring {
+ func loadHistory() async throws -> [MeetingSession] { try await self.loadAll() }
+ func loadTranscript(id: MeetingSessionID) async throws -> MeetingSession? { try await self.load(id: id) }
+}
+
private final nonisolated class MeetingSessionFileSystem: @unchecked Sendable {
let manager: FileManager
@@ -110,7 +118,31 @@ actor MeetingSessionStore: MeetingSessionStoring {
return migrated
}
+ /// Reading transcript/history never opens or hashes audio. Recovery and audio consumers
+ /// keep using load(id:), which still reconciles and validates the requested recording.
+ func loadTranscript(id: MeetingSessionID) async throws -> MeetingSession? {
+ try self.readTranscript(id: id)
+ }
+
+ private func readTranscript(id: MeetingSessionID) throws -> MeetingSession? {
+ guard !self.tombstonedIDs.contains(id) else { return nil }
+ let url = self.sessionManifestURL(for: id)
+ guard self.fileSystem.manager.fileExists(atPath: url.path) else { return nil }
+ let session = try self.decodeSession(at: url)
+ let migrated = self.migrateCanonicalTranscriptTimelineIfNeeded(session)
+ if migrated != session { try self.writeSession(migrated) }
+ return migrated
+ }
+
+ func loadHistory() async throws -> [MeetingSession] {
+ try self.loadCatalog(reconcileAudio: false)
+ }
+
func loadAll() throws -> [MeetingSession] {
+ try self.loadCatalog(reconcileAudio: true)
+ }
+
+ private func loadCatalog(reconcileAudio: Bool) throws -> [MeetingSession] {
try self.prepareRootDirectory()
let indexedIDs = (try? self.readIndex().sessionIDs) ?? []
let discoveredIDs = try self.discoverSessionIDs()
@@ -120,9 +152,8 @@ actor MeetingSessionStore: MeetingSessionStoring {
sessions.reserveCapacity(allIDs.count)
for id in allIDs {
do {
- if let session = try self.load(id: id) {
- sessions.append(session)
- }
+ let session = try reconcileAudio ? self.load(id: id) : self.readTranscript(id: id)
+ if let session { sessions.append(session) }
} catch {
DebugLogger.shared.error(
"Failed to load meeting session \(id): \(error.localizedDescription)",
@@ -134,23 +165,43 @@ actor MeetingSessionStore: MeetingSessionStoring {
}
func loadRecoverable() throws -> [MeetingSession] {
- try self.loadAll().filter {
- guard $0.recoveryResolvedAt == nil, $0.retention.audioDeletedAt == nil else { return false }
- switch $0.state {
- case .preparing, .recording, .recordingDegraded, .stopping, .processing, .interrupted:
- return true
- case .failed:
- return $0.failures.last?.recoverable == true &&
- $0.endedAt != nil &&
- $0.audioTracks.contains { track in
- track.chunks.contains {
- $0.finalizationState == .finalized && $0.byteCount > 0
- }
- }
- case .completed:
- return false
+ // Completed history must not make startup hash the whole audio library. Only the
+ // recordings that may actually be resumed need full audio reconciliation here.
+ var recovered: [MeetingSession] = []
+ for candidate in try self.loadCatalog(reconcileAudio: false) where Self.needsRecoveryInspection(candidate) {
+ do {
+ if let session = try self.load(id: candidate.id), Self.isRecoverable(session) {
+ recovered.append(session)
+ }
+ } catch {
+ DebugLogger.shared.warning(
+ "Could not inspect interrupted meeting \(candidate.id): \(error.localizedDescription)",
+ source: "MeetingSessionStore"
+ )
}
}
+ return recovered
+ }
+
+ private static func needsRecoveryInspection(_ session: MeetingSession) -> Bool {
+ guard session.recoveryResolvedAt == nil, session.retention.audioDeletedAt == nil else { return false }
+ switch session.state {
+ case .preparing, .recording, .recordingDegraded, .stopping, .processing, .interrupted:
+ return true
+ case .failed:
+ // A track manifest can contain finalized audio missing from session.json after a crash.
+ return session.failures.last?.recoverable == true && session.endedAt != nil
+ case .completed:
+ return false
+ }
+ }
+
+ private static func isRecoverable(_ session: MeetingSession) -> Bool {
+ guard self.needsRecoveryInspection(session) else { return false }
+ guard session.state == .failed else { return true }
+ return session.audioTracks.contains { track in
+ track.chunks.contains { $0.finalizationState == .finalized && $0.byteCount > 0 }
+ }
}
func sessionDirectory(for id: MeetingSessionID) throws -> URL {
diff --git a/Sources/Fluid/Services/Meeting/MeetingTranscriptExporter.swift b/Sources/Fluid/Services/Meeting/MeetingTranscriptExporter.swift
index 22ec41e0..bda57931 100644
--- a/Sources/Fluid/Services/Meeting/MeetingTranscriptExporter.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingTranscriptExporter.swift
@@ -1,6 +1,6 @@
import Foundation
-/// Single source of truth for rendering a `MeetingSession` transcript as plain text or JSON.
+/// Single source of truth for rendering a `MeetingSession` transcript as plain text, Markdown or JSON.
nonisolated enum MeetingTranscriptExporter {
static func text(for session: MeetingSession, includeEchoes: Bool = false) -> String {
let speakerNames = Dictionary(uniqueKeysWithValues: session.activeSpeakers.map { ($0.id, $0.displayName) })
@@ -13,6 +13,37 @@ nonisolated enum MeetingTranscriptExporter {
}.joined(separator: "\n\n")
}
+ static func markdown(for session: MeetingSession, includeEchoes: Bool = false) -> String {
+ let speakerNames = Dictionary(uniqueKeysWithValues: session.activeSpeakers.map { ($0.id, $0.displayName) })
+ let title = Self.markdownHeading(session.title)
+ let segments = session.transcriptSegments
+ .filter { includeEchoes || !$0.isEcho }
+ .sorted { $0.start.seconds < $1.start.seconds }
+ var blocks = ["# \(title)", "## Transcript"]
+ for segment in segments {
+ let speaker = Self.markdownHeading(Self.speakerLabel(for: segment, in: session, speakerNames: speakerNames))
+ blocks.append("### [\(Self.timestampText(segment.start.seconds))] \(speaker)")
+ blocks.append(Self.escapeMarkdown(segment.text))
+ }
+ return blocks.joined(separator: "\n\n") + "\n"
+ }
+
+ private static func markdownHeading(_ text: String) -> String {
+ self.escapeMarkdown(text.components(separatedBy: .newlines).joined(separator: " "))
+ }
+
+ private static func escapeMarkdown(_ text: String) -> String {
+ var escaped = ""
+ escaped.reserveCapacity(text.utf8.count)
+ for character in text {
+ if "\\`*_{}[]<>()#+-.!|>".contains(character) {
+ escaped.append("\\")
+ }
+ escaped.append(character)
+ }
+ return escaped
+ }
+
/// Single resolver for the label shown next to a segment, shared by text export and
/// `MeetingTranscriptionView` so the transcript and the UI never disagree on what a segment
/// is called. `attributionState` (when present) is authoritative; its absence means the
diff --git a/Sources/Fluid/Services/MenuBarManager.swift b/Sources/Fluid/Services/MenuBarManager.swift
index 03922dd0..1dfece79 100644
--- a/Sources/Fluid/Services/MenuBarManager.swift
+++ b/Sources/Fluid/Services/MenuBarManager.swift
@@ -94,6 +94,11 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
override init() {
super.init()
+ NotificationCenter.default.publisher(for: .fluidMeetEnabledChanged)
+ .sink { [weak self] _ in
+ MainActor.assumeIsolated { self?.updateMeetingMenuItemsText() }
+ }
+ .store(in: &self.cancellables)
NotificationCenter.default.publisher(for: .openMicrophoneSettingsRequested)
.receive(on: RunLoop.main)
.sink { [weak self] _ in
@@ -1071,6 +1076,18 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
}
private func updateMeetingMenuItemsText(now: Date = Date()) {
+ guard SettingsStore.shared.fluidMeetEnabled else {
+ for item in [
+ self.startMeetingRecordingMenuItem,
+ self.stopMeetingRecordingMenuItem,
+ self.meetingStatusMenuItem,
+ self.openMeetingTranscriptionMenuItem,
+ self.meetingMenuSeparator,
+ ] {
+ item?.isHidden = true
+ }
+ return
+ }
let activity = self.meetingMenuPresentation.activity
let hasMeetingStatus = activity != .inactive || self.meetingStartError != nil
let isRecording = activity == .recording
@@ -1434,6 +1451,25 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
}
}
+ #if DEBUG
+ func configureMeetingShortcutForTesting(state: MeetingCoordinatorState, onStop: @escaping () -> Void) {
+ self.applyMeetingPresentation(state: state, activeSession: nil, latestCompletedSession: nil)
+ self.stopMeetingRecordingHandler = onStop
+ }
+ #endif
+
+ /// Reuse the menu's guarded actions; a meeting shortcut never starts dictation or navigates.
+ func toggleMeetingRecordingFromShortcut(using operation: () async throws -> Void) async {
+ guard SettingsStore.shared.fluidMeetEnabled, !self.meetingStartRequested else { return }
+ if self.meetingMenuPresentation.activity == .recording {
+ self.stopMeetingRecording()
+ return
+ }
+ guard self.asrService?.activeExclusiveActivity == nil,
+ self.asrService?.isRunningOrStarting != true else { return }
+ await self.startMeetingRecordingInBackground(using: operation)
+ }
+
@objc private func startMeetingRecording() {
Task { @MainActor in
await self.startMeetingRecordingInBackground {
@@ -1445,7 +1481,7 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
/// Keep source discovery and permission waits inside the duplicate-click guard.
/// Errors stay in the menu; starting must never request main-window navigation.
func startMeetingRecordingInBackground(using operation: () async throws -> Void) async {
- guard self.canStartMeetingRecording, !self.meetingStartRequested else { return }
+ guard SettingsStore.shared.fluidMeetEnabled, self.canStartMeetingRecording, !self.meetingStartRequested else { return }
self.meetingStartRequested = true
self.meetingStartError = nil
self.updateMeetingMenuItemsText()
diff --git a/Sources/Fluid/Services/ParakeetSpeechModelCatalog.swift b/Sources/Fluid/Services/ParakeetSpeechModelCatalog.swift
index 52a4ae90..f4898848 100644
--- a/Sources/Fluid/Services/ParakeetSpeechModelCatalog.swift
+++ b/Sources/Fluid/Services/ParakeetSpeechModelCatalog.swift
@@ -50,7 +50,7 @@ nonisolated enum ParakeetSpeechModelCatalog {
}
}
- /// Encoder vectors are compatible only with the checkpoint that produced them.
+ /// Retain the source checkpoint identity when sharing dictionary profiles across the family.
/// Legacy v2/v3 keep their established persisted keys byte for byte.
var pronunciationModelKey: String {
switch self.variant {
@@ -253,6 +253,25 @@ nonisolated enum ParakeetSpeechModelCatalog {
static let descriptors = [ParakeetSpeechModelCatalog.v2, ParakeetSpeechModelCatalog.v3, ParakeetSpeechModelCatalog.mini, ParakeetSpeechModelCatalog.pico]
+ /// Dictionary sharing is separate from the exact identity used to load model files.
+ static func sharesPronunciationFamily(_ source: String, with target: String) -> Bool {
+ self.isParakeetPronunciationKey(source) && self.isParakeetPronunciationKey(target)
+ }
+
+ private static func isParakeetPronunciationKey(_ key: String) -> Bool {
+ if key == "parakeet-v2" || key == "parakeet-v3" { return true }
+ for descriptor in [self.mini, self.pico] {
+ if key == descriptor.modelID { return true }
+ let prefix = descriptor.modelID + ":sha256:"
+ guard key.hasPrefix(prefix) else { continue }
+ let hash = key.dropFirst(prefix.count)
+ return hash.utf8.count == 64 && hash.utf8.allSatisfy {
+ (48...57).contains($0) || (97...102).contains($0)
+ }
+ }
+ return false
+ }
+
static func descriptor(forModelID modelID: String) -> Descriptor? {
self.descriptors.first { $0.modelID == modelID }
}
diff --git a/Sources/Fluid/Services/PronunciationDictionaryStore.swift b/Sources/Fluid/Services/PronunciationDictionaryStore.swift
index a7e0e458..00d341a0 100644
--- a/Sources/Fluid/Services/PronunciationDictionaryStore.swift
+++ b/Sources/Fluid/Services/PronunciationDictionaryStore.swift
@@ -39,6 +39,12 @@ struct PronunciationDictionaryProfile: Codable, Equatable, Identifiable, Sendabl
}
}
+ var hasTemporalRecordings: Bool {
+ self.enrollments.count >= 3 && self.enrollments.prefix(3).allSatisfy {
+ $0.inspectionID != nil || $0.originalAudioID != nil
+ }
+ }
+
var id: String { "\(self.dictionaryEntryID.uuidString):\(self.modelKey)" }
}
@@ -99,6 +105,29 @@ actor PronunciationDictionaryStore {
return self.document?.profiles.filter { $0.modelKey == modelKey } ?? []
}
+ /// Read-only family fallback. Keep checkpoint identities and stored recordings intact.
+ /// One profile per word prevents another model's version competing with its own word.
+ func sharedProfiles(modelKey: String, hiddenSize: Int?) -> [PronunciationDictionaryProfile] {
+ self.loadIfNeeded()
+ return Self.selectSharedProfiles(self.document?.profiles ?? [], modelKey: modelKey, hiddenSize: hiddenSize)
+ }
+
+ nonisolated static func selectSharedProfiles(
+ _ profiles: [PronunciationDictionaryProfile], modelKey: String, hiddenSize: Int?
+ ) -> [PronunciationDictionaryProfile] {
+ let eligible = profiles.filter {
+ $0.isEligibleForMatching && (hiddenSize == nil || $0.hiddenSize == hiddenSize) &&
+ ($0.modelKey == modelKey || ParakeetSpeechModelCatalog.sharesPronunciationFamily($0.modelKey, with: modelKey))
+ }.sorted {
+ let leftExact = $0.modelKey == modelKey, rightExact = $1.modelKey == modelKey
+ if leftExact != rightExact { return leftExact }
+ if $0.enrollments.count != $1.enrollments.count { return $0.enrollments.count > $1.enrollments.count }
+ return $0.id < $1.id
+ }
+ var seen = Set<UUID>()
+ return eligible.filter { seen.insert($0.dictionaryEntryID).inserted }
+ }
+
func enrollmentCount(
dictionaryEntryID: UUID,
modelKey: String
diff --git a/Sources/Fluid/Services/SimpleUpdater.swift b/Sources/Fluid/Services/SimpleUpdater.swift
index 1a5d5e5e..3de2272b 100644
--- a/Sources/Fluid/Services/SimpleUpdater.swift
+++ b/Sources/Fluid/Services/SimpleUpdater.swift
@@ -197,7 +197,7 @@ final class SimpleUpdater: ObservableObject {
return self.versionString(for: latest)
}
- func rollbackToLatestBackup() async throws {
+ func rollbackToLatestBackup(expectedVersion: String? = nil) async throws {
guard self.updateOperationGate.begin() else {
throw SimpleUpdateError.updateAlreadyInProgress
}
@@ -218,11 +218,14 @@ final class SimpleUpdater: ObservableObject {
throw SimpleUpdateError.rollbackUnavailable
}
+ if let expectedVersion, self.versionString(for: rollbackBundleURL) != expectedVersion {
+ throw SimpleUpdateError.releaseChanged
+ }
do {
try await self.performSwapAndRelaunch(
installedAppURL: Bundle.main.bundleURL,
downloadedAppURL: rollbackBundleURL,
- expectedVersion: nil,
+ expectedVersion: expectedVersion,
isRollback: true
)
DebugLogger.shared.info(
@@ -235,18 +238,52 @@ final class SimpleUpdater: ObservableObject {
}
}
+ /// Explicit user-selected downgrade. Re-fetch the tag so a stale picker never installs another release.
+ func installPreviousRelease(version: String) async throws {
+ guard self.updateOperationGate.begin() else { throw SimpleUpdateError.updateAlreadyInProgress }
+ self.isUpdateInProgress = true
+ self.cancelUpdateCheck()
+ self.updatePrompts.dismissUpdateOffers()
+ var installed = false
+ defer {
+ if !installed { self.resetUpdateOperation() }
+ }
+ let releases = try await self.fetchReleases(owner: "altic-dev", repo: "Fluid-oss")
+ guard let release = releases.first(where: { $0.tag_name == version }),
+ let target = self.parseSemanticVersion(version),
+ let current = self.parseSemanticVersion(self.currentAppVersion),
+ target < current
+ else { throw SimpleUpdateError.noSuitableRelease }
+ try Task.checkCancellation()
+ #if DEBUG
+ if let simulationInstallHandler {
+ try await simulationInstallHandler()
+ return
+ }
+ #endif
+ try await self.installReleaseArchive(release, repo: "Fluid-oss")
+ installed = true
+ }
+
func fetchRecentReleaseBuildOptions(
owner: String,
repo: String,
limit: Int = 3,
- includePrerelease: Bool = false
+ includePrerelease: Bool = false,
+ olderThanCurrent: Bool = false
) async throws -> [ReleaseBuildOption] {
let releases = try await self.fetchReleases(owner: owner, repo: repo)
let count = max(1, limit)
let candidates = self.sortedCandidateReleases(
releases,
includePrerelease: includePrerelease
- ).prefix(count)
+ ).filter { entry in
+ guard olderThanCurrent else { return true }
+ guard let current = self.parseSemanticVersion(self.currentAppVersion),
+ let target = self.parseSemanticVersion(entry.release.tag_name)
+ else { return false }
+ return target < current && self.releaseArchiveAsset(entry.release, repo: repo)?.name.lowercased().hasSuffix(".zip") == true
+ }.prefix(count)
return candidates.map { entry in
let release = entry.release
@@ -598,7 +635,6 @@ final class SimpleUpdater: ObservableObject {
throw SimpleUpdateError.noSuitableRelease
}
- let currentBundle = Bundle.main
// up to date
if !(latestVersion > current) {
self.availableUpdateVersion = nil
@@ -628,19 +664,27 @@ final class SimpleUpdater: ObservableObject {
}
#endif
+ try await self.installReleaseArchive(latest, repo: repo)
+ shouldKeepOperationActive = true
+ }
+
+ private func releaseArchiveAsset(_ release: GHRelease, repo: String) -> GHRelease.Asset? {
// Find asset matching: "{repo-lower}-{version-from-tag}.*" and zip preferred
- let rawVersion = latestTag.hasPrefix("v") ? String(latestTag.dropFirst()) : latestTag
+ let rawVersion = release.tag_name.hasPrefix("v") ? String(release.tag_name.dropFirst()) : release.tag_name
let prefix = "\(repo.lowercased())-\(rawVersion)"
- let asset = latest.assets.first { asset in
+ return release.assets.first { asset in
let base = (asset.name as NSString).deletingPathExtension.lowercased()
return (base == prefix) &&
(asset.content_type == "application/zip" || asset.content_type == "application/x-zip-compressed")
- } ?? latest.assets.first { asset in
+ } ?? release.assets.first { asset in
let base = (asset.name as NSString).deletingPathExtension.lowercased()
return base == prefix
}
+ }
- guard let asset = asset else { throw SimpleUpdateError.noAsset }
+ private func installReleaseArchive(_ release: GHRelease, repo: String) async throws {
+ let rawVersion = release.tag_name.hasPrefix("v") ? String(release.tag_name.dropFirst()) : release.tag_name
+ guard let asset = self.releaseArchiveAsset(release, repo: repo) else { throw SimpleUpdateError.noAsset }
self.showUpdateInstallStatus(version: rawVersion)
@@ -675,12 +719,11 @@ final class SimpleUpdater: ObservableObject {
}
try await self.performSwapAndRelaunch(
- installedAppURL: currentBundle.bundleURL,
+ installedAppURL: Bundle.main.bundleURL,
downloadedAppURL: extractedBundleURL,
expectedVersion: rawVersion,
isRollback: false
)
- shouldKeepOperationActive = true
}
// MARK: - Helpers
diff --git a/Sources/Fluid/Services/VoiceTrainingAliasFilter.swift b/Sources/Fluid/Services/VoiceTrainingAliasFilter.swift
index 7998aa93..f25adb04 100644
--- a/Sources/Fluid/Services/VoiceTrainingAliasFilter.swift
+++ b/Sources/Fluid/Services/VoiceTrainingAliasFilter.swift
@@ -1,95 +1,43 @@
-import AppKit
+import Foundation
-/// Voice captures may teach pronunciation without creating a global text replacement.
-/// A spell-check miss is only an eligibility guard, not proof that an alias is correct.
+/// User-confirmed voice captures may contain correctly spelled words and phrases.
+/// Only standalone grammatical basics require an explicit manual replacement.
@MainActor
enum VoiceTrainingAliasFilter {
struct Result {
let accepted: [String]
let rejected: [String]
- let lookupAvailable: Bool
- }
-
- enum LookupResult: Sendable {
- case checked([NSRange])
- case unavailable
- }
- typealias Lookup = @MainActor (String) async -> LookupResult
-
- static func filter(_ aliases: [String], lookup: Lookup = checkSpelling) async -> Result {
- // Training already caps captures at 20. Bound this API independently too.
- let bounded = Array(aliases.prefix(20))
- var text = ""
- var candidates: [(alias: String, range: NSRange)] = []
- for alias in bounded {
- guard !alias.isEmpty, alias.count <= 160, alias.contains(where: \.isLetter) else { continue }
- let normalized = alias.lowercased().split(whereSeparator: \.isWhitespace).joined(separator: " ")
- let range = NSRange(location: text.utf16.count, length: normalized.utf16.count)
- text += normalized + "\n"
- candidates.append((alias, range))
- }
- guard !candidates.isEmpty else {
- return Result(accepted: [], rejected: aliases, lookupAvailable: true)
- }
- guard case let .checked(misspellings) = await lookup(text), !Task.isCancelled else {
- // Never interpret an unavailable dictionary as permission to add every alias.
- return Result(accepted: [], rejected: aliases, lookupAvailable: false)
- }
- let accepted = candidates.filter { candidate in
- misspellings.contains { NSIntersectionRange($0, candidate.range).length > 0 }
- }.map(\.alias)
- let acceptedSet = Set(accepted)
- return Result(accepted: accepted, rejected: aliases.filter { !acceptedSet.contains($0) }, lookupAvailable: true)
- }
-
- /// English vocabulary is checked locally; no remote API or model inference.
- /// This is called only when saving voice training, never during live dictation/rendering.
- static func checkSpelling(_ text: String) async -> LookupResult {
- let request = SpellingRequest()
- return await withCheckedContinuation { continuation in
- request.start(text, continuation: continuation)
+ @MainActor var rejectionMessage: String {
+ let words = Set(self.rejected.prefix(20).map(VoiceTrainingAliasFilter.normalizedAlias))
+ .intersection(VoiceTrainingAliasFilter.basicWords).sorted()
+ guard !words.isEmpty else {
+ return "No usable word was captured. Try recording again, or add a manual replacement."
+ }
+ let found = words.map { "“\($0)”" }.joined(separator: ", ")
+ return "Replacement found these words: \(found). These single words are too broad to replace automatically. Add a manual replacement if intended."
}
}
- @MainActor
- private final class SpellingRequest {
- private var continuation: CheckedContinuation<LookupResult, Never>?
- private var timeout: Task<Void, Never>?
- private var documentTag: Int?
+ private static let basicWords: Set<String> = [
+ "a", "an", "the", "there", "and", "or",
+ ]
- func start(_ text: String, continuation: CheckedContinuation<LookupResult, Never>) {
- self.continuation = continuation
- let tag = NSSpellChecker.uniqueSpellDocumentTag()
- self.documentTag = tag
- self.timeout = Task { @MainActor in
- do { try await Task.sleep(for: .seconds(2)) } catch { return }
- self.finish(.unavailable)
- }
- NSSpellChecker.shared.requestChecking(
- of: text,
- range: NSRange(text.startIndex..., in: text),
- types: NSTextCheckingResult.CheckingType.spelling.rawValue,
- options: [.orthography: NSOrthography.defaultOrthography(forLanguage: "en_US")],
- inSpellDocumentWithTag: tag
- ) { @Sendable _, results, _, wordCount in
- let ranges = results.filter { $0.resultType == .spelling }.map(\.range)
- Task { @MainActor in
- self.finish(wordCount > 0 ? .checked(ranges) : .unavailable)
- }
- }
- }
+ private static func normalizedAlias(_ alias: String) -> String {
+ alias.lowercased().trimmingCharacters(in: .punctuationCharacters.union(.whitespacesAndNewlines))
+ }
- private func finish(_ result: LookupResult) {
- guard let continuation = self.continuation else { return }
- self.continuation = nil
- self.timeout?.cancel()
- self.timeout = nil
- if let documentTag = self.documentTag {
- NSSpellChecker.shared.closeSpellDocument(withTag: documentTag)
- self.documentTag = nil
- }
- continuation.resume(returning: result)
+ static func filter(_ aliases: [String], manuallyConfirmed: Set<String> = []) async -> Result {
+ guard !Task.isCancelled else { return Result(accepted: [], rejected: aliases) }
+ // Training already caps captures at twenty; bound this entry point independently.
+ let accepted = aliases.prefix(20).filter { alias in
+ guard !alias.isEmpty, alias.count <= 160 else { return false }
+ if manuallyConfirmed.contains(alias) { return true }
+ guard alias.contains(where: \.isLetter) else { return false }
+ let normalized = self.normalizedAlias(alias)
+ return !self.basicWords.contains(normalized)
}
+ let acceptedSet = Set(accepted)
+ return Result(accepted: accepted, rejected: aliases.filter { !acceptedSet.contains($0) })
}
}
diff --git a/Sources/Fluid/Theme/Components/FluidDisclosureGroupStyle.swift b/Sources/Fluid/Theme/Components/FluidDisclosureGroupStyle.swift
new file mode 100644
index 00000000..166b190c
--- /dev/null
+++ b/Sources/Fluid/Theme/Components/FluidDisclosureGroupStyle.swift
@@ -0,0 +1,43 @@
+import SwiftUI
+
+/// The label, chevron, padding, and blank row space all belong to one native button.
+/// Content actions remain outside that button and retain their own bindings.
+/// Apply to content disclosures directly; native sidebar sections must keep their system style.
+struct FluidDisclosureGroupStyle: DisclosureGroupStyle {
+ @Environment(\.theme) private var theme
+ @Environment(\.accessibilityReduceMotion) private var reduceMotion
+
+ func makeBody(configuration: Configuration) -> some View {
+ VStack(alignment: .leading, spacing: 0) {
+ Button {
+ if self.reduceMotion {
+ configuration.isExpanded.toggle()
+ } else {
+ withAnimation(.easeInOut(duration: 0.16)) {
+ configuration.isExpanded.toggle()
+ }
+ }
+ } label: {
+ HStack(spacing: self.theme.metrics.spacing.sm) {
+ Image(systemName: "chevron.right")
+ .font(.fluidSystem(size: 11, weight: .semibold))
+ .rotationEffect(.degrees(configuration.isExpanded ? 90 : 0))
+ .frame(width: 14)
+ .accessibilityHidden(true)
+ configuration.label
+ Spacer(minLength: 0)
+ }
+ .padding(.vertical, self.theme.metrics.spacing.sm)
+ .frame(maxWidth: .infinity, alignment: .leading)
+ .contentShape(Rectangle())
+ }
+ .buttonStyle(.plain)
+ .accessibilityValue(configuration.isExpanded ? "Expanded" : "Collapsed")
+ .accessibilityHint("Show or hide details")
+
+ if configuration.isExpanded {
+ configuration.content
+ }
+ }
+ }
+}
diff --git a/Sources/Fluid/Theme/Components/FluidGlassControls.swift b/Sources/Fluid/Theme/Components/FluidGlassControls.swift
index 20cefec0..e5c0cd80 100644
--- a/Sources/Fluid/Theme/Components/FluidGlassControls.swift
+++ b/Sources/Fluid/Theme/Components/FluidGlassControls.swift
@@ -54,52 +54,117 @@ struct FluidRecordInvitationStyle: ButtonStyle {
}
}
-/// Only this small background redraws. The label, hit target, and surrounding layout stay fixed.
-private struct FluidRecordInvitationSurface: View {
+/// Core Animation moves the decoration without scheduling SwiftUI layout on every frame.
+private struct FluidRecordInvitationSurface: NSViewRepresentable {
let animate: Bool
let recording: Bool
let reduceTransparency: Bool
- var body: some View {
- ZStack {
- if self.recording {
- Color.red
- } else {
- LinearGradient(
- colors: [Color(red: 0.09, green: 0.34, blue: 0.78), Color(red: 0.15, green: 0.49, blue: 0.94), Color(red: 0.07, green: 0.35, blue: 0.82)],
- startPoint: .leading,
- endPoint: .trailing
- )
- TimelineView(.animation(minimumInterval: 1.0 / 24, paused: !self.animate)) { timeline in
- let time = self.animate ? timeline.date.timeIntervalSinceReferenceDate.truncatingRemainder(dividingBy: 24) : 0
- Canvas { context, size in
- guard size.width > 0, size.height > 0 else { return }
- // One broad highlight travels slowly beneath the dots; no blur or particles to retain.
- if !self.reduceTransparency {
- let centre = CGPoint(x: size.width * (0.5 + 0.28 * sin(time / 24 * .pi * 2)), y: size.height * 0.35)
- context.fill(Path(CGRect(origin: .zero, size: size)), with: .radialGradient(
- Gradient(colors: [.white.opacity(0.12), .clear]),
- center: centre,
- startRadius: 0,
- endRadius: size.width * 0.55
- ))
- }
- for index in 0..<12 {
- let fraction = (Double(index) * 0.61_803_398_875 + time / 24).truncatingRemainder(dividingBy: 1)
- let x = size.width * fraction
- let y = size.height * (0.18 + Double((index * 7) % 13) / 13 * 0.64)
- let diameter = 1.5 + Double(index % 3) * 0.45
- let edgeFade = min(1, min(fraction, 1 - fraction) * 12)
- let opacity = (self.reduceTransparency ? 0.45 : 0.32) * edgeFade
- context.fill(Path(ellipseIn: CGRect(x: x, y: y, width: diameter, height: diameter)), with: .color(.white.opacity(opacity)))
- }
- }
- }
+ func makeNSView(context: Context) -> FluidRecordInvitationLayerView {
+ FluidRecordInvitationLayerView()
+ }
+
+ func updateNSView(_ view: FluidRecordInvitationLayerView, context: Context) {
+ view.configure(animate: self.animate, recording: self.recording, reduceTransparency: self.reduceTransparency)
+ }
+}
+
+private final class FluidRecordInvitationLayerView: NSView {
+ private let gradient = CAGradientLayer()
+ private let glow = CAGradientLayer()
+ private let dots = (0..<12).map { _ in CALayer() }
+ private var animate = false
+ private var recording = false
+ private var reduceTransparency = false
+ private var lastSize: CGSize?
+
+ override init(frame frameRect: NSRect) {
+ super.init(frame: frameRect)
+ self.wantsLayer = true
+ self.layer?.masksToBounds = true
+ self.gradient.startPoint = CGPoint(x: 0, y: 0.5)
+ self.gradient.endPoint = CGPoint(x: 1, y: 0.5)
+ self.layer?.addSublayer(self.gradient)
+ self.glow.type = .radial
+ self.glow.colors = [NSColor.white.withAlphaComponent(0.12).cgColor, NSColor.clear.cgColor]
+ self.glow.startPoint = CGPoint(x: 0.5, y: 0.35)
+ self.glow.endPoint = CGPoint(x: 1, y: 0.9)
+ self.layer?.addSublayer(self.glow)
+ for dot in self.dots {
+ self.layer?.addSublayer(dot)
+ }
+ }
+
+ required init?(coder: NSCoder) {
+ nil
+ }
+
+ override func hitTest(_ point: NSPoint) -> NSView? {
+ nil
+ }
+
+ func configure(animate: Bool, recording: Bool, reduceTransparency: Bool) {
+ guard self.lastSize == nil || self.animate != animate || self.recording != recording || self.reduceTransparency != reduceTransparency else { return }
+ self.animate = animate
+ self.recording = recording
+ self.reduceTransparency = reduceTransparency
+ self.lastSize = nil
+ self.needsLayout = true
+ }
+
+ override func layout() {
+ super.layout()
+ guard self.bounds.width > 0, self.bounds.height > 0, self.lastSize != self.bounds.size else { return }
+ self.lastSize = self.bounds.size
+ CATransaction.begin()
+ CATransaction.setDisableActions(true)
+ self.layer?.cornerRadius = self.bounds.height / 2
+ self.gradient.frame = self.bounds
+ self.gradient.colors = self.recording ? [NSColor.systemRed.cgColor, NSColor.systemRed.cgColor] : [
+ NSColor(srgbRed: 0.09, green: 0.34, blue: 0.78, alpha: 1).cgColor,
+ NSColor(srgbRed: 0.15, green: 0.49, blue: 0.94, alpha: 1).cgColor,
+ NSColor(srgbRed: 0.07, green: 0.35, blue: 0.82, alpha: 1).cgColor,
+ ]
+ self.glow.frame = self.bounds
+ self.glow.isHidden = self.recording || self.reduceTransparency
+ self.glow.removeAllAnimations()
+ if self.animate, !self.glow.isHidden {
+ let drift = CABasicAnimation(keyPath: "startPoint")
+ drift.fromValue = CGPoint(x: 0.22, y: 0.35)
+ drift.toValue = CGPoint(x: 0.78, y: 0.35)
+ drift.duration = 12
+ drift.autoreverses = true
+ drift.repeatCount = .infinity
+ drift.timingFunction = CAMediaTimingFunction(name: .easeInEaseOut)
+ self.glow.add(drift, forKey: "drift")
+ }
+ for (index, dot) in self.dots.enumerated() {
+ dot.removeAllAnimations()
+ dot.isHidden = self.recording
+ let fraction = (Double(index) * 0.61_803_398_875).truncatingRemainder(dividingBy: 1)
+ let diameter = 1.5 + Double(index % 3) * 0.45
+ let opacity = self.reduceTransparency ? 0.45 : 0.32
+ dot.bounds = CGRect(x: 0, y: 0, width: diameter, height: diameter)
+ dot.cornerRadius = diameter / 2
+ dot.backgroundColor = NSColor.white.cgColor
+ dot.position = CGPoint(x: self.bounds.width * fraction, y: self.bounds.height * (0.18 + Double((index * 7) % 13) / 13 * 0.64))
+ dot.opacity = Float(opacity * min(1, min(fraction, 1 - fraction) * 12))
+ if self.animate, !self.recording {
+ let travel = CABasicAnimation(keyPath: "position.x")
+ travel.fromValue = 0
+ travel.toValue = self.bounds.width
+ let fade = CAKeyframeAnimation(keyPath: "opacity")
+ fade.values = [0, opacity, opacity, 0]
+ fade.keyTimes = [0, 0.0833, 0.9167, 1]
+ let motion = CAAnimationGroup()
+ motion.animations = [travel, fade]
+ motion.duration = 24
+ motion.timeOffset = fraction * 24
+ motion.repeatCount = .infinity
+ dot.add(motion, forKey: "motion")
}
}
- .clipShape(Capsule())
- .allowsHitTesting(false)
- .accessibilityHidden(true)
+ CATransaction.commit()
}
}
diff --git a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift
index 0e5b3678..9e17289a 100644
--- a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift
+++ b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift
@@ -169,7 +169,8 @@ extension VoiceEngineSettingsView {
Text(descriptor.creditLine)
.font(self.theme.typography.caption)
.foregroundStyle(self.voiceEngineSecondaryText)
- .fixedSize(horizontal: false, vertical: true)
+ // A fixed vertical size here can push the page above the toolbar.
+ .lineLimit(2)
}
}
diff --git a/Sources/Fluid/UI/AppNavigationState.swift b/Sources/Fluid/UI/AppNavigationState.swift
index 886f40e2..a1c6220c 100644
--- a/Sources/Fluid/UI/AppNavigationState.swift
+++ b/Sources/Fluid/UI/AppNavigationState.swift
@@ -54,6 +54,7 @@ struct AppPagePresentation: Equatable {
enum SettingsSection: String, CaseIterable, Identifiable, Hashable {
case general
+ case fluidMeet
case dictation
case dictationFormatting
case shortcuts
@@ -70,6 +71,7 @@ enum SettingsSection: String, CaseIterable, Identifiable, Hashable {
var title: String {
switch self {
case .general: return "General"
+ case .fluidMeet: return "FluidMeet Settings"
case .dictation: return "Dictation"
case .dictationFormatting: return "Dictation Formatting"
case .shortcuts: return "Shortcuts"
@@ -84,6 +86,7 @@ enum SettingsSection: String, CaseIterable, Identifiable, Hashable {
var systemImage: String {
switch self {
case .general: return "gearshape"
+ case .fluidMeet: return "person.2"
case .dictation: return "mic"
case .dictationFormatting: return "textformat"
case .shortcuts: return "keyboard"
diff --git a/Sources/Fluid/UI/CustomDictionaryView.swift b/Sources/Fluid/UI/CustomDictionaryView.swift
index 456bd6ec..48f869b6 100644
--- a/Sources/Fluid/UI/CustomDictionaryView.swift
+++ b/Sources/Fluid/UI/CustomDictionaryView.swift
@@ -9,6 +9,7 @@
//
import AppKit
+import Combine
import SwiftUI
import UniformTypeIdentifiers
@@ -16,9 +17,10 @@ import UniformTypeIdentifiers
// swiftlint:disable:next type_body_length
struct CustomDictionaryView: View {
var formattingOnly = false
- @State private var isWordDrawerPresented = false
+ @State private var isWordDrawerPresented = true
@State private var wordSearch = ""
@State private var isDrawerActionsPresented = false
+ @State private var newWordFocusRequest = 0
@State private var drawerDeletion: SettingsStore.CustomDictionaryEntry?
@Environment(\.theme) private var theme
@Environment(\.accessibilityReduceMotion) private var reduceMotion
@@ -50,11 +52,13 @@ struct CustomDictionaryView: View {
@State private var trainingReplacement = ""
@State private var trainingSaveID: UUID?
@State private var trainingVariants: [String] = []
+ @State private var manualTrainingSpellings = Set<String>()
@AppStorage("DictionarySharedFeatureMatcherEnabled") private var pronunciationEnabled = false
@State private var pronunciationMatchingEnabled = SettingsStore.shared.pronunciationMatchingEnabled
@State private var trainingPronunciationEnrollments: [PronunciationEnrollmentCapture] = []
@State private var trainingSampleCount = 0
@State private var lastTrainingOutput = ""
+ @State private var trainingAttempts: [String] = []
@State private var lastTrainingOutputIsCovered = false
@State private var consecutiveCoveredCaptures = 0
@State private var trainingStatusMessage = "Type the correct text."
@@ -65,6 +69,7 @@ struct CustomDictionaryView: View {
@State private var trainingStopRequestedDuringStart = false
@State private var isTrainingProcessing = false
@State private var isAutomaticTrainingEnabled = false
+ @StateObject private var trainingRestart = DictionaryTrainingRestart()
@State private var isTrainedReplacementButtonHovered = false
@State private var isTrainedReplacementGlowExpanded = false
@State private var replacementConfirmation: ReplacementConfirmation?
@@ -73,6 +78,8 @@ struct CustomDictionaryView: View {
@State private var manualReturnStep: DictionaryWordWizardStep = .spelling
@State private var manualTriggerDraft = ""
@State private var manualReplacement = ""
+ @State private var manualEntryFocusRequest = 0
+ @FocusState private var manualReplacementFocused: Bool
@State private var isYourDictionaryPresented = false
@State private var isPunctuationDictionaryPresented = false
@State private var punctuationAutoConvertEnabled = SettingsStore.shared.autoConvertPunctuationEnabled
@@ -218,6 +225,10 @@ struct CustomDictionaryView: View {
self.manualDuplicateTriggers.isEmpty
}
+ private var isManualEntryBlocked: Bool {
+ self.isTrainingStarting || self.isTrainingRecording || self.isTrainingProcessing || self.isAutomaticTrainingEnabled
+ }
+
private var punctuationEditorTitle: String {
self.editingPunctuationRuleID == nil ? "Add Rule" : "Edit Rule"
}
@@ -346,7 +357,8 @@ struct CustomDictionaryView: View {
.sheet(item: self.$editingEntry) { entry in
EditDictionaryEntrySheet(
entry: entry,
- existingTriggers: self.allExistingTriggers(excluding: entry.id)
+ existingTriggers: self.allExistingTriggers(excluding: entry.id),
+ isCaptureBusy: self.asr.isRunning || self.isTrainingStarting || self.isTrainingRecording || self.isTrainingProcessing || self.isAutomaticTrainingEnabled
) { updatedEntry in
if let index = self.entries.firstIndex(where: { $0.id == updatedEntry.id }) {
self.entries[index] = updatedEntry
@@ -385,8 +397,19 @@ struct CustomDictionaryView: View {
guard !self.formattingOnly else { return }
self.pronunciationRevisionNoticeRefresh &+= 1
}
+ .onReceive(self.asr.$isDictionaryTrainingCaptureActive.removeDuplicates().dropFirst()) { active in
+ guard !self.formattingOnly, !active else { return }
+ self.handleTrainingCaptureEndedElsewhere()
+ }
+ .onReceive(NotificationCenter.default.publisher(for: NSApplication.didResignActiveNotification)) { _ in
+ guard self.trainingRestart.isWaiting else { return }
+ self.trainingRestart.cancel()
+ self.isAutomaticTrainingEnabled = false
+ }
.onChange(of: self.pronunciationEnabled) { _, enabled in
guard !enabled else { return }
+ self.trainingRestart.cancel()
+ self.isAutomaticTrainingEnabled = false
self.trainingPronunciationEnrollments = []
if self.trainingSaveID != nil {
self.trainingSaveID = nil
@@ -397,6 +420,7 @@ struct CustomDictionaryView: View {
}
.onDisappear {
guard !self.formattingOnly else { return }
+ self.trainingRestart.cancel()
if self.trainingSaveID != nil {
self.trainingSaveID = nil
self.isTrainingProcessing = false
@@ -441,30 +465,43 @@ struct CustomDictionaryView: View {
HStack(alignment: .center, spacing: 8) {
Text("Your dictionary").font(self.theme.typography.sectionTitle)
Spacer()
- Button { self.isDrawerActionsPresented.toggle() } label: {
- Image(systemName: "ellipsis")
- .font(.system(size: 13, weight: .semibold))
- .frame(width: 20, height: 20)
- }
- .fluidGlassAction(circular: true)
- .accessibilityLabel("Dictionary actions")
- .popover(isPresented: self.$isDrawerActionsPresented) {
- VStack(alignment: .leading, spacing: 12) {
- Button("Import…") {
- self.isDrawerActionsPresented = false
- self.importDictionary()
- }.fluidGlassAction()
- Button("Export…") {
- self.isDrawerActionsPresented = false
- self.exportDictionary()
- }.fluidGlassAction()
- Divider()
- Button("Custom Words (Advanced)…") {
- self.isDrawerActionsPresented = false
- self.presentCustomWords()
- }.fluidGlassAction()
+ FluidGlassControlGroup {
+ HStack(spacing: 8) {
+ Button(action: self.openNewManualDictionaryEntry) {
+ Image(systemName: "plus")
+ .font(.system(size: 12, weight: .semibold))
+ .frame(width: 18, height: 18)
+ }
+ .fluidGlassAction(circular: true, quiet: true)
+ .disabled(self.isManualEntryBlocked)
+ .accessibilityLabel("Add word")
+ .help(self.isManualEntryBlocked ? "Finish the current word recording or save first" : "Add a word manually")
+ Button { self.isDrawerActionsPresented.toggle() } label: {
+ Image(systemName: "ellipsis")
+ .font(.system(size: 12, weight: .semibold))
+ .frame(width: 18, height: 18)
+ }
+ .fluidGlassAction(circular: true, quiet: true)
+ .accessibilityLabel("Dictionary actions")
+ .popover(isPresented: self.$isDrawerActionsPresented) {
+ VStack(alignment: .leading, spacing: 12) {
+ Button("Import…") {
+ self.isDrawerActionsPresented = false
+ self.importDictionary()
+ }.fluidGlassAction()
+ Button("Export…") {
+ self.isDrawerActionsPresented = false
+ self.exportDictionary()
+ }.fluidGlassAction()
+ Divider()
+ Button("Custom Words (Advanced)…") {
+ self.isDrawerActionsPresented = false
+ self.presentCustomWords()
+ }.fluidGlassAction()
+ }
+ .padding(16)
+ }
}
- .padding(16)
}
}
HStack(spacing: 8) {
@@ -487,32 +524,25 @@ struct CustomDictionaryView: View {
.font(self.theme.typography.caption)
.foregroundStyle(self.theme.palette.secondaryText)
.lineLimit(3)
- Button("Test word") {
- self.wizardSavedWord = entry.replacement
- self.wizardStep = .saved
- self.isWordDrawerPresented = false
- }.fluidGlassAction()
- .disabled(self.asr.isRunning || self.isTrainingStarting || self.isTrainingProcessing)
- .accessibilityLabel("Test \(entry.replacement)")
}
Spacer()
Button { self.editingEntry = entry } label: {
FluidPencilShape()
.stroke(style: StrokeStyle(lineWidth: 1.7, lineCap: .round, lineJoin: .round))
- .frame(width: 20, height: 20)
- .frame(width: 22, height: 22)
+ .frame(width: 14, height: 14)
+ .frame(width: 18, height: 18)
.contentShape(Rectangle())
}
- .fluidGlassAction(circular: true)
+ .fluidGlassAction(circular: true, quiet: true)
.help("Edit word")
.accessibilityLabel("Edit \(entry.replacement)")
Button(role: .destructive) { self.drawerDeletion = entry } label: {
Image(systemName: "trash")
- .font(.fluidSystem(size: 15, weight: .regular))
- .frame(width: 22, height: 22)
+ .font(.fluidSystem(size: 12, weight: .regular))
+ .frame(width: 18, height: 18)
.contentShape(Rectangle())
}
- .fluidGlassAction(circular: true)
+ .fluidGlassAction(circular: true, quiet: true)
.help("Delete word")
.accessibilityLabel("Delete \(entry.replacement)")
}
@@ -604,7 +634,7 @@ struct CustomDictionaryView: View {
count: self.trainingSampleCount,
heard: self.lastTrainingOutput,
variants: self.trainingVariants,
- busy: self.isTrainingRecording || self.isTrainingProcessing || self.isTrainingStarting,
+ busy: self.isTrainingRecording || self.isTrainingProcessing || self.isTrainingStarting || self.trainingRestart.isWaiting,
recording: self.isTrainingRecording,
processing: self.isTrainingProcessing || self.isTrainingStarting,
starting: self.isTrainingStarting,
@@ -618,7 +648,7 @@ struct CustomDictionaryView: View {
},
onRecord: {
self.wizardStep = .recording
- Task { await self.toggleAutomaticTraining() }
+ Task { await self.startAutomaticTrainingIfIdle() }
},
onSave: {
if self.trainingAlreadyCorrectWithoutReplacement {
@@ -630,7 +660,7 @@ struct CustomDictionaryView: View {
}
},
onBack: { self.wizardStep = .spelling },
- onNewWord: { self.resetTraining(); self.wizardStep = .spelling },
+ onNewWord: self.startNewDictionaryWord,
onManual: {
self.manualReturnStep = self.wizardStep
if self.manualSourceWord != self.normalizedTrainingReplacement || self.manualReplacement.isEmpty {
@@ -649,10 +679,17 @@ struct CustomDictionaryView: View {
self.resetTraining(keepingWord: true)
self.wizardStep = .recording
},
+ wordFocusRequest: self.newWordFocusRequest,
automaticCaptureActive: self.isAutomaticTrainingEnabled,
+ waiting: self.trainingRestart.isWaiting,
+ canSave: self.canAddTrainedReplacement || self.trainingAlreadyCorrectWithoutReplacement,
+ onPauseRecording: { Task { await self.pauseAutomaticTraining() } },
+ onAddSpelling: self.addManualTrainingSpelling,
+ onRemoveSpelling: self.removeTrainingVariant,
audioLevels: self.asr.audioLevelPublisher,
pronunciationNotice: self.trainingProgress.pronunciationNotice,
- pronunciationIncomplete: self.activePronunciationMatching && !self.trainingProgress.pronunciationReady
+ pronunciationIncomplete: self.activePronunciationMatching && !self.trainingProgress.pronunciationReady,
+ attempts: self.trainingAttempts
)
.onChange(of: self.trainingReplacement) { oldValue, newValue in
self.handleTrainingReplacementChange(oldValue: oldValue, newValue: newValue)
@@ -823,6 +860,7 @@ struct CustomDictionaryView: View {
TextField("fluid voice, fluid boys", text: self.$manualTriggerDraft)
.dictionaryInputChrome()
+ .accessibilityLabel("Misheard words")
.onSubmit { self.addManualReplacementIfValid() }
Text("Separate different versions with commas. Enter only commas to replace comma punctuation.")
@@ -837,7 +875,14 @@ struct CustomDictionaryView: View {
.font(self.theme.typography.captionStrong)
TextField("FluidVoice", text: self.$manualReplacement)
.dictionaryInputChrome()
+ .focused(self.$manualReplacementFocused)
+ .accessibilityLabel("Correct word")
.onSubmit { self.addManualReplacementIfValid() }
+ .task(id: self.manualEntryFocusRequest) {
+ await Task.yield()
+ guard !Task.isCancelled else { return }
+ self.manualReplacementFocused = true
+ }
Text("The spelling you want in your transcription.")
.font(self.theme.typography.caption)
.foregroundStyle(self.theme.palette.secondaryText)
@@ -2245,13 +2290,24 @@ struct CustomDictionaryView: View {
private func toggleAutomaticTraining() async {
if self.isAutomaticTrainingEnabled {
- self.isAutomaticTrainingEnabled = false
- if self.isTrainingRecording {
- await self.stopTrainingSample()
- }
+ await self.pauseAutomaticTraining()
return
}
+ await self.startAutomaticTrainingIfIdle()
+ }
+ private func pauseAutomaticTraining() async {
+ self.trainingRestart.cancel()
+ self.isAutomaticTrainingEnabled = false
+ if self.isTrainingRecording {
+ await self.stopTrainingSample()
+ }
+ }
+
+ private func startAutomaticTrainingIfIdle() async {
+ guard !self.isAutomaticTrainingEnabled, !self.isTrainingStarting,
+ !self.isTrainingRecording, !self.isTrainingProcessing, !self.trainingRestart.isWaiting
+ else { return }
if self.canRetryTrainingAfterMaximum {
self.resetTrainingVerificationAttempts()
}
@@ -2307,6 +2363,20 @@ struct CustomDictionaryView: View {
await self.finishTrainingSampleStop()
}
+ /// Our own stop clears `isTrainingRecording` before the recorder stops, so reaching here with it
+ /// still set means something else ended the capture, such as Escape. Leave the screen idle.
+ private func handleTrainingCaptureEndedElsewhere() {
+ guard self.isTrainingRecording, !self.isTrainingStarting, !self.isTrainingProcessing else { return }
+ DictionaryTrainingEndpointMonitor.shared.stop()
+ self.trainingRestart.cancel()
+ self.isAutomaticTrainingEnabled = false
+ self.isTrainingRecording = false
+ self.trainingStopRequestedDuringStart = false
+ self.trainingHasError = true
+ self.trainingStatusMessage = "Recording stopped. Press Record to continue."
+ DebugLogger.shared.info("DICTIONARY_TRAINING capture ended outside the editor; returning to idle", source: "CustomDictionary")
+ }
+
private func handleAutomaticTrainingSpeechEnd() {
guard self.isAutomaticTrainingEnabled, self.isTrainingRecording else { return }
Task { await self.stopTrainingSample() }
@@ -2347,25 +2417,35 @@ struct CustomDictionaryView: View {
self.isAutomaticTrainingEnabled = false
self.wizardStep = .review
}
- await self.continueAutomaticTrainingIfNeeded()
+ self.continueAutomaticTrainingIfNeeded()
}
- private func continueAutomaticTrainingIfNeeded() async {
- guard self.isAutomaticTrainingEnabled,
- !self.trainingFinalOutputIsReady,
- !self.trainingAlreadyCorrectWithoutReplacement,
- self.trainingSampleCount < CustomDictionaryTrainingMerge.maxSamples
- else {
+ private var canContinueAutomaticTraining: Bool {
+ NSApp.isActive && self.isAutomaticTrainingEnabled && !self.trainingFinalOutputIsReady &&
+ !self.trainingAlreadyCorrectWithoutReplacement &&
+ self.trainingSampleCount < CustomDictionaryTrainingMerge.maxSamples
+ }
+
+ private func continueAutomaticTrainingIfNeeded() {
+ guard self.canContinueAutomaticTraining else {
+ self.trainingRestart.cancel()
self.isAutomaticTrainingEnabled = false
return
}
-
- await Task.yield()
- await self.startTrainingSample()
+ self.trainingRestart.schedule(canContinue: {
+ guard self.canContinueAutomaticTraining else {
+ self.isAutomaticTrainingEnabled = false
+ return false
+ }
+ return true
+ }) {
+ await self.startTrainingSample()
+ }
}
private func resetTrainingVerificationAttempts() {
self.trainingSampleCount = 0
+ self.trainingAttempts = []
self.lastTrainingOutput = ""
self.lastTrainingOutputIsCovered = false
self.consecutiveCoveredCaptures = 0
@@ -2385,6 +2465,7 @@ struct CustomDictionaryView: View {
self.lastTrainingOutput = detected
self.trainingSampleCount = min(self.trainingSampleCount + 1, CustomDictionaryTrainingMerge.maxSamples)
+ self.trainingAttempts = Array((self.trainingAttempts + [detected]).suffix(CustomDictionaryTrainingMerge.maxSamples))
if detected.caseInsensitiveCompare(self.normalizedTrainingReplacement) == .orderedSame {
self.lastTrainingOutputIsCovered = true
@@ -2451,17 +2532,15 @@ struct CustomDictionaryView: View {
let enrollments = self.trainingPronunciationEnrollments
let savePronunciation = self.trainingProgress.pronunciationReady
let pronunciationGeneration = DictionaryMatcherExperiment.generation
- let filtered = await VoiceTrainingAliasFilter.filter(self.trainingVariants)
+ let filtered = await VoiceTrainingAliasFilter.filter(self.trainingVariants, manuallyConfirmed: self.manualTrainingSpellings)
guard self.trainingSaveID == saveID, !Task.isCancelled else { return }
DebugLogger.shared.info(
- "VOICE_TRAINING_ALIAS_FILTER accepted=\(filtered.accepted.count) rejected=\(filtered.rejected.count) available=\(filtered.lookupAvailable)",
+ "VOICE_TRAINING_ALIAS_FILTER accepted=\(filtered.accepted.count) rejected=\(filtered.rejected.count)",
source: "CustomDictionary"
)
guard savePronunciation || !filtered.accepted.isEmpty else {
self.trainingHasError = true
- self.trainingStatusMessage = filtered.lookupAvailable
- ? "These recordings contain everyday words. Try again, or add a replacement manually."
- : "Couldn't check these words. Try saving again, or add a replacement manually."
+ self.trainingStatusMessage = filtered.rejectionMessage
return
}
@@ -2518,10 +2597,26 @@ struct CustomDictionaryView: View {
}
private func removeTrainingVariant(_ variant: String) {
+ guard !self.isManualEntryBlocked else { return }
self.trainingVariants.removeAll { $0 == variant }
+ self.manualTrainingSpellings.remove(variant)
self.refreshLastTrainingCoverage()
}
+ private func addManualTrainingSpelling(_ spelling: String) -> String? {
+ guard !self.isManualEntryBlocked else { return "Pause learning before editing spellings." }
+ guard self.trainingVariants.count < CustomDictionaryTrainingMerge.maxSamples else { return "Remove a spelling before adding another." }
+ guard let normalized = CustomDictionaryTrainingMerge.normalizedTrigger(spelling) else { return "Enter the spelling FluidVoice gets wrong." }
+ guard normalized.count <= 160 else { return "Use 160 characters or fewer." }
+ guard normalized.caseInsensitiveCompare(self.normalizedTrainingReplacement) != .orderedSame else { return "That is already the correct spelling." }
+ guard !self.trainingVariants.contains(where: { $0.caseInsensitiveCompare(normalized) == .orderedSame }) else { return "That spelling is already in this list." }
+ guard !self.allExistingTriggers().contains(normalized) else { return "That spelling already belongs to a saved word." }
+ self.trainingVariants.append(normalized)
+ self.manualTrainingSpellings.insert(normalized)
+ self.refreshLastTrainingCoverage()
+ return nil
+ }
+
private func refreshLastTrainingCoverage() {
guard !self.lastTrainingOutput.isEmpty else {
self.lastTrainingOutputIsCovered = false
@@ -2542,14 +2637,37 @@ struct CustomDictionaryView: View {
}
}
+ private func openNewManualDictionaryEntry() {
+ guard !self.isManualEntryBlocked else { return }
+ self.isDrawerActionsPresented = false
+ if self.wizardStep != .manual {
+ self.manualReturnStep = self.wizardStep
+ self.manualSourceWord = ""
+ self.manualReplacement = ""
+ self.manualTriggerDraft = ""
+ self.wizardStep = .manual
+ }
+ self.manualEntryFocusRequest &+= 1
+ }
+
+ private func startNewDictionaryWord() {
+ self.isDrawerActionsPresented = false
+ self.resetTraining()
+ self.wizardStep = .spelling
+ self.newWordFocusRequest &+= 1
+ }
+
private func resetTraining(statusMessage: String = "Type the correct text.", keepingWord: Bool = false) {
+ self.trainingRestart.cancel()
self.trainingSaveID = nil
self.isAutomaticTrainingEnabled = false
DictionaryTrainingEndpointMonitor.shared.stop()
if !keepingWord { self.trainingReplacement = "" }
self.trainingVariants = []
+ self.manualTrainingSpellings = []
self.trainingPronunciationEnrollments = []
self.trainingSampleCount = 0
+ self.trainingAttempts = []
self.lastTrainingOutput = ""
self.lastTrainingOutputIsCovered = false
self.consecutiveCoveredCaptures = 0
@@ -2566,6 +2684,8 @@ struct CustomDictionaryView: View {
let oldKey = CustomDictionaryTrainingMerge.normalizedReplacement(oldValue).lowercased()
let newKey = CustomDictionaryTrainingMerge.normalizedReplacement(newValue).lowercased()
guard oldKey != newKey else { return }
+ self.trainingRestart.cancel()
+ self.isAutomaticTrainingEnabled = false
if self.trainingSaveID != nil {
self.trainingSaveID = nil
@@ -2573,8 +2693,10 @@ struct CustomDictionaryView: View {
}
self.trainingVariants = self.existingTrainingVariants(for: newValue)
+ self.manualTrainingSpellings = []
self.trainingPronunciationEnrollments = []
self.trainingSampleCount = 0
+ self.trainingAttempts = []
self.lastTrainingOutput = ""
self.lastTrainingOutputIsCovered = false
self.consecutiveCoveredCaptures = 0
@@ -2812,10 +2934,12 @@ private extension CustomDictionaryView {
}
func handlePronunciationMatchingChange(enabled: Bool) {
+ self.trainingRestart.cancel()
SettingsStore.shared.pronunciationMatchingEnabled = enabled
self.isAutomaticTrainingEnabled = false
DictionaryTrainingEndpointMonitor.shared.stop()
self.trainingVariants = self.existingTrainingVariants(for: self.trainingReplacement)
+ self.manualTrainingSpellings = []
self.trainingPronunciationEnrollments = []
self.resetTrainingVerificationAttempts()
self.trainingStatusMessage = self.normalizedTrainingReplacement.isEmpty
@@ -3232,7 +3356,7 @@ nonisolated enum DictionaryPronunciationRevisionPolicy {
) -> String? {
guard descriptor.variant == .mini || descriptor.variant == .pico else { return nil }
let current = Set(profiles.filter {
- $0.modelKey == descriptor.pronunciationModelKey && $0.isEligibleForMatching
+ ParakeetSpeechModelCatalog.sharesPronunciationFamily($0.modelKey, with: descriptor.pronunciationModelKey) && $0.isEligibleForMatching
}.map(\.dictionaryEntryID))
let outdated = profiles.filter {
entryIDs.contains($0.dictionaryEntryID) && !current.contains($0.dictionaryEntryID)
@@ -4001,10 +4125,13 @@ struct EditDictionaryEntrySheet: View {
let entry: SettingsStore.CustomDictionaryEntry
let existingTriggers: Set<String>
+ let isCaptureBusy: Bool
let onSave: (SettingsStore.CustomDictionaryEntry) -> Void
@State private var triggersText = ""
@State private var replacement = ""
+ @State private var isTestingWord = false
+ @State private var isTestBusy = false
private var duplicateTriggers: [String] {
self.parseTriggers().filter { self.existingTriggers.contains($0) }
@@ -4016,6 +4143,11 @@ struct EditDictionaryEntrySheet: View {
self.duplicateTriggers.isEmpty
}
+ private var hasUnsavedChanges: Bool {
+ self.parseTriggers() != self.entry.triggers ||
+ CustomDictionaryManualEntry.sanitizedReplacement(self.replacement) != self.entry.replacement
+ }
+
var body: some View {
VStack(alignment: .leading, spacing: 16) {
// Header
@@ -4030,71 +4162,100 @@ struct EditDictionaryEntrySheet: View {
Divider()
- ScrollView {
- VStack(alignment: .leading, spacing: 20) {
- // Triggers input
- VStack(alignment: .leading, spacing: 6) {
- Text("Misheard Words (triggers)")
- .font(.fluidSystem(.subheadline).weight(.medium))
- Text("Add one version per line. Commas can be saved too.")
- .font(.fluidSystem(.caption))
- .foregroundStyle(.secondary)
- TextEditor(text: self.$triggersText)
- .font(.fluidSystem(.body))
- .frame(minHeight: 54, maxHeight: 76)
- .scrollContentBackground(.hidden)
- .dictionaryInputChrome(minHeight: 54)
-
- // Duplicate warning
- if !self.duplicateTriggers.isEmpty {
- HStack(spacing: 4) {
- Image(systemName: "exclamationmark.triangle.fill")
- .foregroundStyle(.orange)
- Text("Duplicate triggers: \(self.duplicateTriggers.joined(separator: ", "))")
- .foregroundStyle(.orange)
+ ScrollViewReader { scroll in
+ ScrollView {
+ VStack(alignment: .leading, spacing: 20) {
+ // Triggers input
+ VStack(alignment: .leading, spacing: 6) {
+ Text("Misheard Words (triggers)")
+ .font(.fluidSystem(.subheadline).weight(.medium))
+ Text("Add one version per line. Commas can be saved too.")
+ .font(.fluidSystem(.caption))
+ .foregroundStyle(.secondary)
+ TextEditor(text: self.$triggersText)
+ .font(.fluidSystem(.body))
+ .frame(minHeight: 54, maxHeight: 76)
+ .scrollContentBackground(.hidden)
+ .dictionaryInputChrome(minHeight: 54)
+
+ // Duplicate warning
+ if !self.duplicateTriggers.isEmpty {
+ HStack(spacing: 4) {
+ Image(systemName: "exclamationmark.triangle.fill")
+ .foregroundStyle(.orange)
+ Text("Duplicate triggers: \(self.duplicateTriggers.joined(separator: ", "))")
+ .foregroundStyle(.orange)
+ }
+ .font(.fluidSystem(.caption))
}
- .font(.fluidSystem(.caption))
}
- }
+ .disabled(self.isTestBusy)
- // Replacement input
- VStack(alignment: .leading, spacing: 6) {
- Text("Correct Spelling (replacement)")
- .font(.fluidSystem(.subheadline).weight(.medium))
- Text("This is what will appear in the final transcription.")
- .font(.fluidSystem(.caption))
- .foregroundStyle(.secondary)
- TextField("FluidVoice", text: self.$replacement)
- .dictionaryInputChrome()
- .onSubmit { self.saveIfValid() }
- }
+ // Replacement input
+ VStack(alignment: .leading, spacing: 6) {
+ Text("Correct Spelling (replacement)")
+ .font(.fluidSystem(.subheadline).weight(.medium))
+ Text("This is what will appear in the final transcription.")
+ .font(.fluidSystem(.caption))
+ .foregroundStyle(.secondary)
+ TextField("FluidVoice", text: self.$replacement)
+ .dictionaryInputChrome()
+ .onSubmit { self.saveIfValid() }
+ }
+ .disabled(self.isTestBusy)
- if !self.parseTriggers().isEmpty && !self.replacement.isEmpty {
- DictionaryReplacementPreview(
- triggers: self.parseTriggers(),
- replacement: self.replacement,
- duplicateTriggers: self.duplicateTriggers
- )
+ if !self.parseTriggers().isEmpty && !self.replacement.isEmpty {
+ DictionaryReplacementPreview(
+ triggers: self.parseTriggers(),
+ replacement: self.replacement,
+ duplicateTriggers: self.duplicateTriggers
+ )
+ }
+
+ if self.isTestingWord {
+ Divider()
+ DictionaryWordPlayground(
+ word: self.entry.replacement,
+ onPracticeMore: nil,
+ busy: self.$isTestBusy
+ )
+ .id("dictionary-word-test")
+ }
}
+ .padding(2)
+ }
+ .task(id: self.isTestingWord) {
+ guard self.isTestingWord else { return }
+ await Task.yield()
+ guard !Task.isCancelled else { return }
+ scroll.scrollTo("dictionary-word-test", anchor: .bottom)
}
- .padding(2)
}
.frame(minHeight: 0, maxHeight: .infinity)
Divider()
// Save button
HStack {
+ Button(self.isTestingWord ? "Hide test" : "Test word") {
+ self.isTestingWord.toggle()
+ }
+ .fluidGlassAction(quiet: true)
+ .disabled(self.hasUnsavedChanges || self.isCaptureBusy || self.isTestBusy)
+ .help(self.hasUnsavedChanges ? "Save your changes before testing this word." : "Try this word with your dictation shortcut.")
Spacer()
Button("Save Changes") { self.saveIfValid() }
.fluidGlassAction(prominent: true, tone: self.theme.palette.accent)
- .disabled(!self.canSave)
+ .disabled(!self.canSave || self.isTestBusy)
.keyboardShortcut(.return, modifiers: [])
}
}
.padding(20)
.frame(minWidth: 400, idealWidth: 480, maxWidth: 560)
- .frame(minHeight: 360, idealHeight: 500, maxHeight: 600)
+ .frame(minHeight: 360, idealHeight: self.isTestingWord ? 680 : 500, maxHeight: self.isTestingWord ? 780 : 600)
.dismissTextFocusOnBackgroundTap()
+ .onChange(of: self.hasUnsavedChanges) { _, changed in
+ if changed { self.isTestingWord = false }
+ }
.onAppear {
self.triggersText = self.entry.triggers.joined(separator: "\n")
self.replacement = self.entry.replacement
@@ -4108,7 +4269,7 @@ struct EditDictionaryEntrySheet: View {
}
private func saveIfValid() {
- guard self.canSave else { return }
+ guard self.canSave, !self.isTestBusy else { return }
let updatedEntry = SettingsStore.CustomDictionaryEntry(
id: self.entry.id,
diff --git a/Sources/Fluid/UI/DictionaryAudioInspectionView.swift b/Sources/Fluid/UI/DictionaryAudioInspectionView.swift
index aead6df6..f402fc1f 100644
--- a/Sources/Fluid/UI/DictionaryAudioInspectionView.swift
+++ b/Sources/Fluid/UI/DictionaryAudioInspectionView.swift
@@ -223,6 +223,7 @@ struct DictionaryAudioInspectionView: View {
)
.padding(.top, self.theme.metrics.spacing.sm)
}
+ .disclosureGroupStyle(FluidDisclosureGroupStyle())
} else {
Text(self.loadingReferences && self.enrollments[index].inspectionID != nil ? "Loading audio…" : self.loadErrors.contains(index)
? "Saved audio couldn't be loaded. Its stored embedding is still compared."
diff --git a/Sources/Fluid/UI/DictionaryMatchDiagnosticsView.swift b/Sources/Fluid/UI/DictionaryMatchDiagnosticsView.swift
index db23650f..27f679e1 100644
--- a/Sources/Fluid/UI/DictionaryMatchDiagnosticsView.swift
+++ b/Sources/Fluid/UI/DictionaryMatchDiagnosticsView.swift
@@ -93,6 +93,7 @@ struct DictionaryMatchDiagnosticsView: View {
.padding(.top, self.theme.metrics.spacing.sm)
}
}
+ .disclosureGroupStyle(FluidDisclosureGroupStyle())
}
Divider()
DisclosureGroup("History", isExpanded: self.$historyExpanded) {
@@ -119,6 +120,7 @@ struct DictionaryMatchDiagnosticsView: View {
}
.padding(.top, self.theme.metrics.spacing.sm)
}
+ .disclosureGroupStyle(FluidDisclosureGroupStyle())
}
.task(id: self.word) {
self.report = nil
diff --git a/Sources/Fluid/UI/DictionaryShortcutTestState.swift b/Sources/Fluid/UI/DictionaryShortcutTestState.swift
index bded93e3..bc63b43e 100644
--- a/Sources/Fluid/UI/DictionaryShortcutTestState.swift
+++ b/Sources/Fluid/UI/DictionaryShortcutTestState.swift
@@ -73,8 +73,7 @@ struct DictionaryShortcutTestState {
}
enum DictionaryCaptureAction {
- static func title(active: Bool, failed: Bool, count: Int) -> String {
- if active { return "Stop recording" }
+ static func title(failed: Bool, count: Int) -> String {
if failed { return "Record again" }
return count < 1 ? "Record" : "Record more"
}
diff --git a/Sources/Fluid/UI/DictionaryVoiceWave.metal b/Sources/Fluid/UI/DictionaryVoiceWave.metal
new file mode 100644
index 00000000..b0ba7343
--- /dev/null
+++ b/Sources/Fluid/UI/DictionaryVoiceWave.metal
@@ -0,0 +1,67 @@
+#include <metal_stdlib>
+#include <SwiftUI/SwiftUI_Metal.h>
+using namespace metal;
+
+// Glowing voice ribbon for the dictionary recording step.
+// phase moves the strands; energy (0...1) is the smoothed microphone level.
+[[ stitchable ]] half4 dictionaryVoiceWave(float2 position, half4 color, float2 size, float phase, float energy, float dark) {
+ float x = position.x / size.x;
+ float y = position.y - size.y * 0.5;
+
+ // Strands fade in and out at both ends instead of stopping at a hard edge.
+ float ends = smoothstep(0.0, 0.18, x) * smoothstep(1.0, 0.82, x);
+ float envelope = sin(M_PI_F * x);
+
+ float3 cyan = float3(0.25, 0.92, 1.0);
+ float3 blue = float3(0.18, 0.48, 1.0);
+ float3 violet = float3(0.58, 0.36, 1.0);
+ float3 tint = x < 0.5 ? mix(cyan, blue, x * 2.0) : mix(blue, violet, (x - 0.5) * 2.0);
+
+ float amplitude = size.y * (0.26 + 0.14 * energy);
+ float glowWidth = 9.0 + 6.0 * energy;
+ float3 light = float3(0.0);
+ float hot = 0.0;
+ float coreSum = 0.0;
+ float glowSum = 0.0;
+
+ for (int i = 0; i < 5; i++) {
+ float f = float(i);
+ float strandPhase = phase * (0.8 + 0.23 * f) + f * 1.9;
+ float frequency = 0.85 + 0.3 * f;
+ float wobble = 1.0 + 0.25 * sin(phase * 0.6 + f * 2.3);
+ float strandY = amplitude * (1.0 - 0.17 * f) * wobble * envelope * sin(2.0 * M_PI_F * frequency * x + strandPhase);
+ float d = abs(y - strandY);
+ float weight = i == 0 ? 1.0 : 0.5 - 0.06 * f;
+ float core = exp(-d * d / (2.0 * 1.1 * 1.1));
+ float glow = exp(-d / glowWidth);
+ light += tint * (core * 1.2 + glow * (0.5 + 0.15 * energy)) * weight;
+ coreSum += core * weight;
+ glowSum += glow * weight;
+ if (i == 0) { hot = core; }
+ }
+
+ // Soft bloom behind the middle of the ribbon, stronger while speaking.
+ float2 centre = float2((x - 0.5) * 2.2, y / (size.y * 0.5));
+ float bloom = exp(-dot(centre, centre) * 2.2) * (0.22 + 0.12 * energy);
+
+ if (dark < 0.5) {
+ // Added light only washes out on a light background. Paint deeper, saturated ink instead:
+ // opacity comes from the strands' strength, so the colour itself can stay rich.
+ float3 teal = float3(0.0, 0.58, 0.82);
+ float3 deepBlue = float3(0.10, 0.33, 0.93);
+ float3 deepViolet = float3(0.45, 0.22, 0.90);
+ float3 ink = x < 0.5 ? mix(teal, deepBlue, x * 2.0) : mix(deepBlue, deepViolet, (x - 0.5) * 2.0);
+ float coverage = saturate(coreSum * 0.95 + glowSum * (0.22 + 0.08 * energy) + bloom * 0.35) * ends;
+ float3 shade = mix(ink, ink * 0.78, saturate(hot));
+ return half4(half3(shade * coverage), half(coverage));
+ }
+
+ light += mix(blue, cyan, 0.4) * bloom;
+
+ light += float3(1.0) * hot * 0.75;
+ light *= ends;
+
+ float alpha = saturate(max(light.r, max(light.g, light.b)));
+ float3 rgb = min(light, float3(alpha));
+ return half4(half3(rgb), half(alpha));
+}
diff --git a/Sources/Fluid/UI/DictionaryWordWizard.swift b/Sources/Fluid/UI/DictionaryWordWizard.swift
index 8f6083ba..568f1455 100644
--- a/Sources/Fluid/UI/DictionaryWordWizard.swift
+++ b/Sources/Fluid/UI/DictionaryWordWizard.swift
@@ -36,17 +36,27 @@ struct DictionaryWordWizard: View {
let onManual: () -> Void
let onPracticeMore: () -> Void
var onRedo: () -> Void = {}
+ var wordFocusRequest = 0
var automaticCaptureActive = false
+ var waiting = false
+ var canSave = true
+ var onPauseRecording: (() -> Void)?
+ var onAddSpelling: ((String) -> String?)?
+ var onRemoveSpelling: ((String) -> Void)?
var audioLevels: AnyPublisher<CGFloat, Never> = Empty().eraseToAnyPublisher()
var pronunciationNotice: String?
var pronunciationIncomplete = false
+ /// What each try transcribed, oldest first.
+ var attempts: [String] = []
@Environment(\.theme) private var theme
@Environment(\.accessibilityReduceMotion) private var reduceMotion
@FocusState private var wordFocused: Bool
@State private var showingRecordingTips = false
@State private var hasViewedTips = false
- @State private var showingCapturedSpellings = false
+ @State private var showingAddSpelling = false
+ @State private var newSpelling = ""
+ @State private var spellingError: String?
@State private var playgroundBusy = false
@State private var confirmingRedo = false
@State private var confirmingWordChange = false
@@ -55,7 +65,13 @@ struct DictionaryWordWizard: View {
var body: some View {
VStack(spacing: self.theme.metrics.spacing.lg) {
HStack {
- if self.step != .spelling, self.step != .saved {
+ if self.step == .recording, self.captureActive, let onPauseRecording {
+ Button("Pause learning", action: onPauseRecording)
+ .fluidGlassAction(quiet: true)
+ .keyboardShortcut(.cancelAction)
+ .disabled(self.processing || self.starting)
+ .help("Pause automatic recording to review or edit spellings (Esc)")
+ } else if self.step != .spelling, self.step != .saved {
Button(action: self.onBack) { Label("Change word", systemImage: "chevron.left") }
.fluidGlassAction()
.disabled(self.busy)
@@ -97,11 +113,18 @@ struct DictionaryWordWizard: View {
Button("Change word", role: .destructive) { self.word = self.pendingWord }
}
.task(id: self.step) { self.wordFocused = self.step == .spelling }
+ .task(id: self.wordFocusRequest) {
+ guard self.wordFocusRequest != 0 else { return }
+ self.wordFocused = false
+ await Task.yield()
+ guard !Task.isCancelled else { return }
+ self.wordFocused = self.step == .spelling
+ }
}
private var spelling: some View {
VStack(spacing: self.theme.metrics.spacing.lg) {
- DictionaryLearningRing(progress: 0, active: false, symbol: "waveform")
+ DictionaryVoiceWave(active: false, height: 96)
Text("Which word does FluidVoice keep getting wrong?")
.font(self.theme.typography.sectionTitle)
.multilineTextAlignment(.center)
@@ -141,12 +164,11 @@ struct DictionaryWordWizard: View {
}
}
self.captureError
- Group {
- if self.step == .review {
- self.manualAlternative.hidden().accessibilityHidden(true)
- } else {
- self.manualAlternative
- }
+ if self.count >= 1 || !self.capturedSpellings.isEmpty {
+ self.capturedVariations
+ }
+ if self.step != .review {
+ self.manualAlternative
}
}
}
@@ -163,26 +185,20 @@ struct DictionaryWordWizard: View {
private var captureStage: some View {
VStack(spacing: self.theme.metrics.spacing.lg) {
- Text(self.step == .review ? (self.pronunciationIncomplete ? "Your spellings are ready" : "Your word is ready") : "Let’s hear your word")
+ Text(self.step == .review ? (self.pronunciationIncomplete ? "Your spellings are ready" : "Your word is ready") : (self.voiceSupported ? "Learn your pronunciation" : "Learn the spelling of"))
.font(self.theme.typography.body)
.foregroundStyle(self.theme.palette.accent)
Text("“\(self.word)”")
.font(self.theme.typography.title)
.fixedSize(horizontal: false, vertical: true)
- DictionaryLearningRing(
- progress: Double(min(self.count, 3)) / 3,
- active: self.recording && !self.starting,
- symbol: self.processing ? "ellipsis" : (self.count >= 3 ? "checkmark" : "waveform"),
- diameter: 264,
- count: self.count,
- processing: self.processing,
+ DictionaryVoiceWave(
+ active: self.isListening,
+ height: self.step == .review ? 100 : 150,
audioLevels: self.audioLevels
)
- .accessibilityElement(children: .ignore)
- .accessibilityLabel(DictionaryLearningEncouragement.messages(for: self.count)[0])
- .accessibilityValue(self.processing ? "Processing recording" : (self.recording ? "Listening" : ""))
- .accessibilityHidden(false)
+ DictionaryCaptureSlots(word: self.word, attempts: self.attempts, pendingStatus: self.pendingSlotStatus)
self.captureControls
+ .padding(.top, self.theme.metrics.spacing.sm)
}
.multilineTextAlignment(.center)
.padding(.vertical, self.theme.metrics.spacing.lg)
@@ -190,13 +206,25 @@ struct DictionaryWordWizard: View {
.accessibilityElement(children: .contain)
}
- private var learningTitle: String {
- if self.step == .review { return self.alreadyCorrect ? "It already sounds right" : "Ready to save" }
- if self.starting { return "Getting ready…" }
- if self.processing { return "Learning your voice…" }
- if self.recording { return self.count < 1 ? "Listening…" : "Say it again" }
- if self.count >= 3 { return "Ready to add" }
- return self.count < 1 ? "Let’s hear your word" : "Keep going—you’re nearly there"
+ /// Shown in the next empty slot while a try is in progress.
+ private var pendingSlotStatus: String? {
+ guard self.step == .recording else { return nil }
+ if self.isListening { return "Listening…" }
+ if self.processing && !self.starting { return "Checking…" }
+ if self.starting || self.waiting { return "Getting ready…" }
+ return nil
+ }
+
+ private var isListening: Bool {
+ self.step == .recording && self.recording && !self.starting && !self.processing && !self.waiting
+ }
+
+ private var captureInstruction: String {
+ if self.waiting { return "Get ready to say it again" }
+ if self.starting { return "Wait for recording to start" }
+ if self.processing { return "Checking what we heard" }
+ if self.isListening { return "Say “\(self.word)” now" }
+ return "Say only the word, then pause."
}
private var guidanceButton: some View {
@@ -248,19 +276,12 @@ struct DictionaryWordWizard: View {
} else {
self.instruction(1, title: "Say only the word", detail: "Use your normal voice. No sentence needed.")
self.instruction(2, title: "Wait when you finish", detail: "Recording stops automatically.")
- self.instruction(3, title: "Wait for the next prompt", detail: "Speak again when you see “Listening…”.")
+ self.instruction(3, title: "Wait for the next recording", detail: "Speak again when “Recording” appears.")
Divider()
Label("Choose a quiet place and keep a comfortable distance from your microphone.", systemImage: "ear")
.font(self.theme.typography.body)
.foregroundStyle(self.theme.palette.secondaryText)
}
- if self.count >= 1, !self.capturedSpellings.isEmpty {
- DisclosureGroup("Captured spellings · \(self.capturedSpellings.count)", isExpanded: self.$showingCapturedSpellings) {
- self.capturedVariations.padding(.top, self.theme.metrics.spacing.sm)
- }
- .font(self.theme.typography.body)
- .tint(self.theme.palette.accent)
- }
}
.fixedSize(horizontal: false, vertical: true)
.padding(.vertical, self.theme.metrics.spacing.lg)
@@ -296,50 +317,48 @@ struct DictionaryWordWizard: View {
private var captureControls: some View {
VStack(spacing: self.theme.metrics.spacing.lg) {
- self.stableInstruction("Say only “\(self.word)” once, then wait.", review: "Ready to add “\(self.word)”")
+ if self.step == .review {
+ if self.pronunciationIncomplete, let pronunciationNotice {
+ Text(pronunciationNotice)
+ .font(self.theme.typography.body)
+ .foregroundStyle(self.theme.palette.secondaryText)
+ .multilineTextAlignment(.center)
+ .frame(maxWidth: .infinity)
+ }
+ self.reviewActions
+ } else {
+ ZStack {
+ Text("Say only the word, then pause.").hidden().accessibilityHidden(true)
+ Text(self.captureInstruction)
+ }
.font(self.theme.typography.title)
.foregroundStyle(self.theme.palette.primaryText)
.fixedSize(horizontal: false, vertical: true)
- self.stableInstruction("We’ll stop automatically and let you know when to speak again.", review: "Add it now, or record another example.")
- .font(self.theme.typography.body)
- .foregroundStyle(self.theme.palette.secondaryText)
- if let pronunciationNotice {
- Text(pronunciationNotice)
+ Text("Recording stops automatically.")
.font(self.theme.typography.body)
.foregroundStyle(self.theme.palette.secondaryText)
- .multilineTextAlignment(.center)
- .frame(maxWidth: .infinity)
- }
- ZStack(alignment: .top) {
- if self.step == .review {
- self.captureRecordActions.hidden().accessibilityHidden(true)
- self.reviewActions
- } else {
- self.reviewActions.hidden().accessibilityHidden(true)
- self.captureRecordActions
- }
+ self.captureRecordActions
}
}
}
- private func stableInstruction(_ recording: String, review: String) -> some View {
- ZStack {
- Text(recording).hidden().accessibilityHidden(true)
- Text(review).hidden().accessibilityHidden(true)
- Text(self.step == .review ? review : recording)
- }
- }
-
private var captureRecordActions: some View {
VStack(spacing: self.theme.metrics.spacing.md) {
- Button(action: self.onRecord) {
- ZStack {
- Text("Stop recording").hidden()
- Text(DictionaryCaptureAction.title(active: self.captureActive, failed: self.error != nil, count: self.count))
+ Group {
+ if self.captureActive {
+ Color.clear
+ } else {
+ Button {
+ guard !self.busy, !self.captureActive else { return }
+ self.onRecord()
+ } label: {
+ Label(DictionaryCaptureAction.title(failed: self.error != nil, count: self.count), systemImage: "mic.fill")
+ }
+ .buttonStyle(FluidRecordInvitationStyle(inviting: self.count < 1, recording: false))
+ .disabled(self.busy)
}
}
- .buttonStyle(FluidRecordInvitationStyle(inviting: !self.captureActive && self.count < 1, recording: self.captureActive))
- .disabled(self.processing || self.starting)
+ .frame(minHeight: FluidButtonSize.large.controlHeight)
self.redoButton
.hidden()
.accessibilityHidden(true)
@@ -352,7 +371,7 @@ struct DictionaryWordWizard: View {
}
private var captureActive: Bool {
- self.automaticCaptureActive || self.recording || self.processing || self.starting
+ self.automaticCaptureActive || self.recording || self.processing || self.starting || self.waiting
}
private var redoButton: some View {
@@ -380,22 +399,24 @@ struct DictionaryWordWizard: View {
.disabled(self.busy)
Button(action: self.onSave) {
ZStack {
- Text("Add “\(self.word)” to dictionary").hidden().accessibilityHidden(true)
- Text(self.busy ? "Saving…" : (self.pronunciationIncomplete ? "Save spelling corrections" : "Add “\(self.word)” to dictionary"))
+ Text(self.pronunciationIncomplete ? "Save spelling corrections" : "Add to dictionary").hidden().accessibilityHidden(true)
+ Text(self.busy ? "Saving…" : (self.pronunciationIncomplete ? "Save spelling corrections" : "Add to dictionary"))
}
}
.buttonStyle(FluidRecordInvitationStyle(inviting: self.step == .review && !self.busy, recording: false, fillsWidth: true))
- .disabled(self.busy)
+ .disabled(self.busy || !self.canSave)
if self.count < CustomDictionaryTrainingMerge.maxSamples {
VStack(spacing: self.theme.metrics.spacing.sm) {
Button("Record more", action: self.onRecord)
.buttonStyle(FluidRecordInvitationStyle(inviting: false, recording: false, prominent: false, fillsWidth: true))
.disabled(self.busy)
- Text(self.pronunciationIncomplete ? "Needed to finish pronunciation learning" : "Optional · Recommended for tricky words")
- .font(self.theme.typography.caption)
- .foregroundStyle(self.theme.palette.secondaryText)
- .frame(maxWidth: 190)
- .fixedSize(horizontal: false, vertical: true)
+ if self.pronunciationIncomplete {
+ Text("Finish pronunciation learning")
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.theme.palette.secondaryText)
+ .frame(maxWidth: 190)
+ .fixedSize(horizontal: false, vertical: true)
+ }
}
}
}
@@ -411,27 +432,94 @@ struct DictionaryWordWizard: View {
private var capturedSpellings: [String] {
var seen = Set<String>()
- return (self.variants + [self.heard]).filter {
+ return self.variants.filter {
!$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && seen.insert($0.lowercased()).inserted
}
}
- @ViewBuilder private var capturedVariations: some View {
- if !self.capturedSpellings.isEmpty {
- VStack(spacing: self.theme.metrics.spacing.sm) {
- Text("Spellings linked to “\(self.word)”")
+ private var capturedVariations: some View {
+ VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) {
+ HStack {
+ Text("Spelling corrections")
+ .font(self.theme.typography.bodyStrong)
+ Spacer()
+ if self.onAddSpelling != nil {
+ Button {
+ self.newSpelling = ""
+ self.spellingError = nil
+ self.showingAddSpelling = true
+ } label: {
+ Label("Add spelling", systemImage: "plus")
+ }
+ .fluidGlassAction(quiet: true)
+ .disabled(self.busy || self.captureActive || self.capturedSpellings.count >= CustomDictionaryTrainingMerge.maxSamples)
+ .help("Pause learning to edit spelling corrections")
+ .popover(isPresented: self.$showingAddSpelling) { self.addSpellingPopover }
+ }
+ }
+ if self.capturedSpellings.isEmpty {
+ Text(self.heard.caseInsensitiveCompare(self.word) == .orderedSame ? "“\(self.word)” was recognised correctly." : "No spelling corrections yet.")
.font(self.theme.typography.caption)
.foregroundStyle(self.theme.palette.secondaryText)
+ } else {
FlowLayout(spacing: 6) {
ForEach(self.capturedSpellings, id: \.self) { variant in
- Text(variant)
- .font(self.theme.typography.caption)
- .padding(.horizontal, 10).padding(.vertical, 6)
- .background(self.theme.palette.accent.opacity(0.08), in: Capsule())
+ HStack(spacing: 4) {
+ Text(variant).font(self.theme.typography.bodySmall)
+ if let onRemoveSpelling {
+ Button { onRemoveSpelling(variant) } label: {
+ Image(systemName: "xmark").font(self.theme.typography.caption)
+ }
+ .fluidGlassAction(circular: true, quiet: true)
+ .disabled(self.busy || self.captureActive)
+ .accessibilityLabel("Remove spelling \(variant)")
+ .help("Remove this spelling correction; keep the voice examples")
+ }
+ }
+ .padding(.leading, 10)
+ .padding(.trailing, self.onRemoveSpelling == nil ? 10 : 0)
+ .padding(.vertical, self.onRemoveSpelling == nil ? 6 : 0)
+ .background(self.theme.palette.accent.opacity(0.08), in: Capsule())
}
}
}
}
+ .padding(self.theme.metrics.spacing.md)
+ .onChange(of: self.captureActive || self.busy) { _, active in
+ if active { self.showingAddSpelling = false }
+ }
+ }
+
+ private var addSpellingPopover: some View {
+ VStack(alignment: .leading, spacing: self.theme.metrics.spacing.md) {
+ Text("Add a spelling correction").font(self.theme.typography.sectionTitle)
+ TextField("What FluidVoice types instead", text: self.$newSpelling)
+ .textFieldStyle(.roundedBorder)
+ .onSubmit { self.addSpelling() }
+ if let spellingError {
+ Text(spellingError)
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.theme.palette.warning)
+ }
+ HStack {
+ Button("Cancel") { self.showingAddSpelling = false }.fluidGlassAction()
+ Spacer()
+ Button("Add") { self.addSpelling() }
+ .fluidGlassAction(prominent: true)
+ .disabled(self.newSpelling.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || self.busy || self.captureActive)
+ }
+ }
+ .padding(self.theme.metrics.spacing.lg)
+ .frame(width: 320)
+ }
+
+ private func addSpelling() {
+ guard !self.busy, !self.captureActive, let onAddSpelling else { return }
+ if let error = onAddSpelling(self.newSpelling) {
+ self.spellingError = error
+ } else {
+ self.showingAddSpelling = false
+ }
}
private func heading(_ title: String, detail: String) -> some View {
@@ -487,307 +575,203 @@ private struct DictionaryReviewActionLayout: Layout {
}
}
-enum DictionaryLearningEncouragement {
- static func messages(for count: Int) -> [String] {
- switch count {
- case ...0: ["Ready when you are"]
- case 1: ["Nice! 2 more to go"]
- case 2: ["One more—you’ve got this"]
- default: ["Got it! Ready to add"]
- }
- }
-}
-
-/// Bounded ambient motion redraws only the artwork; text stays stable and hidden rings stop updating.
-private struct DictionaryLearningRing: View {
- let progress: Double
+/// A glowing ribbon drawn by a Metal shader. It drifts slowly at rest and swells with the voice.
+/// Reduce Motion freezes it; Reduce Transparency swaps the glow for a plain line.
+private struct DictionaryVoiceWave: View {
let active: Bool
- let symbol: String
- var diameter: CGFloat = 176
- var count: Int? = nil
- var processing = false
+ var height: CGFloat = 140
var audioLevels: AnyPublisher<CGFloat, Never> = Empty().eraseToAnyPublisher()
@Environment(\.theme) private var theme
+ @Environment(\.colorScheme) private var colorScheme
@Environment(\.accessibilityReduceMotion) private var reduceMotion
@Environment(\.accessibilityReduceTransparency) private var reduceTransparency
@Environment(\.scenePhase) private var scenePhase
-
- // Match the preview blue explicitly, independent of the selected app accent or appearance.
- private let learningBlue = Color(red: 10 / 255, green: 132 / 255, blue: 1)
-
@State private var isVisible = false
@State private var voiceEnergy = 0.0
- @State private var listening = 0.0
- @State private var pulse = 0.0
- @State private var highlightsProgress = false
- @State private var completionBloom = 1.0
+ @State private var motion = DictionaryWaveMotion()
private var animates: Bool {
- self.isVisible && !self.reduceMotion && self.scenePhase == .active &&
- (self.active || (self.count != nil && self.progress < 1))
+ self.active && self.isVisible && self.scenePhase == .active && !self.reduceMotion
+ }
+
+ private var respondsToAudio: Bool {
+ self.active && self.animates
}
var body: some View {
- ZStack {
- if !self.reduceTransparency {
- Circle()
- .fill(RadialGradient(
- colors: [self.learningBlue.opacity(0.04 + self.progress * 0.14), .clear],
- center: .center,
- startRadius: 10,
- endRadius: self.diameter / 2
- ))
- }
- TimelineView(.animation(minimumInterval: 1.0 / 30, paused: !self.animates)) { context in
- let phase = self.reduceMotion ? 0 : context.date.timeIntervalSinceReferenceDate.truncatingRemainder(dividingBy: 14) / 14 * .pi * 2
- DictionaryRingArtwork(
- phase: phase,
- progress: self.progress,
- energy: self.reduceMotion ? 0 : self.voiceEnergy,
- glow: max(self.listening, self.pulse, self.progress >= 1 ? 0.5 : 0),
- accent: self.learningBlue,
- rest: self.theme.palette.secondaryText,
- additive: !self.reduceTransparency
- )
- }
- .animation(self.reduceMotion ? nil : .easeInOut(duration: 0.65), value: self.progress)
- .animation(self.reduceMotion ? nil : .easeOut(duration: 0.12), value: self.voiceEnergy)
- if let count, count >= 3, !self.reduceMotion {
- DictionaryCompletionBloom(progress: self.completionBloom, color: self.learningBlue)
- .accessibilityHidden(true)
- .allowsHitTesting(false)
- }
- if let count {
- ZStack {
- Text(DictionaryLearningEncouragement.messages(for: count)[0])
- .font(self.theme.typography.bodyStrong)
- .foregroundStyle(self.highlightsProgress ? self.learningBlue : self.theme.palette.primaryText)
- .animation(self.reduceMotion ? nil : .easeInOut(duration: 0.9), value: self.highlightsProgress)
- .multilineTextAlignment(.center)
- .frame(height: 60)
- HStack(spacing: 8) {
- ForEach(0..<3) { index in
- Circle()
- .fill(index < count ? self.learningBlue : self.theme.palette.secondaryText.opacity(0.2))
- .frame(width: 8, height: 8)
- }
- }
- .offset(y: 44)
- ZStack {
- if self.processing { ProgressView().controlSize(.small) }
- }
- .frame(height: 20)
- .offset(y: 66)
- }
- .frame(width: self.diameter * 0.57, height: self.diameter)
- .transaction { $0.animation = nil }
+ TimelineView(.animation(minimumInterval: 1.0 / 30, paused: !self.animates)) { timeline in
+ let frame = self.motion.advance(
+ to: timeline.date,
+ targetEnergy: self.respondsToAudio ? self.voiceEnergy : 0,
+ speed: self.active ? 1.1 : 0.45,
+ animating: self.animates
+ )
+ if self.reduceTransparency {
+ DictionaryWaveShape(phase: frame.phase, amplitude: 0.45 + CGFloat(frame.energy) * 0.4, frequency: 1)
+ .stroke(self.theme.palette.accent, style: StrokeStyle(lineWidth: 2.5, lineCap: .round))
} else {
- Image(systemName: self.symbol)
- .font(.system(size: 27, weight: .light))
- .foregroundStyle(self.theme.palette.secondaryText)
+ Rectangle()
+ .fill(.white)
+ .visualEffect { content, proxy in
+ content.colorEffect(ShaderLibrary.dictionaryVoiceWave(
+ .float2(proxy.size),
+ .float(Float(frame.phase)),
+ .float(Float(frame.energy)),
+ .float(self.colorScheme == .dark ? 1 : 0)
+ ))
+ }
}
}
- .frame(width: self.diameter, height: self.diameter)
- .animation(self.reduceMotion ? nil : .easeInOut(duration: 0.4), value: self.progress)
- .transaction { if self.reduceMotion { $0.animation = nil } }
+ .frame(maxWidth: 620)
+ .frame(height: self.height)
+ .opacity(self.active ? 1 : 0.85)
+ .animation(self.reduceMotion ? nil : .smooth(duration: 0.3), value: self.active)
.accessibilityHidden(true)
.allowsHitTesting(false)
- .onAppear { self.isVisible = true; self.listening = self.active ? 1 : 0 }
+ .onAppear { self.isVisible = true }
.onDisappear { self.isVisible = false; self.voiceEnergy = 0 }
- .task(id: (self.count ?? 0) >= 3) {
- guard (self.count ?? 0) >= 3, !self.reduceMotion else {
- self.completionBloom = 1
- return
- }
- var reset = Transaction()
- reset.disablesAnimations = true
- withTransaction(reset) { self.completionBloom = 0 }
- do { try await Task.sleep(for: .milliseconds(40)) } catch { return }
- withAnimation(.easeOut(duration: 1.1)) { self.completionBloom = 1 }
- }
- .task(id: self.count) {
- var reset = Transaction()
- reset.disablesAnimations = true
- withTransaction(reset) { self.highlightsProgress = false }
- guard let count = self.count, count > 0 else { return }
- if !self.reduceMotion {
- do { try await Task.sleep(for: .milliseconds(80)) } catch { return }
- }
- guard !Task.isCancelled else { return }
- self.highlightsProgress = true
- }
.onChange(of: self.active) { _, active in
if !active { self.voiceEnergy = 0 }
- withAnimation(self.reduceMotion ? nil : .easeInOut(duration: 0.5)) { self.listening = active ? 1 : 0 }
- }
- .onChange(of: self.count) { old, new in
- // A captured word flashes the ring, then settles into the new progress.
- guard let old, let new, new > old, !self.reduceMotion else { return }
- self.pulse = 1
- withAnimation(.easeOut(duration: 1.4)) { self.pulse = 0 }
}
.onChange(of: self.scenePhase) { _, phase in
if phase != .active { self.voiceEnergy = 0 }
}
- .onReceive(self.audioLevels.throttle(for: .milliseconds(50), scheduler: RunLoop.main, latest: true)) { level in
- guard self.active, self.isVisible, self.scenePhase == .active, !self.reduceMotion else { return }
- let target = DictionaryRingResponse.energy(for: level)
- self.voiceEnergy += (target - self.voiceEnergy) * (target > self.voiceEnergy ? 0.75 : 0.35)
+ .onChange(of: self.reduceMotion) { _, reduced in
+ if reduced { self.voiceEnergy = 0 }
+ }
+ .onReceive(self.audioLevels.throttle(for: .milliseconds(100), scheduler: RunLoop.main, latest: true)) { level in
+ guard self.respondsToAudio else { return }
+ self.voiceEnergy = DictionaryGlowResponse.energy(for: level)
}
}
}
-/// A single contained release of light at the third capture, with no change to layout or hit targets.
-private struct DictionaryCompletionBloom: View, Animatable {
- var progress: Double
- let color: Color
-
- var animatableData: Double {
- get { self.progress }
- set { self.progress = newValue }
+/// Per-frame phase and eased energy, so speed changes and level jumps never make the ribbon jump.
+private final class DictionaryWaveMotion {
+ private var lastDate: Date?
+ private var phase = 0.0
+ private var energy = 0.0
+ private static let noiseGate = 0.35
+
+ func advance(to date: Date, targetEnergy: Double, speed: Double, animating: Bool) -> (phase: Double, energy: Double) {
+ let elapsed = self.lastDate.map { min(max(date.timeIntervalSince($0), 0), 0.1) } ?? 0
+ self.lastDate = animating ? date : nil
+ guard animating else {
+ self.energy = 0
+ return (self.phase, 0)
+ }
+ self.phase = (self.phase + elapsed * speed).truncatingRemainder(dividingBy: 2 * .pi * 100)
+ // Room noise stays below the gate; speech eases in and out slowly so the glow never flickers.
+ let gated = max(0, (targetEnergy - Self.noiseGate) / (1 - Self.noiseGate))
+ let rate = gated > self.energy ? 3.0 : 1.2
+ self.energy += (gated - self.energy) * min(1, elapsed * rate)
+ return (self.phase, self.energy)
}
+}
- var body: some View {
- Canvas { context, size in
- let phase = min(1, max(0, self.progress))
- guard phase < 1 else { return }
- let centre = CGPoint(x: size.width / 2, y: size.height / 2)
- let radius = min(size.width, size.height) * (0.27 + phase * 0.19)
- let opacity = sin(phase * .pi)
- context.stroke(
- Path(ellipseIn: CGRect(x: centre.x - radius, y: centre.y - radius, width: radius * 2, height: radius * 2)),
- with: .color(self.color.opacity(opacity * 0.5)),
- lineWidth: 2 * (1 - phase) + 0.5
- )
- for index in 0..<16 {
- let angle = Double(index) / 16 * .pi * 2
- let length = 3 + (1 - phase) * 7
- var ray = Path()
- ray.move(to: CGPoint(x: centre.x + cos(angle) * (radius - length), y: centre.y + sin(angle) * (radius - length)))
- ray.addLine(to: CGPoint(x: centre.x + cos(angle) * radius, y: centre.y + sin(angle) * radius))
- context.stroke(ray, with: .color(self.color.opacity(opacity * 0.8)), style: StrokeStyle(lineWidth: 1.5, lineCap: .round))
- }
- }
+/// One sine line that tapers to nothing at both ends.
+private struct DictionaryWaveShape: Shape {
+ var phase: Double
+ var amplitude: CGFloat
+ let frequency: CGFloat
+
+ var animatableData: CGFloat {
+ get { self.amplitude }
+ set { self.amplitude = newValue }
}
-}
-enum DictionaryRingResponse {
- static func energy(for level: CGFloat) -> Double {
- guard level.isFinite else { return 0 }
- return sqrt(Double(min(1, max(0, level))))
+ func path(in rect: CGRect) -> Path {
+ var path = Path()
+ guard rect.width > 0 else { return path }
+ let step: CGFloat = 3
+ var x: CGFloat = 0
+ while x <= rect.width {
+ let t = x / rect.width
+ let envelope = sin(.pi * t)
+ let y = rect.midY + rect.height / 2 * self.amplitude * envelope * sin(2 * .pi * self.frequency * t + CGFloat(self.phase))
+ if x == 0 { path.move(to: CGPoint(x: rect.minX + x, y: y)) } else { path.addLine(to: CGPoint(x: rect.minX + x, y: y)) }
+ x += step
+ }
+ return path
}
}
-/// Three translucent sheets fan wide and pinch loosely; lines inside a sheet never cross, so the ring reads as silk, not rope.
-private struct DictionaryRingArtwork: View, Animatable {
- let phase: Double
- var progress: Double
- var energy: Double
- var glow: Double
- let accent: Color
- let rest: Color
- let additive: Bool
+/// Three tries, each showing what FluidVoice heard. Past three, the latest tries stay visible.
+private struct DictionaryCaptureSlots: View {
+ let word: String
+ let attempts: [String]
+ let pendingStatus: String?
+ @Environment(\.theme) private var theme
- var animatableData: AnimatablePair<Double, AnimatablePair<Double, Double>> {
- get { AnimatablePair(self.progress, AnimatablePair(self.energy, self.glow)) }
- set {
- self.progress = newValue.first
- self.energy = newValue.second.first
- self.glow = newValue.second.second
- }
+ private var firstVisible: Int {
+ let shown = self.attempts.count + (self.pendingStatus == nil ? 0 : 1)
+ return max(0, shown - 3)
}
var body: some View {
- Canvas(rendersAsynchronously: true) { context, size in
- let rect = CGRect(origin: .zero, size: size)
- let lit = min(1, max(0, self.glow) * 0.7 + self.energy * 0.5)
- let shading = GraphicsContext.Shading.conicGradient(
- Gradient(stops: self.stops),
- center: CGPoint(x: rect.midX, y: rect.midY),
- angle: .degrees(-90)
- )
- let paths = (0..<DictionaryRibbon.sheets).flatMap { sheet in
- (0..<7).map { line in
- DictionaryRibbon(phase: self.phase, strand: Double(line - 3) * 4 / 3, energy: self.energy, sheet: sheet).path(in: rect)
- }
- }
- if self.additive { context.blendMode = .plusLighter }
- context.drawLayer { layer in
- layer.addFilter(.blur(radius: 5 + lit * 5))
- layer.opacity = 0.3 + lit * 0.35
- for path in paths {
- layer.stroke(path, with: shading, lineWidth: 1.6 + lit * 0.8)
- }
- }
- context.opacity = 0.5 + lit * 0.3
- for path in paths {
- context.stroke(path, with: shading, lineWidth: 0.7 + lit * 0.25)
+ HStack(spacing: self.theme.metrics.spacing.sm) {
+ ForEach(0..<3, id: \.self) { offset in
+ self.slot(self.firstVisible + offset)
}
}
+ .frame(maxWidth: 440)
}
- /// The learned arc fades into the resting color instead of ending on a hard edge.
- private var stops: [Gradient.Stop] {
- let learned = self.accent
- let resting = self.rest.opacity(0.55)
- guard self.progress > 0.001 else { return [.init(color: resting, location: 0), .init(color: resting, location: 1)] }
- guard self.progress < 0.999 else { return [.init(color: learned, location: 0), .init(color: learned, location: 1)] }
- let fade = 0.1
- let start = max(0, self.progress - fade / 2)
- let end = min(1, max(start, self.progress + fade / 2))
- let seam = max(end, 1 - fade)
- return [
- .init(color: learned, location: 0),
- .init(color: learned, location: start),
- .init(color: resting, location: end),
- .init(color: resting, location: seam),
- .init(color: learned, location: 1),
- ]
+ private func slot(_ index: Int) -> some View {
+ let heard = self.attempts.indices.contains(index) ? self.attempts[index] : nil
+ let matches = heard.map { $0.caseInsensitiveCompare(self.word.trimmingCharacters(in: .whitespacesAndNewlines)) == .orderedSame } ?? false
+ let pending = heard == nil && index == self.attempts.count ? self.pendingStatus : nil
+ return VStack(spacing: 2) {
+ Text("Try \(index + 1)")
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.theme.palette.secondaryText)
+ HStack(spacing: 4) {
+ if matches {
+ Image(systemName: "checkmark").accessibilityHidden(true)
+ } else if pending == "Listening…" {
+ Circle().fill(Color(nsColor: .systemRed)).frame(width: 7, height: 7).accessibilityHidden(true)
+ }
+ Text(heard ?? pending ?? "—")
+ .lineLimit(1)
+ .truncationMode(.middle)
+ }
+ .font(heard == nil ? self.theme.typography.body : self.theme.typography.bodyStrong)
+ .foregroundStyle(matches ? self.theme.palette.accent : (heard == nil ? self.theme.palette.tertiaryText : self.theme.palette.primaryText))
+ }
+ .padding(.vertical, 10)
+ .padding(.horizontal, 12)
+ .frame(maxWidth: .infinity)
+ .background(
+ matches ? self.theme.palette.accent.opacity(0.1) : self.theme.palette.contentBackground,
+ in: RoundedRectangle(cornerRadius: self.theme.metrics.corners.md)
+ )
+ .overlay {
+ RoundedRectangle(cornerRadius: self.theme.metrics.corners.md)
+ .stroke(matches || pending != nil ? self.theme.palette.accent.opacity(0.5) : self.theme.palette.cardBorder, lineWidth: 1)
+ .allowsHitTesting(false)
+ }
+ .accessibilityElement(children: .ignore)
+ .accessibilityLabel("Try \(index + 1)")
+ .accessibilityValue(heard.map { matches ? "Heard correctly: \($0)" : "Heard: \($0)" } ?? pending ?? "Not recorded yet")
}
}
-struct DictionaryRibbon: Shape {
- static let sheets = 3
-
- let phase: Double
- let strand: Double
- let energy: Double
- var sheet = 0
-
- func path(in rect: CGRect) -> Path {
- let radius = min(rect.width, rect.height) * 0.34
- let offset = Double(self.sheet) * 2.1
- let lobes = Double([3, 2, 4][self.sheet % 3])
- let drift = self.sheet.isMultiple(of: 2) ? self.phase : -self.phase
- let spread = self.strand / 4
- return Path { path in
- for sample in 0...180 {
- let angle = Double(sample) / 180 * .pi * 2
- let center = 1 + sin(angle * 3 + offset + self.phase) * 0.05 + sin(angle * 2 + offset * 0.6 - self.phase) * 0.03 + self.energy * 0.02
- // The sheet twists through a loose pinch, then opens into a wide veil; speech opens it further.
- let width = cos(angle * lobes + offset + drift) * (0.11 + self.energy * 0.05) + 0.018
- let distance = radius * (center + spread * width)
- let point = CGPoint(
- x: rect.midX + CGFloat(cos(angle)) * distance,
- y: rect.midY + CGFloat(sin(angle)) * distance
- )
- if sample == 0 { path.move(to: point) } else { path.addLine(to: point) }
- }
- path.closeSubpath()
- }
+enum DictionaryGlowResponse {
+ static func energy(for level: CGFloat) -> Double {
+ guard level.isFinite else { return 0 }
+ return sqrt(Double(min(1, max(0, level))))
}
}
/// Uses normal shortcut dictation, accepting output only for a capture begun in this editor.
struct DictionaryWordPlayground: View {
let word: String
- let onPracticeMore: () -> Void
+ let onPracticeMore: (() -> Void)?
@Binding var busy: Bool
@EnvironmentObject private var appServices: AppServices
@Environment(\.theme) private var theme
@FocusState private var editorFocused: Bool
@State private var test = DictionaryShortcutTestState()
- @AppStorage("DictionaryPronunciationDebugCapture") private var debugCapture = false
private var shortcut: String {
let value = SettingsStore.shared.primaryDictationShortcutDisplayString
@@ -826,12 +810,14 @@ struct DictionaryWordPlayground: View {
.font(self.theme.typography.bodyStrong)
.foregroundStyle(self.theme.palette.accent)
case .missed:
- Text("We didn’t catch “\(self.word)”. Try again, or add more examples.")
+ Text("We didn’t catch “\(self.word)”. Try your shortcut again.")
.font(self.theme.typography.body)
.foregroundStyle(self.theme.palette.secondaryText)
- Button("Record more examples", action: self.onPracticeMore)
- .fluidGlassAction()
- .disabled(self.busy)
+ if let onPracticeMore {
+ Button("Record more examples", action: onPracticeMore)
+ .fluidGlassAction()
+ .disabled(self.busy)
+ }
case .empty:
Text("No words came through. Try your shortcut again.")
.font(self.theme.typography.body)
@@ -839,16 +825,14 @@ struct DictionaryWordPlayground: View {
case .none:
EmptyView()
}
- if DictionaryPronunciationExperiment.enabled {
- Toggle("Save dictionary debug audio locally", isOn: self.$debugCapture)
- .font(self.theme.typography.caption)
- .help("Saves audio, embeddings, cuts and scores locally. Switch off to stop collection. Keeps at most 50 debug reports / 200 MB.")
- }
}
.task(id: self.word) {
self.test = DictionaryShortcutTestState()
- self.editorFocused = true
self.busy = false
+ self.editorFocused = false
+ await Task.yield()
+ guard !Task.isCancelled else { return }
+ self.editorFocused = true
}
.onReceive(self.appServices.asr.$isRunning.dropFirst()) { running in
if running {
diff --git a/Sources/Fluid/UI/MeetingAssignSpeakersSheet.swift b/Sources/Fluid/UI/MeetingAssignSpeakersSheet.swift
index 72c6e7e3..c391d41b 100644
--- a/Sources/Fluid/UI/MeetingAssignSpeakersSheet.swift
+++ b/Sources/Fluid/UI/MeetingAssignSpeakersSheet.swift
@@ -54,18 +54,24 @@ nonisolated enum MeetingAssignSpeakersValidation {
}
struct MeetingAssignSpeakersSheet: View {
+ private struct MergeRequest {
+ let source: SessionSpeakerID
+ let target: SessionSpeakerID
+ }
+
let speakers: [MeetingSessionSpeaker]
let quotesBySpeaker: [SessionSpeakerID: [String]]
let tints: [SessionSpeakerID: Color]
let accentColor: Color
let focusedSpeakerID: SessionSpeakerID?
- let onSave: ([SessionSpeakerID: String]) async -> String?
+ let onSave: (MeetingSpeakerEdits) async -> String?
let onCancel: () -> Void
@Environment(\.theme) private var theme
- @State private var drafts: [SessionSpeakerID: String]
+ @State private var edits: MeetingSpeakerEdits
@State private var isSaving = false
@State private var errorMessage: String?
+ @State private var pendingMerge: MergeRequest?
@FocusState private var focusedField: SessionSpeakerID?
init(
@@ -74,7 +80,7 @@ struct MeetingAssignSpeakersSheet: View {
tints: [SessionSpeakerID: Color],
accentColor: Color,
focusedSpeakerID: SessionSpeakerID? = nil,
- onSave: @escaping ([SessionSpeakerID: String]) async -> String?,
+ onSave: @escaping (MeetingSpeakerEdits) async -> String?,
onCancel: @escaping () -> Void
) {
self.speakers = speakers
@@ -84,7 +90,7 @@ struct MeetingAssignSpeakersSheet: View {
self.focusedSpeakerID = focusedSpeakerID
self.onSave = onSave
self.onCancel = onCancel
- self._drafts = State(initialValue: Dictionary(uniqueKeysWithValues: speakers.map { ($0.id, $0.displayName) }))
+ self._edits = State(initialValue: MeetingSpeakerEdits(names: Dictionary(uniqueKeysWithValues: speakers.map { ($0.id, $0.displayName) })))
}
var body: some View {
@@ -128,7 +134,7 @@ struct MeetingAssignSpeakersSheet: View {
.keyboardShortcut(.cancelAction)
.disabled(self.isSaving)
Spacer()
- Button(self.isSaving ? "Saving…" : "Save names") { Task { await self.save() } }
+ Button(self.isSaving ? "Saving…" : "Save changes") { Task { await self.save() } }
.meetingGlassAction(prominent: true)
.keyboardShortcut(.defaultAction)
.disabled(self.isSaving)
@@ -139,14 +145,30 @@ struct MeetingAssignSpeakersSheet: View {
.frame(minWidth: 480, idealWidth: 560, maxWidth: 560)
.frame(minHeight: 420, idealHeight: 560)
.background(self.theme.palette.windowBackground)
+ .alert(
+ "Merge speakers?",
+ isPresented: Binding(
+ get: { self.pendingMerge != nil },
+ set: { if !$0 { self.pendingMerge = nil } }
+ ),
+ presenting: self.pendingMerge
+ ) { request in
+ Button("Cancel", role: .cancel) { self.pendingMerge = nil }
+ Button("Confirm merge") {
+ self.edits.setMerge(source: request.source, into: request.target)
+ self.pendingMerge = nil
+ }
+ } message: { request in
+ Text("All of \(self.speakerName(request.source))’s transcript will belong to \(self.speakerName(request.target)) when you save changes.")
+ }
}
private var header: some View {
HStack(alignment: .top, spacing: self.theme.metrics.spacing.md) {
VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) {
- Text("Assign speakers")
+ Text("Edit speakers")
.font(self.theme.typography.title)
- Text("Name each voice and we'll relabel the whole transcript.")
+ Text("Rename voices or merge duplicate speakers. Changes apply when you save.")
.font(self.theme.typography.bodySmall)
.foregroundStyle(self.theme.palette.secondaryText)
}
@@ -164,6 +186,7 @@ struct MeetingAssignSpeakersSheet: View {
@ViewBuilder
private func row(for speaker: MeetingSessionSpeaker) -> some View {
let tint = speaker.isLocalUser ? self.accentColor : (self.tints[speaker.id] ?? self.theme.palette.tertiaryText)
+ let targets = self.mergeTargets(for: speaker)
VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) {
HStack(spacing: self.theme.metrics.spacing.sm) {
Circle()
@@ -199,35 +222,81 @@ struct MeetingAssignSpeakersSheet: View {
}
}
- TextField(
- speaker.displayName,
- text: Binding(
- get: { self.drafts[speaker.id] ?? speaker.displayName },
- set: { self.drafts[speaker.id] = $0 }
- ),
- prompt: Text(speaker.displayName)
- )
- .textFieldStyle(.roundedBorder)
- .controlSize(.large)
- .accessibilityLabel("Name for \(speaker.displayName)")
- .focused(self.$focusedField, equals: speaker.id)
- .disabled(self.isSaving)
+ HStack(spacing: self.theme.metrics.spacing.md) {
+ TextField(
+ speaker.displayName,
+ text: Binding(
+ get: { self.edits.names[speaker.id] ?? speaker.displayName },
+ set: { self.edits.names[speaker.id] = $0 }
+ ),
+ prompt: Text(speaker.displayName)
+ )
+ .textFieldStyle(.roundedBorder)
+ .controlSize(.large)
+ .accessibilityLabel("Name for \(speaker.displayName)")
+ .focused(self.$focusedField, equals: speaker.id)
+ .disabled(self.isSaving || self.edits.merges[speaker.id] != nil)
+
+ if self.edits.merges[speaker.id] == nil, !speaker.isLocalUser, !targets.isEmpty {
+ Menu("Merge…") {
+ ForEach(targets) { target in
+ Button(self.speakerName(target.id)) {
+ self.pendingMerge = MergeRequest(source: speaker.id, target: target.id)
+ }
+ }
+ }
+ .meetingGlassAction()
+ .fixedSize()
+ .disabled(self.isSaving)
+ .accessibilityLabel("Merge \(self.speakerName(speaker.id)) with another speaker")
+ }
+ }
+
+ if let target = self.edits.merges[speaker.id] {
+ HStack(spacing: self.theme.metrics.spacing.md) {
+ Text("Will merge into \(self.speakerName(target))")
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.theme.palette.secondaryText)
+ .fixedSize(horizontal: false, vertical: true)
+ Spacer(minLength: 0)
+ Button("Cancel merge") { self.edits.setMerge(source: speaker.id, into: nil) }
+ .meetingGlassAction()
+ .disabled(self.isSaving)
+ }
+ } else if speaker.isLocalUser || targets.isEmpty {
+ Text(speaker.isLocalUser ? "Your microphone voice is kept separate" :
+ (speaker.trackKind == .microphone ? "No other microphone speakers" : "No other meeting-audio speakers"))
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.theme.palette.tertiaryText)
+ }
}
}
+ private func mergeTargets(for speaker: MeetingSessionSpeaker) -> [MeetingSessionSpeaker] {
+ self.speakers.filter {
+ $0.id != speaker.id && !$0.isLocalUser && $0.trackKind == speaker.trackKind
+ && self.edits.merges[$0.id] == nil
+ }
+ }
+
+ private func speakerName(_ id: SessionSpeakerID) -> String {
+ let name = self.edits.names[id]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+ return name.isEmpty ? (self.speakers.first { $0.id == id }?.displayName ?? "Speaker") : name
+ }
+
private func save() async {
self.errorMessage = nil
if let duplicate = MeetingAssignSpeakersValidation.duplicateName(
- current: self.speakers.map { (id: $0.id, displayName: $0.displayName) },
- drafts: self.drafts
+ current: self.speakers.filter { self.edits.merges[$0.id] == nil }.map { (id: $0.id, displayName: $0.displayName) },
+ drafts: self.edits.names
) {
- self.errorMessage = "Two speakers are both named \"\(duplicate)\". Give them different names, or merge them from the transcript."
+ self.errorMessage = "Two speakers are both named \"\(duplicate)\". Give them different names, or merge them here."
return
}
self.isSaving = true
- let message = await self.onSave(self.drafts)
+ let message = await self.onSave(self.edits)
self.isSaving = false
if let message {
diff --git a/Sources/Fluid/UI/MeetingDetectionPrompt.swift b/Sources/Fluid/UI/MeetingDetectionPrompt.swift
index f9a21eac..d492eb34 100644
--- a/Sources/Fluid/UI/MeetingDetectionPrompt.swift
+++ b/Sources/Fluid/UI/MeetingDetectionPrompt.swift
@@ -72,6 +72,7 @@ final class MeetingDetectionPromptController: ObservableObject {
private init() {}
func present(_ request: MeetingAutoDetector.PromptRequest) {
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
switch Self.incomingPromptPolicy(
isStarting: self.isStarting,
currentEpisodeID: self.request?.episodeID,
@@ -187,6 +188,10 @@ final class MeetingDetectionPromptController: ObservableObject {
self.scheduleAutoDismiss(for: request.episodeID)
}
+ func cancelForFeatureDisable() {
+ self.hide()
+ }
+
private func hide() {
self.startTask?.cancel()
self.startTask = nil
@@ -633,6 +638,7 @@ final class MeetingStillRecordingNudgeController: ObservableObject {
private init() {}
func present() {
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
guard !self.isPresented else { return }
self.isPresented = true
let panel = self.panelOrCreate()
diff --git a/Sources/Fluid/UI/MeetingFloatingCaptionsPanel.swift b/Sources/Fluid/UI/MeetingFloatingCaptionsPanel.swift
index 6f8edda9..bacd34bc 100644
--- a/Sources/Fluid/UI/MeetingFloatingCaptionsPanel.swift
+++ b/Sources/Fluid/UI/MeetingFloatingCaptionsPanel.swift
@@ -103,7 +103,9 @@ final class MeetingFloatingCaptionsController: ObservableObject {
/// Opens the captions window, growing out of `sourceFrame` (the pill) when motion is allowed.
func show(from sourceFrame: NSRect?) {
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
let coordinator = AppServices.shared.meetingSessionCoordinator
+ guard MeetingOverlayVisibility.isVisible(for: coordinator.state) else { return }
let panel = self.panelOrCreate(coordinator: coordinator)
self.subscribeIfNeeded(coordinator: coordinator)
@@ -126,6 +128,17 @@ final class MeetingFloatingCaptionsController: ObservableObject {
self.panel?.orderOut(nil)
}
+ func deactivate() {
+ self.hide()
+ self.stateSubscription?.cancel()
+ self.stateSubscription = nil
+ if let observer = self.screenParametersObserver {
+ NotificationCenter.default.removeObserver(observer)
+ }
+ self.screenParametersObserver = nil
+ self.panel = nil
+ }
+
private func panelOrCreate(coordinator: MeetingSessionCoordinator) -> MeetingFloatingCaptionsPanel {
if let panel { return panel }
diff --git a/Sources/Fluid/UI/MeetingModelSettingsSection.swift b/Sources/Fluid/UI/MeetingModelSettingsSection.swift
index 1ece7bc8..8c0f2c70 100644
--- a/Sources/Fluid/UI/MeetingModelSettingsSection.swift
+++ b/Sources/Fluid/UI/MeetingModelSettingsSection.swift
@@ -40,6 +40,7 @@ struct MeetingModelSettingsSection: View {
Text("Model details")
.meetingHoverFeedback(cornerRadius: self.theme.metrics.corners.sm)
}
+ .disclosureGroupStyle(FluidDisclosureGroupStyle())
.font(self.theme.typography.captionStrong)
.foregroundStyle(self.theme.palette.secondaryText)
}
diff --git a/Sources/Fluid/UI/MeetingRecordingPill.swift b/Sources/Fluid/UI/MeetingRecordingPill.swift
index 6e819613..a8f491dc 100644
--- a/Sources/Fluid/UI/MeetingRecordingPill.swift
+++ b/Sources/Fluid/UI/MeetingRecordingPill.swift
@@ -29,6 +29,7 @@ final class MeetingRecordingPillController: ObservableObject {
private init() {}
func activate(coordinator: MeetingSessionCoordinator) {
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
guard self.stateSubscription == nil else { return }
self.stateSubscription = coordinator.$state
.receive(on: DispatchQueue.main)
@@ -38,6 +39,21 @@ final class MeetingRecordingPillController: ObservableObject {
}
}
+ /// Stop observing recording state and release the meeting overlay while disabled.
+ func deactivate() {
+ self.stateSubscription?.cancel()
+ self.stateSubscription = nil
+ self.hideOverlay()
+ for observer in [self.screenParametersObserver, self.windowMoveObserver].compactMap({ $0 }) {
+ NotificationCenter.default.removeObserver(observer)
+ }
+ self.screenParametersObserver = nil
+ self.windowMoveObserver = nil
+ self.panel = nil
+ self.visibleAnchor = nil
+ MeetingFloatingCaptionsController.shared.deactivate()
+ }
+
/// Collapses back to the smallest pill.
func collapse() {
guard self.reducer.presentation == .captions else { return }
@@ -59,6 +75,12 @@ final class MeetingRecordingPillController: ObservableObject {
}
private func handleCoordinatorState(_ state: MeetingCoordinatorState, coordinator: MeetingSessionCoordinator) {
+ guard SettingsStore.shared.fluidMeetEnabled else {
+ self.hideOverlay()
+ return
+ }
+ // State deliveries are queued; a stale recording event must not reopen the overlay.
+ guard state == coordinator.state else { return }
switch state {
case let .recording(sessionID), let .recordingDegraded(sessionID):
if self.reducer.sessionID != sessionID {
@@ -106,6 +128,7 @@ final class MeetingRecordingPillController: ObservableObject {
}
private func setPresentation(_ presentation: MeetingOverlayPresentation) {
+ guard SettingsStore.shared.fluidMeetEnabled else { return }
guard self.presentation != presentation, let panel else {
self.presentation = presentation
return
diff --git a/Sources/Fluid/UI/MeetingSessionCanvases.swift b/Sources/Fluid/UI/MeetingSessionCanvases.swift
index f00093e4..8c40026b 100644
--- a/Sources/Fluid/UI/MeetingSessionCanvases.swift
+++ b/Sources/Fluid/UI/MeetingSessionCanvases.swift
@@ -51,6 +51,13 @@ struct MeetingRecordingCanvas: View {
Divider()
}
+ if let warning = self.session.microphoneEchoWarning {
+ MeetingAudioTranscriptNotice(
+ text: "Some audio may have an echo.",
+ detail: warning,
+ isSubtle: true
+ )
+ }
MeetingLiveTranscriptCard(snapshot: self.liveTranscript)
Divider()
@@ -350,19 +357,17 @@ struct MeetingResultCanvas: View {
let onReassignSegment: (MeetingTranscriptSegmentID, SessionSpeakerID) -> Void
var onEditTranscriptSegment: (MeetingTranscriptSegmentID, String, Int) async -> String? = { _, _, _ in "Transcript editing is unavailable." }
let onNameUnknownSegment: (MeetingTranscriptSegmentID, String) -> Void
- let onRenameSpeaker: (SessionSpeakerID, String) -> Void
- let onMergeSpeakers: (SessionSpeakerID, SessionSpeakerID) -> Void
- let onUndo: () -> Void
+ let onUndo: () async -> Bool
let onRenameSession: (String) -> Void
- let onAssignSpeakers: ([SessionSpeakerID: String]) async -> String?
+ let onAssignSpeakers: (MeetingSpeakerEdits) async -> String?
let onClose: (() -> Void)?
/// Opens FluidMeet settings on Integrations, so adding an agent is one step from any meeting.
var onOpenIntegrations: (() -> Void)? = nil
@Environment(\.theme) private var theme
+ @Environment(\.undoManager) private var undoManager
+ @StateObject private var correctionUndo = MeetingCorrectionUndoRegistration()
@State private var pendingEditedSegment: MeetingTranscriptSegment?
- @State private var pendingRenameSpeaker: MeetingSessionSpeaker?
- @State private var pendingRenameText = ""
@State private var pendingNameUnknownSegmentID: MeetingTranscriptSegmentID?
@State private var pendingUnknownSpeakerName = ""
@State private var isRenamingSession = false
@@ -441,6 +446,15 @@ struct MeetingResultCanvas: View {
MeetingSummaryView(session: self.session, isQuiescent: self.isQuiescent)
.id("\(self.session.id)-\(self.session.updatedAt)")
} else {
+ if let warning = self.session.microphoneEchoWarning {
+ MeetingAudioTranscriptNotice(
+ text: "Some audio may have an echo.",
+ detail: warning,
+ isSubtle: true
+ )
+ } else if self.session.events.contains(where: { $0.kind == .transcriptCoverageIncomplete }) {
+ MeetingAudioTranscriptNotice(text: "Some recorded audio was not transcribed.")
+ }
if !activeSpeakers.isEmpty {
VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) {
self.speakerStrip(activeSpeakers)
@@ -480,13 +494,9 @@ struct MeetingResultCanvas: View {
speakerNames: speakerNames
),
speakerTint: row.segment.speakerID.flatMap { self.speakerTints[$0] },
- onRenameSpeakerTapped: {
- if let speakerID = row.segment.speakerID {
- self.presentAssignSpeakers(focusing: speakerID)
- } else {
- self.pendingNameUnknownSegmentID = row.segment.id
- self.pendingUnknownSpeakerName = ""
- }
+ onNameUnknown: {
+ self.pendingNameUnknownSegmentID = row.segment.id
+ self.pendingUnknownSpeakerName = ""
},
isLocalUser: row.isLocal,
reassignTargets: activeSpeakers.filter { $0.id != row.segment.speakerID },
@@ -497,7 +507,21 @@ struct MeetingResultCanvas: View {
playback: self.playback,
isLoadingAudio: self.playback.activeSegmentID == row.segment.id && self.playback.isLoading,
onPlayAudio: { self.playback.toggle(sessionID: self.session.id, segmentID: row.segment.id) },
- onEditText: { if self.isQuiescent { self.pendingEditedSegment = row.segment } }
+ isEditingText: self.pendingEditedSegment?.id == row.segment.id,
+ onEditText: { if self.isQuiescent { self.pendingEditedSegment = row.segment } },
+ onSaveText: { text in
+ guard let editing = self.pendingEditedSegment, editing.id == row.segment.id else {
+ return "This paragraph is no longer being edited."
+ }
+ let error = await self.onEditTranscriptSegment(editing.id, text, editing.revision)
+ if error == nil, self.pendingEditedSegment?.id == editing.id {
+ self.pendingEditedSegment = nil
+ }
+ return error
+ },
+ onCancelEditText: {
+ if self.pendingEditedSegment?.id == row.segment.id { self.pendingEditedSegment = nil }
+ }
)
}
}
@@ -509,19 +533,19 @@ struct MeetingResultCanvas: View {
.frame(maxWidth: .infinity, alignment: .center)
}
.id(self.session.id)
- .sheet(item: self.$pendingEditedSegment) { segment in
- MeetingTranscriptTextEditor(
- text: segment.text,
- onSave: { text in
- let error = await self.onEditTranscriptSegment(segment.id, text, segment.revision)
- if error == nil { self.pendingEditedSegment = nil }
- return error
- },
- onCancel: { self.pendingEditedSegment = nil }
- )
+ .onAppear { self.refreshUndoRegistration() }
+ .onDisappear {
+ self.playback.stop()
+ self.correctionUndo.detach()
+ }
+ .onChange(of: self.canRegisterUndo) { _, _ in self.refreshUndoRegistration() }
+ .onChange(of: self.session.updatedAt) { _, _ in self.refreshUndoRegistration() }
+ .onChange(of: self.undoManager) { _, _ in self.refreshUndoRegistration() }
+ .onChange(of: self.session.id) { _, _ in
+ self.playback.stop()
+ self.pendingEditedSegment = nil
+ self.refreshUndoRegistration()
}
- .onDisappear { self.playback.stop() }
- .onChange(of: self.session.id) { _, _ in self.playback.stop() }
.onChange(of: self.session.retention.audioDeletedAt) { _, _ in self.playback.stop() }
.onChange(of: self.isQuiescent) { _, value in if !value { self.playback.stop() } }
.onChange(of: self.documentSection) { _, _ in self.playback.stop() }
@@ -531,24 +555,6 @@ struct MeetingResultCanvas: View {
do { try await Task.sleep(for: .seconds(2)) } catch { return }
self.copied = false
}
- .alert(
- "Rename Speaker",
- isPresented: Binding(
- get: { self.pendingRenameSpeaker != nil },
- set: { if !$0 { self.pendingRenameSpeaker = nil } }
- )
- ) {
- TextField("Speaker name", text: self.$pendingRenameText)
- Button("Cancel", role: .cancel) { self.pendingRenameSpeaker = nil }
- Button("Rename") {
- if let speaker = self.pendingRenameSpeaker {
- self.onRenameSpeaker(speaker.id, self.pendingRenameText)
- }
- self.pendingRenameSpeaker = nil
- }
- } message: {
- Text("Enter a new name for this speaker.")
- }
.alert(
"Name Unknown Speaker",
isPresented: Binding(
@@ -636,7 +642,7 @@ struct MeetingResultCanvas: View {
static func speakerAccuracyNote(speakerCount: Int) -> String? {
guard speakerCount > 0 else { return nil }
guard speakerCount > self.reliableSpeakerLimit else {
- return "Speaker labels are automatic and may not be exact. Click a name to fix it."
+ return "Speaker labels are automatic. Name speakers above; double-click a paragraph to edit its text."
}
return "\(speakerCount) speakers found. Above \(self.reliableSpeakerLimit), labels are less reliable: one person may show up twice, or two people as one."
}
@@ -653,10 +659,18 @@ struct MeetingResultCanvas: View {
}
private func speakerStrip(_ speakers: [MeetingSessionSpeaker]) -> some View {
- HStack(spacing: self.theme.metrics.spacing.md) {
- Text("Speakers")
- .font(self.theme.typography.caption)
- .foregroundStyle(self.theme.palette.tertiaryText)
+ VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) {
+ HStack(spacing: self.theme.metrics.spacing.sm) {
+ Text("Speakers")
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.theme.palette.tertiaryText)
+ Spacer(minLength: 0)
+ Button("Edit speakers", systemImage: "pencil") {
+ self.presentAssignSpeakers(focusing: nil)
+ }
+ .fluidGlassAction()
+ .disabled(!self.isQuiescent)
+ }
ScrollView(.horizontal, showsIndicators: false) {
HStack(spacing: self.theme.metrics.spacing.lg) {
ForEach(speakers) { speaker in
@@ -668,7 +682,6 @@ struct MeetingResultCanvas: View {
action: { self.presentAssignSpeakers(focusing: speaker.id) }
)
.disabled(!self.isQuiescent)
- .contextMenu { self.speakerChipMenu(for: speaker) }
}
}
.padding(.vertical, self.theme.metrics.spacing.xs)
@@ -691,13 +704,12 @@ struct MeetingResultCanvas: View {
Menu {
Menu("Export", systemImage: "square.and.arrow.up") {
Button("Text…") { self.onExportTranscript(self.session, .text, self.showsProbableEchoes) }
+ Button("Markdown…") { self.onExportTranscript(self.session, .markdown, self.showsProbableEchoes) }
Button("JSON…") { self.onExportTranscript(self.session, .json, self.showsProbableEchoes) }
}
.disabled(!hasText || self.documentSection != .transcript)
Divider()
self.renameMeetingAction
- Button("Undo correction", systemImage: "arrow.uturn.backward", action: self.onUndo)
- .disabled(!self.canUndo || !self.isQuiescent)
if self.hasEchoSegments {
Divider()
Toggle("Show probable echo", isOn: self.$showsProbableEchoes)
@@ -728,28 +740,20 @@ struct MeetingResultCanvas: View {
.disabled(!self.isQuiescent)
}
- @ViewBuilder
- private func speakerChipMenu(for speaker: MeetingSessionSpeaker) -> some View {
- Button("Rename…", systemImage: "pencil") {
- self.pendingRenameSpeaker = speaker
- self.pendingRenameText = speaker.displayName
- }
- .disabled(!self.isQuiescent)
- if !speaker.isLocalUser {
- let eligibleTargets = self.session.activeSpeakers.filter {
- $0.id != speaker.id && !$0.isLocalUser && $0.trackKind == speaker.trackKind
- }
- if !eligibleTargets.isEmpty {
- Menu("Merge into") {
- ForEach(eligibleTargets) { target in
- Button(target.displayName) {
- self.onMergeSpeakers(speaker.id, target.id)
- }
- }
- }
- .disabled(!self.isQuiescent)
- }
- }
+ private var canRegisterUndo: Bool {
+ self.canUndo && self.isQuiescent && self.documentSection == .transcript
+ && self.pendingEditedSegment == nil
+ && self.pendingNameUnknownSegmentID == nil && !self.isRenamingSession
+ && !self.isShowingAssignSpeakers
+ }
+
+ private func refreshUndoRegistration() {
+ self.correctionUndo.refresh(
+ manager: self.undoManager,
+ sessionID: self.session.id,
+ isEnabled: self.canRegisterUndo,
+ action: self.onUndo
+ )
}
private func displayName(for speaker: MeetingSessionSpeaker) -> String {
@@ -962,6 +966,21 @@ struct MeetingFailureCanvas: View {
}
}
+private struct MeetingAudioTranscriptNotice: View {
+ let text: String
+ var detail: String? = nil
+ var isSubtle = false
+ @Environment(\.theme) private var theme
+
+ var body: some View {
+ Label(self.text, systemImage: self.isSubtle ? "info.circle" : "exclamationmark.triangle")
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.isSubtle ? self.theme.palette.secondaryText : self.theme.palette.warning)
+ .fixedSize(horizontal: false, vertical: true)
+ .help(self.detail ?? self.text)
+ }
+}
+
private struct MeetingTrackHealthRow: View {
let title: String
let systemImage: String
@@ -1053,7 +1072,7 @@ private struct MeetingTranscriptSegmentRow: View {
let segment: MeetingTranscriptSegment
let speakerName: String
let speakerTint: Color?
- let onRenameSpeakerTapped: (() -> Void)?
+ let onNameUnknown: () -> Void
let isLocalUser: Bool
let reassignTargets: [MeetingSessionSpeaker]
let isQuiescent: Bool
@@ -1063,7 +1082,10 @@ private struct MeetingTranscriptSegmentRow: View {
let playback: MeetingSegmentPlayback
let isLoadingAudio: Bool
let onPlayAudio: () -> Void
+ let isEditingText: Bool
let onEditText: () -> Void
+ let onSaveText: (String) async -> String?
+ let onCancelEditText: () -> Void
@Environment(\.theme) private var theme
@@ -1078,10 +1100,24 @@ private struct MeetingTranscriptSegmentRow: View {
HStack(spacing: self.theme.metrics.spacing.sm) {
self.speakerLabel
Spacer(minLength: self.theme.metrics.spacing.md)
- Button("Edit", systemImage: "pencil", action: self.onEditText)
+ if self.segment.speakerID == nil {
+ Menu {
+ ForEach(self.reassignTargets) { target in
+ Button(target.displayName) {
+ self.playback.stop()
+ self.onReassign(target.id)
+ }
+ }
+ Divider()
+ Button("New speaker…", action: self.onNameUnknown)
+ } label: {
+ Label("Assign speaker", systemImage: "tag")
+ }
.fluidGlassAction()
- .disabled(!self.isQuiescent)
- .help("Edit text, or double-click this paragraph")
+ .disabled(!self.isQuiescent || self.isEditingText)
+ .help("Tag this unknown paragraph with a speaker")
+ .accessibilityLabel("Assign speaker to unknown paragraph")
+ }
Button {
if self.isAudioExpanded { self.playback.stop() } else { self.onPlayAudio() }
} label: {
@@ -1094,14 +1130,23 @@ private struct MeetingTranscriptSegmentRow: View {
.accessibilityLabel(self.isAudioExpanded ? "Close audio player" : "Play audio chunk for \(self.speakerName)")
}
.frame(minHeight: 28)
- Text(self.segment.text)
- .font(self.theme.typography.body)
- .foregroundStyle(self.theme.palette.primaryText)
- .lineSpacing(5)
- .textSelection(.enabled)
- .frame(maxWidth: .infinity, alignment: .leading)
- .simultaneousGesture(TapGesture(count: 2).onEnded { if self.isQuiescent { self.onEditText() } })
- .accessibilityAction(named: Text("Edit text")) { if self.isQuiescent { self.onEditText() } }
+ if self.isEditingText {
+ MeetingTranscriptTextEditor(
+ text: self.segment.text,
+ onSave: self.onSaveText,
+ onCancel: self.onCancelEditText
+ )
+ } else {
+ Text(self.segment.text)
+ .font(self.theme.typography.body)
+ .foregroundStyle(self.theme.palette.primaryText)
+ .lineSpacing(5)
+ .textSelection(.enabled)
+ .frame(maxWidth: .infinity, alignment: .leading)
+ .contentShape(Rectangle())
+ .simultaneousGesture(TapGesture(count: 2).onEnded { self.beginEditing() })
+ .accessibilityAction(named: Text("Edit text")) { self.beginEditing() }
+ }
if self.segment.isEcho {
Text("Probable echo").font(self.theme.typography.caption).foregroundStyle(.secondary)
}
@@ -1114,14 +1159,18 @@ private struct MeetingTranscriptSegmentRow: View {
}
.opacity(self.segment.isEcho ? 0.6 : 1)
.contextMenu {
- Menu("Reassign to") {
+ Button("Edit text", systemImage: "pencil") { self.beginEditing() }
+ .disabled(!self.isQuiescent || self.isEditingText)
+ if self.segment.speakerID == nil {
+ Divider()
ForEach(self.reassignTargets) { target in
- Button(target.displayName) {
+ Button("Assign to \(target.displayName)") {
+ self.playback.stop()
self.onReassign(target.id)
}
+ .disabled(!self.isQuiescent)
}
}
- .disabled(!self.isQuiescent || self.reassignTargets.isEmpty)
}
.accessibilityElement(children: .contain)
.accessibilityLabel(
@@ -1129,34 +1178,17 @@ private struct MeetingTranscriptSegmentRow: View {
)
}
+ private func beginEditing() {
+ guard self.isQuiescent else { return }
+ self.playback.stop()
+ self.onEditText()
+ }
+
private var speakerLabel: some View {
- let name = Text(self.speakerName)
+ Text(self.speakerName)
.foregroundColor(self.nameColor)
.fontWeight(.semibold)
.font(self.theme.typography.bodySmallStrong)
-
- return HStack(spacing: 8) {
- if let onRenameSpeakerTapped = self.onRenameSpeakerTapped {
- Button(action: onRenameSpeakerTapped) {
- HStack(spacing: 3) {
- name
- // Always shown, not hover-only, so the affordance survives an unfocused window.
- Image(systemName: "chevron.right")
- .font(.fluidSystem(size: 10, weight: .semibold))
- .foregroundColor(self.nameColor)
- .opacity(0.65)
- }
- }
- .buttonStyle(.plain)
- .meetingHoverFeedback(cornerRadius: self.theme.metrics.corners.sm)
- .disabled(!self.isQuiescent)
- // No .help here: the tooltip lands on top of the turn's first line of text.
- .accessibilityLabel("Rename \(self.speakerName)")
- .pointerStyle(.link)
- } else {
- name
- }
- }
}
private var nameColor: Color {
@@ -1165,6 +1197,8 @@ private struct MeetingTranscriptSegmentRow: View {
}
}
+/// Inline paragraph editing: no sheet and no recording/model work. Keep the draft on save
+/// failure; the caller holds the starting segment revision for stale-edit protection.
struct MeetingTranscriptTextEditor: View {
let onSave: (String) async -> String?
let onCancel: () -> Void
@@ -1172,6 +1206,7 @@ struct MeetingTranscriptTextEditor: View {
@State private var text: String
@State private var isSaving = false
@State private var errorMessage: String?
+ @FocusState private var isFocused: Bool
init(text: String, onSave: @escaping (String) async -> String?, onCancel: @escaping () -> Void) {
self._text = State(initialValue: text)
@@ -1180,51 +1215,66 @@ struct MeetingTranscriptTextEditor: View {
}
var body: some View {
- VStack(alignment: .leading, spacing: self.theme.metrics.spacing.lg) {
- Text("Edit transcript")
- .font(self.theme.typography.title)
- .accessibilityAddTraits(.isHeader)
- Text("Correct this paragraph. Its timestamps, speaker, and recording stay the same.")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.theme.palette.secondaryText)
- TextEditor(text: self.$text)
+ VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) {
+ TextField("Transcript text", text: self.$text, axis: .vertical)
+ .textFieldStyle(.plain)
.font(self.theme.typography.body)
- .scrollContentBackground(.hidden)
+ .foregroundStyle(self.theme.palette.primaryText)
+ .lineSpacing(5)
+ .lineLimit(1...20)
.padding(self.theme.metrics.spacing.sm)
.background(self.theme.palette.cardBackground, in: RoundedRectangle(cornerRadius: self.theme.metrics.corners.sm))
- .frame(minHeight: 200)
+ .overlay {
+ RoundedRectangle(cornerRadius: self.theme.metrics.corners.sm)
+ .strokeBorder(self.theme.palette.accent.opacity(0.45), lineWidth: 1)
+ }
+ .focused(self.$isFocused)
.disabled(self.isSaving)
- .accessibilityLabel("Transcript text")
+ .accessibilityLabel("Edit transcript paragraph")
+ .onSubmit { self.save() }
+ .onKeyPress(.return) {
+ self.save()
+ return .handled
+ }
+ .onExitCommand { if !self.isSaving { self.onCancel() } }
+ .task {
+ await Task.yield()
+ guard !Task.isCancelled else { return }
+ self.isFocused = true
+ }
if let errorMessage {
Text(errorMessage)
.font(self.theme.typography.caption)
.foregroundStyle(self.theme.palette.warning)
}
- FluidGlassControlGroup {
- HStack {
- Button("Cancel", action: self.onCancel)
- .meetingGlassAction()
- .keyboardShortcut(.cancelAction)
- .disabled(self.isSaving)
- Spacer()
- Button(self.isSaving ? "Saving…" : "Save") {
- guard !self.isSaving else { return }
- self.isSaving = true
- Task { @MainActor in
- self.errorMessage = await self.onSave(self.text)
- self.isSaving = false
- }
+ HStack(spacing: self.theme.metrics.spacing.sm) {
+ Text("Enter to save · Escape to cancel")
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.theme.palette.secondaryText)
+ Spacer(minLength: 0)
+ FluidGlassControlGroup {
+ HStack(spacing: self.theme.metrics.spacing.sm) {
+ Button("Cancel", action: self.onCancel)
+ .fluidGlassAction()
+ .disabled(self.isSaving)
+ Button(self.isSaving ? "Saving…" : "Save", action: self.save)
+ .fluidGlassAction(prominent: true)
+ .keyboardShortcut(.return, modifiers: .command)
+ .disabled(self.isSaving || self.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
}
- .meetingGlassAction(prominent: true)
- .keyboardShortcut(.defaultAction)
- .disabled(self.isSaving || self.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
}
}
}
- .padding(self.theme.metrics.spacing.xl)
- .frame(width: 560)
- .frame(minHeight: 400)
- .background(self.theme.palette.windowBackground)
- .interactiveDismissDisabled(self.isSaving)
+ .frame(maxWidth: .infinity, alignment: .leading)
+ }
+
+ private func save() {
+ guard !self.isSaving, !self.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return }
+ self.isSaving = true
+ Task { @MainActor in
+ self.errorMessage = await self.onSave(self.text)
+ self.isSaving = false
+ if self.errorMessage != nil { self.isFocused = true }
+ }
}
}
diff --git a/Sources/Fluid/UI/MeetingTranscriptionView.swift b/Sources/Fluid/UI/MeetingTranscriptionView.swift
index 300a921a..07bf3b87 100644
--- a/Sources/Fluid/UI/MeetingTranscriptionView.swift
+++ b/Sources/Fluid/UI/MeetingTranscriptionView.swift
@@ -104,6 +104,62 @@ struct MeetingSetupReadiness: Equatable {
)
}
+/// Registers only this meeting's saved corrections with the window's native Undo command.
+@MainActor
+final class MeetingCorrectionUndoRegistration: ObservableObject {
+ private weak var manager: UndoManager?
+ private var sessionID: MeetingSessionID?
+ private var isEnabled = false
+ private var action: (@MainActor () async -> Bool)?
+ private var task: Task<Void, Never>?
+ private var generation = UUID()
+
+ func refresh(manager: UndoManager?, sessionID: MeetingSessionID, isEnabled: Bool, action: @escaping @MainActor () async -> Bool) {
+ if self.manager !== manager || self.sessionID != sessionID { self.detach() }
+ self.manager = manager
+ self.sessionID = sessionID
+ self.isEnabled = isEnabled
+ self.action = action
+ if self.task == nil { self.install() }
+ }
+
+ func detach() {
+ self.manager?.removeAllActions(withTarget: self)
+ self.manager = nil
+ self.sessionID = nil
+ self.action = nil
+ self.isEnabled = false
+ self.generation = UUID()
+ // An authorized save may finish after navigation, but must never register Undo in
+ // the next meeting or another page. Do not cancel persistence mid-write.
+ self.task = nil
+ }
+
+ private func install() {
+ guard let manager = self.manager else { return }
+ manager.removeAllActions(withTarget: self)
+ guard self.isEnabled else { return }
+ manager.beginUndoGrouping()
+ manager.registerUndo(withTarget: self) { $0.performUndo() }
+ manager.setActionName("Meeting Correction")
+ manager.endUndoGrouping()
+ }
+
+ private func performUndo() {
+ guard self.isEnabled, self.task == nil, let action = self.action else { return }
+ let generation = self.generation
+ self.task = Task { @MainActor in
+ let hasMore = await action()
+ guard self.generation == generation else { return }
+ self.task = nil
+ self.isEnabled = self.isEnabled && hasMore
+ // Register after the async save, outside UndoManager's undo callback. This is
+ // another correction to undo, not a redo. A failed save remains retryable.
+ self.install()
+ }
+ }
+}
+
/// Owned by the window, so leaving FluidMeet does not discard its last loaded history.
@MainActor
final class MeetingHistorySnapshot: ObservableObject {
@@ -115,11 +171,23 @@ final class MeetingHistorySnapshot: ObservableObject {
private var refreshRequested = false
init(load: @escaping @Sendable () async throws -> [MeetingSession] = {
- try await MeetingSessionStore.shared.loadAll()
+ try await MeetingSessionStore.shared.loadHistory()
}) {
self.load = load
}
+ /// A correction already returned the saved session. Publish it without rereading every
+ /// meeting and probing all their audio chunks. An older in-flight read must not undo it.
+ func applySavedSession(_ session: MeetingSession) {
+ if let index = self.sessions.firstIndex(where: { $0.id == session.id }) {
+ self.sessions[index] = session
+ } else {
+ self.sessions.append(session)
+ self.sessions.sort { $0.startedAt > $1.startedAt }
+ }
+ if self.refreshTask != nil { self.refreshRequested = true }
+ }
+
func refresh() async {
self.refreshRequested = true
if let refreshTask {
@@ -205,8 +273,20 @@ struct MeetingTranscriptionView: View {
}
var body: some View {
- VStack(spacing: 0) {
- GeometryReader { geometry in
+ GeometryReader { geometry in
+ VStack(spacing: 0) {
+ MeetingTranscriptionHeader(
+ state: self.canvasState,
+ isMeetingHistoryVisible: self.isMeetingHistoryVisible,
+ isSummaryRunning: self.summaryActivity.selectionLock != nil,
+ onNewMeeting: { self.startNewMeeting(hideHistory: geometry.size.width < 900) },
+ onOpenMeetingSettings: self.openMeetingSettings,
+ onToggleMeetingHistory: self.toggleMeetingHistory
+ )
+ .padding(.horizontal, FluidPageLayout.inset)
+ .padding(.vertical, 12)
+ Divider()
+
ZStack(alignment: .trailing) {
MeetingTranscriptionCanvas(
setupDraft: self.$setupDraft,
@@ -225,8 +305,6 @@ struct MeetingTranscriptionView: View {
onReassignSegment: self.reassignSegment,
onEditTranscriptSegment: self.editTranscriptSegment,
onNameUnknownSegment: self.nameUnknownSegment,
- onRenameSpeaker: self.renameSpeaker,
- onMergeSpeakers: self.mergeSpeakers,
onUndoCorrection: self.undoTranscriptCorrection,
onRenameSession: self.renameMeetingSession,
onAssignSpeakers: self.assignSpeakers,
@@ -293,23 +371,6 @@ struct MeetingTranscriptionView: View {
}
}
.background(self.theme.palette.contentBackground)
- .fluidPageActions {
- MeetingTranscriptionHeader(
- state: self.canvasState,
- isMeetingHistoryVisible: self.isMeetingHistoryVisible,
- onNewMeeting: self.startNewMeeting,
- onOpenMeetingSettings: self.openMeetingSettings,
- onToggleMeetingHistory: {
- let willShowHistory = !self.isMeetingHistoryVisible
- withAnimation(self.accessibilityReduceMotion ? nil : .easeInOut(duration: 0.2)) {
- self.isMeetingHistoryVisible.toggle()
- }
- if willShowHistory, self.historySnapshot.sessions.isEmpty {
- Task { await self.loadMeetingHistory() }
- }
- }
- )
- }
.clipped()
.task {
MeetingDiarizationModelStore.shared.prepareInBackground()
@@ -791,11 +852,11 @@ struct MeetingTranscriptionView: View {
self.actionErrorMessage = nil
Task {
do {
- _ = try await self.coordinator.reassignSegment(sessionID: sessionID, segmentID: segmentID, to: speakerID)
+ let saved = try await self.coordinator.reassignSegment(sessionID: sessionID, segmentID: segmentID, to: speakerID)
+ self.historySnapshot.applySavedSession(saved)
} catch {
self.actionErrorMessage = error.localizedDescription
}
- await self.loadMeetingHistory()
}
}
@@ -807,86 +868,61 @@ struct MeetingTranscriptionView: View {
self.actionErrorMessage = nil
Task {
do {
- _ = try await self.coordinator.nameUnknownSegment(
+ let saved = try await self.coordinator.nameUnknownSegment(
sessionID: sessionID,
segmentID: segmentID,
displayName: displayName
)
+ self.historySnapshot.applySavedSession(saved)
} catch {
self.actionErrorMessage = error.localizedDescription
}
- await self.loadMeetingHistory()
- }
- }
-
- private func renameSpeaker(sessionID: MeetingSessionID, speakerID: SessionSpeakerID, to displayName: String) {
- self.actionErrorMessage = nil
- Task {
- do {
- _ = try await self.coordinator.renameSpeaker(sessionID: sessionID, speakerID: speakerID, to: displayName)
- } catch {
- self.actionErrorMessage = error.localizedDescription
- }
- await self.loadMeetingHistory()
}
}
private func editTranscriptSegment(sessionID: MeetingSessionID, segmentID: MeetingTranscriptSegmentID, text: String, revision: Int) async -> String? {
do {
- _ = try await self.coordinator.editTranscriptText(sessionID: sessionID, segmentID: segmentID, text: text, expectedRevision: revision)
- await self.loadMeetingHistory()
+ let saved = try await self.coordinator.editTranscriptText(sessionID: sessionID, segmentID: segmentID, text: text, expectedRevision: revision)
+ self.historySnapshot.applySavedSession(saved)
return nil
} catch {
return error.localizedDescription
}
}
- private func assignSpeakers(sessionID: MeetingSessionID, names: [SessionSpeakerID: String]) async -> String? {
+ private func assignSpeakers(sessionID: MeetingSessionID, edits: MeetingSpeakerEdits) async -> String? {
self.actionErrorMessage = nil
var errorMessage: String?
do {
- _ = try await self.coordinator.renameSpeakers(sessionID: sessionID, names: names)
+ let saved = try await self.coordinator.editSpeakers(sessionID: sessionID, edits: edits)
+ self.historySnapshot.applySavedSession(saved)
} catch {
errorMessage = error.localizedDescription
}
- await self.loadMeetingHistory()
return errorMessage
}
- private func mergeSpeakers(sessionID: MeetingSessionID, source: SessionSpeakerID, into targetID: SessionSpeakerID) {
- self.actionErrorMessage = nil
- Task {
- do {
- _ = try await self.coordinator.mergeSpeakers(sessionID: sessionID, source: source, into: targetID)
- } catch {
- self.actionErrorMessage = error.localizedDescription
- }
- await self.loadMeetingHistory()
- }
- }
-
private func renameMeetingSession(sessionID: MeetingSessionID, to title: String) {
self.actionErrorMessage = nil
Task {
do {
- _ = try await self.coordinator.renameSession(sessionID: sessionID, to: title)
+ let saved = try await self.coordinator.renameSession(sessionID: sessionID, to: title)
+ self.historySnapshot.applySavedSession(saved)
} catch {
self.actionErrorMessage = error.localizedDescription
}
- await self.loadMeetingHistory()
}
}
- private func undoTranscriptCorrection(sessionID: MeetingSessionID) {
+ private func undoTranscriptCorrection(sessionID: MeetingSessionID) async -> Bool {
self.actionErrorMessage = nil
- Task {
- do {
- _ = try await self.coordinator.undoTranscriptCorrection(sessionID: sessionID)
- } catch {
- self.actionErrorMessage = error.localizedDescription
- }
- await self.loadMeetingHistory()
+ do {
+ let saved = try await self.coordinator.undoTranscriptCorrection(sessionID: sessionID)
+ self.historySnapshot.applySavedSession(saved)
+ } catch {
+ self.actionErrorMessage = error.localizedDescription
}
+ return self.coordinator.canUndoCorrection(sessionID: sessionID)
}
private func deleteSession(id: MeetingSessionID) {
@@ -1112,11 +1148,22 @@ struct MeetingTranscriptionView: View {
return candidate
}
- private func startNewMeeting() {
+ private func toggleMeetingHistory() {
+ let willShowHistory = !self.isMeetingHistoryVisible
+ withAnimation(self.accessibilityReduceMotion ? nil : .easeInOut(duration: 0.2)) {
+ self.isMeetingHistoryVisible.toggle()
+ }
+ if willShowHistory, self.historySnapshot.sessions.isEmpty {
+ Task { await self.loadMeetingHistory() }
+ }
+ }
+
+ private func startNewMeeting(hideHistory: Bool = false) {
guard self.summaryActivity.selectionLock == nil else { return }
do {
try self.coordinator.resetForNewMeeting()
self.selectedHistorySessionID = nil
+ if hideHistory { self.isMeetingHistoryVisible = false }
self.setupDraft.usesAutomaticApplication = true
self.selectPreferredApplication(from: self.applications.map(\.identity))
self.resetDraftTitleToDefault()
@@ -1270,6 +1317,7 @@ struct MeetingTranscriptionView: View {
let data: Data
switch format {
case .text: data = Data(MeetingTranscriptExporter.text(for: session, includeEchoes: includeEchoes).utf8)
+ case .markdown: data = Data(MeetingTranscriptExporter.markdown(for: session, includeEchoes: includeEchoes).utf8)
case .json: data = try MeetingTranscriptExporter.json(for: session)
}
try data.write(to: url, options: .atomic)
@@ -1334,11 +1382,13 @@ struct MeetingTranscriptionView: View {
enum MeetingTranscriptExportFormat {
case text
+ case markdown
case json
var fileExtension: String {
switch self {
case .text: return "txt"
+ case .markdown: return "md"
case .json: return "json"
}
}
@@ -1379,11 +1429,9 @@ struct MeetingTranscriptionCanvas: View {
let onReassignSegment: (MeetingSessionID, MeetingTranscriptSegmentID, SessionSpeakerID) -> Void
var onEditTranscriptSegment: (MeetingSessionID, MeetingTranscriptSegmentID, String, Int) async -> String? = { _, _, _, _ in "Transcript editing is unavailable." }
let onNameUnknownSegment: (MeetingSessionID, MeetingTranscriptSegmentID, String) -> Void
- let onRenameSpeaker: (MeetingSessionID, SessionSpeakerID, String) -> Void
- let onMergeSpeakers: (MeetingSessionID, SessionSpeakerID, SessionSpeakerID) -> Void
- let onUndoCorrection: (MeetingSessionID) -> Void
+ let onUndoCorrection: (MeetingSessionID) async -> Bool
let onRenameSession: (MeetingSessionID, String) -> Void
- let onAssignSpeakers: (MeetingSessionID, [SessionSpeakerID: String]) async -> String?
+ let onAssignSpeakers: (MeetingSessionID, MeetingSpeakerEdits) async -> String?
let canUndoCorrection: (MeetingSessionID) -> Bool
let isQuiescent: Bool
let onRepairSetup: () -> Void
@@ -1483,13 +1531,7 @@ struct MeetingTranscriptionCanvas: View {
onNameUnknownSegment: { segmentID, name in
self.onNameUnknownSegment(session.id, segmentID, name)
},
- onRenameSpeaker: { speakerID, name in
- self.onRenameSpeaker(session.id, speakerID, name)
- },
- onMergeSpeakers: { source, target in
- self.onMergeSpeakers(session.id, source, target)
- },
- onUndo: { self.onUndoCorrection(session.id) },
+ onUndo: { await self.onUndoCorrection(session.id) },
onRenameSession: { title in self.onRenameSession(session.id, title) },
onAssignSpeakers: { names in await self.onAssignSpeakers(session.id, names) },
onClose: self.onCloseSelection,
@@ -1511,59 +1553,70 @@ struct MeetingTranscriptionCanvas: View {
}
}
+struct MeetingPageActionAvailability: Equatable {
+ let showsNewMeeting: Bool
+ let canStartNewMeeting: Bool
+ let canEditSetup: Bool
+
+ init(state: MeetingTranscriptionCanvasState, isSummaryRunning: Bool) {
+ switch state {
+ case .result, .failed:
+ self.showsNewMeeting = true
+ self.canEditSetup = true
+ case let .setup(isStarting, _):
+ self.showsNewMeeting = false
+ self.canEditSetup = !isStarting
+ case .recording, .stopping, .processing:
+ self.showsNewMeeting = false
+ self.canEditSetup = false
+ }
+ self.canStartNewMeeting = self.showsNewMeeting && !isSummaryRunning
+ }
+}
+
private struct MeetingTranscriptionHeader: View {
let state: MeetingTranscriptionCanvasState
let isMeetingHistoryVisible: Bool
+ let isSummaryRunning: Bool
let onNewMeeting: () -> Void
let onOpenMeetingSettings: () -> Void
let onToggleMeetingHistory: () -> Void
+ @Environment(\.fluidPageToolbarVisible) private var isPageVisible
var body: some View {
- Group {
- if self.canStartNewMeeting {
+ let availability = MeetingPageActionAvailability(state: self.state, isSummaryRunning: self.isSummaryRunning)
+ FluidGlassControlGroup {
+ HStack(spacing: 12) {
+ if availability.showsNewMeeting {
+ Button(action: self.onNewMeeting) {
+ Label("New Meeting", systemImage: "plus")
+ }
+ .fluidGlassAction()
+ .disabled(!availability.canStartNewMeeting)
+ .keyboardShortcut("n", modifiers: .command)
+ .help(self.isSummaryRunning ? "Finish or cancel the summary before starting a new meeting" : "Return to recording setup")
+ .accessibilityHint("Return to recording setup; the current meeting stays saved")
+ }
+ Spacer(minLength: 8)
MeetingHeaderIconButton(
- systemImage: "plus",
- label: "New Meeting",
- action: self.onNewMeeting
+ systemImage: "gearshape",
+ label: "FluidMeet settings",
+ action: self.onOpenMeetingSettings
)
- .keyboardShortcut("n", modifiers: .command)
- .accessibilityHint("Clear the current meeting and return to recording setup")
- }
+ .disabled(!availability.canEditSetup)
+ .accessibilityHint("Change the saved recording application, microphone, and meeting defaults")
- MeetingHeaderIconButton(
- systemImage: "gearshape",
- label: "FluidMeet settings",
- action: self.onOpenMeetingSettings
- )
- .disabled(!self.canEditSetup)
- .accessibilityHint("Change the saved recording application, microphone, and meeting defaults")
-
- MeetingHeaderIconButton(
- systemImage: self.isMeetingHistoryVisible
- ? "rectangle.righthalf.inset.filled"
- : "sidebar.right",
- label: self.isMeetingHistoryVisible ? "Hide meeting history" : "Show meeting history",
- isSelected: self.isMeetingHistoryVisible,
- action: self.onToggleMeetingHistory
- )
- }
- }
-
- private var canStartNewMeeting: Bool {
- switch self.state {
- case .result, .failed:
- return true
- case .setup, .recording, .stopping, .processing:
- return false
- }
- }
-
- private var canEditSetup: Bool {
- switch self.state {
- case let .setup(isStarting, _): !isStarting
- case .result, .failed: true
- case .recording, .stopping, .processing: false
+ MeetingHeaderIconButton(
+ systemImage: self.isMeetingHistoryVisible
+ ? "rectangle.righthalf.inset.filled"
+ : "sidebar.right",
+ label: self.isMeetingHistoryVisible ? "Hide meeting history" : "Show meeting history",
+ isSelected: self.isMeetingHistoryVisible,
+ action: self.onToggleMeetingHistory
+ )
+ }
}
+ .disabled(!self.isPageVisible)
}
}
@@ -1579,7 +1632,8 @@ private struct MeetingHeaderIconButton: View {
Label(self.label, systemImage: self.systemImage)
.foregroundStyle(self.isSelected ? self.theme.palette.accent : self.theme.palette.primaryText)
}
- .buttonStyle(.automatic)
+ .labelStyle(.iconOnly)
+ .fluidGlassAction(circular: true)
.help(self.label)
.accessibilityLabel(self.label)
.accessibilityAddTraits(self.isSelected ? .isSelected : [])
@@ -1822,6 +1876,10 @@ private struct MeetingHistoryInspector: View {
self.onExportTranscript(session, .text)
}
.disabled(!session.transcriptSegments.contains(where: { !$0.isEcho }))
+ Button("Export Transcript (Markdown)…", systemImage: "doc.richtext") {
+ self.onExportTranscript(session, .markdown)
+ }
+ .disabled(!session.transcriptSegments.contains(where: { !$0.isEcho }))
Button("Export Transcript (JSON)…", systemImage: "curlybraces") {
self.onExportTranscript(session, .json)
}
diff --git a/Sources/Fluid/UI/SettingsSearch.swift b/Sources/Fluid/UI/SettingsSearch.swift
index 27cdb43e..b00edb51 100644
--- a/Sources/Fluid/UI/SettingsSearch.swift
+++ b/Sources/Fluid/UI/SettingsSearch.swift
@@ -12,6 +12,7 @@ import SwiftUI
enum SettingsSearchTarget: Hashable {
case general
+ case fluidMeet
case launchAtStartup
case showWindowAtLogin
case dockVisibility
@@ -29,6 +30,7 @@ enum SettingsSearchTarget: Hashable {
case editModeShortcut
case cancelRecordingShortcut
case pasteLastTranscriptionShortcut
+ case meetingRecordingShortcut
case activationMode
case holdSpaceToMute
case copyToClipboard
@@ -74,6 +76,9 @@ enum SettingsSearchTarget: Hashable {
var section: SettingsSection {
switch self {
+ case .fluidMeet:
+ return .fluidMeet
+
case .general,
.launchAtStartup,
.showWindowAtLogin,
@@ -106,7 +111,7 @@ enum SettingsSearchTarget: Hashable {
case .shortcuts, .globalHotkey, .primaryDictationShortcuts,
.commandModeShortcut, .editModeShortcut, .cancelRecordingShortcut,
- .pasteLastTranscriptionShortcut:
+ .pasteLastTranscriptionShortcut, .meetingRecordingShortcut:
return .shortcuts
case .notifications, .aiEnhancementFailures, .microphoneChanges, .pasteCheck:
@@ -137,6 +142,7 @@ extension SettingsSection {
var searchTarget: SettingsSearchTarget {
switch self {
case .general: return .general
+ case .fluidMeet: return .fluidMeet
case .dictation: return .dictation
case .dictationFormatting: return .textFormatting
case .shortcuts: return .shortcuts
@@ -191,6 +197,7 @@ struct SettingsSearchAvailability {
.editModeShortcut,
.cancelRecordingShortcut,
.pasteLastTranscriptionShortcut,
+ .meetingRecordingShortcut,
.activationMode,
.copyToClipboard,
.textInsertionMode,
@@ -230,6 +237,7 @@ enum SettingsSearchIndex {
}
private static let entries: [Entry] = [
+ .init(target: .fluidMeet, title: "Enable FluidMeet", terms: ["meetings recording disable turn off enable"]),
.init(target: .general, title: "General", terms: ["app settings preferences"]),
.init(
target: .launchAtStartup,
@@ -288,6 +296,7 @@ enum SettingsSearchIndex {
title: "Paste Last Transcription",
terms: ["reinsert recent text hotkey shortcut clipboard"]
),
+ .init(target: .meetingRecordingShortcut, title: "Meeting Recording Shortcut", terms: ["FluidMeet start stop toggle recording keyboard hotkey"]),
.init(target: .activationMode, title: "Activation Mode", terms: ["hold toggle hotkey behavior"]),
.init(target: .holdSpaceToMute, title: "Hold Space to Mute", terms: ["pause microphone spacebar resume silence"]),
.init(
diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift
index 0e7c93b2..ce922c29 100644
--- a/Sources/Fluid/UI/SettingsView.swift
+++ b/Sources/Fluid/UI/SettingsView.swift
@@ -67,6 +67,11 @@ struct SettingsView: View {
@State private var showDetailedAnalyticsConfirmation: Bool = false
@State private var rollbackVersion: String = ""
@State private var isRollingBack: Bool = false
+ @State private var rollbackOptions: [SimpleUpdater.ReleaseBuildOption] = []
+ @State private var selectedRollbackVersion = ""
+ @State private var isLoadingRollbackVersions = false
+ @State private var rollbackVersionsLoaded = false
+ @State private var rollbackVersionsError: String?
@State private var audioHistoryBudgetText: String = Self.audioBudgetText(for: SettingsStore.shared.audioHistoryBudgetGB)
@State private var audioHistoryUsageBytes: Int64 = 0
@State private var draggedMicrophoneUID: String?
@@ -221,6 +226,29 @@ struct SettingsView: View {
searchScrollRequest: self.searchScrollRequest
) {
VStack(spacing: 16) {
+ if self.selectedSection == .fluidMeet {
+ ThemedCard(style: .standard) {
+ VStack(alignment: .leading, spacing: 14) {
+ Label("FluidMeet Settings", systemImage: "person.2")
+ .font(.fluidSystem(.headline))
+ .foregroundStyle(.primary)
+ self.settingsToggleRow(
+ title: "Enable FluidMeet",
+ description: "Record and transcribe meetings. Turning this off hides FluidMeet and stops meeting detection. Your saved meetings are kept.",
+ footnote: self.appServices.hasActiveMeetingSessionActivity
+ ? "Finish the current meeting recording or processing before turning FluidMeet off." : nil,
+ isOn: Binding(
+ get: { self.settings.fluidMeetEnabled },
+ set: { self.appServices.setFluidMeetEnabled($0) }
+ )
+ )
+ .disabled(self.appServices.hasActiveMeetingSessionActivity)
+ }
+ .padding(16)
+ }
+ .settingsSearchTarget(.fluidMeet)
+ }
+
// App Settings Card
ThemedCard(style: .standard) {
VStack(alignment: .leading, spacing: 14) {
@@ -377,205 +405,15 @@ struct SettingsView: View {
.frame(width: 150)
}
}
-
- Divider().opacity(0.2)
-
- // Automatic Updates
- VStack(alignment: .leading, spacing: 6) {
- HStack(alignment: .center) {
- VStack(alignment: .leading, spacing: 2) {
- Text("Automatic Updates")
- .font(self.theme.typography.bodyStrong)
- .foregroundStyle(self.settingsTitleText)
- Text("Check for updates automatically once per hour")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
- }
-
- Spacer()
-
- Toggle("", isOn: Binding(
- get: { SettingsStore.shared.autoUpdateCheckEnabled },
- set: { SettingsStore.shared.autoUpdateCheckEnabled = $0 }
- ))
- .toggleStyle(.switch)
- .tint(self.theme.palette.accent)
- .labelsHidden()
- }
-
- HStack(alignment: .center) {
- VStack(alignment: .leading, spacing: 2) {
- Text("Show update pop-ups")
- .font(self.theme.typography.bodyStrong)
- .foregroundStyle(self.settingsTitleText)
- Text("When off, available updates appear in the top bar and here without automatic pop-ups.")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
- }
-
- Spacer()
-
- Toggle("Show update pop-ups", isOn: Binding(
- get: { self.settings.showUpdatePopups },
- set: { self.settings.showUpdatePopups = $0 }
- ))
- .toggleStyle(.switch)
- .tint(self.theme.palette.accent)
- .labelsHidden()
- }
-
- HStack(alignment: .center) {
- VStack(alignment: .leading, spacing: 2) {
- Text("Beta Releases")
- .font(self.theme.typography.bodyStrong)
- .foregroundStyle(self.settingsTitleText)
- Text("Opt in to preview builds that may be unstable")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
- }
-
- Spacer()
-
- Toggle("", isOn: Binding(
- get: { SettingsStore.shared.betaReleasesEnabled },
- set: { SettingsStore.shared.betaReleasesEnabled = $0 }
- ))
- .toggleStyle(.switch)
- .tint(self.theme.palette.accent)
- .labelsHidden()
- }
-
- if SettingsStore.shared.betaReleasesEnabled {
- Text("Beta opt-in enabled. Update checks include both stable and beta builds.")
- .font(.fluidSystem(.caption))
- .foregroundStyle(self.theme.palette.warning)
- }
-
- if let lastCheck = SettingsStore.shared.lastUpdateCheckDate {
- Text("Last checked: \(lastCheck.formatted(date: .abbreviated, time: .shortened))")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
- }
-
- Text("Current version: \(self.currentAppVersion)")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
- }
- .settingsSearchTarget(.automaticUpdates)
-
- if let version = self.updater.availableUpdateVersion {
- HStack(spacing: 10) {
- Label("Update available · \(version)", systemImage: "arrow.down.circle")
- .font(self.theme.typography.bodyStrong)
- .foregroundStyle(self.theme.palette.accent)
- .fixedSize(horizontal: false, vertical: true)
-
- Spacer()
-
- Button("View Update") {
- self.updater.showAvailableUpdate()
- }
- .fluidOutlinedButton()
- .controlSize(.regular)
- .disabled(self.updater.isCheckingForUpdates || self.updater.isUpdateInProgress)
- }
- }
-
- // Update Buttons
- HStack(spacing: 10) {
- Button("Check for Updates") {
- self.updater.checkForUpdatesManually()
- }
- .buttonStyle(.borderedProminent)
- .tint(self.theme.palette.accent)
- .controlSize(.regular)
- .disabled(self.updater.isCheckingForUpdates || self.updater.isUpdateInProgress)
-
- Button("Release Notes") {
- if let url = URL(string: "https://github.com/altic-dev/Fluid-oss/releases") {
- NSWorkspace.shared.open(url)
- }
- }
- .fluidOutlinedButton()
- .controlSize(.regular)
-
- Button(self.rollbackVersion.isEmpty ? "Rollback" : "Rollback to \(self.rollbackVersion)") {
- guard !self.isRollingBack else { return }
-
- let infoText = self.rollbackVersion.isEmpty ? "your previously installed version" : self.rollbackVersion
- let targetVersion = self.rollbackVersion
- let confirm = NSAlert()
- confirm.messageText = "Rollback to \(infoText)?"
- confirm.informativeText = "This will restore a previous app version and relaunch FluidVoice."
- confirm.alertStyle = .warning
- confirm.addButton(withTitle: "Rollback")
- confirm.addButton(withTitle: "Cancel")
-
- guard confirm.runModal() == .alertFirstButtonReturn else { return }
-
- self.isRollingBack = true
- Task {
- defer {
- Task { @MainActor in
- self.isRollingBack = false
- }
- }
-
- do {
- try await SimpleUpdater.shared.rollbackToLatestBackup()
- await MainActor.run {
- let success = NSAlert()
- success.messageText = "Rollback Successful"
- success.informativeText = "Rolled back to \(targetVersion). FluidVoice will relaunch shortly."
- success.alertStyle = .informational
- success.addButton(withTitle: "Report Bug")
- success.addButton(withTitle: "OK")
- let response = success.runModal()
- if response == .alertFirstButtonReturn {
- self.openIssueReportingPage()
- }
- }
- } catch {
- await MainActor.run {
- let fail = NSAlert()
- fail.messageText = "Rollback Failed"
- fail.informativeText = error.localizedDescription
- fail.alertStyle = .critical
- fail.addButton(withTitle: "OK")
- fail.runModal()
- self.refreshRollbackState()
- }
- }
- }
- }
- .fluidOutlinedButton()
- .controlSize(.regular)
- .disabled(self.rollbackVersion.isEmpty || self.isRollingBack)
- .opacity(self.isRollingBack ? 0.7 : 1.0)
-
- Button("Get Previous Builds") {
- self.openPreviousBuildPicker()
- }
- .fluidOutlinedButton()
- .controlSize(.regular)
- }
- .padding(.top, 12)
-
- if self.rollbackVersion.isEmpty {
- Text("No rollback backup found.")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
- } else {
- Text("Rollback target: \(self.rollbackVersion)")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
- }
}
}
.padding(16)
}
.shownInSettingsSection(.general, selectedSection: self.selectedSection)
+ self.updateSettingsCard
+ .shownInSettingsSection(.general, selectedSection: self.selectedSection)
+
if self.asr.micStatus != .authorized {
// Only surface microphone permission when the user needs to act.
ThemedCard(style: .standard) {
@@ -799,6 +637,33 @@ struct SettingsView: View {
}
)
.settingsSearchTarget(.pasteLastTranscriptionShortcut)
+ self.shortcutGroupDivider
+ self.shortcutRow(
+ content: .init(
+ icon: "person.2.wave.2",
+ iconColor: .secondary,
+ title: "Start / Stop Meeting",
+ description: self.settings.fluidMeetEnabled
+ ? "Toggle meeting recording using your saved source"
+ : "Enable FluidMeet in FluidMeet Settings to use this shortcut"
+ ),
+ shortcut: self.settings.meetingRecordingHotkeyShortcut,
+ isRecording: self.isRecording(.meeting),
+ isAnyRecordingActive: self.isRecordingAnyShortcut,
+ recordingMessage: self.isRecording(.meeting) ? self.shortcutRecordingMessage : nil,
+ onChangePressed: {
+ self.shortcutRecordingMessage = nil
+ self.activeShortcutRecordingTarget = .meeting
+ },
+ onRemovePressed: {
+ if self.activeShortcutRecordingTarget == .meeting {
+ self.shortcutRecordingMessage = nil
+ self.activeShortcutRecordingTarget = nil
+ }
+ self.settings.meetingRecordingHotkeyShortcut = nil
+ }
+ )
+ .settingsSearchTarget(.meetingRecordingShortcut)
}
}
}
@@ -1671,10 +1536,6 @@ struct SettingsView: View {
}
}
- private func refreshRollbackState() {
- self.rollbackVersion = SimpleUpdater.shared.latestRollbackVersion() ?? ""
- }
-
private func openIssueReportingPage() {
guard let url = URL(string: "https://github.com/altic-dev/Fluid-oss/issues/new/choose") else { return }
NSWorkspace.shared.open(url)
@@ -1862,57 +1723,6 @@ struct SettingsView: View {
alert.runModal()
}
- private func openPreviousBuildPicker() {
- Task { @MainActor in
- do {
- let options = try await SimpleUpdater.shared.fetchRecentReleaseBuildOptions(
- owner: "altic-dev",
- repo: "Fluid-oss",
- limit: 3,
- includePrerelease: SettingsStore.shared.betaReleasesEnabled
- )
- self.presentPreviousBuildPicker(options)
- } catch {
- self.openAllReleasesPage()
- }
- }
- }
-
- private func presentPreviousBuildPicker(_ options: [SimpleUpdater.ReleaseBuildOption]) {
- guard !options.isEmpty else {
- self.openAllReleasesPage()
- return
- }
-
- let picker = NSAlert()
- picker.messageText = "Download Previous Build"
- picker.informativeText = "No local rollback backup was found. Choose a recent release build:"
- picker.alertStyle = .informational
-
- for option in options {
- picker.addButton(withTitle: option.version)
- }
- picker.addButton(withTitle: "All Releases")
- picker.addButton(withTitle: "Cancel")
-
- let response = picker.runModal()
- let first = NSApplication.ModalResponse.alertFirstButtonReturn.rawValue
- let index = response.rawValue - first
-
- if index >= 0, index < options.count {
- NSWorkspace.shared.open(options[index].url)
- return
- }
- if index == options.count {
- self.openAllReleasesPage()
- }
- }
-
- private func openAllReleasesPage() {
- guard let url = URL(string: "https://github.com/altic-dev/Fluid-oss/releases") else { return }
- NSWorkspace.shared.open(url)
- }
-
private func applyAnalyticsConsentChange(_ enabled: Bool) {
SettingsStore.shared.shareDetailedAnalytics = enabled
AnalyticsService.shared.setDetailedAnalyticsEnabled(enabled)
@@ -2708,11 +2518,12 @@ private extension SettingsView {
case .general:
self.refreshRollbackState()
self.settings.refreshLaunchAtStartupStatus(clearError: true, logMismatch: false)
+ await self.loadRollbackVersions()
case .audio:
await self.prepareAudioSettings()
case .dictation:
await self.refreshAudioHistoryUsageInBackground()
- case .dictationFormatting, .shortcuts, .notifications, .overlay, .dataAndDiagnostics, .experimental:
+ case .fluidMeet, .dictationFormatting, .shortcuts, .notifications, .overlay, .dataAndDiagnostics, .experimental:
break
}
}
@@ -3198,3 +3009,276 @@ private extension SettingsView {
.settingsSearchTarget(.returnToStartingField)
}
}
+
+private extension SettingsView {
+ private var updateSettingsCard: some View {
+ ThemedCard(style: .standard) {
+ VStack(alignment: .leading, spacing: 14) {
+ Label("Updates", systemImage: "arrow.triangle.2.circlepath")
+ .font(.fluidSystem(.headline))
+ .foregroundStyle(.primary)
+
+ ViewThatFits(in: .horizontal) {
+ HStack(alignment: .center, spacing: 24) {
+ self.installedVersionLabel.fixedSize()
+ Spacer(minLength: 16)
+ self.updateActions
+ }
+ VStack(alignment: .leading, spacing: 14) {
+ self.installedVersionLabel
+ self.updateActions
+ }
+ }
+
+ self.updateCheckStatus
+
+ Divider().opacity(0.2)
+ VStack(spacing: 16) {
+ self.settingsToggleRow(
+ title: "Automatic update checks",
+ description: "Check for new versions once per hour.",
+ isOn: Binding(
+ get: { self.settings.autoUpdateCheckEnabled },
+ set: { self.settings.autoUpdateCheckEnabled = $0 }
+ )
+ )
+ Divider().opacity(0.2)
+ self.settingsToggleRow(
+ title: "Show update pop-ups",
+ description: "When off, available updates still appear in the top bar and here.",
+ isOn: Binding(
+ get: { self.settings.showUpdatePopups },
+ set: { self.settings.showUpdatePopups = $0 }
+ )
+ )
+ Divider().opacity(0.2)
+ self.settingsToggleRow(
+ title: "Include beta releases",
+ description: "Check stable and preview releases. Preview builds may be less reliable.",
+ isOn: Binding(
+ get: { self.settings.betaReleasesEnabled },
+ set: { self.settings.betaReleasesEnabled = $0 }
+ )
+ )
+ }
+
+ Divider().opacity(0.2)
+ self.rollbackControls
+ }
+ .frame(maxWidth: .infinity, alignment: .leading)
+ .padding(16)
+ }
+ .settingsSearchTarget(.automaticUpdates)
+ }
+
+ private var installedVersionLabel: some View {
+ VStack(alignment: .leading, spacing: 2) {
+ Text("Version \(self.currentAppVersion)")
+ .font(self.theme.typography.bodyStrong)
+ .foregroundStyle(self.settingsTitleText)
+ .textSelection(.enabled)
+ Text(self.lastUpdateCheckLabel)
+ .font(self.theme.typography.bodySmall)
+ .foregroundStyle(self.settingsSecondaryText)
+ }
+ .accessibilityElement(children: .combine)
+ }
+
+ private var lastUpdateCheckLabel: String {
+ self.settings.lastUpdateCheckDate.map {
+ "Checked " + $0.formatted(.dateTime.month(.abbreviated).day().hour().minute())
+ } ?? "Not checked yet"
+ }
+
+ private var updateActions: some View {
+ FluidGlassControlGroup {
+ HStack(spacing: 8) {
+ Button(self.updateActionTitle) {
+ if self.updater.availableUpdateVersion != nil {
+ self.updater.showAvailableUpdate()
+ } else {
+ self.updater.checkForUpdatesManually()
+ }
+ }
+ .fluidGlassAction(prominent: true)
+ .disabled(self.updater.isCheckingForUpdates || self.updater.isUpdateInProgress || self.isRollingBack)
+
+ if self.updater.availableUpdateVersion != nil {
+ Button("Check Again") { self.updater.checkForUpdatesManually() }
+ .fluidGlassAction()
+ .disabled(self.updater.isCheckingForUpdates || self.updater.isUpdateInProgress || self.isRollingBack)
+ }
+
+ Button("Release Notes") {
+ if let url = URL(string: "https://github.com/altic-dev/Fluid-oss/releases") {
+ NSWorkspace.shared.open(url)
+ }
+ }
+ .fluidGlassAction()
+ }
+ }
+ }
+
+ private var updateActionTitle: String {
+ if self.updater.isUpdateInProgress { return "Installing…" }
+ if self.updater.isCheckingForUpdates { return "Checking…" }
+ return self.updater.availableUpdateVersion == nil ? "Check for Updates" : "View Update"
+ }
+
+ @ViewBuilder private var updateCheckStatus: some View {
+ if self.updater.isCheckingForUpdates || self.updater.isUpdateInProgress {
+ HStack(spacing: 8) {
+ ProgressView().controlSize(.small)
+ Text(self.updater.isUpdateInProgress ? "Installing an update…" : "Checking for updates…")
+ .font(self.theme.typography.bodySmall)
+ }
+ } else if let version = self.updater.availableUpdateVersion {
+ Label("Version \(version) is available", systemImage: "arrow.down.circle")
+ .font(self.theme.typography.bodySmallStrong)
+ .foregroundStyle(self.theme.palette.accent)
+ .fixedSize(horizontal: false, vertical: true)
+ }
+ }
+
+ private var rollbackControls: some View {
+ VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) {
+ FluidManagementRow(title: "Roll back", detail: "Choose an earlier version. FluidVoice will restart.") {
+ HStack(spacing: self.theme.metrics.spacing.sm) {
+ FluidDropdownPicker(
+ "Rollback version",
+ selectedTitle: self.selectedRollbackTitle,
+ selection: self.$selectedRollbackVersion
+ ) {
+ Text("Choose a version").tag("")
+ if !self.rollbackVersion.isEmpty {
+ Text("\(self.rollbackVersion) · Saved on this Mac").tag("local")
+ }
+ ForEach(self.rollbackOptions, id: \.version) { option in
+ Text(option.version).tag(option.version)
+ }
+ }
+ .fluidDropdownStyle()
+ .disabled(self.isRollingBack || self.updater.isUpdateInProgress)
+ Button(self.isRollingBack ? "Rolling Back…" : "Roll Back") {
+ self.restorePreviousVersion()
+ }
+ .fluidGlassAction()
+ .disabled(self.selectedRollbackVersion.isEmpty || self.isRollingBack || self.updater.isUpdateInProgress)
+ }
+ }
+ if self.isLoadingRollbackVersions {
+ HStack(spacing: self.theme.metrics.spacing.sm) {
+ ProgressView().controlSize(.mini)
+ Text("Loading versions…").font(self.theme.typography.caption)
+ }
+ .foregroundStyle(self.settingsSecondaryText)
+ } else if let error = self.rollbackVersionsError {
+ HStack {
+ Text(error).font(self.theme.typography.caption)
+ .foregroundStyle(self.settingsSecondaryText)
+ Button("Try Again") { Task { await self.loadRollbackVersions() } }
+ .buttonStyle(.link)
+ }
+ } else if self.rollbackOptions.isEmpty, self.rollbackVersion.isEmpty {
+ Text("No earlier versions are available.")
+ .font(self.theme.typography.caption)
+ .foregroundStyle(self.settingsSecondaryText)
+ }
+ }
+ }
+
+ private var selectedRollbackTitle: String {
+ if self.selectedRollbackVersion == "local" { return "\(self.rollbackVersion) · Saved on this Mac" }
+ return self.selectedRollbackVersion.isEmpty ? "Choose a version" : self.selectedRollbackVersion
+ }
+
+ private func loadRollbackVersions() async {
+ guard !self.isLoadingRollbackVersions, !self.rollbackVersionsLoaded else { return }
+ self.isLoadingRollbackVersions = true
+ self.rollbackVersionsError = nil
+ defer { self.isLoadingRollbackVersions = false }
+ do {
+ let options = try await self.updater.fetchRecentReleaseBuildOptions(
+ owner: "altic-dev",
+ repo: "Fluid-oss",
+ limit: 12,
+ includePrerelease: true,
+ olderThanCurrent: true
+ )
+ try Task.checkCancellation()
+ self.rollbackOptions = options.filter {
+ $0.version.trimmingCharacters(in: CharacterSet(charactersIn: "v")) != self.rollbackVersion
+ }
+ self.rollbackVersionsLoaded = true
+ } catch {
+ self.rollbackVersionsError = self.rollbackVersion.isEmpty
+ ? "Couldn't load versions. Check your connection and try again."
+ : "Couldn't load online versions. Your saved version is still available."
+ }
+ }
+
+ private func restorePreviousVersion() {
+ guard !self.isRollingBack, !self.updater.isUpdateInProgress,
+ !self.selectedRollbackVersion.isEmpty else { return }
+ let useLocalBackup = self.selectedRollbackVersion == "local"
+ let targetVersion = useLocalBackup ? self.rollbackVersion : self.selectedRollbackVersion
+ guard !targetVersion.isEmpty else { return }
+ let infoText = targetVersion
+ let confirm = NSAlert()
+ confirm.messageText = "Roll back to \(infoText)?"
+ confirm.informativeText = "This will restore a previous app version and relaunch FluidVoice."
+ confirm.alertStyle = .warning
+ confirm.addButton(withTitle: "Roll Back")
+ confirm.addButton(withTitle: "Cancel")
+
+ guard confirm.runModal() == .alertFirstButtonReturn else { return }
+
+ self.isRollingBack = true
+ Task {
+ defer {
+ Task { @MainActor in
+ self.isRollingBack = false
+ }
+ }
+
+ do {
+ if useLocalBackup {
+ try await self.updater.rollbackToLatestBackup(expectedVersion: targetVersion)
+ } else {
+ try await self.updater.installPreviousRelease(version: targetVersion)
+ }
+ await MainActor.run {
+ let success = NSAlert()
+ success.messageText = "Rollback Successful"
+ success.informativeText = "Rolled back to \(targetVersion). FluidVoice will relaunch shortly."
+ success.alertStyle = .informational
+ success.addButton(withTitle: "Report Bug")
+ success.addButton(withTitle: "OK")
+ let response = success.runModal()
+ if response == .alertFirstButtonReturn {
+ self.openIssueReportingPage()
+ }
+ }
+ } catch {
+ await MainActor.run {
+ let fail = NSAlert()
+ fail.messageText = "Rollback Failed"
+ fail.informativeText = error.localizedDescription
+ fail.alertStyle = .critical
+ fail.addButton(withTitle: "OK")
+ fail.runModal()
+ self.refreshRollbackState()
+ }
+ }
+ }
+ }
+
+ private func refreshRollbackState() {
+ self.rollbackVersion = SimpleUpdater.shared.latestRollbackVersion() ?? ""
+ if self.selectedRollbackVersion.isEmpty, !self.rollbackVersion.isEmpty {
+ self.selectedRollbackVersion = "local"
+ } else if self.selectedRollbackVersion == "local", self.rollbackVersion.isEmpty {
+ self.selectedRollbackVersion = ""
+ }
+ }
+}
diff --git a/Tests/DictionaryShortcutTestStateTests.swift b/Tests/DictionaryShortcutTestStateTests.swift
index ba50d057..e62add42 100644
--- a/Tests/DictionaryShortcutTestStateTests.swift
+++ b/Tests/DictionaryShortcutTestStateTests.swift
@@ -7,11 +7,10 @@ enum DictionaryShortcutTestStateTests {
precondition(condition(), message)
}
for count in [0, 1, 2, 3] {
- expect(DictionaryCaptureAction.title(active: false, failed: true, count: count) == "Record again", "Every failed capture offers retry")
- expect(DictionaryCaptureAction.title(active: true, failed: true, count: count) == "Stop recording", "Active capture retains Stop")
+ expect(DictionaryCaptureAction.title(failed: true, count: count) == "Record again", "Every failed capture offers retry")
}
- expect(DictionaryCaptureAction.title(active: false, failed: false, count: 0) == "Record", "First capture")
- expect(DictionaryCaptureAction.title(active: false, failed: false, count: 1) == "Record more", "Additional successful capture")
+ expect(DictionaryCaptureAction.title(failed: false, count: 0) == "Record", "First capture")
+ expect(DictionaryCaptureAction.title(failed: false, count: 1) == "Record more", "Additional successful capture")
var test = DictionaryShortcutTestState()
test.receive("ModelOpt", word: "ModelOpt")
diff --git a/Tests/FluidASRBaselineTests/MeetingLiveP0DiagnosticsTests.swift b/Tests/FluidASRBaselineTests/MeetingLiveP0DiagnosticsTests.swift
index 2ec177f5..e4c5e650 100644
--- a/Tests/FluidASRBaselineTests/MeetingLiveP0DiagnosticsTests.swift
+++ b/Tests/FluidASRBaselineTests/MeetingLiveP0DiagnosticsTests.swift
@@ -21,6 +21,7 @@ final class MeetingLiveP0DiagnosticsTests: XCTestCase {
func prepareModels() async throws { XCTFail("P0 deterministic tests must not load models") }
func appendAudio(_ buffer: AVAudioPCMBuffer) async throws {}
func processBufferedAudio() async throws { self.processed += 1; try self.process() }
+ func finish() async throws -> String { "" }
func reset() async { self.resets += 1 }
func setEouCallback(_ callback: @escaping @Sendable (String) -> Void) async {}
func setPartialTranscriptCallback(_ callback: @escaping @Sendable (String) -> Void) async {}
@@ -32,8 +33,8 @@ final class MeetingLiveP0DiagnosticsTests: XCTestCase {
sampleRate: 16_000,
channels: 1,
interleaved: false
- // Fixed test fixture: missing required audio storage or evidence is a setup failure.
- // swiftlint:disable:next force_unwrapping
+ // Fixed test fixture: missing required audio storage or evidence is a setup failure.
+ // swiftlint:disable:next force_unwrapping
)!
// Fixed test fixture: missing required audio storage or evidence is a setup failure.
// swiftlint:disable:next force_unwrapping
@@ -71,7 +72,7 @@ final class MeetingLiveP0DiagnosticsTests: XCTestCase {
XCTAssertEqual(resets, 0)
}
- func testForcedBoundaryThen38SecondsWithoutPartialDoesNotRecover() async {
+ func testLongPhraseDoesNotResetWhileTextGapIsBelowTwoMinutes() async {
let clock = Clock()
let recognizer = Recognizer()
let engine = MeetingLiveTrackEngine(
@@ -82,24 +83,23 @@ final class MeetingLiveP0DiagnosticsTests: XCTestCase {
)
await engine.diagnosticConsume(self.sample(at: 0))
await engine.diagnosticPartial("synthetic fixture")
- // Advance audio time while keeping wall-clock partial stall below its threshold, so
- // the hard duration boundary (not a quiet timeout) initiates the reset.
+ // Advancing audio time alone must not reset an active phrase.
clock.set(1)
await engine.diagnosticConsume(self.sample(at: 21))
var snapshot = await engine.diagnosticSnapshotValue()
- XCTAssertEqual(snapshot.resetInvocations, 1)
+ XCTAssertEqual(snapshot.resetInvocations, 0)
for index in 1...380 {
clock.set(1 + Double(index) / 10)
await engine.diagnosticConsume(self.sample(at: 21 + Double(index) / 10))
}
snapshot = await engine.diagnosticSnapshotValue()
XCTAssertEqual(snapshot.processCompletions, 382)
- XCTAssertEqual(snapshot.resetInvocations, 1, "Continued decode completion does not restart the watchdog")
+ XCTAssertEqual(snapshot.resetInvocations, 0, "The text gap is still below the recovery timeout")
XCTAssertNil(snapshot.firstPartialAfterResetSeconds)
await engine.diagnosticPartial("synthetic resumed fixture")
snapshot = await engine.diagnosticSnapshotValue()
- XCTAssertEqual(snapshot.firstPartialAfterResetSeconds ?? -1, 38, accuracy: 0.0001)
- XCTAssertEqual(snapshot.wrapperGeneration, 1)
+ XCTAssertNil(snapshot.firstPartialAfterResetSeconds)
+ XCTAssertEqual(snapshot.wrapperGeneration, 0)
}
func testDiagnosticsCanBeExplicitlyDisabled() async {
diff --git a/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift b/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift
index 76e84987..3243f584 100644
--- a/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift
+++ b/Tests/FluidDictationIntegrationTests/CustomDictionaryManualEntryTests.swift
@@ -23,51 +23,166 @@ final class CustomDictionaryManualEntryTests: XCTestCase {
super.tearDown()
}
- func testIncompletePronunciationNoticeRendersWithoutRecordingOrSaving() async throws {
+ func testTrainingRestartWaitsBeforeStartingAndClearsWaitingState() async {
+ let restart = DictionaryTrainingRestart()
+ let started = expectation(description: "Next example starts after the break")
+ var calls = 0
+ let notBefore = ContinuousClock.now.advanced(by: .milliseconds(80))
+ restart.schedule(delay: .milliseconds(80), canContinue: { true }) {
+ XCTAssertGreaterThanOrEqual(ContinuousClock.now, notBefore)
+ calls += 1
+ started.fulfill()
+ }
+ XCTAssertTrue(restart.isWaiting)
+ XCTAssertEqual(calls, 0)
+ await fulfillment(of: [started], timeout: 1)
+ XCTAssertEqual(calls, 1)
+ XCTAssertFalse(restart.isWaiting)
+ }
+
+ func testCancelledAndSupersededTrainingRestartsCannotStartCapture() async {
+ let restart = DictionaryTrainingRestart()
+ let stale = expectation(description: "Cancelled examples cannot restart")
+ stale.isInverted = true
+ for _ in 0..<10 {
+ restart.schedule(delay: .milliseconds(30), canContinue: { true }) { stale.fulfill() }
+ }
+ restart.cancel()
+ XCTAssertFalse(restart.isWaiting)
+ let current = expectation(description: "Only the current example starts")
+ restart.schedule(delay: .milliseconds(20), canContinue: { true }) { current.fulfill() }
+ await fulfillment(of: [current, stale], timeout: 0.15)
+ XCTAssertFalse(restart.isWaiting)
+ }
+
+ func testTrainingRestartRechecksWhetherThePageStillWantsCapture() async {
+ let restart = DictionaryTrainingRestart()
+ var canContinue = true
+ let stopped = expectation(description: "Changed word cannot restart the old capture")
+ stopped.isInverted = true
+ restart.schedule(delay: .milliseconds(20), canContinue: { canContinue }) { stopped.fulfill() }
+ canContinue = false
+ await fulfillment(of: [stopped], timeout: 0.1)
+ XCTAssertFalse(restart.isWaiting)
+ }
+
+ func testSplitWordsRemainValidVoiceTrainingCorrections() async {
+ let result = await VoiceTrainingAliasFilter.filter(["fluid voice", "john turner", "there", "the"])
+ XCTAssertEqual(result.accepted, ["fluid voice", "john turner"])
+ XCTAssertEqual(result.rejected, ["there", "the"])
+ }
+
+ func testOnlyExplicitManualSpellingsBypassTheBasicWordGuard() async {
+ let result = await VoiceTrainingAliasFilter.filter(["the", "and", "fluid voice"], manuallyConfirmed: ["the", "absent"])
+ XCTAssertEqual(result.accepted, ["the", "fluid voice"])
+ XCTAssertEqual(result.rejected, ["and"])
+ let invalid = await VoiceTrainingAliasFilter.filter(["", String(repeating: "x", count: 161)], manuallyConfirmed: ["", String(repeating: "x", count: 161)])
+ XCTAssertTrue(invalid.accepted.isEmpty)
+ }
+
+ func testTrainingPauseAndVisibleSpellingsRenderWithoutInvokingActions() async throws {
+ let entries = SettingsStore.shared.customDictionaryEntries
var actions = 0
- for width in [420.0, 760.0] {
- for scheme in [ColorScheme.light, .dark] {
- let progress = DictionaryTrainingProgress(spellingCount: 3, pronunciationCount: 1, pronunciationEnabled: true)
- let view = DictionaryWordWizard(
- word: .constant("Palermo"),
- step: .review,
- count: 3,
- heard: "pal ermo",
- variants: ["pal ermo"],
- busy: false,
- recording: false,
- processing: false,
- starting: false,
- error: nil,
- voiceSupported: true,
- alreadyCorrect: false,
- savedWord: "",
- onContinue: { actions += 1 },
- onRecord: { actions += 1 },
- onSave: { actions += 1 },
- onBack: { actions += 1 },
- onNewWord: { actions += 1 },
- onManual: { actions += 1 },
- onPracticeMore: { actions += 1 },
- pronunciationNotice: progress.pronunciationNotice,
- pronunciationIncomplete: true
- )
- .padding(20).frame(width: width)
- .background(scheme == .dark ? Color(white: 0.12) : Color(white: 0.96))
- .appTheme(.adaptive(accent: FluidBrandColors.blue, colorScheme: scheme))
- .environment(\.colorScheme, scheme)
- let host = NSHostingView(rootView: view)
- let window = NSWindow(contentRect: NSRect(x: 0, y: 0, width: width, height: 1050), styleMask: [.titled], backing: .buffered, defer: false)
- window.contentView = host
- window.appearance = NSAppearance(named: scheme == .dark ? .darkAqua : .aqua)
- window.orderFrontRegardless()
- try await Task.sleep(for: .milliseconds(150))
- host.layoutSubtreeIfNeeded()
- let bitmap = try XCTUnwrap(host.bitmapImageRepForCachingDisplay(in: host.bounds))
- host.cacheDisplay(in: host.bounds, to: bitmap)
- let png = try XCTUnwrap(bitmap.representation(using: .png, properties: [:]))
- try png.write(to: URL(fileURLWithPath: "/tmp/dictionary-wizard-review-\(Int(width))-\(scheme).png"))
- window.orderOut(nil)
+ for phase in ["starting", "listening", "processing", "waiting"] {
+ for width in [420.0, 900.0] {
+ for scheme in [ColorScheme.light, .dark] {
+ let view = DictionaryWordWizard(
+ word: .constant(width < 500 ? "Supercalifragilisticexpialidocious" : "FluidVoice"),
+ step: .recording,
+ count: 1,
+ heard: "fluid voice",
+ variants: ["fluid voice", "fluid boys"],
+ busy: phase != "listening",
+ recording: phase == "listening" || phase == "starting",
+ processing: phase == "processing",
+ starting: phase == "starting",
+ error: nil,
+ voiceSupported: true,
+ alreadyCorrect: false,
+ savedWord: "",
+ onContinue: { actions += 1 },
+ onRecord: { actions += 1 },
+ onSave: { actions += 1 },
+ onBack: { actions += 1 },
+ onNewWord: { actions += 1 },
+ onManual: { actions += 1 },
+ onPracticeMore: { actions += 1 },
+ automaticCaptureActive: true,
+ waiting: phase == "waiting",
+ onPauseRecording: { actions += 1 },
+ onAddSpelling: { _ in actions += 1; return nil },
+ onRemoveSpelling: { _ in actions += 1 }
+ )
+ .padding(20).frame(width: width)
+ .background(scheme == .dark ? Color(white: 0.12) : Color(white: 0.96))
+ .appTheme(.adaptive(accent: FluidBrandColors.blue, colorScheme: scheme))
+ .environment(\.colorScheme, scheme)
+ let host = NSHostingView(rootView: view)
+ let window = NSWindow(contentRect: NSRect(x: 0, y: 0, width: width, height: 1100), styleMask: [.titled], backing: .buffered, defer: false)
+ window.contentView = host
+ window.appearance = NSAppearance(named: scheme == .dark ? .darkAqua : .aqua)
+ window.orderFrontRegardless()
+ try await Task.sleep(for: .milliseconds(100))
+ host.layoutSubtreeIfNeeded()
+ let bitmap = try XCTUnwrap(host.bitmapImageRepForCachingDisplay(in: host.bounds))
+ host.cacheDisplay(in: host.bounds, to: bitmap)
+ let png = try XCTUnwrap(bitmap.representation(using: .png, properties: [:]))
+ try png.write(to: URL(fileURLWithPath: "/tmp/dictionary-recording-\(phase)-\(Int(width))-\(scheme).png"))
+ window.orderOut(nil)
+ }
+ }
+ }
+ XCTAssertEqual(actions, 0)
+ XCTAssertEqual(SettingsStore.shared.customDictionaryEntries, entries)
+ }
+
+ func testReviewStatesRenderWithoutRecordingOrSaving() async throws {
+ var actions = 0
+ for pronunciationCount in [1, 3] {
+ for width in [420.0, 760.0] {
+ for scheme in [ColorScheme.light, .dark] {
+ let progress = DictionaryTrainingProgress(spellingCount: 3, pronunciationCount: pronunciationCount, pronunciationEnabled: true)
+ let view = DictionaryWordWizard(
+ word: .constant("Palermo"),
+ step: .review,
+ count: 3,
+ heard: "pal ermo",
+ variants: ["pal ermo"],
+ busy: false,
+ recording: false,
+ processing: false,
+ starting: false,
+ error: nil,
+ voiceSupported: true,
+ alreadyCorrect: false,
+ savedWord: "",
+ onContinue: { actions += 1 },
+ onRecord: { actions += 1 },
+ onSave: { actions += 1 },
+ onBack: { actions += 1 },
+ onNewWord: { actions += 1 },
+ onManual: { actions += 1 },
+ onPracticeMore: { actions += 1 },
+ pronunciationNotice: progress.pronunciationNotice,
+ pronunciationIncomplete: !progress.pronunciationReady
+ )
+ .padding(20).frame(width: width)
+ .background(scheme == .dark ? Color(white: 0.12) : Color(white: 0.96))
+ .appTheme(.adaptive(accent: FluidBrandColors.blue, colorScheme: scheme))
+ .environment(\.colorScheme, scheme)
+ let host = NSHostingView(rootView: view)
+ let window = NSWindow(contentRect: NSRect(x: 0, y: 0, width: width, height: 1050), styleMask: [.titled], backing: .buffered, defer: false)
+ window.contentView = host
+ window.appearance = NSAppearance(named: scheme == .dark ? .darkAqua : .aqua)
+ window.orderFrontRegardless()
+ try await Task.sleep(for: .milliseconds(150))
+ host.layoutSubtreeIfNeeded()
+ let bitmap = try XCTUnwrap(host.bitmapImageRepForCachingDisplay(in: host.bounds))
+ host.cacheDisplay(in: host.bounds, to: bitmap)
+ let png = try XCTUnwrap(bitmap.representation(using: .png, properties: [:]))
+ try png.write(to: URL(fileURLWithPath: "/tmp/dictionary-wizard-review-\(pronunciationCount)-\(Int(width))-\(scheme).png"))
+ window.orderOut(nil)
+ }
}
}
XCTAssertEqual(actions, 0)
@@ -133,6 +248,8 @@ final class CustomDictionaryManualEntryTests: XCTestCase {
for toggleDuringCapture in [false, true] {
DictionaryMatcherExperiment.setEnabled(toggleDuringCapture)
let monitor = DictionaryTrainingEndpointMonitor()
+ // Measure speech-end handling after the model is ready, as the word wizard does.
+ await monitor.prepare()
let stopped = expectation(description: "Speech ends independently of pronunciation")
var callbacks = 0
monitor.start(
@@ -285,8 +402,8 @@ final class CustomDictionaryManualEntryTests: XCTestCase {
} catch {}
}
- func testVoiceTrainingKeepsPronunciationWithoutEverydayTextAliases() async {
- let filtered = await VoiceTrainingAliasFilter.filter(["but", "now", "right now"]) { _ in .checked([]) }
+ func testVoiceTrainingKeepsPronunciationWithoutBasicTextAliases() async {
+ let filtered = await VoiceTrainingAliasFilter.filter(["a", "the", "and"])
let entries = CustomDictionaryTrainingMerge.mergedEntries(
current: [], replacement: "Lyft", triggers: filtered.accepted, savePronunciation: true
)
@@ -298,7 +415,7 @@ final class CustomDictionaryManualEntryTests: XCTestCase {
func testVoiceTrainingFilterPreservesExistingManualRules() async {
let manual = SettingsStore.CustomDictionaryEntry(triggers: ["now"], replacement: "Lyft")
let unrelated = SettingsStore.CustomDictionaryEntry(triggers: ["but"], replacement: "ManualChoice")
- let filtered = await VoiceTrainingAliasFilter.filter(["now", "hello"]) { _ in .checked([]) }
+ let filtered = await VoiceTrainingAliasFilter.filter(["an", "the"])
let entries = CustomDictionaryTrainingMerge.mergedEntries(
current: [manual, unrelated], replacement: "Lyft", triggers: filtered.accepted, savePronunciation: true
)
@@ -306,14 +423,23 @@ final class CustomDictionaryManualEntryTests: XCTestCase {
XCTAssertEqual(CustomDictionaryManualEntry.normalizedTriggers(["but", "now", "right now"]), ["but", "now", "right now"])
}
- func testUnavailableVocabularyCannotCreateTextOnlyTrainingRules() async {
- let filtered = await VoiceTrainingAliasFilter.filter(["now"]) { _ in .unavailable }
+ func testBasicWordsCannotCreateTextOnlyTrainingRules() async {
+ let filtered = await VoiceTrainingAliasFilter.filter(["the"])
let entries = CustomDictionaryTrainingMerge.mergedEntries(
current: [], replacement: "Lyft", triggers: filtered.accepted
)
XCTAssertTrue(entries.isEmpty)
}
+ func testVoiceTrainingCanSaveCorrectlySpelledWordsAndPhrasesWithoutPronunciation() async {
+ let filtered = await VoiceTrainingAliasFilter.filter(["hello", "right now", "fluid voice"])
+ let entries = CustomDictionaryTrainingMerge.mergedEntries(
+ current: [], replacement: "FluidVoice", triggers: filtered.accepted
+ )
+ XCTAssertEqual(entries.count, 1)
+ XCTAssertEqual(Set(entries.first?.triggers ?? []), Set(["hello", "right now", "fluid voice"]))
+ }
+
func testKeepsWhitespaceOnlyReplacements() {
XCTAssertEqual(CustomDictionaryManualEntry.sanitizedReplacement("\n"), "\n")
XCTAssertEqual(CustomDictionaryManualEntry.sanitizedReplacement(" "), " ")
@@ -836,3 +962,45 @@ extension CustomDictionaryManualEntryTests {
XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: root.path).count, 1)
}
}
+
+#if arch(arm64)
+extension CustomDictionaryManualEntryTests {
+ /// Opt-in local replay uses saved audio read-only; never changes the user's dictionary.
+ func testParakeetFamilySavedAudioReplay() async throws {
+ guard ProcessInfo.processInfo.environment["FV_DICTIONARY_FAMILY_REPLAY"] == "1" else {
+ throw XCTSkip("Set FV_DICTIONARY_FAMILY_REPLAY=1 to replay local dictionary audio")
+ }
+ let profiles = await PronunciationDictionaryStore.shared.allProfiles()
+ let source = try XCTUnwrap(profiles.first {
+ $0.modelKey == "parakeet-v2" && $0.hasTemporalRecordings && $0.enrollments.count >= 3
+ })
+ let audioID = try XCTUnwrap(source.enrollments.last?.inspectionID)
+ let audio = try await PronunciationDictionaryStore.shared.inspection(for: audioID)
+ let samples = Array(audio.samples.prefix(audio.recordedSampleCount))
+ var entryIDs = Set<UUID>()
+ let entries = profiles.filter { entryIDs.insert($0.dictionaryEntryID).inserted }.map {
+ SettingsStore.CustomDictionaryEntry(id: $0.dictionaryEntryID, triggers: [], replacement: $0.label)
+ }
+ let provider = FluidAudioProvider(
+ modelOverride: .fluidParakeetMini,
+ configureWordBoosting: false,
+ enhancementOptions: FluidAudioProviderEnhancementOptions(experimentalUnifiedFinalEnabled: false, pronunciationMatchingEnabled: true, customDictionaryEntries: entries)
+ )
+ try await provider.prepare()
+ await provider.pronunciationReferencePreparationTask?.value
+ let fixture = URL(fileURLWithPath: #filePath).deletingLastPathComponent().appendingPathComponent("Resources/dictation_fixture.wav")
+ let negativeSamples = try AudioConverter().resampleAudioFile(path: fixture.path)
+ DictionaryMatcherExperiment.setEnabled(false)
+ let baselinePositive = try await provider.transcribeFinal(samples)
+ let baselineNegative = try await provider.transcribeFinal(negativeSamples)
+ DictionaryMatcherExperiment.setEnabled(true)
+ provider.resetStreamingPreviewCache()
+ let positive = try await provider.transcribeFinal(samples)
+ print("FAMILY_REPLAY source=\(source.modelKey) target=\(provider.pronunciationModelKeyForTesting) expected=\(source.label) baseline=\(baselinePositive.text) output=\(positive.text)")
+ XCTAssertTrue(positive.text.localizedCaseInsensitiveContains(source.label))
+ let negative = try await provider.transcribeFinal(negativeSamples)
+ print("FAMILY_REPLAY negative=\(negative.text)")
+ XCTAssertEqual(negative.text, baselineNegative.text, "The complete saved dictionary must leave unrelated speech unchanged")
+ }
+}
+#endif
diff --git a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
index 6980a0e6..bc2ee62a 100644
--- a/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
+++ b/Tests/FluidDictationIntegrationTests/DictationE2ETests.swift
@@ -764,27 +764,13 @@ extension DictationE2ETests {
XCTAssertEqual(SettingsStore.shared.customDictionaryEntries, entries)
}
- func testDictionaryRingAudioResponseIsBoundedAndGeometryStaysInsideItsFrame() {
- XCTAssertEqual(DictionaryRingResponse.energy(for: 0), 0)
- XCTAssertEqual(DictionaryRingResponse.energy(for: -1), 0)
- XCTAssertEqual(DictionaryRingResponse.energy(for: .nan), 0)
- XCTAssertEqual(DictionaryRingResponse.energy(for: .infinity), 0)
- XCTAssertEqual(DictionaryRingResponse.energy(for: 2), 1)
- XCTAssertGreaterThan(DictionaryRingResponse.energy(for: 0.7), DictionaryRingResponse.energy(for: 0.1))
- for size in [176.0, 240.0] {
- let frame = CGRect(x: 0, y: 0, width: size, height: size)
- for tick in 0..<32 {
- let phase = Double(tick) / 32 * .pi * 2
- for sheet in 0..<DictionaryRibbon.sheets {
- for strand in -4...4 {
- let quiet = DictionaryRibbon(phase: phase, strand: Double(strand), energy: 0, sheet: sheet).path(in: frame)
- let speech = DictionaryRibbon(phase: phase, strand: Double(strand), energy: 1, sheet: sheet).path(in: frame)
- XCTAssertNotEqual(quiet, speech)
- XCTAssertTrue(frame.contains(speech.boundingRect), "Speech motion must not clip")
- }
- }
- }
- }
+ func testDictionaryGlowAudioResponseIsBoundedAndHandlesInvalidLevels() {
+ XCTAssertEqual(DictionaryGlowResponse.energy(for: 0), 0)
+ XCTAssertEqual(DictionaryGlowResponse.energy(for: -1), 0)
+ XCTAssertEqual(DictionaryGlowResponse.energy(for: .nan), 0)
+ XCTAssertEqual(DictionaryGlowResponse.energy(for: .infinity), 0)
+ XCTAssertEqual(DictionaryGlowResponse.energy(for: 2), 1)
+ XCTAssertGreaterThan(DictionaryGlowResponse.energy(for: 0.7), DictionaryGlowResponse.energy(for: 0.1))
}
func testDictionaryPlaygroundMatchesWholeWordsWithoutTreatingSimilarWordsAsSuccess() {
@@ -4010,7 +3996,8 @@ extension DictationE2ETests {
onNewWord: { actions += 1 },
onManual: { actions += 1 },
onPracticeMore: { actions += 1 },
- onRedo: { actions += 1 }
+ onRedo: { actions += 1 },
+ attempts: Array(["boot tea", "Booptie", "FluidVoice"].prefix(state.0))
)
.padding(24)
.frame(width: width, height: 950, alignment: .top)
@@ -4282,3 +4269,501 @@ extension DictationE2ETests {
}
#endif
}
+
+extension DictationE2ETests {
+ private func stampFrames(_ rows: [Int], hidden: Int = 8) -> DictionaryMatchFrames {
+ var values: [Float] = []
+ for row in rows {
+ var frame = [Float](repeating: 0.05, count: hidden)
+ frame[row] = 1
+ values += frame
+ }
+ return DictionaryMatchFrames(hiddenSize: hidden, values: values)
+ }
+
+ func testStampMatcherFindsWordSaidSlowerInsideSentence() {
+ let word = self.stampFrames([0, 1, 2, 3, 4, 5])
+ let sentence = self.stampFrames([6, 7, 6, 7, 6] + [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5] + [7, 6, 7, 6, 7])
+ let hits = DictionaryStampMatcher.hits(references: [word], in: sentence)
+ XCTAssertEqual(hits.count, 1)
+ // Either copy of a repeated frame is an equally exact match at the edges.
+ XCTAssertTrue((5...6).contains(hits.first?.frames.lowerBound ?? -1))
+ XCTAssertTrue((16...17).contains(hits.first?.frames.upperBound ?? -1))
+ XCTAssertLessThan(hits.first?.distance ?? 1, 0.05)
+ }
+
+ func testStampMatcherIgnoresSentenceWithoutTheWord() {
+ let word = self.stampFrames([0, 1, 2, 3, 4, 5])
+ let sentence = self.stampFrames([6, 7, 6, 7, 6, 7, 6, 7, 6, 7])
+ XCTAssertTrue(DictionaryStampMatcher.hits(references: [word], in: sentence).isEmpty)
+ }
+
+ func testStampMatcherFindsEachOccurrenceOnce() {
+ let word = self.stampFrames([0, 1, 2, 3])
+ let sentence = self.stampFrames([6, 0, 1, 2, 3, 7, 6, 7, 0, 1, 2, 3, 6])
+ let hits = DictionaryStampMatcher.hits(references: [word], in: sentence)
+ XCTAssertEqual(hits.map(\.frames).sorted { $0.lowerBound < $1.lowerBound }, [1..<5, 8..<12])
+ }
+
+ func testStampMatcherRejectsMismatchedOrInvalidInput() {
+ let word = self.stampFrames([0, 1, 2, 3])
+ XCTAssertTrue(DictionaryStampMatcher.hits(references: [word], in: self.stampFrames([0, 1, 2, 3], hidden: 9)).isEmpty)
+ XCTAssertTrue(DictionaryStampMatcher.hits(references: [], in: self.stampFrames([0, 1, 2, 3])).isEmpty)
+ XCTAssertTrue(DictionaryStampMatcher.hits(references: [word], in: DictionaryMatchFrames(hiddenSize: 8, values: [])).isEmpty)
+ }
+
+ func testWordRangeDropsSilenceAndKeepsFinalPop() {
+ let rate = 16_000
+ var samples = [Float](repeating: 0.0005, count: rate / 2)
+ samples += (0..<(rate * 6 / 10)).map { Float(sin(Double($0) * 0.2) * 0.3) }
+ samples += [Float](repeating: 0.0005, count: rate * 8 / 100)
+ samples += (0..<(rate * 3 / 100)).map { Float(sin(Double($0) * 1.3) * 0.05) }
+ samples += [Float](repeating: 0.0005, count: rate / 2)
+ // Recognizer word times are loose: they start early and end before the final pop.
+ let words = [DictionaryAudioInspection.Word(text: "Grasel", start: 0.40, end: 0.80), .init(text: "Wump", start: 0.80, end: 1.0)]
+ let range = DictionaryPronunciationExperiment.wordRange(samples, words: words)
+ XCTAssertNotNil(range)
+ let start = Double(range?.lowerBound ?? 0) / Double(rate)
+ let end = Double(range?.upperBound ?? 0) / Double(rate)
+ XCTAssertEqual(start, 0.47, accuracy: 0.02, "Leading silence is dropped")
+ XCTAssertGreaterThanOrEqual(end, 1.21, "The final pop after a short gap is kept")
+ XCTAssertLessThan(end, 1.30, "Trailing silence is dropped")
+ XCTAssertNil(DictionaryPronunciationExperiment.wordRange(samples, words: []))
+ }
+}
+
+extension DictationE2ETests {
+ func testSpokenFramesDropPausesInsideASlowlySpokenWord() {
+ let loud = (0..<1280).map { Float(sin(Double($0) * 0.3) * 0.2) }
+ let quiet = [Float](repeating: 0.0005, count: 1280)
+ let samples = loud + loud + quiet + quiet + loud + loud
+ XCTAssertEqual(DictionaryPronunciationExperiment.spokenFrames(samples, frames: 0..<6), [0, 1, 4, 5])
+ // Too little speech to trust: keep every frame rather than an almost empty reference.
+ XCTAssertEqual(DictionaryPronunciationExperiment.spokenFrames(loud + quiet + quiet, frames: 0..<3), [0, 1, 2])
+ }
+}
+
+extension DictationE2ETests {
+ func testTextClaimedCoversSavedSpellingsAndCorrectWords() {
+ let entry = SettingsStore.CustomDictionaryEntry(triggers: ["sneak them"], replacement: "Snigdham")
+ let words = ["What", "is", "sneak", "them?", "Snigdham", "said"]
+ XCTAssertEqual(DictionaryAudioSpan.textClaimed(words: words, entries: [entry]), [2, 3, 4])
+ XCTAssertTrue(DictionaryAudioSpan.textClaimed(words: words, entries: []).isEmpty)
+ }
+
+ func testAudioSpanReplacesWholeMishearingButNotNeighbours() {
+ let targets = ["Snigdham", "snick them"]
+ // The match touched only "them"; the run grows to the whole mishearing.
+ XCTAssertEqual(DictionaryAudioSpan.span(words: ["What", "do", "you", "mean", "by", "sneak", "them"], touched: [6], claimed: [], targets: targets), 5...6)
+ // The match spilled onto "is"; the run drops it and takes "Snick" instead.
+ XCTAssertEqual(DictionaryAudioSpan.span(words: ["Snick", "them", "is", "a", "weird", "name"], touched: [1, 2], claimed: [], targets: targets), 0...1)
+ // A new single-word spelling stays a single word.
+ XCTAssertEqual(DictionaryAudioSpan.span(words: ["Who", "the", "fuck", "is", "Nickdom?"], touched: [4], claimed: [], targets: targets), 4...4)
+ }
+
+ func testAudioSpanNeverTouchesWordsTheTextLayerOwns() {
+ let words = ["Snick", "them", "met", "Snick", "them"]
+ XCTAssertNil(DictionaryAudioSpan.span(words: words, touched: [3, 4], claimed: [3, 4], targets: ["Snigdham"]))
+ // The same word said twice: the text layer owns one, the audio match may still take the other.
+ XCTAssertEqual(DictionaryAudioSpan.span(words: words, touched: [0, 1], claimed: [3, 4], targets: ["Snigdham"]), 0...1)
+ XCTAssertNil(DictionaryAudioSpan.span(words: words, touched: [], claimed: [], targets: ["Snigdham"]))
+ }
+
+ #if arch(arm64)
+ func testRecordedWordMatchLeavesSavedSpellingsToTheTextLayer() {
+ let id = UUID()
+ let profile = PronunciationDictionaryProfile(dictionaryEntryID: id, label: "Snigdham", modelKey: "v3", hiddenSize: 1, enrollments: [])
+ let result = ASRResult(text: "what do you mean by sneak them", confidence: 1, duration: 3, processingTime: 0, tokenTimings: [
+ TokenTiming(token: "▁what", tokenId: 1, startTime: 0.16, endTime: 0.40, confidence: 1),
+ TokenTiming(token: "▁do", tokenId: 2, startTime: 0.40, endTime: 0.56, confidence: 1),
+ TokenTiming(token: "▁you", tokenId: 3, startTime: 0.56, endTime: 0.64, confidence: 1),
+ TokenTiming(token: "▁mean", tokenId: 4, startTime: 0.64, endTime: 0.80, confidence: 1),
+ TokenTiming(token: "▁by", tokenId: 5, startTime: 0.80, endTime: 0.96, confidence: 1),
+ TokenTiming(token: "▁sneak", tokenId: 6, startTime: 0.96, endTime: 1.44, confidence: 1),
+ TokenTiming(token: "▁them", tokenId: 7, startTime: 1.44, endTime: 1.60, confidence: 1),
+ ])
+ // The stamp match covers only the end of the word, as recorded in the app (frames 15-20).
+ let hit = PronunciationWindowMatch(prototypeIndex: 0, score: 0.94, frameRange: 15..<20)
+ let temporal: Set<FluidAudioProvider.TemporalMatchKey> = [.init(prototypeIndex: 0, frameRange: 15..<20)]
+ let unknownSpelling = SettingsStore.CustomDictionaryEntry(id: id, triggers: ["snick them"], replacement: "Snigdham")
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: result, matches: [hit], profiles: [profile], labels: [id: "Snigdham"], temporalMatches: temporal, entries: [unknownSpelling]),
+ "what do you mean by Snigdham"
+ )
+ // A saved spelling already covers it: the audio layer leaves the text for the text layer.
+ let knownSpelling = SettingsStore.CustomDictionaryEntry(id: id, triggers: ["sneak them"], replacement: "Snigdham")
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: result, matches: [hit], profiles: [profile], labels: [id: "Snigdham"], temporalMatches: temporal, entries: [knownSpelling]),
+ result.text
+ )
+ }
+ #endif
+}
+
+extension DictationE2ETests {
+ private func voiceLearner(
+ entries: [SettingsStore.CustomDictionaryEntry],
+ saved: @escaping ([SettingsStore.CustomDictionaryEntry]) -> Void,
+ canWrite: @escaping () -> Bool = { true }
+ ) -> DictionaryVoiceSpellingLearner {
+ DictionaryVoiceSpellingLearner(window: .milliseconds(50), isEnabled: { true }, canWrite: canWrite, load: { entries }, save: saved)
+ }
+
+ func testVoiceSpellingIsSavedWhenTheUserKeepsTheReplacement() async throws {
+ let entry = SettingsStore.CustomDictionaryEntry(triggers: ["snick them"], replacement: "Snigdham")
+ var saves: [[SettingsStore.CustomDictionaryEntry]] = []
+ let learner = self.voiceLearner(entries: [entry]) { saves.append($0) }
+ learner.propose([DictionaryVoiceSpelling(entryID: entry.id, label: "Snigdham", spelling: "Nickdom?")])
+ try await Task.sleep(for: .milliseconds(300))
+ XCTAssertEqual(saves.first?.first?.triggers, ["snick them", "nickdom"])
+ XCTAssertEqual(saves.first?.first?.id, entry.id, "The same dictionary word gains the spelling")
+ XCTAssertEqual(learner.pendingCount, 0)
+ }
+
+ func testVoiceSpellingIsDroppedWhenTheUserEditsTheWord() async throws {
+ let entry = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Snigdham")
+ var saves: [[SettingsStore.CustomDictionaryEntry]] = []
+ let learner = self.voiceLearner(entries: [entry]) { saves.append($0) }
+ learner.propose([DictionaryVoiceSpelling(entryID: entry.id, label: "Snigdham", spelling: "Nickdom")])
+ learner.reject(editedText: "Snigdham")
+ try await Task.sleep(for: .milliseconds(300))
+ XCTAssertTrue(saves.isEmpty)
+ XCTAssertEqual(learner.pendingCount, 0)
+ }
+
+ func testVoiceSpellingNeverLearnsCommonWordsTheWordItselfOrForAChangedEntry() async throws {
+ let entry = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Snigdham")
+ XCTAssertFalse(DictionaryVoiceSpellingLearner.isLearnable(.init(entryID: entry.id, label: "Snigdham", spelling: "the")))
+ XCTAssertFalse(DictionaryVoiceSpellingLearner.isLearnable(.init(entryID: entry.id, label: "Snigdham", spelling: "snigdham.")))
+ var saves: [[SettingsStore.CustomDictionaryEntry]] = []
+ let renamed = SettingsStore.CustomDictionaryEntry(id: entry.id, triggers: [], replacement: "Snigdha")
+ let learner = self.voiceLearner(entries: [renamed]) { saves.append($0) }
+ learner.propose([DictionaryVoiceSpelling(entryID: entry.id, label: "Snigdham", spelling: "Nickdom")])
+ try await Task.sleep(for: .milliseconds(300))
+ XCTAssertTrue(saves.isEmpty, "A renamed word must not inherit a spelling learned for its old name")
+ }
+}
+
+#if arch(arm64)
+extension DictationE2ETests {
+ func testGreyZoneVoiceMatchNeedsTheWordsToLookLikeTheWord() {
+ let id = UUID()
+ let profile = PronunciationDictionaryProfile(dictionaryEntryID: id, label: "Snigdham", modelKey: "v3", hiddenSize: 1, enrollments: [])
+ let entry = SettingsStore.CustomDictionaryEntry(id: id, triggers: ["snick them"], replacement: "Snigdham")
+ func result(_ a: String, _ b: String) -> ASRResult {
+ ASRResult(text: "what do you mean by \(a) \(b)", confidence: 1, duration: 3, processingTime: 0, tokenTimings: [
+ TokenTiming(token: "▁what", tokenId: 1, startTime: 0.16, endTime: 0.40, confidence: 1),
+ TokenTiming(token: "▁do", tokenId: 2, startTime: 0.40, endTime: 0.56, confidence: 1),
+ TokenTiming(token: "▁you", tokenId: 3, startTime: 0.56, endTime: 0.64, confidence: 1),
+ TokenTiming(token: "▁mean", tokenId: 4, startTime: 0.64, endTime: 0.80, confidence: 1),
+ TokenTiming(token: "▁by", tokenId: 5, startTime: 0.80, endTime: 0.96, confidence: 1),
+ TokenTiming(token: "▁\(a)", tokenId: 6, startTime: 0.96, endTime: 1.44, confidence: 1),
+ TokenTiming(token: "▁\(b)", tokenId: 7, startTime: 1.44, endTime: 1.60, confidence: 1),
+ ])
+ }
+ let temporal: Set<FluidAudioProvider.TemporalMatchKey> = [.init(prototypeIndex: 0, frameRange: 15..<20)]
+ // Distance 0.17: past the sure cutoff, inside the grey zone.
+ let grey = PronunciationWindowMatch(prototypeIndex: 0, score: 0.83, frameRange: 15..<20)
+ let lookalike = result("sneak", "them")
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: lookalike, matches: [grey], profiles: [profile], labels: [id: "Snigdham"], temporalMatches: temporal, entries: [entry]),
+ "what do you mean by Snigdham"
+ )
+ let unrelated = result("banana", "bread")
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: unrelated, matches: [grey], profiles: [profile], labels: [id: "Snigdham"], temporalMatches: temporal, entries: [entry]),
+ unrelated.text,
+ "A grey-zone sound match must not replace words that look nothing like the word"
+ )
+ // Distance 0.10: sure enough on sound alone.
+ let sure = PronunciationWindowMatch(prototypeIndex: 0, score: 0.90, frameRange: 15..<20)
+ XCTAssertEqual(
+ FluidAudioProvider.applyPronunciationMatches(result: unrelated, matches: [sure], profiles: [profile], labels: [id: "Snigdham"], temporalMatches: temporal, entries: [entry]),
+ "what do you mean by Snigdham"
+ )
+ }
+}
+#endif
+
+extension DictationE2ETests {
+ func testBuiltInDictionaryAppliesWithoutUserEntriesAndUserSpellingsWin() {
+ defer { ASRService.invalidateDictionaryCache() }
+ self.withRestoredDefaults(keys: [self.customDictionaryEntriesKey]) {
+ SettingsStore.shared.customDictionaryEntries = []
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(
+ ASRService.applyCustomDictionary("Ask claud to log it in post hoc for fluid voice."),
+ "Ask Claude to log it in PostHog for FluidVoice."
+ )
+
+ // A user mapping for the same spelling replaces the built-in one.
+ SettingsStore.shared.customDictionaryEntries = [SettingsStore.CustomDictionaryEntry(triggers: ["post hoc"], replacement: "post hoc")]
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(ASRService.applyCustomDictionary("an analysis done post hoc"), "an analysis done post hoc")
+ XCTAssertEqual(ASRService.applyCustomDictionary("ask claud"), "ask Claude", "Other built-ins still apply")
+ }
+ }
+}
+
+extension DictationE2ETests {
+ func testUserPhraseOverridesOnlyTheBuiltInOccurrencesItCovers() {
+ defer { ASRService.invalidateDictionaryCache() }
+ self.withRestoredDefaults(keys: [self.customDictionaryEntriesKey]) {
+ SettingsStore.shared.customDictionaryEntries = [.init(triggers: ["post hoc analysis"], replacement: "post hoc analysis")]
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(
+ ASRService.applyCustomDictionary("🙂 post hoc analysis; log it in post hoc for fluid voice."),
+ "🙂 post hoc analysis; log it in PostHog for FluidVoice."
+ )
+ }
+ }
+
+ func testUserReplacementAndCanonicalSpellingWinOverBuiltIns() {
+ defer { ASRService.invalidateDictionaryCache() }
+ self.withRestoredDefaults(keys: [self.customDictionaryEntriesKey]) {
+ SettingsStore.shared.customDictionaryEntries = [.init(triggers: ["my spelling"], replacement: "clawed")]
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(ASRService.applyCustomDictionary("my spelling; clawed; claud"), "clawed; clawed; Claude")
+
+ SettingsStore.shared.customDictionaryEntries = [.init(triggers: ["hoc"], replacement: "HOC")]
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(ASRService.applyCustomDictionary("post hoc; claud"), "post HOC; Claude")
+ }
+ }
+
+ func testUserRulesCanOverrideTheResultOfABuiltInCorrection() {
+ defer { ASRService.invalidateDictionaryCache() }
+ self.withRestoredDefaults(keys: [self.customDictionaryEntriesKey]) {
+ SettingsStore.shared.customDictionaryEntries = [.init(triggers: ["Claude Code"], replacement: "My Assistant")]
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(ASRService.applyCustomDictionary("open claud code and fluid voice"), "open My Assistant and FluidVoice")
+ }
+ }
+
+ func testOverlappingUserPhrasesBothStayProtectedFromBuiltIns() {
+ defer { ASRService.invalidateDictionaryCache() }
+ self.withRestoredDefaults(keys: [self.customDictionaryEntriesKey]) {
+ SettingsStore.shared.customDictionaryEntries = [
+ .init(triggers: ["first phrase"], replacement: "post hoc analysis"),
+ .init(triggers: ["second phrase"], replacement: "analysis claud"),
+ ]
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(ASRService.applyCustomDictionary("post hoc analysis claud; claud"), "post hoc analysis claud; Claude")
+ }
+ }
+
+ func testRemovingUserRulesRestoresBuiltInDefaultsAfterCacheInvalidation() {
+ defer { ASRService.invalidateDictionaryCache() }
+ self.withRestoredDefaults(keys: [self.customDictionaryEntriesKey]) {
+ SettingsStore.shared.customDictionaryEntries = [.init(triggers: ["post hoc analysis"], replacement: "post hoc analysis")]
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(ASRService.applyCustomDictionary("post hoc analysis"), "post hoc analysis")
+ SettingsStore.shared.customDictionaryEntries = []
+ ASRService.invalidateDictionaryCache()
+ XCTAssertEqual(ASRService.applyCustomDictionary("post hoc analysis"), "PostHog analysis")
+ }
+ }
+
+ func testPendingVoiceLearningYieldsToANewerManualRule() async throws {
+ let target = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Snigdham")
+ var entries = [target]
+ var saves: [[SettingsStore.CustomDictionaryEntry]] = []
+ let learner = DictionaryVoiceSpellingLearner(
+ window: .milliseconds(50),
+ isEnabled: { true },
+ canWrite: { true },
+ load: { entries },
+ save: { saves.append($0) }
+ )
+ defer { learner.cancelAll() }
+ learner.propose([.init(entryID: target.id, label: "Snigdham", spelling: "Nickdom?")])
+ entries.append(.init(triggers: ["NICKDOM"], replacement: "Nick Dom"))
+ try await Task.sleep(for: .milliseconds(300))
+ XCTAssertTrue(saves.isEmpty)
+ XCTAssertEqual(learner.pendingCount, 0)
+ }
+
+ func testVoiceLearningDoesNotClaimAnotherUsersCanonicalWordOrPhrase() async throws {
+ let target = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Snigdham")
+ let manual = SettingsStore.CustomDictionaryEntry(triggers: ["nick dome"], replacement: "Nickdom")
+ var saves: [[SettingsStore.CustomDictionaryEntry]] = []
+ let learner = self.voiceLearner(entries: [target, manual]) { saves.append($0) }
+ defer { learner.cancelAll() }
+ learner.propose([
+ .init(entryID: target.id, label: "Snigdham", spelling: "Nickdom"),
+ .init(entryID: target.id, label: "Snigdham", spelling: "nick dome now"),
+ ])
+ try await Task.sleep(for: .milliseconds(300))
+ XCTAssertTrue(saves.isEmpty)
+ XCTAssertEqual(learner.pendingCount, 0)
+ }
+
+ func testVoiceLearningCanStillSaveAnUnclaimedSpellingWithOtherUserRules() async throws {
+ let target = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Snigdham")
+ let other = SettingsStore.CustomDictionaryEntry(triggers: ["cloud"], replacement: "Claude")
+ var saves: [[SettingsStore.CustomDictionaryEntry]] = []
+ let learner = self.voiceLearner(entries: [target, other]) { saves.append($0) }
+ defer { learner.cancelAll() }
+ learner.propose([.init(entryID: target.id, label: "Snigdham", spelling: "Nickdom")])
+ try await Task.sleep(for: .milliseconds(300))
+ XCTAssertEqual(saves.count, 1)
+ XCTAssertEqual(saves.first?.first?.triggers, ["nickdom"])
+ XCTAssertEqual(saves.first?.last, other)
+ }
+}
+
+#if arch(arm64)
+extension DictationE2ETests {
+ func testExpandedTemporalMatchKeepsItsCorrectionEvidence() {
+ let id = UUID()
+ let profile = PronunciationDictionaryProfile(dictionaryEntryID: id, label: "Snigdham", modelKey: "v3", hiddenSize: 1, enrollments: [])
+ let result = ASRResult(text: "what do you mean by sneak them", confidence: 1, duration: 3, processingTime: 0, tokenTimings: [
+ TokenTiming(token: "▁what", tokenId: 1, startTime: 0.16, endTime: 0.40, confidence: 1),
+ TokenTiming(token: "▁do", tokenId: 2, startTime: 0.40, endTime: 0.56, confidence: 1),
+ TokenTiming(token: "▁you", tokenId: 3, startTime: 0.56, endTime: 0.64, confidence: 1),
+ TokenTiming(token: "▁mean", tokenId: 4, startTime: 0.64, endTime: 0.80, confidence: 1),
+ TokenTiming(token: "▁by", tokenId: 5, startTime: 0.80, endTime: 0.96, confidence: 1),
+ TokenTiming(token: "▁sneak", tokenId: 6, startTime: 0.96, endTime: 1.44, confidence: 1),
+ TokenTiming(token: "▁them", tokenId: 7, startTime: 1.44, endTime: 1.60, confidence: 1),
+ ])
+ let hit = PronunciationWindowMatch(prototypeIndex: 0, score: 0.94, frameRange: 18..<20)
+ let evidence = DictionaryAcousticEvidence(
+ id: UUID(),
+ entryID: id,
+ label: "Snigdham",
+ profileKey: "fixture",
+ modelKey: "v3",
+ sourceWordRange: 6..<7,
+ frames: .init(hiddenSize: 1, values: [1, 2]),
+ matchFrameRange: hit.frameRange
+ )
+ var accepted: [DictionaryAcousticEvidence] = []
+ let text = FluidAudioProvider.applyPronunciationMatches(
+ result: result,
+ matches: [hit],
+ profiles: [profile],
+ labels: [id: "Snigdham"],
+ temporalMatches: [.init(prototypeIndex: 0, frameRange: hit.frameRange)],
+ entries: [.init(id: id, triggers: ["snick them"], replacement: "Snigdham")],
+ onAccepted: { entryID, label, range, frames in
+ accepted += FluidAudioProvider.acceptedAcousticEvidence(
+ [evidence],
+ entryID: entryID,
+ label: label,
+ wordRange: range,
+ frameRange: frames
+ )
+ }
+ )
+ XCTAssertEqual(text, "what do you mean by Snigdham")
+ XCTAssertEqual(accepted.count, 1)
+ XCTAssertEqual(accepted.first?.sourceWordRange, 5..<7)
+ }
+}
+#endif
+
+extension DictationE2ETests {
+ func testVoiceSpellingObservationCoversConfirmationAndEditDebounce() {
+ XCTAssertGreaterThan(
+ AutomaticDictionaryCorrectionTracker.observationDuration(hasVoiceSpellings: true),
+ DictionaryVoiceSpellingLearner.confirmationWindow
+ )
+ XCTAssertEqual(AutomaticDictionaryCorrectionTracker.observationDuration(hasVoiceSpellings: false), .seconds(30))
+ }
+
+ func testEndingObservationCancelsOnlyThatRecordingsUnconfirmedSpellings() async throws {
+ let first = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Snigdham")
+ let second = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Velora")
+ let firstRecording = UUID()
+ let secondRecording = UUID()
+ var entries = [first, second]
+ var saves = 0
+ let learner = DictionaryVoiceSpellingLearner(
+ window: .milliseconds(50),
+ isEnabled: { true },
+ canWrite: { true },
+ load: { entries },
+ save: { entries = $0; saves += 1 }
+ )
+ defer { learner.cancelAll() }
+ learner.propose([.init(entryID: first.id, label: first.replacement, spelling: "Nickdom")], recordingID: firstRecording)
+ learner.propose([.init(entryID: second.id, label: second.replacement, spelling: "velorra")], recordingID: secondRecording)
+ learner.cancel(recordingID: firstRecording)
+ learner.cancel(recordingID: firstRecording)
+ XCTAssertEqual(learner.pendingCount, 1)
+ try await Task.sleep(for: .milliseconds(300))
+ XCTAssertEqual(saves, 1)
+ XCTAssertTrue(entries[0].triggers.isEmpty)
+ XCTAssertEqual(entries[1].triggers, ["velorra"])
+ XCTAssertEqual(learner.pendingCount, 0)
+ }
+
+ func testEndingObservationCannotSaveARuleWhileWaitingForCaptureToFinish() async throws {
+ let entry = SettingsStore.CustomDictionaryEntry(triggers: [], replacement: "Snigdham")
+ let recordingID = UUID()
+ var canWrite = false
+ var saves = 0
+ let learner = self.voiceLearner(entries: [entry], saved: { _ in saves += 1 }, canWrite: { canWrite })
+ defer { learner.cancelAll() }
+ learner.propose([.init(entryID: entry.id, label: entry.replacement, spelling: "Nickdom")], recordingID: recordingID)
+ try await Task.sleep(for: .milliseconds(100))
+ learner.cancel(recordingID: recordingID)
+ canWrite = true
+ try await Task.sleep(for: .milliseconds(100))
+ XCTAssertEqual(learner.pendingCount, 0)
+ XCTAssertEqual(saves, 0)
+ }
+}
+
+#if arch(arm64)
+extension DictationE2ETests {
+ func testCorrectionEvidenceAlignsBothGrowingAndShrinkingWordRangesWithoutMixingHits() {
+ let id = UUID()
+ let chosen = DictionaryAcousticEvidence(
+ id: UUID(),
+ entryID: id,
+ label: "Name",
+ profileKey: "profile",
+ modelKey: "v3",
+ sourceWordRange: 1..<3,
+ frames: .init(hiddenSize: 1, values: [1, 2]),
+ matchFrameRange: 4..<6
+ )
+ let otherHit = DictionaryAcousticEvidence(
+ id: UUID(),
+ entryID: id,
+ label: "Name",
+ profileKey: "profile",
+ modelKey: "v3",
+ sourceWordRange: 1..<3,
+ frames: .init(hiddenSize: 1, values: [3, 4]),
+ matchFrameRange: 8..<10
+ )
+ let otherWord = DictionaryAcousticEvidence(
+ id: UUID(),
+ entryID: UUID(),
+ label: "Other",
+ profileKey: "other",
+ modelKey: "v3",
+ sourceWordRange: 1..<3,
+ frames: .init(hiddenSize: 1, values: [5, 6]),
+ matchFrameRange: 4..<6
+ )
+ for finalRange in [1..<2, 0..<4] {
+ let result = FluidAudioProvider.acceptedAcousticEvidence(
+ [chosen, otherHit, otherWord],
+ entryID: id,
+ label: "Name's",
+ wordRange: finalRange,
+ frameRange: 4..<6
+ )
+ XCTAssertEqual(result.count, 1)
+ XCTAssertEqual(result.first?.id, chosen.id)
+ XCTAssertEqual(result.first?.sourceWordRange, finalRange)
+ XCTAssertEqual(result.first?.label, "Name's")
+ XCTAssertEqual(result.first?.frames.values, chosen.frames.values)
+ XCTAssertEqual(result.first?.profileKey, chosen.profileKey)
+ }
+ }
+}
+#endif
diff --git a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift
index f3d1f7b6..87974690 100644
--- a/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift
+++ b/Tests/FluidDictationIntegrationTests/HotkeyShortcutTests.swift
@@ -1167,7 +1167,7 @@ final class HotkeyShortcutTests: XCTestCase {
XCTAssertTrue(optional.contains("(.command, self.commandModeHotkeyShortcut)"), "The saved binding must remain reserved when Command Mode is toggled off")
XCTAssertFalse(conflicts.contains("if self.isCommandModeShortcutEnabled"))
XCTAssertTrue(conflicts.contains("configuredShortcut == shortcut"))
- XCTAssertTrue(conflicts.contains("shortcut.conflictsWith(configuredShortcut)"))
+ XCTAssertTrue(conflicts.contains("shortcut.conflictsWith(configuredShortcut,"))
XCTAssertTrue(conflicts.contains("otherTarget != target"), "Editing the Command binding must not conflict with itself")
}
@@ -2148,6 +2148,7 @@ final class HotkeyShortcutTests: XCTestCase {
asr: ASRService,
onStart: @escaping () async -> Void,
onStop: @escaping () async -> Void = {},
+ isShortcutCaptureActive: @escaping () -> Bool = { false },
isSessionLocked: @escaping () -> Bool = { false }
) -> GlobalHotkeyManager {
let manager = GlobalHotkeyManager(
@@ -2163,6 +2164,7 @@ final class HotkeyShortcutTests: XCTestCase {
dictationModeCallback: onStart,
stopAndProcessCallback: { _ in await onStop() },
isDictateRecordingProvider: { true },
+ isShortcutCaptureActiveProvider: isShortcutCaptureActive,
isSessionLockedProvider: isSessionLocked
)
manager.setHotkeyMode(.toggle)
@@ -4402,3 +4404,110 @@ private final class ModifierOnlyFlagsReplay {
self.keyDown()
}
}
+
+@MainActor
+extension HotkeyShortcutTests {
+ func testMeetingShortcutIsOptInPersistsAndRejectsUnsafeBindings() throws {
+ try self.withRestoredDefaults(keys: ["MeetingRecordingHotkeyShortcut"]) {
+ UserDefaults.standard.removeObject(forKey: "MeetingRecordingHotkeyShortcut")
+ let settings = SettingsStore.shared
+ XCTAssertNil(settings.meetingRecordingHotkeyShortcut)
+ let shortcut = HotkeyShortcut(keyCode: 46, modifierFlags: [.control, .option, .command])
+ settings.meetingRecordingHotkeyShortcut = shortcut
+ XCTAssertEqual(settings.meetingRecordingHotkeyShortcut, shortcut)
+ let payload = settings.makeBackupPayload()
+ let decoded = try JSONDecoder().decode(SettingsBackupPayload.self, from: JSONEncoder().encode(payload))
+ XCTAssertEqual(decoded.meetingRecordingHotkeyShortcut, shortcut)
+ XCTAssertEqual(decoded.meetingRecordingShortcutEnabled, true)
+ for invalid in [HotkeyShortcut(keyCode: 46, modifierFlags: []), HotkeyShortcut(keyCode: 58, modifierFlags: []), HotkeyShortcut(mouseButton: 3, modifierFlags: [.option])] {
+ settings.meetingRecordingHotkeyShortcut = invalid
+ XCTAssertNil(settings.meetingRecordingHotkeyShortcut)
+ }
+ var legacy = try XCTUnwrap(JSONSerialization.jsonObject(with: JSONEncoder().encode(payload)) as? [String: Any])
+ legacy.removeValue(forKey: "meetingRecordingHotkeyShortcut")
+ legacy.removeValue(forKey: "meetingRecordingShortcutEnabled")
+ let oldPayload = try JSONDecoder().decode(SettingsBackupPayload.self, from: JSONSerialization.data(withJSONObject: legacy))
+ XCTAssertNil(oldPayload.meetingRecordingShortcutEnabled)
+ XCTAssertNil(oldPayload.meetingRecordingHotkeyShortcut)
+ settings.meetingRecordingHotkeyShortcut = nil
+ XCTAssertEqual(settings.makeBackupPayload().meetingRecordingShortcutEnabled, false)
+ }
+ }
+
+ func testMeetingShortcutTogglesOnceInEveryActivationModeWithoutDictation() async throws {
+ let defaults = UserDefaults.standard
+ let oldShortcut = defaults.object(forKey: "MeetingRecordingHotkeyShortcut")
+ let oldEnabled = defaults.object(forKey: "FluidMeetEnabled")
+ defer {
+ defaults.set(oldShortcut, forKey: "MeetingRecordingHotkeyShortcut")
+ defaults.set(oldEnabled, forKey: "FluidMeetEnabled")
+ }
+ let settings = SettingsStore.shared
+ settings.fluidMeetEnabled = true
+ settings.meetingRecordingHotkeyShortcut = HotkeyShortcut(keyCode: 46, modifierFlags: [.control, .option, .command])
+ var locked = false
+ var capturingShortcut = false
+ var dictationCalls = 0
+ let asr = ASRService()
+ let manager = self.makePrimaryReleaseTestManager(asr: asr, onStart: { dictationCalls += 1 }, isShortcutCaptureActive: { capturingShortcut }, isSessionLocked: { locked })
+ var meetingCalls = 0
+ manager.setMeetingRecordingCallback { meetingCalls += 1 }
+ let down = try XCTUnwrap(CGEvent(keyboardEventSource: nil, virtualKey: 46, keyDown: true))
+ down.flags = [.maskControl, .maskAlternate, .maskCommand]
+ let up = try XCTUnwrap(CGEvent(keyboardEventSource: nil, virtualKey: 46, keyDown: false))
+ up.flags = []
+ for mode in [HotkeyActivationMode.hold, .automatic, .toggle] {
+ manager.setHotkeyMode(mode)
+ let before = meetingCalls
+ down.setIntegerValueField(.keyboardEventAutorepeat, value: 0)
+ XCTAssertNil(manager.handleKeyEvent(type: .keyDown, event: down))
+ down.setIntegerValueField(.keyboardEventAutorepeat, value: 1)
+ XCTAssertNil(manager.handleKeyEvent(type: .keyDown, event: down))
+ XCTAssertNil(manager.handleKeyEvent(type: .keyUp, event: up))
+ XCTAssertEqual(meetingCalls, before + 1)
+ }
+ down.setIntegerValueField(.keyboardEventAutorepeat, value: 0)
+ let synthesized = try XCTUnwrap(CGEvent(keyboardEventSource: nil, virtualKey: 46, keyDown: true))
+ synthesized.flags = down.flags
+ synthesized.setIntegerValueField(.eventSourceUserData, value: TypingService.synthesizedEventUserData)
+ XCTAssertNotNil(manager.handleKeyEvent(type: .keyDown, event: synthesized))
+ XCTAssertFalse(GlobalHotkeyManager.isSynthesizedTypingEvent(down))
+ capturingShortcut = true
+ XCTAssertNotNil(manager.handleKeyEvent(type: .keyDown, event: down))
+ capturingShortcut = false
+ locked = true
+ XCTAssertNotNil(manager.handleKeyEvent(type: .keyDown, event: down))
+ locked = false
+ settings.fluidMeetEnabled = false
+ XCTAssertNotNil(manager.handleKeyEvent(type: .keyDown, event: down))
+ settings.fluidMeetEnabled = true
+ settings.meetingRecordingHotkeyShortcut = nil
+ XCTAssertNotNil(manager.handleKeyEvent(type: .keyDown, event: down))
+ XCTAssertEqual(meetingCalls, 3)
+ settings.meetingRecordingHotkeyShortcut = HotkeyShortcut(keyCode: 46, modifierFlags: [.control, .option, .command])
+ XCTAssertNil(manager.handleKeyEvent(type: .keyDown, event: down))
+ manager.resetModifierOnlyShortcutTracking()
+ XCTAssertNotNil(manager.handleKeyEvent(type: .keyUp, event: up), "A reset must clear stale meeting key ownership")
+ XCTAssertEqual(meetingCalls, 4)
+ for _ in 0..<10 {
+ await Task.yield()
+ }
+ XCTAssertEqual(dictationCalls, 0)
+ XCTAssertFalse(asr.isRunningOrStarting)
+ XCTAssertNil(asr.activeExclusiveActivity)
+ }
+}
+
+extension HotkeyShortcutTests {
+ func testMeetingChordRejectsModifierPrefixesWithoutChangingOtherShortcutRules() {
+ let option = HotkeyShortcut(keyCode: 61, modifierFlags: [])
+ let chord = HotkeyShortcut(keyCode: 46, modifierFlags: [.control, .option, .command])
+ let safeChord = HotkeyShortcut(keyCode: 46, modifierFlags: [.control, .command])
+ XCTAssertTrue(chord.conflictsWith(option, includingKeyboardModifierPrefixes: true))
+ XCTAssertTrue(option.conflictsWith(chord, includingKeyboardModifierPrefixes: true))
+ XCTAssertFalse(safeChord.conflictsWith(option, includingKeyboardModifierPrefixes: true))
+ XCTAssertFalse(chord.conflictsWith(option), "Existing dictation shortcut rules remain unchanged")
+ XCTAssertFalse(safeChord.conflictsWith(HotkeyShortcut(keyCode: 56, modifierFlags: [.control]), includingKeyboardModifierPrefixes: true))
+ XCTAssertTrue(safeChord.conflictsWith(HotkeyShortcut(keyCode: 59, modifierFlags: []), includingKeyboardModifierPrefixes: true))
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/LLMClientRequestBodyTests.swift b/Tests/FluidDictationIntegrationTests/LLMClientRequestBodyTests.swift
index 9224a33d..f76ab020 100644
--- a/Tests/FluidDictationIntegrationTests/LLMClientRequestBodyTests.swift
+++ b/Tests/FluidDictationIntegrationTests/LLMClientRequestBodyTests.swift
@@ -709,3 +709,44 @@ private class Issue445StreamURLProtocol: URLProtocol {
override func stopLoading() {}
}
+
+extension LLMClientRequestBodyTests {
+ func testModeSwitchesPreserveStartingFieldAndFormattingUntilANewCapture() {
+ let first = TypingService.RecordingTargetContext(id: UUID(), pid: 1, bundleIdentifier: "first", window: nil, element: nil)
+ let second = TypingService.RecordingTargetContext(id: UUID(), pid: 2, bundleIdentifier: "second", window: nil, element: nil)
+ var original: TypingService.RecordingTargetContext?
+ var preceding = ""
+ var captures = 0
+ DictationStopSnapshot.captureStartingContext(reusingCapture: false) {
+ original = first
+ preceding = "First field "
+ captures += 1
+ }
+ // Command, Edit and returning to Dictation share the same original capture.
+ for _ in 0..<3 {
+ DictationStopSnapshot.captureStartingContext(reusingCapture: true) {
+ original = second
+ preceding = "Second field "
+ captures += 1
+ }
+ }
+ XCTAssertEqual(captures, 1)
+ XCTAssertEqual(preceding, "First field ")
+ XCTAssertEqual(
+ DictationStopSnapshot.selectTarget(current: second, original: original, returnToStartingField: true, ownOverlayFocused: false)?.id,
+ first.id
+ )
+ XCTAssertEqual(
+ DictationStopSnapshot.selectTarget(current: second, original: original, returnToStartingField: false, ownOverlayFocused: false)?.id,
+ second.id
+ )
+ DictationStopSnapshot.captureStartingContext(reusingCapture: false) {
+ original = second
+ preceding = "Second field "
+ captures += 1
+ }
+ XCTAssertEqual(captures, 2)
+ XCTAssertEqual(original?.id, second.id)
+ XCTAssertEqual(preceding, "Second field ")
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAEC3Tests.swift b/Tests/FluidDictationIntegrationTests/MeetingAEC3Tests.swift
index 02b8cb52..0e92f82d 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingAEC3Tests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingAEC3Tests.swift
@@ -10,6 +10,70 @@ import XCTest
final class MeetingAEC3Tests: XCTestCase {
private let token = MeetingAECStreamToken(generation: 7, identity: 11)
+ func testFailedAECRecordsAndTranscribesEveryFollowingSampleWithoutRetrying() async throws {
+ let directory = FileManager.default.temporaryDirectory.appendingPathComponent("aec-fallback-\(UUID())")
+ defer { try? FileManager.default.removeItem(at: directory) }
+ let writer = try MeetingAudioChunkWriter(
+ track: Self.microphoneTrack(), sessionDirectory: directory, chunkDuration: 60, eventHandler: { _ in }
+ )
+ let gate = MeetingMicrophoneTranscriptGate(.unprotected)
+ let live = AECLiveRecorder()
+ let failures = AECTerminalRecorder()
+ let committer = MeetingAECOutputCommitter(
+ writer: writer,
+ transcriptGate: gate,
+ liveAudioHandler: { _, sample in live.append(sample) },
+ terminalFailure: { failures.append($0) }
+ )
+ let processor = AECFakeProcessor()
+ processor.failNext = true
+ let pipeline = MeetingAECPipeline(processor: processor, committer: committer)
+ for frame in 0..<25 {
+ if frame == 1 { processor.failNext = true }
+ let pts = CMTime(value: Int64(frame * 480), timescale: 48_000)
+ let render = try XCTUnwrap(MeetingAECPCMAdapter.synthesize(samples: [Float](repeating: 0.2, count: 480), presentationTime: pts))
+ let capture = try XCTUnwrap(MeetingAECPCMAdapter.synthesize(samples: [Float](repeating: 0.125, count: 480), presentationTime: pts))
+ pipeline.consumeRender(render, stream: self.token)
+ pipeline.consumeCapture(capture, stream: self.token)
+ }
+ await committer.waitForPendingCommits()
+ XCTAssertEqual(live.count, 25)
+ XCTAssertTrue(processor.failNext, "the failed route must not repeatedly restart AEC and withhold speech")
+ XCTAssertTrue(failures.values.isEmpty)
+ let current = await writer.snapshot()
+ XCTAssertEqual(current.captureEras?.last?.echoProtection, .unfiltered)
+ XCTAssertNil(current.captureEras?.last?.aecProvenance)
+ pipeline.stop(boundary: gate.invalidate())
+ await committer.waitForPendingCommits()
+ let stopped = await writer.stop()
+ XCTAssertFalse(gate.admitsTranscript(), "Stop must leave live delivery closed")
+ let chunk = try XCTUnwrap(stopped.chunks.first)
+ let audio = try AVAudioFile(forReading: chunk.fileURL(relativeTo: directory))
+ XCTAssertEqual(audio.length, 25 * 480)
+ let buffer = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: audio.processingFormat, frameCapacity: AVAudioFrameCount(audio.length)))
+ try audio.read(into: buffer)
+ let channel = try XCTUnwrap(buffer.floatChannelData?[0])
+ XCTAssertTrue((0..<Int(buffer.frameLength)).allSatisfy { abs(channel[$0] - 0.125) < 0.000_001 })
+ }
+
+ func testFallbackCannotReopenAfterExternalInvalidation() async throws {
+ let directory = FileManager.default.temporaryDirectory.appendingPathComponent("aec-stale-fallback-\(UUID())")
+ defer { try? FileManager.default.removeItem(at: directory) }
+ let writer = try MeetingAudioChunkWriter(
+ track: Self.microphoneTrack(), sessionDirectory: directory, chunkDuration: 60, eventHandler: { _ in }
+ )
+ let gate = MeetingMicrophoneTranscriptGate(.unprotected)
+ let live = AECLiveRecorder()
+ let committer = MeetingAECOutputCommitter(writer: writer, transcriptGate: gate, liveAudioHandler: { _, sample in live.append(sample) }, terminalFailure: { _ in })
+ let sample = try XCTUnwrap(MeetingAECPCMAdapter.synthesize(samples: [Float](repeating: 0.1, count: 480), presentationTime: .zero))
+ committer.commitUnfiltered(sample)
+ committer.invalidateForExternalBoundary(gate.invalidate())
+ await committer.waitForPendingCommits()
+ XCTAssertFalse(gate.admitsTranscript())
+ let track = await writer.stop()
+ XCTAssertEqual(track.captureEras?.last?.echoProtection, .unprotected)
+ }
+
private func block(
_ kind: MeetingAECInputKind,
start: Int64,
@@ -174,13 +238,13 @@ final class MeetingAEC3Tests: XCTestCase {
), .ambiguous)
XCTAssertEqual(MeetingAECOutputRouteClassifier.classify(
first: external, second: external, revisionStayedStable: true
- ), .ambiguous)
+ ), .supportedSpeaker)
XCTAssertEqual(MeetingAECOutputRouteClassifier.classify(
first: missing, second: missing, revisionStayedStable: true
), .ambiguous)
XCTAssertEqual(MeetingAECOutputRouteClassifier.classify(
first: bluetoothSpeaker, second: bluetoothSpeaker, revisionStayedStable: true
- ), .ambiguous)
+ ), .supportedSpeaker)
XCTAssertEqual(MeetingAECOutputRouteClassifier.classify(
first: bluetoothHeadphones, second: bluetoothHeadphones, revisionStayedStable: true
), .physicallyClosed)
@@ -511,6 +575,57 @@ final class MeetingAEC3Tests: XCTestCase {
XCTAssertThrowsError(try processor.process(render: [.nan] + [Float](repeating: 0, count: 479), capture: capture))
}
+ func testQuietVoiceSurvivesLoudEchoWithoutCountingEchoAsSpeech() throws {
+ let rate = 48_000
+ let frames = 1200
+ let count = frames * 480
+ let echoDelay = 2346
+ var render = [Float](repeating: 0, count: count)
+ var voice = render
+ var capture = render
+ var output = render
+ var renderSeed: UInt32 = 0x12345678
+ var voiceSeed: UInt32 = 0x87654321
+ var filteredRender: Float = 0
+ var filteredVoice: Float = 0
+ for index in 0..<count {
+ renderSeed = renderSeed &* 1_664_525 &+ 1_013_904_223
+ let noise = Float(Int32(bitPattern: renderSeed)) / 2_147_483_648 * 0.22
+ filteredRender = 0.72 * filteredRender + 0.28 * noise
+ render[index] = filteredRender
+ if (900 * 480..<1100 * 480).contains(index) {
+ voiceSeed = voiceSeed &* 1_103_515_245 &+ 12_345
+ let breath = Float(Int32(bitPattern: voiceSeed)) / 2_147_483_648 * 0.04
+ filteredVoice = 0.82 * filteredVoice + 0.18 * breath
+ let t = Double(index) / Double(rate)
+ let phase = 2 * Double.pi * (170 * t + 4 * sin(2 * Double.pi * 1.3 * t))
+ let envelope = Float(0.4 + 0.6 * pow(sin(2 * Double.pi * 2.1 * t), 2))
+ let harmonics = Float(sin(phase) + 0.5 * sin(3 * phase) + 0.25 * sin(5 * phase))
+ voice[index] = envelope * (0.008 * harmonics + 0.2 * filteredVoice)
+ }
+ capture[index] = voice[index] + (index >= echoDelay ? 0.62 * render[index - echoDelay] : 0)
+ }
+ let processor = try MeetingAECProcessor()
+ for frame in 0..<frames {
+ let range = frame * 480..<(frame + 1) * 480
+ let result = try processor.process(render: Array(render[range]), capture: Array(capture[range]))
+ XCTAssertTrue(result.samples.allSatisfy { $0.isFinite && (-1...1).contains($0) })
+ output.replaceSubrange(range, with: result.samples)
+ }
+ func energy(_ samples: [Float], _ range: Range<Int>) -> Double {
+ range.reduce(0) { $0 + Double(samples[$1]) * Double(samples[$1]) }
+ }
+ let echoOnly = 700 * 480..<900 * 480
+ let echoReduction = 10 * log10(energy(capture, echoOnly) / energy(output, echoOnly))
+ XCTAssertGreaterThan(echoReduction, 20, "Video-only echo must remain suppressed")
+ // The pinned 48 kHz processor has a 433-sample filter-bank delay. Correlate
+ // against the independent known voice so residual echo cannot pass this test.
+ let speech = 900 * 480 + 4800..<1100 * 480 - 4800
+ let dot = speech.reduce(0.0) { $0 + Double(output[$1 + 433]) * Double(voice[$1]) }
+ let voiceGain = 20 * log10(abs(dot / energy(voice, speech)))
+ XCTAssertGreaterThan(voiceGain, -8, "Quiet local speech must survive overlapping playback")
+ }
+
func testAECProvenanceIsBackwardCompatibleAndSoftwareProtectionAdmits() throws {
XCTAssertTrue(MeetingMicrophoneEchoProtection.softwareEchoCancelled.admitsTranscript)
let legacy = Data("""
@@ -568,7 +683,7 @@ final class MeetingAEC3Tests: XCTestCase {
))
}
- func testPipelinePromotesOnlyOnFrame201AndBridgeFailureDemotes() async throws {
+ func testPipelinePromotesOnlyOnFrame201AndBridgeFailureKeepsSpeech() async throws {
let directory = FileManager.default.temporaryDirectory
.appendingPathComponent("fluid-aec3-test-\(UUID().uuidString)", isDirectory: true)
defer { try? FileManager.default.removeItem(at: directory) }
@@ -630,14 +745,14 @@ final class MeetingAEC3Tests: XCTestCase {
)), stream: self.token)
await committer.waitForPendingCommits()
track = await writer.snapshot()
- XCTAssertEqual(track.captureEras?.last?.echoProtection, .unprotected)
+ XCTAssertEqual(track.captureEras?.last?.echoProtection, .unfiltered)
XCTAssertNil(track.captureEras?.last?.aecProvenance)
XCTAssertEqual(
track.captureEras?.map(\.startSeconds),
[0, 2, 2.01],
"the demotion era must be ordered after the protected era on the sample timeline"
)
- XCTAssertEqual(live.count, 1, "the bridge-failure raw frame must not reach live ASR")
+ XCTAssertEqual(live.count, 2, "the bridge-failure frame must reach live ASR with honest unfiltered provenance")
XCTAssertTrue(failures.values.isEmpty)
pipeline.stop(boundary: gate.invalidate())
@@ -740,6 +855,27 @@ final class MeetingAEC3Tests: XCTestCase {
XCTAssertFalse(controller.armReadyCandidate())
}
+ func testMissingAECInputTimesOutButActiveOrNewRouteDoesNot() throws {
+ for kind in [MeetingAECInputKind.render, .capture] {
+ var controller = MeetingAECActivationController()
+ let token = try XCTUnwrap(controller.reserveConstruction())
+ XCTAssertTrue(controller.commitCandidate(token))
+ XCTAssertTrue(controller.armReadyCandidate(now: 10))
+ XCTAssertFalse(controller.observeValidatedFormat(kind: kind, token: token))
+ XCTAssertFalse(controller.formatValidationTimedOut(now: 11.99))
+ XCTAssertTrue(controller.formatValidationTimedOut(now: 12))
+ controller.invalidate()
+ XCTAssertFalse(controller.formatValidationTimedOut(now: 20))
+ let next = try XCTUnwrap(controller.reserveConstruction())
+ XCTAssertTrue(controller.commitCandidate(next))
+ XCTAssertTrue(controller.armReadyCandidate(now: 20))
+ XCTAssertFalse(controller.formatValidationTimedOut(now: 20.01))
+ XCTAssertFalse(controller.observeValidatedFormat(kind: .render, token: next))
+ XCTAssertTrue(controller.observeValidatedFormat(kind: .capture, token: next))
+ XCTAssertFalse(controller.formatValidationTimedOut(now: 1000), "the startup deadline cannot reset an active AEC")
+ }
+ }
+
/// Route/stream/lifecycle invalidation immediately before or after every startup step must
/// drop the handshake to raw and reject the stale token's next transition.
func testActivationControllerInvalidationBetweenEveryStartupStep() {
@@ -799,6 +935,68 @@ final class MeetingAEC3Tests: XCTestCase {
XCTAssertTrue(gate.admitsTranscript())
}
+ func testOldLiveSamplesStayRejectedAfterNewRouteReopensGate() throws {
+ let gate = MeetingMicrophoneTranscriptGate(.unfiltered)
+ let oldEpoch = gate.invalidationEpoch
+ let sample = try XCTUnwrap(MeetingAECPCMAdapter.synthesize(
+ samples: [Float](repeating: 0.1, count: 480), presentationTime: .zero
+ ))
+ XCTAssertTrue(gate.observeAndAdmit(sample, since: oldEpoch))
+ _ = gate.invalidate()
+ let newEpoch = gate.invalidationEpoch
+ XCTAssertTrue(gate.updateIfNotInvalidated(.acousticallyClosed, since: newEpoch))
+ XCTAssertFalse(gate.observeAndAdmit(sample, since: oldEpoch))
+ XCTAssertTrue(gate.observeAndAdmit(sample, since: newEpoch))
+ }
+
+ func testRapidFallbackPreservesQueuedFilteredAndRawLiveSamplesInOrder() async throws {
+ let directory = FileManager.default.temporaryDirectory.appendingPathComponent("aec-rapid-fallback-\(UUID())")
+ defer { try? FileManager.default.removeItem(at: directory) }
+ let writer = try MeetingAudioChunkWriter(
+ track: Self.microphoneTrack(), sessionDirectory: directory, chunkDuration: 60, eventHandler: { _ in }
+ )
+ let gate = MeetingMicrophoneTranscriptGate(.unprotected)
+ let live = AECLiveRecorder()
+ let failures = AECTerminalRecorder()
+ let committer = MeetingAECOutputCommitter(
+ writer: writer,
+ transcriptGate: gate,
+ liveAudioHandler: { _, sample in live.append(sample) },
+ terminalFailure: { failures.append($0) }
+ )
+ let entered = DispatchSemaphore(value: 0)
+ let release = DispatchSemaphore(value: 0)
+ defer { release.signal() }
+ let blocker = Task {
+ try await writer.updateTrackMetadata { _ in
+ entered.signal()
+ _ = release.wait(timeout: .now() + 5)
+ }
+ }
+ XCTAssertEqual(entered.wait(timeout: .now() + 2), .success)
+ for index in 1...4 {
+ let sample = try XCTUnwrap(MeetingAECPCMAdapter.synthesize(
+ samples: [Float](repeating: 0.1, count: 480),
+ presentationTime: CMTime(value: Int64(index * 480), timescale: 48_000)
+ ))
+ if index <= 2 {
+ committer.commitProcessed(sample, mayPromote: true)
+ } else {
+ committer.commitUnfiltered(sample)
+ }
+ }
+ XCTAssertEqual(live.count, 0, "live audio waits for its provenance to be saved")
+ release.signal()
+ try await blocker.value
+ await committer.waitForPendingCommits()
+ XCTAssertEqual(live.presentationValues, [480, 960, 1440, 1920])
+ XCTAssertTrue(failures.values.isEmpty)
+ XCTAssertTrue(gate.admitsTranscript())
+ let track = await writer.stop()
+ XCTAssertEqual(track.captureEras?.last?.echoProtection, .unfiltered)
+ XCTAssertNil(track.captureEras?.last?.aecProvenance)
+ }
+
func testFlushPreservesAlreadyProcessedPairsAndEmitsRawTailWithExactPTS() {
var joiner = MeetingAECStreamJoiner()
var frames: [MeetingAECJoinedFrame] = []
@@ -1070,7 +1268,7 @@ final class MeetingAEC3Tests: XCTestCase {
try committer.commitProcessed(sample(1440), mayPromote: false)
_ = await writer.snapshot()
- XCTAssertEqual(recorder.presentationValues, [480, 1440])
+ XCTAssertEqual(recorder.presentationValues, [480, 960, 1440])
_ = await writer.stop()
}
@@ -1283,6 +1481,10 @@ private final nonisolated class AECLiveRecorder: @unchecked Sendable {
var count: Int { self.lock.withLock { self.samples.count } }
+ var presentationValues: [CMTimeValue] {
+ self.lock.withLock { self.samples.map { CMSampleBufferGetPresentationTimeStamp($0).value } }
+ }
+
func append(_ sample: CMSampleBuffer) {
self.lock.withLock { self.samples.append(sample) }
}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAnalysisManifestTests.swift b/Tests/FluidDictationIntegrationTests/MeetingAnalysisManifestTests.swift
index 2e8d7b5e..67c406c0 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingAnalysisManifestTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingAnalysisManifestTests.swift
@@ -20,6 +20,23 @@ private nonisolated struct ManifestFixtureObserver: MeetingChunkAudioObserving {
final class MeetingAnalysisManifestTests: XCTestCase {
private let backendID = MeetingBackendID(rawValue: "fixture.analysis-manifest")
+ func testUnfilteredSpeechIsAdmittedWithoutClaimingEchoProtection() throws {
+ let audio = self.chunk(sequence: 0, start: 100, end: 110)
+ let mic = self.track(kind: .microphone, chunks: [audio], eras: [self.era(protection: .unfiltered, start: 0)])
+ let plan = self.plan(mode: .onlineCall, tracks: [mic])
+ let manifest = try MeetingAnalysisManifestBuilder(
+ plan: plan, observer: self.observer([(mic.id, audio, self.observed(audio, duration: 10))])
+ ).build()
+ let result = try XCTUnwrap(manifest.track(mic.id))
+ XCTAssertTrue(result.gaps.isEmpty)
+ XCTAssertEqual(result.spans.count, 1)
+ XCTAssertEqual(result.spans.first?.admission.rationale, .onlineMicrophoneUnfiltered)
+ XCTAssertEqual(result.spans.first?.captureEra.echoProtection, .unfiltered)
+ XCTAssertNotNil(MeetingMicrophoneEchoProtection.unfiltered.transcriptWarning)
+ XCTAssertNil(MeetingMicrophoneEchoProtection.softwareEchoCancelled.transcriptWarning)
+ XCTAssertEqual(MeetingCanonicalAdmission.rationale(captureMode: .onlineCall, trackKind: .microphone, echoProtection: .unprotected).decision, .inadmissible)
+ }
+
func testOptInSavedRecordingManifestWithoutChangingSession() throws {
guard let path = ProcessInfo.processInfo.environment["FLUIDVOICE_MEETING_MANIFEST_SESSION"] else {
throw XCTSkip("Set FLUIDVOICE_MEETING_MANIFEST_SESSION for read-only real-recording validation.")
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAudioChunkLedgerRecoveryTests.swift b/Tests/FluidDictationIntegrationTests/MeetingAudioChunkLedgerRecoveryTests.swift
index a71972e2..9a54e56e 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingAudioChunkLedgerRecoveryTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingAudioChunkLedgerRecoveryTests.swift
@@ -73,7 +73,20 @@ final class MeetingAudioChunkLedgerRecoveryTests: XCTestCase {
XCTAssertEqual(loaded?.audioTracks.first?.chunks.first?.finalizationState, .finalized)
XCTAssertEqual(loaded?.audioTracks.first?.chunks.first?.captureAnalysisAsset?.presence, .ready)
XCTAssertEqual(loaded?.audioTracks.first?.chunks.first?.playbackArchiveAsset?.presence, .failed)
- try Data([9]).write(to: pcmURL)
+ try Data([9, 9, 9, 9]).write(to: pcmURL) // Same size: only hashing can detect this corruption.
+ var completed = try XCTUnwrap(loaded)
+ completed.state = .completed
+ completed.endedAt = completed.startedAt.addingTimeInterval(1)
+ try await store.save(completed)
+ let manifestURL = directory.appendingPathComponent("session.json")
+ let beforeHistory = try Data(contentsOf: manifestURL)
+ let history = try await store.loadHistory()
+ let transcript = try await store.loadTranscript(id: sessionID)
+ XCTAssertEqual(history.first?.audioTracks.first?.chunks.first?.finalizationState, .finalized)
+ XCTAssertEqual(transcript?.audioTracks.first?.chunks.first?.captureAnalysisAsset?.presence, .ready)
+ let recoverable = try await store.loadRecoverable()
+ XCTAssertTrue(recoverable.isEmpty, "startup must not validate completed recordings")
+ XCTAssertEqual(try Data(contentsOf: manifestURL), beforeHistory, "browsing never rewrites audio integrity state")
let mismatched = try await store.load(id: sessionID)
XCTAssertEqual(mismatched?.audioTracks.first?.chunks.first?.finalizationState, .failed)
XCTAssertEqual(mismatched?.audioTracks.first?.chunks.first?.byteCount, 0)
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAudioChunkSinkTests.swift b/Tests/FluidDictationIntegrationTests/MeetingAudioChunkSinkTests.swift
index b129be6f..d63c7c8b 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingAudioChunkSinkTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingAudioChunkSinkTests.swift
@@ -570,3 +570,145 @@ final class MeetingAudioChunkSinkTests: XCTestCase {
return sampleBuffer
}
}
+
+extension MeetingAudioChunkSinkTests {
+ func testNativeIntegerMicrophonesConvertWithoutChangingFramesChannelsOrTiming() throws {
+ let normalizer = MeetingMicrophonePCMNormalizer()
+ // One producer changes between USB/Bluetooth-style rates, widths and channel layouts.
+ for rate in [16_000.0, 44_100.0, 48_000.0, 96_000.0] {
+ for bits in [16, 24, 32] {
+ for channels in [1, 2, 3] {
+ for planar in [false, true] {
+ let (source, expected) = try self.integerMicrophoneSample(bits: bits, channels: channels, rate: rate, planar: planar)
+ XCTAssertThrowsError(try MeetingPCMFormatContract(formatDescription: XCTUnwrap(CMSampleBufferGetFormatDescription(source))))
+ let normalized = normalizer.normalize(source)
+ XCTAssertFalse(normalized === source)
+ let description = try XCTUnwrap(CMSampleBufferGetFormatDescription(normalized))
+ let contract = try MeetingPCMFormatContract(formatDescription: description)
+ XCTAssertEqual(contract.sampleRate, rate)
+ XCTAssertEqual(contract.channelCount, channels)
+ XCTAssertEqual(MeetingPCMFormatContract.layoutData(description), try MeetingPCMFormatContract.layoutData(XCTUnwrap(CMSampleBufferGetFormatDescription(source))))
+ XCTAssertEqual(CMSampleBufferGetNumSamples(normalized), CMSampleBufferGetNumSamples(source))
+ XCTAssertEqual(CMSampleBufferGetPresentationTimeStamp(normalized), CMSampleBufferGetPresentationTimeStamp(source))
+ XCTAssertEqual(CMSampleBufferGetDuration(normalized), CMSampleBufferGetDuration(source))
+ XCTAssertEqual(CMSampleBufferGetDecodeTimeStamp(normalized), CMSampleBufferGetDecodeTimeStamp(source))
+ let format = try MeetingPCMFormatResolver.resolve(description)
+ let pcm = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 12))
+ pcm.frameLength = 12
+ XCTAssertEqual(CMSampleBufferCopyPCMDataIntoAudioBufferList(normalized, at: 0, frameCount: 12, into: pcm.mutableAudioBufferList), noErr)
+ let samples = try XCTUnwrap(pcm.floatChannelData?[0])
+ for index in expected.indices {
+ XCTAssertEqual(samples[index], expected[index], accuracy: 0.000_001)
+ }
+ // The callback's storage and integer format must remain untouched.
+ XCTAssertThrowsError(try MeetingPCMFormatContract(formatDescription: XCTUnwrap(CMSampleBufferGetFormatDescription(source))))
+ }
+ }
+ }
+ }
+ }
+
+ func testFloat32MicrophonePassesThroughWithoutCopyingOrChangingEchoAdmission() throws {
+ let source = try self.makeRawSampleBuffer(channelCount: 3, layoutTag: nil, frameCount: 480, pts: 1)
+ let normalizer = MeetingMicrophonePCMNormalizer()
+ XCTAssertTrue(normalizer.normalize(source) === source)
+ let gate = MeetingMicrophoneTranscriptGate(.unprotected)
+ XCTAssertFalse(gate.observeAndAdmit(normalizer.normalize(source)))
+ XCTAssertFalse(gate.admitsTranscript(), "Format conversion must not bless unprotected speaker echo")
+ let (integer, _) = try self.integerMicrophoneSample(bits: 16, channels: 1, rate: 48_000, planar: false)
+ XCTAssertFalse(gate.observeAndAdmit(normalizer.normalize(integer)))
+ }
+
+ func testNormalizedIntegerMicrophoneReachesWriterAndLiveCopyWithNoLostOpeningFrames() async throws {
+ let root = try self.makeDirectory()
+ defer { try? FileManager.default.removeItem(at: root) }
+ let track = MeetingAudioTrack(
+ id: UUID(),
+ kind: .microphone,
+ sourceIdentifier: "integer-mic",
+ sourceDisplayName: "Integer microphone",
+ format: nil,
+ timebase: MeetingTimebaseMetadata(
+ startedHostTime: 0,
+ machTimebaseNumerator: 1,
+ machTimebaseDenominator: 1,
+ firstPresentationTime: nil
+ ),
+ health: .waiting,
+ chunks: []
+ )
+ let events = MeetingWriterEventRecorder()
+ let writer = try MeetingAudioChunkWriter(track: track, sessionDirectory: root, chunkDuration: 60) { events.record($0) }
+ let normalizer = MeetingMicrophonePCMNormalizer()
+ for index in 0..<100 {
+ let (source, _) = try self.integerMicrophoneSample(bits: index.isMultiple(of: 2) ? 16 : 24, channels: 1, rate: 48_000, planar: false, pts: Double(index * 12) / 48_000)
+ let normalized = normalizer.normalize(source)
+ let live = try XCTUnwrap(MeetingLiveSampleCopy.copy(normalized))
+ XCTAssertEqual(live.buffer.frameLength, 12)
+ XCTAssertEqual(try XCTUnwrap(live.buffer.floatChannelData?[0])[0], -0.5, accuracy: 0.000_001)
+ XCTAssertTrue(writer.enqueue(normalized))
+ _ = await writer.snapshot()
+ }
+ let saved = await writer.stop()
+ XCTAssertEqual(saved.chunks.count, 1, "Changing native bit width must not split the normalized capture")
+ let chunk = try XCTUnwrap(saved.chunks.first)
+ XCTAssertEqual(chunk.finalizationState, .finalized)
+ XCTAssertEqual(chunk.captureAnalysisAsset?.frameCount, 1200)
+ let file = try AVAudioFile(forReading: root.appendingPathComponent(chunk.relativeFilePath), commonFormat: .pcmFormatFloat32, interleaved: true)
+ let recorded = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: 1200))
+ try file.read(into: recorded)
+ XCTAssertEqual(recorded.frameLength, 1200)
+ let values = try XCTUnwrap(recorded.floatChannelData?[0])
+ for index in 0..<1200 {
+ XCTAssertEqual(values[index], Float((index % 3) - 1) * 0.5, accuracy: 0.000_001)
+ }
+ XCTAssertFalse(events.snapshot().contains { if case .interrupted(.writerFailure, _, _) = $0 { return true }; return false })
+ }
+
+ func testUnsupportedAndMalformedBuffersRemainReportableAndDoNotPoisonLaterConversion() throws {
+ let normalizer = MeetingMicrophonePCMNormalizer()
+ let integerFormat = try XCTUnwrap(AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48_000, channels: 1, interleaved: true))
+ let shortPayload = try self.makeSampleBuffer(format: integerFormat, bytes: Data([0]), frameCount: 12, pts: 0)
+ XCTAssertTrue(normalizer.normalize(shortPayload) === shortPayload)
+ let oversized = try self.makeSampleBuffer(format: integerFormat, bytes: Data([0]), frameCount: 1_048_577, pts: 0)
+ XCTAssertTrue(normalizer.normalize(oversized) === oversized)
+ let doubleFormat = try XCTUnwrap(AVAudioFormat(commonFormat: .pcmFormatFloat64, sampleRate: 48_000, channels: 1, interleaved: true))
+ let unsupported = try self.makeSampleBuffer(format: doubleFormat, bytes: Data(repeating: 0, count: 96), frameCount: 12, pts: 0)
+ XCTAssertTrue(normalizer.normalize(unsupported) === unsupported)
+ let (valid, _) = try self.integerMicrophoneSample(bits: 16, channels: 1, rate: 48_000, planar: false)
+ let first = normalizer.normalize(valid)
+ XCTAssertNoThrow(try MeetingPCMFormatContract(formatDescription: XCTUnwrap(CMSampleBufferGetFormatDescription(first))))
+ let (invalid, _) = try self.integerMicrophoneSample(bits: 16, channels: 1, rate: 48_000, planar: false)
+ CMSampleBufferInvalidate(invalid)
+ XCTAssertTrue(normalizer.normalize(invalid) === invalid)
+ let second = normalizer.normalize(valid)
+ XCTAssertFalse(first === second, "Each asynchronously retained callback must own its output bytes")
+ XCTAssertNoThrow(try MeetingPCMFormatContract(formatDescription: XCTUnwrap(CMSampleBufferGetFormatDescription(second))))
+ }
+
+ private func integerMicrophoneSample(bits: Int, channels: Int, rate: Double, planar: Bool, pts: Double = 1.25) throws -> (CMSampleBuffer, [Float]) {
+ var asbd = AudioStreamBasicDescription(
+ mSampleRate: rate,
+ mFormatID: kAudioFormatLinearPCM,
+ mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked | (planar ? kAudioFormatFlagIsNonInterleaved : 0),
+ mBytesPerPacket: UInt32(bits / 8 * (planar ? 1 : channels)),
+ mFramesPerPacket: 1,
+ mBytesPerFrame: UInt32(bits / 8 * (planar ? 1 : channels)),
+ mChannelsPerFrame: UInt32(channels),
+ mBitsPerChannel: UInt32(bits),
+ mReserved: 0
+ )
+ let layout = try XCTUnwrap(AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_DiscreteInOrder | UInt32(channels)))
+ let format = try XCTUnwrap(AVAudioFormat(streamDescription: &asbd, channelLayout: layout))
+ let expected: [Float] = (0..<(12 * channels)).map { Float(($0 % 3) - 1) * 0.5 }
+ var bytes = Data()
+ for offset in expected.indices {
+ let index = planar ? (offset % 12) * channels + offset / 12 : offset
+ let value = Int64(Double(expected[index]) * pow(2, Double(bits - 1)))
+ for byte in 0..<(bits / 8) {
+ bytes.append(UInt8(truncatingIfNeeded: value >> (byte * 8)))
+ }
+ }
+ return try (self.makeSampleBuffer(format: format, bytes: bytes, frameCount: 12, pts: pts), expected)
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAudioComparisonProbeTests.swift b/Tests/FluidDictationIntegrationTests/MeetingAudioComparisonProbeTests.swift
new file mode 100644
index 00000000..4a853f05
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/MeetingAudioComparisonProbeTests.swift
@@ -0,0 +1,247 @@
+import AVFoundation
+import CoreMedia
+@testable import FluidVoice_Debug
+import Foundation
+import XCTest
+
+private final class ProbeTestClock: @unchecked Sendable {
+ private let lock = NSLock()
+ private var time: TimeInterval = 0
+ func now() -> TimeInterval { self.lock.withLock { self.time } }
+ func advance() { self.lock.withLock { self.time = 121 } }
+}
+
+final class MeetingAudioComparisonProbeTests: XCTestCase {
+ func testOptInReplaySavedMicrophoneThroughAECAndFallback() async throws {
+ let request = (try? Data(contentsOf: URL(fileURLWithPath: "/tmp/fluidvoice-aec-replay-request.json")))
+ .flatMap { try? JSONDecoder().decode([String: String].self, from: $0) }
+ guard let input = ProcessInfo.processInfo.environment["FLUIDVOICE_AEC_REPLAY_INPUT"] ?? request?["input"],
+ let output = ProcessInfo.processInfo.environment["FLUIDVOICE_AEC_REPLAY_OUTPUT"] ?? request?["output"]
+ else { throw XCTSkip("Set saved PCM input/output paths for a local diagnostic replay.") }
+ for useFallback in [false, true] {
+ let source = try AVAudioFile(forReading: URL(fileURLWithPath: input))
+ XCTAssertEqual(source.processingFormat.sampleRate, 48_000)
+ let directory = URL(fileURLWithPath: output).appendingPathComponent(useFallback ? "fallback" : "aec")
+ let track = MeetingAudioTrack(
+ id: UUID(),
+ kind: .microphone,
+ sourceIdentifier: "replay",
+ sourceDisplayName: "Saved microphone",
+ format: nil,
+ timebase: .init(startedHostTime: 0, machTimebaseNumerator: 1, machTimebaseDenominator: 1, firstPresentationTime: nil),
+ health: .waiting,
+ chunks: [],
+ captureMethod: .screenCaptureKit,
+ captureEras: [.init(method: .screenCaptureKit, deviceUID: nil, deviceName: nil, roleAtElection: .unknown, echoProtection: .unprotected, startSeconds: 0)]
+ )
+ let writer = try MeetingAudioChunkWriter(track: track, sessionDirectory: directory, chunkDuration: 60, eventHandler: { _ in })
+ let gate = MeetingMicrophoneTranscriptGate(.unprotected)
+ let counter = ReplaySampleCounter()
+ let committer = MeetingAECOutputCommitter(
+ writer: writer, transcriptGate: gate, liveAudioHandler: { _, sample in counter.add(CMSampleBufferGetNumSamples(sample)) }, terminalFailure: { _ in }
+ )
+ let processor: any MeetingAECProcessing
+ if useFallback {
+ processor = ReplayFailingAEC()
+ } else {
+ processor = try MeetingAECProcessor()
+ }
+ let pipeline = MeetingAECPipeline(processor: processor, committer: committer)
+ let token = MeetingAECStreamToken(generation: 1, identity: 1)
+ let buffer = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: source.processingFormat, frameCapacity: 480))
+ var offset: Int64 = 0
+ var frames = 0
+ while source.framePosition < source.length {
+ try source.read(into: buffer, frameCount: 480)
+ let count = Int(buffer.frameLength)
+ let channel = try XCTUnwrap(buffer.floatChannelData?[0])
+ let samples = Array(UnsafeBufferPointer(start: channel, count: count))
+ let pts = CMTime(value: offset, timescale: 48_000)
+ let capture = try XCTUnwrap(MeetingAECPCMAdapter.synthesize(samples: samples, presentationTime: pts))
+ let render = try XCTUnwrap(MeetingAECPCMAdapter.synthesize(samples: [Float](repeating: 0, count: count), presentationTime: pts))
+ pipeline.consumeRender(render, stream: token)
+ pipeline.consumeCapture(capture, stream: token)
+ offset += Int64(count)
+ frames += 1
+ if frames.isMultiple(of: 8) { _ = await writer.snapshot() }
+ }
+ await committer.waitForPendingCommits()
+ pipeline.stop(boundary: gate.invalidate())
+ await committer.waitForPendingCommits()
+ let saved = await writer.stop()
+ let chunk = try XCTUnwrap(saved.chunks.first)
+ let file = try AVAudioFile(forReading: chunk.fileURL(relativeTo: directory))
+ XCTAssertEqual(file.length, source.length, "filtering must not discard captured PCM")
+ XCTAssertGreaterThan(counter.samples, 0)
+ if useFallback { XCTAssertEqual(counter.samples, Int(source.length)) }
+ let encoder = JSONEncoder()
+ encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
+ try encoder.encode(saved).write(to: directory.appendingPathComponent("replay-track.json"))
+ print("AEC_REPLAY fallback=\(useFallback) recordedSamples=\(file.length) liveSamples=\(counter.samples) path=\(chunk.fileURL(relativeTo: directory).path)")
+ }
+ }
+
+ func testRequestSurvivesWrongModeAndRetriesButExpires() throws {
+ let name = "MeetingAudioComparisonProbeTests.\(UUID())"
+ let defaults = try XCTUnwrap(UserDefaults(suiteName: name))
+ defer { defaults.removePersistentDomain(forName: name) }
+ XCTAssertNil(MeetingAudioComparisonProbe.requestExpiration(mode: .onlineCall, defaults: defaults, now: 100))
+ XCTAssertNil(defaults.object(forKey: MeetingAudioComparisonProbe.requestExpirationKey))
+ defaults.set(true, forKey: MeetingAudioComparisonProbe.nextRecordingKey)
+ XCTAssertNil(MeetingAudioComparisonProbe.requestExpiration(mode: .inRoom, defaults: defaults, now: 100))
+ let expiration = try XCTUnwrap(MeetingAudioComparisonProbe.requestExpiration(mode: .onlineCall, defaults: defaults, now: 101))
+ XCTAssertEqual(expiration, 1900)
+ XCTAssertEqual(MeetingAudioComparisonProbe.requestExpiration(mode: .onlineCall, defaults: defaults, now: 102), expiration)
+ XCTAssertTrue(defaults.bool(forKey: MeetingAudioComparisonProbe.nextRecordingKey))
+ MeetingAudioComparisonProbe.completeRequest(expiration: expiration - 1, defaults: defaults)
+ XCTAssertTrue(defaults.bool(forKey: MeetingAudioComparisonProbe.nextRecordingKey), "A stale probe cannot disarm a newer request")
+ XCTAssertNil(MeetingAudioComparisonProbe.requestExpiration(mode: .onlineCall, defaults: defaults, now: expiration))
+ XCTAssertNil(defaults.object(forKey: MeetingAudioComparisonProbe.nextRecordingKey))
+ XCTAssertNil(defaults.object(forKey: MeetingAudioComparisonProbe.requestExpirationKey))
+ defaults.set(true, forKey: MeetingAudioComparisonProbe.nextRecordingKey)
+ let renewed = try XCTUnwrap(MeetingAudioComparisonProbe.requestExpiration(mode: .onlineCall, defaults: defaults, now: 2000))
+ MeetingAudioComparisonProbe.completeRequest(expiration: renewed, defaults: defaults)
+ XCTAssertNil(defaults.object(forKey: MeetingAudioComparisonProbe.nextRecordingKey))
+ }
+
+ func testThreeStagesKeepIndependentPCMAndTimingOutsideNormalHistory() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let probe = try MeetingAudioComparisonProbe(session: self.session(), directory: root)
+ try probe.offer(self.sample(value: 0.25), stage: .browser)
+ try probe.offer(self.sample(value: 0.5), stage: .microphoneInput)
+ try probe.offer(self.sample(value: 0.1), stage: .microphoneOutput)
+ try probe.offer(self.sample(value: 0.1), stage: .microphoneRecognizerInput)
+ try probe.offer(self.sample(value: 0.25), stage: .browserRecognizerInput)
+ probe.recordEvent("test-partial", text: "Known local test phrase")
+ await probe.stop()
+ await probe.stop()
+ for (stage, value) in [(MeetingAudioComparisonProbe.Stage.browser, Float(0.25)), (.microphoneInput, 0.5), (.microphoneOutput, 0.1)] {
+ let track = try self.track(stage, root: root)
+ let chunk = try XCTUnwrap(track.chunks.first)
+ XCTAssertEqual(chunk.captureAnalysisAsset?.frameCount, 480)
+ XCTAssertEqual(chunk.presentationStart.seconds, 42, accuracy: 0.000_001)
+ let url = root.appendingPathComponent(stage.rawValue).appendingPathComponent(chunk.relativeFilePath)
+ let file = try AVAudioFile(forReading: url)
+ let pcm = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: file.processingFormat, frameCapacity: 480))
+ try file.read(into: pcm)
+ let data = try XCTUnwrap(pcm.floatChannelData?[0])
+ XCTAssertEqual(data[0], value, accuracy: 0.000_001)
+ XCTAssertEqual(data[479], value, accuracy: 0.000_001)
+ }
+ XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("session.json").path))
+ let report = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(contentsOf: root.appendingPathComponent("comparison.json"))) as? [String: Any])
+ XCTAssertEqual(report["hasAllStages"] as? Bool, true)
+ XCTAssertEqual(report["completed"] as? Bool, true)
+ var tracks: [String: MeetingAudioTrack] = [:]
+ for stage in MeetingAudioComparisonProbe.Stage.allCases {
+ tracks[stage.rawValue] = try self.track(stage, root: root)
+ }
+ XCTAssertFalse(MeetingAudioComparisonProbe.hasUsefulComparison(tracks), "A short attempt must leave diagnostics armed")
+ let lines = try String(contentsOf: root.appendingPathComponent("events.jsonl"), encoding: .utf8).split(separator: "\n")
+ XCTAssertEqual(lines.count, 1)
+ let event = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(lines[0].utf8)) as? [String: Any])
+ XCTAssertEqual(event["step"] as? String, "test-partial")
+ XCTAssertEqual(event["text"] as? String, "Known local test phrase")
+ }
+
+ func testFullComparisonCompletesRequestOnlyAfterReportIsSaved() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let completed = expectation(description: "Usable comparison saved")
+ completed.assertForOverFulfill = true
+ let probe = try MeetingAudioComparisonProbe(session: self.session(), directory: root, onUsefulComparison: {
+ XCTAssertTrue(FileManager.default.fileExists(atPath: root.appendingPathComponent("comparison.json").path))
+ completed.fulfill()
+ })
+ for stage in MeetingAudioComparisonProbe.Stage.allCases {
+ try probe.offer(self.sample(value: 0.1, frames: 1_440_000), stage: stage)
+ }
+ await probe.stop()
+ await probe.stop()
+ await fulfillment(of: [completed], timeout: 5)
+ }
+
+ func testProbeDeadlineAndStopDoNotDropOrChangeNormalRecording() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let clock = ProbeTestClock()
+ let probe = try MeetingAudioComparisonProbe(session: self.session(), directory: root.appendingPathComponent("probe"), maximumSeconds: 120, uptime: { clock.now() })
+ XCTAssertTrue(probe.isRecordingDiagnostics)
+ let normalTrack = MeetingAudioTrack(
+ id: UUID(),
+ kind: .microphone,
+ sourceIdentifier: "test",
+ sourceDisplayName: "Test",
+ format: nil,
+ timebase: self.session().timebase,
+ health: .waiting,
+ chunks: []
+ )
+ let writer = try MeetingAudioChunkWriter(
+ track: normalTrack,
+ sessionDirectory: root.appendingPathComponent("normal"),
+ chunkDuration: 60,
+ eventHandler: { _ in },
+ diagnosticSampleObserver: { sample, epoch in probe.offer(sample, stage: .microphoneOutput, producerEpoch: epoch) }
+ )
+ XCTAssertTrue(try writer.enqueue(self.sample(value: 0.5)))
+ _ = await writer.snapshot()
+ clock.advance()
+ XCTAssertFalse(probe.isRecordingDiagnostics)
+ probe.recordEvent("expired-event", text: "Must not be retained")
+ XCTAssertTrue(try writer.enqueue(self.sample(value: 0.5, pts: 42.01)))
+ let normal = await writer.stop()
+ await probe.stop()
+ let diagnostic = try self.track(.microphoneOutput, root: root.appendingPathComponent("probe"))
+ XCTAssertEqual(diagnostic.chunks.first?.captureAnalysisAsset?.frameCount, 480)
+ XCTAssertEqual(try Data(contentsOf: root.appendingPathComponent("probe/events.jsonl")).count, 0)
+ XCTAssertEqual(normal.chunks.first?.captureAnalysisAsset?.frameCount, 960)
+ XCTAssertEqual(normal.health.droppedSampleCount, 0)
+ let report = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(contentsOf: root.appendingPathComponent("probe/comparison.json"))) as? [String: Any])
+ XCTAssertEqual(report["hasAllStages"] as? Bool, false, "Missing raw input must never look like a valid comparison")
+ try probe.offer(self.sample(value: 1), stage: .microphoneOutput)
+ await probe.stop()
+ XCTAssertEqual(try self.track(.microphoneOutput, root: root.appendingPathComponent("probe")).chunks.first?.captureAnalysisAsset?.frameCount, 480)
+ }
+
+ private func session() -> MeetingSession {
+ MeetingSession(
+ configuration: MeetingCaptureConfiguration(mode: .inRoom, title: "Probe", microphone: MeetingMicrophoneIdentity(captureDeviceID: "test", displayName: "Test")),
+ timebase: MeetingTimebaseMetadata(startedHostTime: 0, machTimebaseNumerator: 1, machTimebaseDenominator: 1, firstPresentationTime: nil)
+ )
+ }
+
+ private func sample(value: Float, pts: Double = 42, frames: AVAudioFrameCount = 480) throws -> CMSampleBuffer {
+ let format = try XCTUnwrap(AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48_000, channels: 1, interleaved: false))
+ let pcm = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames))
+ pcm.frameLength = frames
+ try XCTUnwrap(pcm.floatChannelData?[0]).update(repeating: value, count: Int(frames))
+ return try XCTUnwrap(meetingMicrophoneSynthesizeSampleBuffer(from: pcm, presentationTime: CMTime(seconds: pts, preferredTimescale: 48_000)))
+ }
+
+ private func track(_ stage: MeetingAudioComparisonProbe.Stage, root: URL) throws -> MeetingAudioTrack {
+ let kind = stage == .browser || stage == .browserRecognizerInput ? "applicationAudio" : "microphone"
+ let path = root.appendingPathComponent("\(stage.rawValue)/tracks/\(kind)/track.json")
+ let decoder = JSONDecoder()
+ decoder.dateDecodingStrategy = .iso8601
+ return try decoder.decode(MeetingAudioTrack.self, from: Data(contentsOf: path))
+ }
+}
+
+private final nonisolated class ReplayFailingAEC: MeetingAECProcessing, @unchecked Sendable {
+ let upstreamRevision = MeetingAECConstants.provenance.upstreamRevision
+ let configurationID = MeetingAECConstants.provenance.bridgeConfigurationID
+ func process(render: [Float], capture: [Float]) throws -> (samples: [Float], statistics: MeetingAECBridgeStatistics) {
+ throw MeetingAECFailure.bridgeProcessing
+ }
+
+ func reset() throws {}
+}
+
+private final nonisolated class ReplaySampleCounter: @unchecked Sendable {
+ private let lock = NSLock()
+ private var count = 0
+ var samples: Int { self.lock.withLock { self.count } }
+ func add(_ value: Int) { self.lock.withLock { self.count += value } }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift b/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
index 8872576d..47c9f168 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
@@ -62,13 +62,23 @@ private final class FakeAudioProcessActivity: AudioProcessActivityProviding {
@MainActor
private final class FakeWindowSnapshotProvider: WindowSnapshotProviding {
var axTitles: [String] = []
- func snapshot(interestPIDs: Set<Int32>) -> [WindowSnapshot] { [] }
+ var lastInterestPIDs: Set<Int32> = []
+ func snapshot(interestPIDs: Set<Int32>) -> [WindowSnapshot] {
+ self.lastInterestPIDs = interestPIDs
+ return []
+ }
+
func titles(processID: Int32) async -> [String] { self.axTitles }
}
@MainActor
private final class FakeBrowserTabReader: BrowserTabReading {
- func frontmostTabURL(bundleIdentifier: String, processID: Int32) async -> BrowserTabURL? { nil }
+ var result: BrowserTabURL?
+ var readPIDs: [Int32] = []
+ func frontmostTabURL(bundleIdentifier: String, processID: Int32) async -> BrowserTabURL? {
+ self.readPIDs.append(processID)
+ return self.result
+ }
}
@MainActor
@@ -95,6 +105,7 @@ private final class DetectorHarness {
let audioProcessActivity = FakeAudioProcessActivity()
let windowProvider = FakeWindowSnapshotProvider()
let workspace = FakeWorkspaceEvents()
+ let browserReader = FakeBrowserTabReader()
var prompts: [MeetingAutoDetector.PromptRequest] = []
var nudges = 0
var withdrawals = 0
@@ -118,7 +129,7 @@ private final class DetectorHarness {
micActivity: FakeMicActivity(),
audioProcessActivity: self.audioProcessActivity,
windowSnapshotProvider: self.windowProvider,
- browserTabReader: FakeBrowserTabReader(),
+ browserTabReader: self.browserReader,
activityGate: self.gate,
clock: self.clock,
isNativeDetectionEnabled: { flags.nativeEnabled },
@@ -1952,3 +1963,481 @@ final class MeetingAutoDetectorTests: XCTestCase {
XCTAssertEqual(selected?.windowID, 1)
}
}
+
+@MainActor
+final class MeetingManualRecordingEndTests: XCTestCase {
+ private func session(bundle: String? = "us.zoom.xos", pid: Int32? = 100) -> MeetingSession {
+ var session = MeetingSession(
+ configuration: MeetingCaptureConfiguration(
+ mode: bundle == nil ? .inRoom : .onlineCall,
+ title: "End detection fixture",
+ application: bundle.map { MeetingApplicationIdentity(bundleIdentifier: $0, processID: pid, displayName: "Call") },
+ microphone: MeetingMicrophoneIdentity(captureDeviceID: "fixture-mic", displayName: "Microphone")
+ ),
+ timebase: MeetingTimebaseMetadata(startedHostTime: 0, machTimebaseNumerator: 1, machTimebaseDenominator: 1)
+ )
+ session.state = .recording
+ return session
+ }
+
+ private func synchronize(_ h: DetectorHarness, _ session: MeetingSession, state: MeetingCoordinatorState? = nil) {
+ AppServices.synchronizeMeetingEndWatch(state: state ?? .recording(session.id), session: session, detector: h.detector)
+ }
+
+ func testManualNativeRecordingAsksWithoutAnyStartPromptOrWindowEvidence() {
+ for bundle in ["us.zoom.xos", "com.microsoft.teams2", "com.cisco.webexmeetingsapp"] {
+ let h = DetectorHarness()
+ let session = self.session(bundle: bundle)
+ h.gate.isIdle = false
+ self.synchronize(h, session)
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1, bundle)
+ XCTAssertTrue(h.prompts.isEmpty, "observing a manually selected app must not create a start prompt")
+ XCTAssertNil(h.detector.automaticTarget, "end monitoring must not select a source")
+ XCTAssertFalse(h.gate.isIdle, "observing an end must not stop capture")
+ XCTAssertEqual(session.state, .recording)
+ }
+ }
+
+ func testManualBrowserRecordingAsksAfterObservedMicrophoneRelease() async {
+ let h = DetectorHarness()
+ let session = self.session(bundle: "com.google.Chrome", pid: 8)
+ h.gate.isIdle = false
+ self.synchronize(h, session)
+ await h.pollChrome(microphoneInUse: true)
+ await h.pollChrome(microphoneInUse: false)
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ XCTAssertTrue(h.prompts.isEmpty)
+ XCTAssertNil(h.detector.automaticTarget)
+ XCTAssertFalse(h.gate.isIdle)
+ }
+
+ func testManualRecordingAsksAfterCapturedAppQuits() {
+ let h = DetectorHarness()
+ let session = self.session()
+ self.synchronize(h, session)
+ h.detector.handleWorkspaceEvent(.init(kind: .terminated, bundleIdentifier: "us.zoom.xos", processID: 100), at: h.clock.now())
+ h.detector.tick(at: h.clock.now())
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ }
+
+ func testRepeatedHealthStatesPreserveAudioEvidenceQuestionAndDismissal() {
+ let h = DetectorHarness()
+ let session = self.session()
+ self.synchronize(h, session)
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ h.advance(10)
+ self.synchronize(h, session, state: .recordingDegraded(session.id))
+ h.advance(11)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ self.synchronize(h, session)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1, "a health-state notification must not repeat the question")
+ h.detector.stillRecordingNudgeDismissed()
+ self.synchronize(h, session, state: .recordingDegraded(session.id))
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1, "the user's dismissal survives subsequent health states")
+ }
+
+ func testCommonObserverDoesNotResetAnAdoptedAutomaticRecording() throws {
+ let h = DetectorHarness()
+ let episode = try XCTUnwrap(h.confirmZoom())
+ XCTAssertTrue(h.detector.startTapped(episodeID: episode))
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ let session = self.session()
+ self.synchronize(h, session)
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1, "binding the real session preserves previously observed audio")
+ h.detector.stillRecordingNudgeDismissed()
+ h.detector.adoptStartedEpisode(episodeID: episode)
+ self.synchronize(h, session, state: .recordingDegraded(session.id))
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ }
+
+ func testManualObserverBeforeAutomaticAdoptionKeepsItsSessionIdentity() throws {
+ let h = DetectorHarness()
+ let episode = try XCTUnwrap(h.confirmZoom())
+ let session = self.session()
+ self.synchronize(h, session)
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.adoptStartedEpisode(episodeID: episode)
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ self.synchronize(h, session, state: .recordingDegraded(session.id))
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ XCTAssertEqual(h.prompts.count, 1)
+ }
+
+ func testPreparingAndMismatchedSnapshotsNeverArmAnEndWatch() {
+ for state in [MeetingCoordinatorState.preparing(UUID()), .recording(UUID())] {
+ let h = DetectorHarness()
+ self.synchronize(h, self.session(), state: state)
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ XCTAssertTrue(h.prompts.isEmpty)
+ }
+ }
+
+ func testInRoomUnsupportedAndMissingProcessRecordingsNeverGuessAnApp() {
+ let sessions = [self.session(bundle: nil), self.session(bundle: "com.example.recorder"), self.session(pid: nil)]
+ for session in sessions {
+ let h = DetectorHarness()
+ self.synchronize(h, session)
+ h.detector.handleWorkspaceEvent(.init(kind: .terminated, bundleIdentifier: "us.zoom.xos", processID: 100), at: h.clock.now())
+ h.detector.tick(at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ XCTAssertNil(h.detector.automaticTarget)
+ XCTAssertTrue(h.prompts.isEmpty)
+ }
+ }
+
+ func testStoppingOrFinishingRecordingClearsItsWatch() {
+ let session = self.session()
+ let failure = MeetingSessionFailure(id: UUID(), occurredAt: Date(), domain: .capture, code: "fixture", message: "fixture", recoverable: true)
+ for state in [MeetingCoordinatorState.stopping(session.id), .processing(session.id, .transcribing), .completed(session.id), .interrupted(session.id), .failed(session.id, failure), .idle] {
+ let h = DetectorHarness()
+ self.synchronize(h, session)
+ self.synchronize(h, session, state: state)
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ }
+ }
+
+ func testEndWatchRejectsAProcessIDThatNowBelongsToAnotherApp() {
+ let h = DetectorHarness()
+ h.detector.handleBackfill([.init(kind: .launched, bundleIdentifier: "com.microsoft.teams2", processID: 100)])
+ self.synchronize(h, self.session())
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ XCTAssertTrue(h.prompts.isEmpty)
+ }
+
+ func testNewRecordingGetsItsOwnQuestionAfterEarlierDismissal() {
+ let h = DetectorHarness()
+ let first = self.session()
+ self.synchronize(h, first)
+ h.detector.stillRecordingNudgeDismissed()
+ self.synchronize(h, first, state: .stopping(first.id))
+ let second = self.session()
+ self.synchronize(h, second)
+ h.detector.handleAudioProcessActivity(true, pid: 100, at: h.clock.now())
+ h.detector.handleAudioProcessActivity(false, pid: 100, at: h.clock.now())
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ }
+
+ func testNeverObservedOrUnreadableBrowserMicrophoneDoesNotInventAnEnd() async {
+ let h = DetectorHarness()
+ self.synchronize(h, self.session(bundle: "com.google.Chrome", pid: 8))
+ await h.pollChrome(microphoneInUse: false)
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ await h.pollChrome(microphoneInUse: true)
+ await h.pollChrome(microphoneInUse: false, readable: false)
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ }
+}
+
+@MainActor
+final class MeetingBrowserPostCallTests: XCTestCase {
+ private let safariPID: Int32 = 42
+ private let room = "/abc-defg-hij"
+
+ private func report(_ h: DetectorHarness, state: MeetingBrowserCallState = .unknown, path: String? = nil) {
+ h.detector.handleBrowserTabURL(
+ BrowserTabURL(host: "meet.google.com", path: path ?? self.room, callState: state),
+ pid: self.safariPID,
+ bundleIdentifier: "com.apple.Safari",
+ at: h.clock.now()
+ )
+ }
+
+ private func safariRecording() throws -> DetectorHarness {
+ let h = DetectorHarness()
+ h.workspace.frontmostProcessID = self.safariPID
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "com.apple.Safari", processID: self.safariPID), at: h.clock.now())
+ self.report(h)
+ h.detector.handleMicEdge(.init(isActive: true), at: h.clock.now())
+ let episode = try XCTUnwrap(h.prompts.last?.episodeID)
+ XCTAssertTrue(h.detector.startTapped(episodeID: episode))
+ h.gate.isIdle = false
+ return h
+ }
+
+ func testPostCallPageAsksWhenSafarisMicrophoneBelongsToASystemHelper() async throws {
+ let h = try self.safariRecording()
+ let safari = MeetingProcessOwner(processID: self.safariPID, bundleIdentifier: "com.apple.Safari", bundlePath: "/Applications/Safari.app")
+ h.audioProcessActivity.snapshotOverride = .init(
+ processes: [.init(
+ processID: 4242,
+ bundleIdentifier: "com.apple.WebKit.GPU",
+ executablePath: "/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.GPU.xpc/Contents/MacOS/com.apple.WebKit.GPU",
+ isInputRunning: true,
+ isOutputRunning: true
+ )],
+ owners: [safari],
+ queryState: .valid
+ )
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ self.report(h, state: .ended)
+ h.advance(1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0, "one brief rejoin opportunity remains")
+ h.advance(1.1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1, "an explicit post-call page is evidence independent of WebKit microphone ownership")
+ XCTAssertFalse(h.gate.isIdle, "the end question must not stop or trim capture")
+ XCTAssertEqual(h.prompts.count, 1)
+ }
+
+ func testPostCallPageNeverOffersAStartAndInvalidatesAnUnacceptedStart() throws {
+ let h = DetectorHarness()
+ h.workspace.frontmostProcessID = self.safariPID
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "com.apple.Safari", processID: self.safariPID), at: h.clock.now())
+ self.report(h, state: .ended)
+ h.detector.handleMicEdge(.init(isActive: true), at: h.clock.now())
+ XCTAssertTrue(h.prompts.isEmpty)
+ self.report(h, state: .inCall)
+ let episode = try XCTUnwrap(h.prompts.last?.episodeID)
+ self.report(h, state: .ended)
+ XCTAssertEqual(h.invalidated, [episode])
+ XCTAssertFalse(h.detector.canStart(episodeID: episode))
+ h.detector.handleMicEdge(.init(isActive: true), at: h.clock.now())
+ XCTAssertEqual(h.prompts.count, 1)
+ }
+
+ func testTabSwitchAndMuteAloneDoNotCountAsAnEnd() throws {
+ let h = try self.safariRecording()
+ self.report(h, state: .inCall)
+ h.detector.handleBrowserTabURL(.init(host: "docs.google.com", path: "/document"), pid: self.safariPID, bundleIdentifier: "com.apple.Safari", at: h.clock.now())
+ h.advance(300)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ self.report(h, state: .inCall)
+ h.detector.handleMicEdge(.init(isActive: false), at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ }
+
+ func testDifferentRoomAndDifferentProcessCannotEndOurCall() throws {
+ let h = try self.safariRecording()
+ self.report(h, state: .ended, path: "/xyz-abcd-uvw")
+ h.detector.handleBrowserTabURL(.init(host: "meet.google.com", path: self.room, callState: .ended), pid: 99, bundleIdentifier: "com.apple.Safari", at: h.clock.now())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ }
+
+ func testUnknownReadsAndTabChangesPreserveAPositiveEndWithoutRestartingItsTimer() throws {
+ let h = try self.safariRecording()
+ self.report(h, state: .ended)
+ h.advance(1)
+ self.report(h, state: .ended)
+ self.report(h)
+ h.detector.handleBrowserTabURL(nil, pid: self.safariPID, bundleIdentifier: "com.apple.Safari", at: h.clock.now())
+ self.report(h, state: .inCall, path: "/xyz-abcd-uvw")
+ h.advance(1.1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ }
+
+ func testExplicitRejoinCancelsThePendingEndAndWithdrawsAnAskedQuestion() throws {
+ let h = try self.safariRecording()
+ self.report(h, state: .ended)
+ h.advance(1)
+ self.report(h, state: .inCall)
+ h.advance(5)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ self.report(h, state: .ended)
+ h.advance(2.1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ self.report(h, state: .inCall)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.withdrawals, 1)
+ }
+
+ func testPostCallPageDoesNotWaitForCaptionQuietAndStillHonorsDismissal() throws {
+ let h = try self.safariRecording()
+ self.report(h, state: .ended)
+ h.flags.lastSpeechAt = h.clock.now()
+ h.advance(2.1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ h.detector.stillRecordingNudgeDismissed()
+ self.report(h, state: .inCall)
+ self.report(h, state: .ended)
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ }
+
+ func testMicrophoneReleaseWithoutPostCallProofKeepsTheGraceAndSpeechQuietRules() async throws {
+ let h = DetectorHarness()
+ let episode = try XCTUnwrap(h.confirmChromeMeet())
+ XCTAssertTrue(h.detector.startTapped(episodeID: episode))
+ await h.pollChrome(microphoneInUse: true)
+ await h.pollChrome(microphoneInUse: false)
+ h.advance(3)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0, "the short post-call grace must not apply to microphone release")
+ h.flags.lastSpeechAt = h.clock.now()
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0, "fresh speech still defers an ambiguous disconnect")
+ h.advance(MeetingAutoDetector.speechQuietSeconds)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ }
+
+ func testMeetEvidenceRequiresMatchingControlsAndIgnoresOrdinaryPageText() {
+ var evidence = MeetingBrowserCallEvidence()
+ evidence.observe(role: "AXStaticText", label: "You left the meeting")
+ evidence.observe(role: "AXStaticText", label: "Rejoin")
+ XCTAssertEqual(evidence.state, .unknown)
+ evidence.observe(role: "AXHeading", label: "You left the meeting")
+ XCTAssertEqual(evidence.state, .unknown)
+ evidence.observe(role: "AXButton", label: " Rejoin ")
+ XCTAssertEqual(evidence.state, .ended)
+ evidence.observe(role: "AXButton", label: "Leave call")
+ XCTAssertEqual(evidence.state, .unknown, "contradictory transitional controls are not proof")
+ XCTAssertTrue(MeetingBrowserCallEvidence.supports(.init(host: "meet.google.com", path: self.room)))
+ XCTAssertFalse(MeetingBrowserCallEvidence.supports(.init(host: "meet.google.com", path: "/home")))
+ XCTAssertFalse(MeetingBrowserCallEvidence.supports(.init(host: "example.com", path: self.room)))
+ var unknownLanguage = MeetingBrowserCallEvidence()
+ unknownLanguage.observe(role: "AXHeading", label: "Unrecognised post-call copy")
+ unknownLanguage.observe(role: "AXButton", label: "Unrecognised action")
+ XCTAssertEqual(unknownLanguage.state, .unknown)
+ }
+}
+
+@MainActor
+extension MeetingManualRecordingEndTests {
+ func testCapturedAppQuitIsObservedWithAutomaticDetectionDisabled() {
+ let h = DetectorHarness()
+ h.nativeEnabled = false
+ h.browserEnabled = false
+ h.gate.isIdle = false
+ self.synchronize(h, self.session())
+ h.detector.handleWorkspaceEvent(.init(kind: .terminated, bundleIdentifier: "us.zoom.xos", processID: 100), at: h.clock.now())
+ h.detector.tick(at: h.clock.now())
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ XCTAssertTrue(h.prompts.isEmpty)
+ XCTAssertNil(h.detector.automaticTarget)
+ }
+
+ func testPollingWatchesOnlyTheCapturedNativeAppWhenStartDetectionIsOff() async {
+ let h = DetectorHarness()
+ h.nativeEnabled = false
+ h.browserEnabled = false
+ h.gate.isIdle = false
+ h.detector.start()
+ defer { h.detector.stop() }
+ await h.detector.pollTick()
+ XCTAssertEqual(h.audioProcessActivity.snapshotCount, 0)
+ self.synchronize(h, self.session())
+ h.detector.handleBackfill([.init(kind: .launched, bundleIdentifier: "com.microsoft.teams2", processID: 300)])
+ let owner = MeetingProcessOwner(processID: 100, bundleIdentifier: "us.zoom.xos", bundlePath: "/Applications/zoom.us.app")
+ h.audioProcessActivity.snapshotOverride = .init(
+ processes: [.init(processID: 100, bundleIdentifier: "us.zoom.xos", executablePath: "/Applications/zoom.us.app/Contents/MacOS/zoom.us", isInputRunning: true, isOutputRunning: true)],
+ owners: [owner],
+ queryState: .valid
+ )
+ await h.detector.pollTick()
+ XCTAssertEqual(h.audioProcessActivity.snapshotCount, 1)
+ XCTAssertEqual(h.windowProvider.lastInterestPIDs, [100])
+ h.audioProcessActivity.snapshotOverride = .init(processes: [], owners: [owner], queryState: .valid)
+ await h.detector.pollTick()
+ h.advance(MeetingAutoDetector.endSignalGraceSeconds + 1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 1)
+ XCTAssertTrue(h.prompts.isEmpty)
+ XCTAssertNil(h.detector.automaticTarget)
+ h.detector.recordingStopped()
+ let count = h.audioProcessActivity.snapshotCount
+ await h.detector.pollTick()
+ XCTAssertEqual(h.audioProcessActivity.snapshotCount, count)
+ }
+
+ func testManualBrowserPostCallIsPolledWithBothStartDetectionOptionsOff() async {
+ let h = DetectorHarness()
+ h.nativeEnabled = false
+ h.browserEnabled = false
+ h.gate.isIdle = false
+ h.detector.start()
+ defer { h.detector.stop() }
+ self.synchronize(h, self.session(bundle: "com.apple.Safari", pid: 42))
+ h.detector.handleBackfill([.init(kind: .launched, bundleIdentifier: "com.google.Chrome", processID: 8)])
+ h.browserReader.result = .init(host: "meet.google.com", path: "/abc-defg-hij", callState: .inCall)
+ await h.detector.pollTick()
+ h.advance(2)
+ h.browserReader.result?.callState = .ended
+ await h.detector.pollTick()
+ h.advance(2.1)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.browserReader.readPIDs, [42, 42])
+ XCTAssertEqual(h.nudges, 1)
+ XCTAssertTrue(h.prompts.isEmpty)
+ XCTAssertNil(h.detector.automaticTarget)
+ }
+}
+
+@MainActor
+extension MeetingManualRecordingEndTests {
+ func testStoppedManualWatchIgnoresAnAudioSnapshotAlreadyInFlight() async {
+ let h = DetectorHarness()
+ h.nativeEnabled = false
+ h.browserEnabled = false
+ h.gate.isIdle = false
+ h.detector.start()
+ defer { h.detector.stop() }
+ self.synchronize(h, self.session())
+ h.audioProcessActivity.beforeSnapshot = { h.detector.recordingStopped() }
+ await h.detector.pollTick()
+ h.audioProcessActivity.beforeSnapshot = nil
+ self.synchronize(h, self.session())
+ h.advance(120)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.nudges, 0)
+ XCTAssertTrue(h.prompts.isEmpty)
+ XCTAssertNil(h.detector.automaticTarget)
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift b/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift
index e5d0288e..d58d7b7e 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift
@@ -3,6 +3,9 @@ import CoreMedia
@testable import FluidVoice_Debug
import Foundation
import XCTest
+#if arch(arm64)
+import FluidAudio
+#endif
/// Opt-in replay harness: drives the REAL `MeetingLiveTranscriptionCoordinator` with a real,
/// previously-recorded Zoom meeting's decoded microphone/application-audio WAVs, to see what the
@@ -16,6 +19,16 @@ final class MeetingLiveReplayTests: XCTestCase {
private static let sampleRate: Double = 16_000
private static let trailingSilenceSeconds: Double = 3
+ func testReplayBufferUsesPerFrameTiming() throws {
+ let format = try XCTUnwrap(AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.sampleRate, channels: 1, interleaved: false))
+ let pcm = try XCTUnwrap(Self.makeBuffer(format: format, samples: [Float](repeating: 0, count: Self.chunkFrames)[...]))
+ let description = try XCTUnwrap(Self.makeFormatDescription(format: format))
+ let sample = try XCTUnwrap(Self.makeSampleBuffer(pcm: pcm, formatDescription: description, pts: CMTime(seconds: 7, preferredTimescale: 16_000)))
+ XCTAssertEqual(CMSampleBufferGetDuration(sample).seconds, 0.02, accuracy: 0.000_001)
+ XCTAssertEqual(CMSampleBufferGetPresentationTimeStamp(sample).seconds, 7, accuracy: 0.000_001)
+ XCTAssertEqual(CMSampleBufferGetNumSamples(sample), Self.chunkFrames)
+ }
+
func testReplayRealMeetingThroughLiveCoordinator() async throws {
guard let replayFlag = ProcessInfo.processInfo.environment["FLUIDVOICE_LIVE_REPLAY"], !replayFlag.isEmpty else {
throw XCTSkip("Set FLUIDVOICE_LIVE_REPLAY=1 to run the live-transcription replay harness.")
@@ -100,12 +113,9 @@ final class MeetingLiveReplayTests: XCTestCase {
// window since the snapshot only reports "available" on the FIRST track's readiness.
try await Task.sleep(nanoseconds: 5_000_000_000)
- // The bounded queue holds only ~480ms per track (capacity 24 * 20ms). Blasting all ~19.5k
- // chunks in a tight loop (sub-second) overwhelms it before the drain loop can ever get
- // scheduled, which would test harness timing, not the pipeline. Cap the feed rate at
- // `accelerationFactor`x real time instead — still far from a real 192s sleep, but paced
- // enough for the actor's drain loop to keep up with genuine ASR inference.
- let accelerationFactor = 1.5
+ // Feed at real time: accelerated input can manufacture queue pressure that never occurred
+ // in the recording, confusing a caption-quality investigation with harness overload.
+ let accelerationFactor = 1.0
let feedStart = Date()
var bufferFailures = 0
var sampleBufferFailures = 0
@@ -195,6 +205,73 @@ final class MeetingLiveReplayTests: XCTestCase {
// MARK: - Helpers
+ #if arch(arm64)
+ func testCompareBeforeAndAfterCancellationWithFinalASR() async throws {
+ let env = ProcessInfo.processInfo.environment
+ guard let directory = env["FLUIDVOICE_COMPARISON_CLIPS"],
+ let modelPath = env["FLUIDVOICE_COMPARISON_TDT_MODELS"]
+ else {
+ throw XCTSkip("Set local comparison clips and cached TDT models for this experiment.")
+ }
+ let models = try await AsrModels.loadLocalOnly(from: URL(fileURLWithPath: modelPath), version: .v2)
+ let manager = AsrManager(config: .default)
+ try await manager.initialize(models: models)
+ for clip in ["voice-alone", "voice-with-video", "short-comment"] {
+ for stage in ["microphone-before-aec", "microphone-after-aec"] {
+ let path = URL(fileURLWithPath: directory).appendingPathComponent("\(clip)-\(stage).wav").path
+ let samples = try Self.load16kMonoFloatSamples(path: path)
+ try await manager.resetDecoderState()
+ let result = try await manager.transcribe(samples, source: .microphone)
+ print("[comparison-asr] \(clip) \(stage): \(result.text)")
+ }
+ }
+ }
+
+ /// Isolates the model from the app's callbacks, text filter and forced-reset policy.
+ func testReplayDirectRecognizerWithoutForcedResets() async throws {
+ let env = ProcessInfo.processInfo.environment
+ guard env["FLUIDVOICE_DIRECT_LIVE_REPLAY"] == "1",
+ let path = env["FLUIDVOICE_LIVE_REPLAY_APP_WAV"]
+ else {
+ throw XCTSkip("Set FLUIDVOICE_DIRECT_LIVE_REPLAY and an audio path for this experiment.")
+ }
+ let samples = try Self.load16kMonoFloatSamples(path: path)
+ let format = try XCTUnwrap(AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.sampleRate, channels: 1, interleaved: false))
+ let manager = StreamingEouAsrManager(chunkSize: .ms160)
+ try await manager.loadModelsFromHuggingFace()
+ let silence = try XCTUnwrap(Self.makeBuffer(format: format, samples: [Float](repeating: 0, count: 16_000)[...]))
+ try await manager.appendAudio(silence)
+ try await manager.processBufferedAudio()
+ await manager.reset()
+ final class Transcript: @unchecked Sendable {
+ let lock = NSLock()
+ var latest = ""
+ func record(_ text: String) { self.lock.withLock { self.latest = text } }
+ func read() -> String { self.lock.withLock { self.latest } }
+ }
+ let transcript = Transcript()
+ await manager.setPartialTranscriptCallback { transcript.record($0) }
+ let seconds = min(30, max(1, Double(env["FLUIDVOICE_DIRECT_LIVE_SECONDS"] ?? "25") ?? 25))
+ let limit = min(samples.count, Int(seconds * 16_000))
+ for offset in stride(from: 0, to: limit, by: Self.chunkFrames) {
+ let end = min(offset + Self.chunkFrames, limit)
+ let pcm = try XCTUnwrap(Self.makeBuffer(format: format, samples: samples[offset..<end]))
+ try await manager.appendAudio(pcm)
+ try await manager.processBufferedAudio()
+ if end % 16_000 == 0 { print("[direct] audioSecond=\(end / 16_000) text=\(transcript.read())") }
+ }
+ print("[direct] before-finish=\(transcript.read())")
+ if env["FLUIDVOICE_DIRECT_PAD_FINISH"] == "1" {
+ await manager.injectSilence(1.28)
+ try await manager.processBufferedAudio()
+ print("[direct] after-padding=\(transcript.read())")
+ }
+ let finished = try await manager.finish()
+ print("[direct] finish=\(finished)")
+ await manager.reset()
+ }
+ #endif
+
private static func appendChunks(
_ events: inout [(kind: MeetingAudioTrackKind, samples: ArraySlice<Float>, ptsSamples: Int)],
kind: MeetingAudioTrackKind,
@@ -321,7 +398,7 @@ final class MeetingLiveReplayTests: XCTestCase {
guard copyStatus == kCMBlockBufferNoErr else { return nil }
var timing = CMSampleTimingInfo(
- duration: CMTime(value: CMTimeValue(frameCount), timescale: CMTimeScale(self.sampleRate)),
+ duration: CMTime(value: 1, timescale: CMTimeScale(self.sampleRate)),
presentationTimeStamp: pts,
decodeTimeStamp: .invalid
)
diff --git a/Tests/FluidDictationIntegrationTests/MeetingLiveTranscriptionTests.swift b/Tests/FluidDictationIntegrationTests/MeetingLiveTranscriptionTests.swift
index 7de86cbb..d95975ed 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingLiveTranscriptionTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingLiveTranscriptionTests.swift
@@ -532,6 +532,7 @@ final class MeetingLiveStartupCancellationTests: XCTestCase {
func prepareModels() async throws { self.didPrepare() }
func appendAudio(_ buffer: AVAudioPCMBuffer) async throws {}
func processBufferedAudio() async throws {}
+ func finish() async throws -> String { "" }
func reset() async {}
}
@@ -603,3 +604,184 @@ final class MeetingLiveStartupCancellationTests: XCTestCase {
}
}
#endif
+
+#if arch(arm64)
+final class MeetingLiveBoundaryTests: XCTestCase {
+ private final class Clock: @unchecked Sendable {
+ private let lock = NSLock()
+ private var value: Double = 0
+ func now() -> Double { self.lock.withLock { self.value } }
+ func set(_ value: Double) { self.lock.withLock { self.value = value } }
+ func advance() { self.set(120) }
+ }
+
+ private actor Recognizer: MeetingLiveRecognizer {
+ private var partial: (@Sendable (String) -> Void)?
+ private var end: (@Sendable (String) -> Void)?
+ private var processing = false
+ private var calls = 0
+ private(set) var resetDuringProcessing = false
+ private(set) var finishes = 0
+ let naturalEnd: Bool
+ let failFinish: Bool
+ init(naturalEnd: Bool = false, failFinish: Bool = false) {
+ self.naturalEnd = naturalEnd
+ self.failFinish = failFinish
+ }
+
+ func prepareModels() async throws {}
+ func appendAudio(_ buffer: AVAudioPCMBuffer) async throws {}
+ func processBufferedAudio() async throws {
+ self.processing = true
+ self.calls += 1
+ if self.calls == 1 {
+ self.partial?("hello")
+ if self.naturalEnd { self.end?("hello") }
+ }
+ for _ in 0..<20 {
+ await Task.yield()
+ }
+ self.processing = false
+ }
+
+ func finish() async throws -> String {
+ self.finishes += 1
+ if self.failFinish { throw NSError(domain: "InjectedFinishFailure", code: 1) }
+ self.partial?("hello final word")
+ self.end?("hello final word")
+ return "hello final word"
+ }
+
+ func reset() async { self.resetDuringProcessing = self.resetDuringProcessing || self.processing }
+ func resumePartial() { self.partial?("hello resumed") }
+ func setEouCallback(_ callback: @escaping @Sendable (String) -> Void) async { self.end = callback }
+ func setPartialTranscriptCallback(_ callback: @escaping @Sendable (String) -> Void) async { self.partial = callback }
+ }
+
+ func testNaturalEndpointWaitsUntilInferenceReturns() async throws {
+ let recognizer = Recognizer(naturalEnd: true)
+ let engine = MeetingLiveTrackEngine(kind: .microphone, recognizer: recognizer)
+ let final = expectation(description: "One completed phrase")
+ final.assertForOverFulfill = true
+ await engine.configure(
+ onPartial: { _, _, _, _, _ in },
+ onUtterance: { _, _, text, _, _ in
+ XCTAssertEqual(text, "hello")
+ final.fulfill()
+ },
+ onDegraded: { _, _ in XCTFail("No degradation expected") },
+ onReady: { _ in }
+ )
+ await engine.diagnosticInstallCallbacks()
+ try await engine.diagnosticConsume(self.sample(at: 0))
+ await fulfillment(of: [final], timeout: 1)
+ let raced = await recognizer.resetDuringProcessing
+ XCTAssertFalse(raced, "An EOU callback must not reset a suspended inference")
+ }
+
+ func testForcedEndpointFlushesTailWithoutDuplicateCallbacks() async throws {
+ try await self.checkForcedEndpoint(failFinish: false)
+ }
+
+ func testTemporaryModelPauseCanResumeWithoutReset() async throws {
+ let recognizer = Recognizer()
+ let clock = Clock()
+ let engine = MeetingLiveTrackEngine(kind: .microphone, recognizer: recognizer, uptime: { clock.now() }, diagnosticsEnabled: true)
+ let resumed = expectation(description: "Partial resumes after a long gap without premature reset")
+ await engine.configure(
+ onPartial: { _, _, text, _, _ in if text == "hello resumed" { resumed.fulfill() } },
+ onUtterance: { _, _, _, _, _ in XCTFail("A temporary pause must not close the phrase") },
+ onDegraded: { _, _ in XCTFail("A temporary pause is not an error") },
+ onReady: { _ in }
+ )
+ await engine.diagnosticInstallCallbacks()
+ try await engine.diagnosticConsume(self.sample(at: 0))
+ clock.set(6)
+ try await engine.diagnosticConsume(self.sample(at: 21))
+ clock.set(119.999)
+ try await engine.diagnosticConsume(self.sample(at: 121))
+ await recognizer.resumePartial()
+ try await engine.diagnosticConsume(self.sample(at: 1.1))
+ await fulfillment(of: [resumed], timeout: 1)
+ let snapshot = await engine.diagnosticSnapshotValue()
+ XCTAssertEqual(snapshot.resetInvocations, 0)
+ XCTAssertEqual(snapshot.textPauses, 1)
+ XCTAssertEqual(snapshot.textResumptions, 1)
+ await engine.stop()
+ }
+
+ func testFailedFlushPreservesPartialAndReportsDegradation() async throws {
+ try await self.checkForcedEndpoint(failFinish: true)
+ }
+
+ func testPauseAndResumeEventsAreSavedWithoutDebugDiagnostics() async throws {
+ let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let session = MeetingSession(
+ configuration: MeetingCaptureConfiguration(mode: .inRoom, title: "Diagnostic test", microphone: MeetingMicrophoneIdentity(captureDeviceID: "test", displayName: "Test")),
+ timebase: MeetingTimebaseMetadata(startedHostTime: 0, machTimebaseNumerator: 1, machTimebaseDenominator: 1, firstPresentationTime: nil)
+ )
+ let probe = try MeetingAudioComparisonProbe(session: session, directory: root)
+ let clock = Clock()
+ let recognizer = Recognizer()
+ let engine = MeetingLiveTrackEngine(kind: .microphone, recognizer: recognizer, uptime: { clock.now() }, diagnosticsEnabled: false)
+ await engine.attachComparisonProbe(probe)
+ await engine.diagnosticInstallCallbacks()
+ try await engine.diagnosticConsume(self.sample(at: 0))
+ clock.set(6)
+ try await engine.diagnosticConsume(self.sample(at: 6))
+ clock.set(7)
+ await recognizer.resumePartial()
+ try await engine.diagnosticConsume(self.sample(at: 7))
+ clock.set(127)
+ try await engine.diagnosticConsume(self.sample(at: 127))
+ await engine.stop()
+ await probe.stop()
+ let lines = try String(contentsOf: root.appendingPathComponent("events.jsonl"), encoding: .utf8).split(separator: "\n")
+ let events = try lines.map { try XCTUnwrap(JSONSerialization.jsonObject(with: Data($0.utf8)) as? [String: Any]) }
+ XCTAssertTrue(events.contains { $0["step"] as? String == "model-text-pause" })
+ XCTAssertTrue(events.contains { $0["step"] as? String == "model-text-resumed" })
+ XCTAssertTrue(events.contains { $0["step"] as? String == "model-flow" && ($0["detail"] as? String)?.contains("decodeCompletions=") == true })
+ XCTAssertTrue(events.contains { $0["step"] as? String == "recognizer-reset" && $0["detail"] as? String == "noTextTimeout" })
+ XCTAssertTrue(events.contains { $0["step"] as? String == "recognizer-reset" && $0["detail"] as? String == "stopped" })
+ }
+
+ private func checkForcedEndpoint(failFinish: Bool) async throws {
+ let recognizer = Recognizer(failFinish: failFinish)
+ let clock = Clock()
+ let engine = MeetingLiveTrackEngine(kind: .microphone, recognizer: recognizer, uptime: { clock.now() })
+ let final = expectation(description: "One completed phrase")
+ final.assertForOverFulfill = true
+ let degraded = expectation(description: "Flush failure reported")
+ degraded.isInverted = !failFinish
+ await engine.configure(
+ onPartial: { _, _, _, _, _ in },
+ onUtterance: { _, _, text, _, _ in
+ XCTAssertEqual(text, failFinish ? "hello" : "hello final word")
+ final.fulfill()
+ },
+ onDegraded: { _, _ in degraded.fulfill() },
+ onReady: { _ in }
+ )
+ await engine.diagnosticInstallCallbacks()
+ try await engine.diagnosticConsume(self.sample(at: 0))
+ clock.advance()
+ try await engine.diagnosticConsume(self.sample(at: 1))
+ try await engine.diagnosticConsume(self.sample(at: 1.1))
+ await fulfillment(of: [final, degraded], timeout: 0.1)
+ let finishes = await recognizer.finishes
+ XCTAssertEqual(finishes, 1)
+ let raced = await recognizer.resetDuringProcessing
+ XCTAssertFalse(raced)
+ await engine.stop()
+ }
+
+ private func sample(at seconds: Double) throws -> MeetingLiveSampleCopy.Sample {
+ let format = try XCTUnwrap(AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 16_000, channels: 1, interleaved: false))
+ let pcm = try XCTUnwrap(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 320))
+ pcm.frameLength = 320
+ try XCTUnwrap(pcm.floatChannelData?[0]).update(repeating: 0, count: 320)
+ return .init(buffer: pcm, pts: CMTime(seconds: seconds, preferredTimescale: 16_000), duration: CMTime(value: 320, timescale: 16_000))
+ }
+}
+#endif
diff --git a/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift b/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift
index fa0d6efb..facdb635 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift
@@ -374,6 +374,39 @@ final class MeetingOverlayPresentationReducerTests: XCTestCase {
@MainActor
final class MeetingMenuBarStartTests: XCTestCase {
+ func testDisabledFluidMeetCannotReopenOverlaysOrCreateMeetingServices() {
+ let original = UserDefaults.standard.object(forKey: "FluidMeetEnabled")
+ defer { UserDefaults.standard.set(original, forKey: "FluidMeetEnabled") }
+ let services = AppServices.shared
+ SettingsStore.shared.fluidMeetEnabled = false
+ let hadCoordinator = services.isMeetingSessionCoordinatorMaterialized
+ let windows = NSApp.windows.count
+ MeetingFloatingCaptionsController.shared.show(from: nil)
+ MeetingRecordingPillController.shared.expand()
+ MeetingStillRecordingNudgeController.shared.present()
+ MeetingDetectionPromptController.shared.present(.init(
+ episodeID: UUID(), bundleIdentifier: "us.zoom.xos", pid: 42, tier: .nativeTier1
+ ))
+ XCTAssertEqual(services.isMeetingSessionCoordinatorMaterialized, hadCoordinator)
+ XCTAssertEqual(NSApp.windows.count, windows)
+ XCTAssertNil(MeetingRecordingPillController.shared.currentOverlayFrame)
+ XCTAssertFalse(MeetingStillRecordingNudgeController.shared.isPresented)
+ XCTAssertNil(MeetingDetectionPromptController.shared.request)
+ }
+
+ func testDisabledFluidMeetIgnoresMenuStartWithoutNavigationOrError() async {
+ let original = UserDefaults.standard.object(forKey: "FluidMeetEnabled")
+ defer { UserDefaults.standard.set(original, forKey: "FluidMeetEnabled") }
+ SettingsStore.shared.fluidMeetEnabled = false
+ let manager = MenuBarManager()
+ var called = false
+ await manager.startMeetingRecordingInBackground { called = true }
+ XCTAssertFalse(called)
+ XCTAssertFalse(manager.meetingStartRequested)
+ XCTAssertNil(manager.meetingStartError)
+ XCTAssertNil(manager.requestedNavigationDestination)
+ }
+
func testRepeatedStartIsIgnoredAndNeverNavigates() async {
let manager = MenuBarManager()
var calls = 0
@@ -457,6 +490,69 @@ final class MeetingHistorySnapshotTests: XCTestCase {
)
}
+ func testSavedCorrectionUpdatesOnlyItsMeetingWithoutDiskReload() async {
+ let loader = HistorySnapshotLoader()
+ let snapshot = MeetingHistorySnapshot { try await loader.load() }
+ let original = self.session()
+ let other = self.session()
+ let initial = Task { await snapshot.refresh() }
+ await loader.waitForCall(1)
+ loader.finish(.success([original, other]))
+ await initial.value
+ var corrected = original
+ corrected.title = "Corrected"
+ snapshot.applySavedSession(corrected)
+ XCTAssertEqual(snapshot.sessions, [corrected, other])
+ XCTAssertEqual(loader.calls, 1, "publishing a saved correction performs no history read")
+ snapshot.applySavedSession(original) // Undo uses the same publication path.
+ XCTAssertEqual(snapshot.sessions, [original, other])
+ XCTAssertEqual(loader.calls, 1)
+ }
+
+ func testRapidCorrectionsCannotBeOverwrittenByAnOlderHistoryRead() async {
+ let loader = HistorySnapshotLoader()
+ let snapshot = MeetingHistorySnapshot { try await loader.load() }
+ let original = self.session()
+ let initial = Task { await snapshot.refresh() }
+ await loader.waitForCall(1)
+ loader.finish(.success([original]))
+ await initial.value
+ let refresh = Task { await snapshot.refresh() }
+ await loader.waitForCall(2)
+ var first = original
+ first.title = "First correction"
+ snapshot.applySavedSession(first)
+ var second = first
+ second.title = "Second correction"
+ snapshot.applySavedSession(second)
+ XCTAssertEqual(snapshot.sessions, [second])
+ loader.finish(.success([original]))
+ await loader.waitForCall(3)
+ XCTAssertEqual(snapshot.sessions, [second], "stale rows never replace the saved result")
+ loader.finish(.failure(NSError(domain: "test", code: 1)))
+ await refresh.value
+ XCTAssertEqual(snapshot.sessions, [second], "refresh failure keeps the saved correction")
+ XCTAssertEqual(loader.calls, 3, "rapid corrections coalesce into one follow-up read")
+ }
+
+ func testSavedSessionDuringInitialLoadIsNotLostOrDuplicated() async {
+ let loader = HistorySnapshotLoader()
+ let snapshot = MeetingHistorySnapshot { try await loader.load() }
+ let saved = self.session()
+ let initial = Task { await snapshot.refresh() }
+ await loader.waitForCall(1)
+ snapshot.applySavedSession(saved)
+ snapshot.applySavedSession(saved)
+ XCTAssertEqual(snapshot.sessions, [saved])
+ loader.finish(.success([]))
+ await loader.waitForCall(2)
+ XCTAssertEqual(snapshot.sessions, [saved])
+ loader.finish(.success([saved]))
+ await initial.value
+ XCTAssertEqual(snapshot.sessions, [saved])
+ XCTAssertTrue(snapshot.hasLoaded)
+ }
+
func testSnapshotKeepsRowsDuringRefreshAndFailureThenRecovers() async {
let loader = HistorySnapshotLoader()
let snapshot = MeetingHistorySnapshot { try await loader.load() }
@@ -512,6 +608,105 @@ final class MeetingHistorySnapshotTests: XCTestCase {
}
}
+@MainActor
+final class MeetingCorrectionUndoRegistrationTests: XCTestCase {
+ func testNativeUndoWalksCorrectionsAndStopsWithoutCreatingRedo() async {
+ let manager = UndoManager()
+ manager.groupsByEvent = false
+ let registration = MeetingCorrectionUndoRegistration()
+ defer { registration.detach() }
+ var count = 0
+ let calls = (0..<3).map { self.expectation(description: "Undo \($0)") }
+ registration.refresh(manager: manager, sessionID: UUID(), isEnabled: true) {
+ calls[count].fulfill()
+ count += 1
+ return count < 3
+ }
+ XCTAssertEqual(manager.undoActionName, "Meeting Correction")
+ for call in calls {
+ XCTAssertTrue(manager.canUndo)
+ manager.undo()
+ await self.fulfillment(of: [call], timeout: 2)
+ }
+ XCTAssertEqual(count, 3)
+ XCTAssertFalse(manager.canUndo)
+ XCTAssertFalse(manager.canRedo)
+ }
+
+ func testEditingAndLeavingOnlyRemoveMeetingUndoNotOtherTextUndo() {
+ let manager = UndoManager()
+ manager.groupsByEvent = false
+ let registration = MeetingCorrectionUndoRegistration()
+ let other = NativeUndoCounter()
+ manager.beginUndoGrouping()
+ manager.registerUndo(withTarget: other) { $0.calls += 1 }
+ manager.endUndoGrouping()
+ let sessionID = UUID()
+ var corrections = 0
+ let action: @MainActor () async -> Bool = { corrections += 1; return false }
+ registration.refresh(manager: manager, sessionID: sessionID, isEnabled: true, action: action)
+ registration.refresh(manager: manager, sessionID: sessionID, isEnabled: false, action: action)
+ manager.undo()
+ XCTAssertEqual(other.calls, 1, "native text undo remains available while editing")
+ XCTAssertEqual(corrections, 0)
+ registration.refresh(manager: manager, sessionID: sessionID, isEnabled: true, action: action)
+ registration.detach()
+ XCTAssertFalse(manager.canUndo, "leaving the transcript cannot undo an invisible meeting")
+ }
+
+ func testFailedSaveKeepsNativeUndoAvailableForRetry() async {
+ let manager = UndoManager()
+ manager.groupsByEvent = false
+ let registration = MeetingCorrectionUndoRegistration()
+ defer { registration.detach() }
+ let failed = self.expectation(description: "Failed save")
+ let retried = self.expectation(description: "Retried save")
+ var calls = 0
+ registration.refresh(manager: manager, sessionID: UUID(), isEnabled: true) {
+ calls += 1
+ if calls == 1 { failed.fulfill(); return true }
+ retried.fulfill()
+ return false
+ }
+ manager.undo()
+ await self.fulfillment(of: [failed], timeout: 2)
+ XCTAssertTrue(manager.canUndo)
+ manager.undo()
+ await self.fulfillment(of: [retried], timeout: 2)
+ XCTAssertEqual(calls, 2)
+ XCTAssertFalse(manager.canUndo)
+ }
+
+ func testPendingSaveCannotRegisterUndoInAnotherMeeting() async {
+ let manager = UndoManager()
+ manager.groupsByEvent = false
+ let registration = MeetingCorrectionUndoRegistration()
+ defer { registration.detach() }
+ let started = self.expectation(description: "First save started")
+ let finished = self.expectation(description: "First save finished")
+ let next = self.expectation(description: "Next meeting undone")
+ var continuation: CheckedContinuation<Bool, Never>?
+ registration.refresh(manager: manager, sessionID: UUID(), isEnabled: true) {
+ let result = await withCheckedContinuation { continuation = $0; started.fulfill() }
+ finished.fulfill()
+ return result
+ }
+ manager.undo()
+ await self.fulfillment(of: [started], timeout: 2)
+ registration.refresh(manager: manager, sessionID: UUID(), isEnabled: true) { next.fulfill(); return false }
+ continuation?.resume(returning: true)
+ await self.fulfillment(of: [finished], timeout: 2)
+ manager.undo()
+ await self.fulfillment(of: [next], timeout: 2)
+ XCTAssertFalse(manager.canUndo)
+ }
+}
+
+@MainActor
+private final class NativeUndoCounter {
+ var calls = 0
+}
+
@MainActor
private final class HistorySnapshotLoader {
private(set) var calls = 0
@@ -540,3 +735,142 @@ private final class HistorySnapshotLoader {
pending?.resume(with: result)
}
}
+
+@MainActor
+extension MeetingMenuBarStartTests {
+ func testMeetingShortcutIgnoresBusyStatesAndDuplicateStops() async throws {
+ let previous = UserDefaults.standard.object(forKey: "FluidMeetEnabled")
+ defer { UserDefaults.standard.set(previous, forKey: "FluidMeetEnabled") }
+ SettingsStore.shared.fluidMeetEnabled = true
+ let manager = MenuBarManager()
+ let id = UUID()
+ var starts = 0
+ var stops = 0
+ for state in [MeetingCoordinatorState.preparing(id), .stopping(id), .processing(id, .transcribing)] {
+ manager.configureMeetingShortcutForTesting(state: state) { stops += 1 }
+ await manager.toggleMeetingRecordingFromShortcut { starts += 1 }
+ }
+ XCTAssertEqual(starts, 0)
+ XCTAssertEqual(stops, 0)
+ manager.configureMeetingShortcutForTesting(state: .recording(id)) { stops += 1 }
+ await manager.toggleMeetingRecordingFromShortcut { starts += 1 }
+ await manager.toggleMeetingRecordingFromShortcut { starts += 1 }
+ XCTAssertEqual(stops, 1)
+ XCTAssertEqual(starts, 0)
+ manager.configureMeetingShortcutForTesting(state: .completed(id)) { stops += 1 }
+ await manager.toggleMeetingRecordingFromShortcut {
+ starts += 1
+ await manager.toggleMeetingRecordingFromShortcut { starts += 1 }
+ }
+ XCTAssertEqual(starts, 1)
+ XCTAssertFalse(manager.meetingStartRequested)
+ XCTAssertNil(manager.requestedNavigationDestination)
+ }
+
+ func testMeetingShortcutCannotInterruptDictationAndFailureAllowsRetry() async throws {
+ let previous = UserDefaults.standard.object(forKey: "FluidMeetEnabled")
+ defer { UserDefaults.standard.set(previous, forKey: "FluidMeetEnabled") }
+ SettingsStore.shared.fluidMeetEnabled = true
+ let asr = ASRService()
+ let manager = MenuBarManager()
+ manager.configure(asrService: asr)
+ let lease = try asr.acquireExclusiveActivity(.dictation)
+ var starts = 0
+ await manager.toggleMeetingRecordingFromShortcut { starts += 1 }
+ XCTAssertEqual(starts, 0)
+ XCTAssertEqual(asr.activeExclusiveActivity, .dictation)
+ asr.releaseExclusiveActivity(lease)
+ await manager.toggleMeetingRecordingFromShortcut { throw MeetingCaptureError.microphoneUnavailable }
+ XCTAssertFalse(manager.meetingStartRequested)
+ XCTAssertNotNil(manager.meetingStartError)
+ await manager.toggleMeetingRecordingFromShortcut { starts += 1 }
+ XCTAssertEqual(starts, 1)
+ XCTAssertNil(manager.meetingStartError)
+ XCTAssertNil(manager.requestedNavigationDestination)
+ }
+}
+
+@MainActor
+final class MeetingPageActionAvailabilityTests: XCTestCase {
+ private func session() -> MeetingSession {
+ MeetingSession(
+ configuration: MeetingCaptureConfiguration(mode: .inRoom, title: "Meeting", microphone: MeetingMicrophoneIdentity(captureDeviceID: "mic", displayName: "Mic")),
+ timebase: MeetingTimebaseMetadata(startedHostTime: 0, machTimebaseNumerator: 1, machTimebaseDenominator: 1, firstPresentationTime: nil)
+ )
+ }
+
+ func testCompletedAndFailedMeetingsKeepNewMeetingVisibleWhileSummaryBlocksItsAction() {
+ for state in [MeetingTranscriptionCanvasState.result(self.session()), .failed(session: nil, message: "Failed")] {
+ for summaryRunning in [false, true] {
+ let actions = MeetingPageActionAvailability(state: state, isSummaryRunning: summaryRunning)
+ XCTAssertTrue(actions.showsNewMeeting)
+ XCTAssertEqual(actions.canStartNewMeeting, !summaryRunning)
+ XCTAssertTrue(actions.canEditSetup)
+ }
+ }
+ }
+
+ func testSetupAndActiveCaptureNeverOfferAnotherMeeting() {
+ let session = self.session()
+ let states: [MeetingTranscriptionCanvasState] = [
+ .setup(isStarting: false, recentSession: nil),
+ .setup(isStarting: true, recentSession: session),
+ .recording(session: session, trackHealth: [:], liveTranscript: .empty),
+ .stopping(session: session, trackHealth: [:], liveTranscript: .empty),
+ .processing(session: session, stage: .transcribing),
+ ]
+ for (index, state) in states.enumerated() {
+ let actions = MeetingPageActionAvailability(state: state, isSummaryRunning: false)
+ XCTAssertFalse(actions.showsNewMeeting)
+ XCTAssertFalse(actions.canStartNewMeeting)
+ XCTAssertEqual(actions.canEditSetup, index == 0)
+ }
+ }
+}
+
+@MainActor
+extension MeetingMenuBarStartTests {
+ func testSavedOnlineApplicationIsUsedWithoutAutomaticDetection() throws {
+ var defaults = MeetingRecordingDefaults.unconfigured
+ defaults.isConfigured = true
+ defaults.applicationBundleIdentifier = "us.zoom.xos"
+ let app = MeetingApplicationIdentity(bundleIdentifier: "us.zoom.xos", processID: 42, displayName: "Zoom")
+ let config = try self.savedSourceConfiguration(defaults: defaults, applications: [app])
+ XCTAssertEqual(config.mode, .onlineCall)
+ XCTAssertEqual(config.application, app)
+ XCTAssertEqual(config.platform?.identifier, app.bundleIdentifier)
+ }
+
+ func testUnavailableSavedApplicationCannotSilentlyBecomeMicrophoneOnly() {
+ var defaults = MeetingRecordingDefaults.unconfigured
+ defaults.isConfigured = true
+ defaults.applicationBundleIdentifier = "us.zoom.xos"
+ XCTAssertThrowsError(try self.savedSourceConfiguration(defaults: defaults, applications: [
+ MeetingApplicationIdentity(bundleIdentifier: "com.microsoft.teams2", processID: 43, displayName: "Teams"),
+ ]))
+ }
+
+ func testSavedInRoomModeDoesNotCaptureASavedApplication() throws {
+ var defaults = MeetingRecordingDefaults.unconfigured
+ defaults.isConfigured = true
+ defaults.mode = .inRoom
+ defaults.applicationBundleIdentifier = "us.zoom.xos"
+ let app = MeetingApplicationIdentity(bundleIdentifier: "us.zoom.xos", processID: 42, displayName: "Zoom")
+ let config = try self.savedSourceConfiguration(defaults: defaults, applications: [app])
+ XCTAssertEqual(config.mode, .inRoom)
+ XCTAssertNil(config.application)
+ }
+
+ private func savedSourceConfiguration(
+ defaults: MeetingRecordingDefaults,
+ applications: [MeetingApplicationIdentity]
+ ) throws -> MeetingCaptureConfiguration {
+ try AppServices.menuBarMeetingConfiguration(
+ defaults: defaults,
+ target: nil,
+ microphones: .init(identities: [.init(captureDeviceID: "mic", displayName: "Mic")], defaultCoreAudioUID: nil),
+ applications: applications,
+ preferredInputUID: nil
+ )
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingParakeetNemotronBackendTests.swift b/Tests/FluidDictationIntegrationTests/MeetingParakeetNemotronBackendTests.swift
index 26b546fb..dd92cce9 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingParakeetNemotronBackendTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingParakeetNemotronBackendTests.swift
@@ -360,11 +360,10 @@ final class MeetingParakeetNemotronBackendTests: XCTestCase {
XCTAssertEqual(turns.map(\.speakerID), [speakerA, speakerB, speakerA])
}
- func testCanonicalMergeKeepsAAmbiguousAAsThreeTurns() {
+ func testCanonicalMergeAttachesUnknownWordsToPreviousSpeaker() {
let trackID = UUID()
let speakerA = UUID()
- // The ambiguous segment has no resolved speaker and a different overlap state, so its
- // key differs from both surrounding A segments even though all three are on one track.
+ // Product transcript joins the Unknown word to the preceding speaker's turn.
let input = [
self.canonicalSegment(trackID: trackID, start: 0, end: 1.0, speakerID: speakerA, text: "First"),
self.canonicalSegment(
@@ -375,9 +374,73 @@ final class MeetingParakeetNemotronBackendTests: XCTestCase {
let turns = MeetingProcessingPipeline.mergeCanonicalSegments(input)
- XCTAssertEqual(turns.count, 3)
- XCTAssertEqual(turns.map(\.text), ["First", "Maybe", "Second"])
- XCTAssertEqual(turns.map(\.overlap), [.none, .ambiguous, .none])
+ XCTAssertEqual(turns.count, 1)
+ XCTAssertEqual(turns.map(\.text), ["First Maybe Second"])
+ XCTAssertEqual(turns.map(\.speakerID), [speakerA])
+ XCTAssertEqual(turns.map(\.overlap), [.none])
+ }
+
+ func testUnknownInheritanceKeepsOpeningUnknownAndUsesPreviousNotNextSpeaker() {
+ let track = UUID()
+ let first = UUID()
+ let second = UUID()
+ var ambiguous = self.canonicalSegment(trackID: track, start: 1.2, end: 1.4, speakerID: nil, text: "tail", overlap: .ambiguous)
+ ambiguous.attributionState = .overlappingSpeakers
+ var uncertain = self.canonicalSegment(trackID: track, start: 1.4, end: 1.6, speakerID: second, text: "two")
+ uncertain.attributionState = .timingUncertain
+ let input = [
+ self.canonicalSegment(trackID: track, start: 0, end: 0.2, speakerID: nil, text: "opening"),
+ self.canonicalSegment(trackID: track, start: 0.5, end: 1, speakerID: first, text: "first"),
+ ambiguous, uncertain,
+ self.canonicalSegment(trackID: track, start: 2, end: 3, speakerID: second, text: "second"),
+ self.canonicalSegment(trackID: track, start: 3, end: 3.2, speakerID: nil, text: "ending"),
+ ]
+ let output = MeetingProcessingPipeline.inheritPreviousSpeaker(input.reversed())
+ XCTAssertEqual(output.map(\.speakerID), [nil, first, first, first, second, second])
+ for (before, after) in zip(input, output) {
+ XCTAssertEqual(before.id, after.id)
+ XCTAssertEqual(before.text, after.text)
+ XCTAssertEqual(before.start, after.start)
+ XCTAssertEqual(before.end, after.end)
+ XCTAssertEqual(before.sourceTrackID, after.sourceTrackID)
+ XCTAssertEqual(before.revision, after.revision)
+ }
+ XCTAssertEqual(MeetingProcessingPipeline.inheritPreviousSpeaker(output), output)
+ XCTAssertNil(input[2].speakerID, "raw input evidence is unchanged")
+ }
+
+ func testUnknownInheritanceUsesChronologicalSpeakerAcrossTracksWithoutMovingAudio() {
+ let mic = UUID()
+ let app = UUID()
+ let first = UUID()
+ let second = UUID()
+ let input = [
+ self.canonicalSegment(trackID: mic, start: 0, end: 1, speakerID: first, text: "mic"),
+ self.canonicalSegment(trackID: app, start: 1, end: 2, speakerID: second, text: "app"),
+ self.canonicalSegment(trackID: mic, start: 2, end: 2.3, speakerID: nil, text: "tail"),
+ ]
+ let output = MeetingProcessingPipeline.inheritPreviousSpeaker(input)
+ XCTAssertEqual(output.last?.speakerID, second)
+ XCTAssertEqual(output.last?.sourceTrackID, mic)
+ let turns = MeetingProcessingPipeline.mergeCanonicalSegments(input)
+ XCTAssertEqual(turns.count, 3, "separate audio tracks keep separate playback ranges")
+ }
+
+ func testUnknownInheritanceSkipsEchoesAsBothTargetsAndAnchors() {
+ let track = UUID()
+ let first = UUID()
+ let second = UUID()
+ let input = [
+ self.canonicalSegment(trackID: track, start: 0, end: 1, speakerID: first, text: "first"),
+ self.canonicalSegment(trackID: track, start: 1, end: 1.2, speakerID: second, text: "echo", isLikelyEcho: true),
+ self.canonicalSegment(trackID: track, start: 1.3, end: 1.4, speakerID: nil, text: "unknown echo", isLikelyEcho: true),
+ self.canonicalSegment(trackID: track, start: 1.4, end: 1.6, speakerID: nil, text: "tail"),
+ ]
+ let output = MeetingProcessingPipeline.inheritPreviousSpeaker(input)
+ XCTAssertEqual(output[1], input[1])
+ XCTAssertEqual(output[2], input[2])
+ XCTAssertEqual(output.last?.speakerID, first)
+ XCTAssertEqual(MeetingProcessingPipeline.inheritPreviousSpeaker([]), [])
}
func testCanonicalMergeCombinesAdjacentSameKeySegments() {
@@ -557,7 +620,7 @@ final class MeetingParakeetNemotronBackendTests: XCTestCase {
let runtime = FakeRuntime()
let backend = self.makeBackend(runtime: runtime)
let configuration = MeetingFinalProcessingConfiguration(languageCode: "de")
- let request = self.makeRequest(session: session, directory: try self.makeTempSessionDirectory(), configuration: configuration)
+ let request = try self.makeRequest(session: session, directory: self.makeTempSessionDirectory(), configuration: configuration)
let plan = try backend.plan(request)
let manifest = try self.makeManifest(plan: plan, observations: fixture.observations)
runtime.asrSession.responses = [
@@ -950,16 +1013,16 @@ final class MeetingParakeetNemotronBackendTests: XCTestCase {
}
XCTAssertEqual(tokenA.label, "slot-0")
- guard case let .assigned(tokenB) = try XCTUnwrap(unitsByText["b"]).speaker else {
- return XCTFail("word b should inherit the continuing incumbent in the handoff pass")
+ guard case let .ambiguous(tokensB) = try XCTUnwrap(unitsByText["b"]).speaker else {
+ return XCTFail("two overlapping voices must stay ambiguous")
}
- XCTAssertEqual(tokenB.label, "slot-0")
+ XCTAssertEqual(tokensB.map(\.label), ["slot-0", "slot-1"])
guard case .unassigned = try XCTUnwrap(unitsByText["c"]).speaker else {
return XCTFail("word c overlaps no activity")
}
- // Assembly keeps the resolved handoff on the existing product speaker.
+ // The overlapping word stays visible, without borrowing the previous speaker.
let assembly = try MeetingTranscriptAssembler().assemble(MeetingAssemblyInput(
plan: plan,
manifest: manifest,
@@ -969,11 +1032,114 @@ final class MeetingParakeetNemotronBackendTests: XCTestCase {
XCTAssertEqual(assembly.segments.map(\.text), ["a", "b"])
XCTAssertEqual(assembly.speakers.count, 1, "only the unambiguous assigned slot mints a speaker")
let resolved = try XCTUnwrap(assembly.sidecar.dispositions.first { $0.unitID == unitsByText["b"]?.id })
- XCTAssertEqual(resolved.disposition, .emitted)
+ XCTAssertEqual(resolved.disposition, .ambiguousUnassigned)
+ XCTAssertNil(assembly.segments.first { $0.text == "b" }?.speakerID)
let outside = try XCTUnwrap(assembly.sidecar.dispositions.first { $0.unitID == unitsByText["c"]?.id })
XCTAssertEqual(outside.disposition, .outsideActivity)
}
+ /// These are the actual final-word/previous-word times returned by TDT and Nemotron for
+ /// eight Samantha/Daniel sentences plus a saved meeting endpoint with Float32 rounding.
+ func testKnownSentenceEndingsSurviveBackendAssemblyAndTurnMerge() async throws {
+ let timings: [(anchor: Double, start: Double, end: Double, activityEnd: Double)] = [
+ (1.20, 1.52, 2.16, 1.4_399_999_380_111_694),
+ (1.28, 1.60, 2.24, 1.5_199_999_809_265_137),
+ (1.04, 1.36, 2.00, 1.2_799_999_713_897_705),
+ (1.12, 1.44, 2.08, 1.3_600_000_143_051_147),
+ (1.28, 1.60, 2.00, 1.6_799_999_475_479_126),
+ (1.44, 1.76, 2.80, 1.7_599_999_904_632_568),
+ (1.20, 1.52, 2.16, 1.4_399_999_380_111_694),
+ (1.20, 1.52, 2.16, 1.4_399_999_380_111_694),
+ (18.96, 19.28, 19.60, 19.19_999_885_559_082),
+ ]
+ for (kind, mode) in [
+ (MeetingAudioTrackKind.microphone, MeetingCaptureMode.inRoom),
+ (.microphone, .onlineCall), (.applicationAudio, .onlineCall),
+ ] {
+ for timing in timings {
+ let duration = max(4, timing.end + 1)
+ let chunk = self.makeChunk(sequence: 0, start: 0, end: duration)
+ let track = kind == .microphone
+ ? self.makeMicTrack(chunks: [chunk], eraStart: 0)
+ : self.makeAppTrack(chunks: [chunk])
+ let session = self.makeSession(mode: mode, tracks: [track])
+ let runtime = FakeRuntime()
+ let backend = self.makeBackend(runtime: runtime, materializer: FakeMaterializer())
+ let plan = try backend.plan(self.makeRequest(
+ session: session, directory: self.makeTempSessionDirectory()
+ ))
+ let manifest = try self.makeManifest(plan: plan, observations: [
+ MeetingAnalysisChunkKey(trackID: track.id, chunkID: chunk.id): self.makeObserved(chunk, duration: duration),
+ ])
+ let epoch = try XCTUnwrap(manifest.track(track.id)?.epochs.first)
+ runtime.diarizerFactory.segmentsByEpoch = [epoch.id: [
+ MeetingNemotronSpeakerSegment(slotIndex: 0, start: 0, end: timing.activityEnd),
+ ]]
+ runtime.asrSession.responses = [.init(text: "been there?", words: [
+ ASRWordTiming(text: "been", start: timing.anchor, end: timing.start),
+ ASRWordTiming(text: "there?", start: timing.start, end: timing.end),
+ ])]
+ let outcome = try await backend.execute(plan: plan, manifest: manifest, progress: { _ in })
+ guard case let .canonicalEvidence(bundle) = outcome else { return XCTFail("missing evidence") }
+ let verdicts = try await MeetingTextOverlapEchoVerdictProvider().echoVerdicts(
+ for: bundle.evidence, manifest: manifest, plan: plan
+ )
+ let result = try MeetingTranscriptAssembler().assemble(MeetingAssemblyInput(
+ plan: plan,
+ manifest: manifest,
+ evidence: bundle.evidence,
+ coverageReceipts: bundle.coverageReceipts,
+ echoVerdicts: verdicts
+ ))
+ XCTAssertEqual(result.segments.map(\.text), ["been", "there?"])
+ XCTAssertEqual(result.segments.map(\.attributionState), [.assigned, .assigned])
+ XCTAssertEqual(Set(result.segments.compactMap(\.speakerID)).count, 1)
+ XCTAssertEqual(result.sidecar.dispositions.map(\.disposition), [.emitted, .emitted])
+ let tail = try XCTUnwrap(bundle.evidence.units.last)
+ XCTAssertEqual(tail.analysisStart, timing.start, accuracy: 0.000_001)
+ XCTAssertEqual(tail.analysisEnd, timing.end, accuracy: 0.000_001)
+ let merged = MeetingProcessingPipeline.mergeCanonicalSegments(result.segments)
+ XCTAssertEqual(merged.map(\.text), ["been there?"])
+ }
+ }
+ }
+
+ func testNearbyCompetingVoiceKeepsRecoveredEndingUnknown() async throws {
+ let chunk = self.makeChunk(sequence: 0, start: 0, end: 4)
+ let track = self.makeMicTrack(chunks: [chunk], eraStart: 0)
+ let runtime = FakeRuntime()
+ let backend = self.makeBackend(runtime: runtime)
+ let plan = try backend.plan(self.makeRequest(
+ session: self.makeSession(mode: .inRoom, tracks: [track]),
+ directory: self.makeTempSessionDirectory()
+ ))
+ let manifest = try self.makeManifest(plan: plan, observations: [
+ MeetingAnalysisChunkKey(trackID: track.id, chunkID: chunk.id): self.makeObserved(chunk, duration: 4),
+ ])
+ let epoch = try XCTUnwrap(manifest.track(track.id)?.epochs.first)
+ runtime.diarizerFactory.segmentsByEpoch = [epoch.id: [
+ MeetingNemotronSpeakerSegment(slotIndex: 0, start: 0, end: 1.44),
+ MeetingNemotronSpeakerSegment(slotIndex: 1, start: 1.47, end: 1.50),
+ ]]
+ runtime.asrSession.responses = [.init(text: "been there?", words: [
+ ASRWordTiming(text: "been", start: 1.2, end: 1.52),
+ ASRWordTiming(text: "there?", start: 1.52, end: 2.16),
+ ])]
+ let outcome = try await backend.execute(plan: plan, manifest: manifest, progress: { _ in })
+ guard case let .canonicalEvidence(bundle) = outcome else { return XCTFail("missing evidence") }
+ XCTAssertEqual(bundle.evidence.units.last?.speaker, .unassigned)
+ let result = try MeetingTranscriptAssembler().assemble(MeetingAssemblyInput(
+ plan: plan,
+ manifest: manifest,
+ evidence: bundle.evidence,
+ coverageReceipts: bundle.coverageReceipts
+ ))
+ XCTAssertEqual(result.segments.map(\.text), ["been", "there?"])
+ XCTAssertNotNil(result.segments.first?.speakerID)
+ XCTAssertNil(result.segments.last?.speakerID)
+ XCTAssertEqual(result.segments.last?.attributionState, .unassigned)
+ }
+
func testCoverageUnionDoesNotDoubleCountDuplicateSpeakerIntervals() {
let (slot0, slot1) = self.attributionTokens()
let activity = [
@@ -1107,61 +1273,19 @@ final class MeetingParakeetNemotronBackendTests: XCTestCase {
XCTAssertEqual(tokens.map(\.label), ["slot-0", "slot-1"])
}
- func testHandoffAssignsNewEntrantWhoseActivityStartsWithShortWord() {
+ func testOverlappingWordDoesNotInheritIncumbentOrNewEntrant() {
let (slot0, slot1) = self.attributionTokens()
- let decision = MeetingParakeetNemotronBackend.handoffAssignment(
- .ambiguous([slot0, slot1]),
- start: 1,
- end: 1.4,
- activity: [
- MeetingBackendSpeakerActivity(token: slot0, start: 0, end: 1.4),
- MeetingBackendSpeakerActivity(token: slot1, start: 1, end: 1.4),
- ],
- previousAssignedWord: (slot0, 0.95)
- )
- guard case let .assigned(token) = decision else { return XCTFail("new onset should win the handoff") }
- XCTAssertEqual(token, slot1)
- }
-
- func testHandoffKeepsIncumbentWhenOtherSpeakerDidNotStartAtBoundary() {
- let (slot0, slot1) = self.attributionTokens()
- let decision = MeetingParakeetNemotronBackend.handoffAssignment(
- .ambiguous([slot0, slot1]),
- start: 1,
- end: 1.4,
- activity: [
- MeetingBackendSpeakerActivity(token: slot0, start: 0, end: 1.4),
- MeetingBackendSpeakerActivity(token: slot1, start: 0.2, end: 1.4),
- ],
- previousAssignedWord: (slot0, 0.95)
- )
- guard case let .assigned(token) = decision else { return XCTFail("continuing incumbent should own the word") }
- XCTAssertEqual(token, slot0)
- }
-
- func testHandoffVetoesMissingContextLongWordsLargeGapsAndMoreThanTwoCandidates() {
- let (slot0, slot1) = self.attributionTokens()
- let epoch = slot0.analysisEpochID
- let slot2 = MeetingBackendSpeakerToken(analysisEpochID: epoch, label: "slot-2")
- let activity = [
- MeetingBackendSpeakerActivity(token: slot0, start: 0, end: 2),
- MeetingBackendSpeakerActivity(token: slot1, start: 1, end: 2),
- MeetingBackendSpeakerActivity(token: slot2, start: 1, end: 2),
- ]
-
- for (assignment, start, end, previous) in [
- (MeetingBackendSpeakerAssignment.ambiguous([slot0, slot1]), 1.0, 1.4, nil),
- (.ambiguous([slot0, slot1]), 1.0, 2.0, (token: slot0, end: 0.95)),
- (.ambiguous([slot0, slot1]), 1.0, 1.4, (token: slot0, end: 0.0)),
- (.ambiguous([slot0, slot1, slot2]), 1.0, 1.4, (token: slot0, end: 0.95)),
- ] {
- guard case .ambiguous = MeetingParakeetNemotronBackend.handoffAssignment(
- assignment,
- start: start,
- end: end,
- activity: activity,
- previousAssignedWord: previous
- ) else { return XCTFail("handoff veto must preserve ambiguity") }
+ for onset in [0.2, 1.0] {
+ let decision = MeetingParakeetNemotronBackend.assignment(
+ start: 1,
+ end: 1.4,
+ activity: [
+ MeetingBackendSpeakerActivity(token: slot0, start: 0, end: 1.4),
+ MeetingBackendSpeakerActivity(token: slot1, start: onset, end: 1.4),
+ ],
+ allowDominance: true
+ )
+ XCTAssertEqual(decision, .ambiguous([slot0, slot1]))
}
}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingPhase4Tests.swift b/Tests/FluidDictationIntegrationTests/MeetingPhase4Tests.swift
index 78119e8c..4c7edd25 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingPhase4Tests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingPhase4Tests.swift
@@ -196,8 +196,8 @@ final class MeetingPhase4ClassificationTests: XCTestCase {
// MARK: - 7. pipelineVersion bump
- func testPipelineVersionIsThirteen() {
- XCTAssertEqual(MeetingProcessingPipeline.pipelineVersion, 13)
+ func testPipelineVersionIsFifteen() {
+ XCTAssertEqual(MeetingProcessingPipeline.pipelineVersion, 15)
}
// MARK: - 9. Store round-trip: a decoded track drives the classification branch
diff --git a/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift b/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
index 32b387de..f0c7c882 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
@@ -12,6 +12,65 @@ import XCTest
// Existing recovery suite shares setup across crash and corruption scenarios.
// swiftlint:disable:next type_body_length
final class MeetingRecoveryTests: XCTestCase {
+ func testDisabledFluidMeetRejectsRecordingBeforeTouchingCaptureOrStorage() async throws {
+ let settings = SettingsStore.shared
+ let original = UserDefaults.standard.object(forKey: "FluidMeetEnabled")
+ defer { UserDefaults.standard.set(original, forKey: "FluidMeetEnabled") }
+ settings.fluidMeetEnabled = false
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let track = self.makeMicrophoneTrack(chunks: [])
+ var saved = self.makeSession(state: .completed, audioTracks: [track])
+ saved.transcriptSegments = [self.makeTranscriptSegment(sourceTrackID: track.id, speakerID: nil)]
+ try await store.create(saved)
+ let capture = StubCaptureController()
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: capture, processing: StubProcessingController(), audioArbiter: StubArbiter()
+ )
+ do {
+ _ = try await coordinator.startRecording(configuration: self.makeConfiguration())
+ XCTFail("Disabled FluidMeet must reject recording")
+ } catch {
+ guard case MeetingCoordinatorError.featureDisabled = error else {
+ return XCTFail("Unexpected error: \(error)")
+ }
+ }
+ XCTAssertEqual(capture.startCount, 0)
+ XCTAssertEqual(capture.preflightCount, 0)
+ XCTAssertFalse(coordinator.hasPendingStart)
+ let sessions = try await store.loadAll()
+ XCTAssertEqual(sessions.map(\.id), [saved.id])
+ XCTAssertEqual(sessions.first?.transcriptSegments, saved.transcriptSegments)
+ }
+
+ func testDisablingFluidMeetDuringModelPreparationDoesNotStartCapture() async throws {
+ let original = UserDefaults.standard.object(forKey: "FluidMeetEnabled")
+ defer { UserDefaults.standard.set(original, forKey: "FluidMeetEnabled") }
+ SettingsStore.shared.fluidMeetEnabled = true
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let capture = StubCaptureController()
+ let coordinator = MeetingSessionCoordinator(
+ store: MeetingSessionStore(rootDirectory: dir),
+ capture: capture,
+ processing: StubProcessingController(),
+ audioArbiter: StubArbiter(),
+ validateRecordingModels: { SettingsStore.shared.fluidMeetEnabled = false }
+ )
+ do {
+ _ = try await coordinator.startRecording(configuration: self.makeConfiguration())
+ XCTFail("A stale start must not revive FluidMeet")
+ } catch {
+ guard case MeetingCoordinatorError.featureDisabled = error else {
+ return XCTFail("Unexpected error: \(error)")
+ }
+ }
+ XCTAssertEqual(capture.preflightCount, 0)
+ XCTAssertEqual(capture.startCount, 0)
+ XCTAssertFalse(coordinator.hasPendingStart)
+ }
+
func testUnexpectedStopPublishesQuiescenceAfterDelayedAudioHandback() async throws {
let dir = self.makeTempDirectory()
defer { try? FileManager.default.removeItem(at: dir) }
@@ -56,6 +115,56 @@ final class MeetingRecoveryTests: XCTestCase {
XCTAssertTrue(saved?.failures.isEmpty == true)
}
+ func testEchoWarningRendersWithoutInvokingRecordingActions() throws {
+ var session = self.makeSession(state: .recording)
+ session.mode = .onlineCall
+ session.events = [.init(id: UUID(), occurredAt: Date(), kind: .echoFilteringUnavailable, trackID: nil, detail: nil)]
+ var actions = 0
+ for (width, scheme): (CGFloat, ColorScheme) in [(640, .dark), (960, .light)] {
+ let canvas = MeetingRecordingCanvas(
+ session: session, trackHealth: [:], liveTranscript: .empty, isStopping: false, onStop: { actions += 1 }
+ )
+ try self.renderMeetingUI(canvas, name: "echo-warning", width: width, height: 700, scheme: scheme)
+ }
+ XCTAssertEqual(actions, 0)
+ }
+
+ func testEchoWarningDoesNotRestartCaptureOrChangeItsSource() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let capture = StubCaptureController()
+ let microphone = self.makeMicrophoneTrack(chunks: [])
+ var application = self.makeMicrophoneTrack(chunks: [])
+ application.kind = .applicationAudio
+ capture.startResult = MeetingCaptureStartResult(tracks: [microphone, application], firstPresentationTime: nil)
+ let coordinator = MeetingSessionCoordinator(
+ store: MeetingSessionStore(rootDirectory: dir),
+ capture: capture,
+ processing: StubProcessingController(),
+ audioArbiter: StubArbiter()
+ )
+ var configuration = self.makeConfiguration()
+ configuration.mode = .onlineCall
+ configuration.application = MeetingApplicationIdentity(bundleIdentifier: "com.apple.Safari", displayName: "Safari")
+ let started = try await coordinator.startRecording(configuration: configuration)
+ let originalSource = coordinator.activeSession?.selectedMicrophone
+ for _ in 0..<3 {
+ capture.emit(.microphoneEchoProtectionChanged(.unfiltered))
+ }
+ await self.drainCaptureEvents()
+ XCTAssertEqual(coordinator.state, .recording(started.id))
+ XCTAssertEqual(capture.startCount, 1)
+ XCTAssertEqual(coordinator.activeSession?.selectedMicrophone, originalSource)
+ XCTAssertEqual(coordinator.activeSession?.events.filter { $0.kind == .echoFilteringUnavailable }.count, 1)
+ XCTAssertNotNil(coordinator.activeSession?.microphoneEchoWarning)
+ capture.emit(.microphoneEchoProtectionChanged(.softwareEchoCancelled))
+ await self.drainCaptureEvents()
+ XCTAssertNil(coordinator.activeSession?.microphoneEchoWarning)
+ XCTAssertEqual(coordinator.state, .recording(started.id))
+ XCTAssertEqual(capture.startCount, 1)
+ await coordinator.shutdownForTermination()
+ }
+
func testVoiceProcessingFallbackDoesNotDegradeHealthyRecording() async throws {
for deliverDuringStart in [false, true] {
let dir = self.makeTempDirectory()
@@ -633,9 +742,7 @@ final class MeetingRecoveryTests: XCTestCase {
onExportTranscript: { _, _, _ in action() },
onReassignSegment: { _, _, _ in action() },
onNameUnknownSegment: { _, _, _ in action() },
- onRenameSpeaker: { _, _, _ in action() },
- onMergeSpeakers: { _, _, _ in action() },
- onUndoCorrection: { _ in action() },
+ onUndoCorrection: { _ in action(); return false },
onRenameSession: { _, _ in action() },
onAssignSpeakers: { _, _ in action(); return nil },
canUndoCorrection: { _ in false },
@@ -1081,9 +1188,7 @@ final class MeetingRecoveryTests: XCTestCase {
onExportTranscript: { _, _, _ in actions.record() },
onReassignSegment: { _, _, _ in actions.record() },
onNameUnknownSegment: { _, _, _ in actions.record() },
- onRenameSpeaker: { _, _, _ in actions.record() },
- onMergeSpeakers: { _, _, _ in actions.record() },
- onUndoCorrection: { _ in actions.record() },
+ onUndoCorrection: { _ in actions.record(); return false },
onRenameSession: { _, _ in actions.record() },
onAssignSpeakers: { _, _ in actions.record(); return nil },
canUndoCorrection: { _ in false },
@@ -1410,6 +1515,38 @@ final class MeetingRecoveryTests: XCTestCase {
// MARK: - Test 2: multi-session restore
+ func testRecoveryInspectsFinalizedTrackMissingFromFailedSessionSnapshot() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let session = self.makeSession(
+ state: .failed,
+ endedAt: Date(),
+ failures: [MeetingSessionFailure(
+ id: UUID(),
+ occurredAt: Date(),
+ domain: .capture,
+ code: "interrupted",
+ message: "Interrupted",
+ recoverable: true
+ )]
+ )
+ try await store.create(session)
+ let directory = try await store.sessionDirectory(for: session.id)
+ let chunk = self.makeFinalizedChunk()
+ let track = self.makeMicrophoneTrack(chunks: [chunk])
+ let fileURL = chunk.fileURL(relativeTo: directory)
+ try FileManager.default.createDirectory(at: fileURL.deletingLastPathComponent(), withIntermediateDirectories: true)
+ try Data([1, 2, 3, 4]).write(to: fileURL)
+ try JSONEncoder().encode(track).write(to: fileURL.deletingLastPathComponent().appendingPathComponent("track.json"))
+
+ let history = try await store.loadHistory()
+ XCTAssertEqual(history.first?.audioTracks, [], "browsing does not reconcile crash-window audio")
+ let recovered = try await store.loadRecoverable()
+ XCTAssertEqual(recovered.map(\.id), [session.id])
+ XCTAssertEqual(recovered.first?.audioTracks.first?.chunks.first?.byteCount, 4)
+ }
+
func testMultiSessionRestoreSelectsNewestViableAndDefersOthers() async throws {
let dir = self.makeTempDirectory()
defer { try? FileManager.default.removeItem(at: dir) }
@@ -3011,6 +3148,75 @@ final class MeetingRecoveryTests: XCTestCase {
XCTAssertNil(coordinator.latestCompletedSession)
}
+ func testTaggingUnknownMergesMatchingNeighborsAndUndoRestoresAllWords() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ var (session, speakerA, speakerB, _) = self.makeCorrectionSession(state: .completed)
+ let base = session.transcriptSegments[0]
+ func part(_ text: String, _ start: Int64, _ speaker: SessionSpeakerID?) -> MeetingTranscriptSegment {
+ var value = base
+ value.id = UUID()
+ value.text = text
+ value.start = .init(value: start, timescale: 1000)
+ value.end = .init(value: start + 200, timescale: 1000)
+ value.speakerID = speaker
+ value.attributionState = speaker == nil ? .unassigned : .assigned
+ return value
+ }
+ let left = part("the", 0, speakerA)
+ let unknown = part("home.", 200, nil)
+ let right = part("Another", 400, speakerA)
+ let different = part("speaker", 600, speakerB)
+ session.transcriptSegments = [left, unknown, right, different]
+ try await store.create(session)
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: StubCaptureController(), processing: StubProcessingController(), audioArbiter: StubArbiter()
+ )
+ let tagged = try await coordinator.reassignSegment(sessionID: session.id, segmentID: unknown.id, to: speakerA)
+ XCTAssertEqual(tagged.transcriptSegments.map(\.text), ["the home. Another", "speaker"])
+ XCTAssertEqual(tagged.transcriptSegments[0].id, unknown.id)
+ XCTAssertEqual(tagged.transcriptSegments[0].speakerID, speakerA)
+ XCTAssertEqual(tagged.transcriptSegments[0].start, left.start)
+ XCTAssertEqual(tagged.transcriptSegments[0].end, right.end)
+ XCTAssertEqual(tagged.audioTracks, session.audioTracks)
+ XCTAssertEqual(tagged.speakers, session.speakers)
+ let loaded = try await store.load(id: session.id)
+ XCTAssertEqual(loaded?.transcriptSegments, tagged.transcriptSegments)
+ let undone = try await coordinator.undoTranscriptCorrection(sessionID: session.id)
+ XCTAssertEqual(undone.transcriptSegments, session.transcriptSegments)
+ XCTAssertEqual(undone.audioTracks, session.audioTracks)
+ }
+
+ func testTaggingUnknownDoesNotMergeAcrossAnotherSpeakerOrAudioTrack() throws {
+ var (session, speakerA, speakerB, _) = self.makeCorrectionSession(state: .completed)
+ let base = session.transcriptSegments[0]
+ func part(_ text: String, _ start: Int64, _ speaker: SessionSpeakerID?, track: MeetingAudioTrackID? = nil) -> MeetingTranscriptSegment {
+ var value = base
+ value.id = UUID()
+ value.text = text
+ value.start = .init(value: start, timescale: 1000)
+ value.end = .init(value: start + 200, timescale: 1000)
+ value.speakerID = speaker
+ value.attributionState = speaker == nil ? .unassigned : .assigned
+ if let track { value.sourceTrackID = track }
+ return value
+ }
+ let earlier = part("Earlier", 0, speakerA)
+ let interruption = part("Interruption", 200, speakerB)
+ let unknown = part("word", 400, nil)
+ let differentTrack = part("Other track", 600, speakerA, track: UUID())
+ session.transcriptSegments = [earlier, interruption, unknown, differentTrack]
+ try session.reassignSegment(id: unknown.id, to: speakerA)
+ let assigned = session.transcriptSegments
+ XCTAssertEqual(session.mergeTaggedSegmentWithNeighbors(id: unknown.id), [unknown.id])
+ XCTAssertEqual(session.transcriptSegments, assigned)
+ // A right-hand match joins even if the preceding speaker is different.
+ session.transcriptSegments[3].sourceTrackID = unknown.sourceTrackID
+ _ = session.mergeTaggedSegmentWithNeighbors(id: unknown.id)
+ XCTAssertEqual(session.transcriptSegments.map(\.text), ["Earlier", "Interruption", "word Other track"])
+ }
+
func testUndoRestoresExactPriorStateForEachCorrectionType() async throws {
let dir = self.makeTempDirectory()
defer { try? FileManager.default.removeItem(at: dir) }
@@ -3605,6 +3811,116 @@ final class MeetingRecoveryTests: XCTestCase {
XCTAssertNotNil(MeetingSpeakerPalette.tints(for: [speakerA, renamedPlaceholder, speakerC])[renamedPlaceholder.id])
}
+ func testSpeakerEditDraftFlattensMergesAndRejectsCycles() throws {
+ let first = UUID(); let second = UUID(); let third = UUID()
+ var edits = MeetingSpeakerEdits(names: [:])
+ edits.setMerge(source: first, into: second)
+ edits.setMerge(source: second, into: third)
+ XCTAssertEqual(try edits.resolvedMerges(), [first: third, second: third])
+ edits.setMerge(source: first, into: nil)
+ XCTAssertEqual(edits.merges, [second: third])
+ edits.merges = [first: second, second: first]
+ XCTAssertThrowsError(try edits.resolvedMerges())
+ }
+
+ func testSpeakerEditorRendersSmallAndLargeListsWithoutSaving() throws {
+ var actions = 0
+ let variants: [(count: Int, width: CGFloat, scheme: ColorScheme)] = [
+ (3, 560, .dark), (8, 480, .light), (20, 480, .dark),
+ ]
+ for variant in variants {
+ let speakers = (1...variant.count).map {
+ self.makeSpeaker(name: "Speaker \($0)", trackKind: $0 == variant.count ? .microphone : .applicationAudio)
+ }
+ let editor = MeetingAssignSpeakersSheet(
+ speakers: speakers,
+ quotesBySpeaker: Dictionary(uniqueKeysWithValues: speakers.map { ($0.id, ["A short sample of this speaker's words."]) }),
+ tints: [:],
+ accentColor: FluidBrandColors.blue,
+ onSave: { _ in actions += 1; return nil },
+ onCancel: { actions += 1 }
+ )
+ try self.renderMeetingUI(
+ editor,
+ name: "speaker-editor-\(variant.count)",
+ width: variant.width,
+ height: 560,
+ scheme: variant.scheme
+ )
+ }
+ XCTAssertEqual(actions, 0, "opening or rendering the speaker editor must not save a draft")
+ }
+
+ func testSpeakerEditorSavesNamesAndMergesTogetherAndUndoesOnce() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let (session, first, second, _) = self.makeCorrectionSession(state: .completed)
+ try await store.create(session)
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: StubCaptureController(), processing: StubProcessingController(), audioArbiter: StubArbiter()
+ )
+ let edits = MeetingSpeakerEdits(names: [first: "Removed voice", second: "Combined"], merges: [first: second])
+ let saved = try await coordinator.editSpeakers(sessionID: session.id, edits: edits)
+ XCTAssertEqual(saved.activeSpeakers.map(\.displayName), ["Combined"])
+ XCTAssertEqual(saved.transcriptSegments.map(\.speakerID), [second])
+ XCTAssertEqual(saved.transcriptSegments.map(\.text), session.transcriptSegments.map(\.text))
+ XCTAssertEqual(saved.transcriptSegments.map(\.sourceTrackID), session.transcriptSegments.map(\.sourceTrackID))
+ XCTAssertEqual(saved.audioTracks, session.audioTracks)
+ let undone = try await coordinator.undoTranscriptCorrection(sessionID: session.id)
+ XCTAssertEqual(undone.speakers, session.speakers)
+ XCTAssertEqual(undone.transcriptSegments, session.transcriptSegments)
+ XCTAssertEqual(undone.audioTracks, session.audioTracks)
+ XCTAssertFalse(coordinator.canUndoCorrection(sessionID: session.id), "one editor save is one correction")
+ }
+
+ func testSpeakerEditorInvalidMergeDoesNotSaveNamesOrTouchAudio() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ var (session, first, second, _) = self.makeCorrectionSession(state: .completed)
+ session.speakers[1].isLocalUser = true
+ try await store.create(session)
+ let before = try await store.loadTranscript(id: session.id)
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: StubCaptureController(), processing: StubProcessingController(), audioArbiter: StubArbiter()
+ )
+ do {
+ _ = try await coordinator.editSpeakers(sessionID: session.id, edits: .init(names: [second: "Wrong"], merges: [first: second]))
+ XCTFail("protected speakers cannot be merged")
+ } catch {}
+ let after = try await store.loadTranscript(id: session.id)
+ XCTAssertEqual(after, before)
+ XCTAssertFalse(coordinator.canUndoCorrection(sessionID: session.id))
+ XCTAssertTrue(coordinator.isQuiescent)
+ }
+
+ func testSpeakerEditorFailedSaveCanRetryAndNoOpDoesNotAddUndo() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let realStore = MeetingSessionStore(rootDirectory: dir)
+ let (session, first, second, _) = self.makeCorrectionSession(state: .completed)
+ try await realStore.create(session)
+ let store = GatedThrowingSaveStore(inner: realStore, recorder: EventRecorder())
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: StubCaptureController(), processing: StubProcessingController(), audioArbiter: StubArbiter()
+ )
+ _ = try await coordinator.editSpeakers(sessionID: session.id, edits: .init(names: [:]))
+ XCTAssertFalse(coordinator.canUndoCorrection(sessionID: session.id))
+ await store.throwOnNextSave()
+ let edits = MeetingSpeakerEdits(names: [second: "Combined"], merges: [first: second])
+ do {
+ _ = try await coordinator.editSpeakers(sessionID: session.id, edits: edits)
+ XCTFail("expected save failure")
+ } catch {}
+ let unchanged = try await realStore.loadTranscript(id: session.id)
+ XCTAssertEqual(unchanged?.speakers, session.speakers)
+ XCTAssertEqual(unchanged?.transcriptSegments, session.transcriptSegments)
+ XCTAssertFalse(coordinator.canUndoCorrection(sessionID: session.id))
+ let retried = try await coordinator.editSpeakers(sessionID: session.id, edits: edits)
+ XCTAssertEqual(retried.activeSpeakers.map(\.displayName), ["Combined"])
+ }
+
// MARK: - MeetingAssignSpeakersValidation.duplicateName
func testDuplicateNameIgnoresACollisionTheUserDidNotCreate() {
diff --git a/Tests/FluidDictationIntegrationTests/MeetingTextOverlapEchoVerdictProviderTests.swift b/Tests/FluidDictationIntegrationTests/MeetingTextOverlapEchoVerdictProviderTests.swift
index ca523437..ba649dde 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingTextOverlapEchoVerdictProviderTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingTextOverlapEchoVerdictProviderTests.swift
@@ -245,6 +245,62 @@ final class MeetingTextOverlapEchoVerdictProviderTests: XCTestCase {
XCTAssertEqual(verdicts["mic-local"], .notEcho)
}
+ func testRecoveredApplicationEndingRemainsAnEchoTarget() async throws {
+ let fixture = try self.makeFixture()
+ let epoch = fixture.appEpochs[0]
+ let token = MeetingBackendSpeakerToken(analysisEpochID: epoch.id, label: "slot-0")
+ var units: [MeetingFinalTextUnit] = []
+ for (index, text) in ["quick", "brown", "fox", "jumps"].enumerated() {
+ let start = 0.2 + Double(index) * 0.25
+ units.append(MeetingFinalTextUnit(
+ id: "app-\(index)",
+ trackID: fixture.appTrackID,
+ analysisEpochID: epoch.id,
+ precision: .word,
+ text: text,
+ analysisStart: start,
+ analysisEnd: start + 0.25,
+ speaker: .assigned(token),
+ analysisSpanIDs: epoch.spanIDs
+ ))
+ units.append(self.unit(
+ "mic-\(index)",
+ trackID: fixture.micTrackID,
+ epoch: fixture.micEpoch,
+ text: text,
+ analysisStart: start + 0.05,
+ analysisEnd: start + 0.30
+ ))
+ }
+ let evidence = MeetingFinalTranscriptEvidence(
+ backendID: .parakeetNemotron,
+ attemptID: fixture.plan.attemptID,
+ units: units,
+ speakerActivity: [MeetingBackendSpeakerActivity(token: token, start: 0, end: 0.9)]
+ )
+ let verdicts = try await MeetingTextOverlapEchoVerdictProvider().echoVerdicts(
+ for: evidence, manifest: fixture.manifest, plan: fixture.plan
+ )
+ XCTAssertEqual(verdicts["mic-3"], .echoSuppressed(duplicateOfUnitID: "app-3"))
+ let result = try MeetingTranscriptAssembler().assemble(MeetingAssemblyInput(
+ plan: fixture.plan,
+ manifest: fixture.manifest,
+ evidence: evidence,
+ coverageReceipts: fixture.manifest.allSpans.map {
+ MeetingSpanCoverageReceipt(
+ id: "receipt:\($0.id)",
+ spanID: $0.id,
+ analysisStart: $0.analysisInterval.start,
+ analysisEnd: $0.analysisInterval.end,
+ status: .processed
+ )
+ },
+ echoVerdicts: verdicts
+ ))
+ XCTAssertEqual(result.segments.map(\.text), ["quick", "brown", "fox", "jumps"])
+ XCTAssertTrue(result.segments.allSatisfy { $0.sourceTrackID == fixture.appTrackID })
+ }
+
func testEchoTextWithoutTemporalOverlapIsNotSuppressed() async throws {
let fixture = try self.makeFixture()
let echoText = "the quick brown fox jumps over the lazy dog"
diff --git a/Tests/FluidDictationIntegrationTests/MeetingTranscriptAssemblerTests.swift b/Tests/FluidDictationIntegrationTests/MeetingTranscriptAssemblerTests.swift
index 7d98199b..36dbfe30 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingTranscriptAssemblerTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingTranscriptAssemblerTests.swift
@@ -826,6 +826,112 @@ final class MeetingTranscriptAssemblerTests: XCTestCase {
XCTAssertEqual(result.segments.map(\.text), ["inside"])
}
+ func testBoundaryRetentionKeepsTailButEchoVetoStillWins() throws {
+ let (plan, manifest, span) = try self.onlineMicFixture()
+ let token = MeetingBackendSpeakerToken(analysisEpochID: span.analysisEpochID, label: "slot-0")
+ let activity = [MeetingBackendSpeakerActivity(token: token, start: 0, end: 1.439_999_938)]
+ let anchor = self.unit(id: "anchor", span: span, analysisStart: 1.2, analysisEnd: 1.52)
+ let tail = self.unit(id: "tail", span: span, text: "ending", analysisStart: 1.52, analysisEnd: 2.16)
+ for suppressed in [false, true] {
+ let result = try MeetingTranscriptAssembler().assemble(MeetingAssemblyInput(
+ plan: plan,
+ manifest: manifest,
+ evidence: self.evidence(plan: plan, units: [anchor, tail], activity: activity),
+ coverageReceipts: self.receipts(for: manifest),
+ echoVerdicts: [anchor.id: .notEcho, tail.id: suppressed ? .echoSuppressed(duplicateOfUnitID: nil) : .notEcho]
+ ))
+ XCTAssertEqual(self.dispositions(result)[tail.id]?.disposition, suppressed ? .echoSuppressed : .emitted)
+ XCTAssertEqual(result.segments.map(\.text).contains("ending"), !suppressed)
+ }
+ }
+
+ func testBoundaryRecoveryRetainsCompetingTailWithoutAssigningPreviousSpeaker() throws {
+ let (_, _, span) = try self.onlineMicFixture()
+ let token = MeetingBackendSpeakerToken(analysisEpochID: span.analysisEpochID, label: "slot-0")
+ let other = MeetingBackendSpeakerToken(analysisEpochID: span.analysisEpochID, label: "slot-1")
+ let anchor = self.unit(id: "anchor", span: span, analysisStart: 1.2, analysisEnd: 1.52)
+ let tail = self.unit(id: "tail", span: span, analysisStart: 1.52, analysisEnd: 2.16, speaker: .unassigned)
+ for otherInterval in [(1.4, 1.48), (1.55, 1.56), (2.17, 2.3)] {
+ let activity = [
+ MeetingBackendSpeakerActivity(token: token, start: 0, end: 1.439_999_938),
+ MeetingBackendSpeakerActivity(token: other, start: otherInterval.0, end: otherInterval.1),
+ ]
+ let assignments = MeetingWordBoundaryRecovery.assignments(
+ units: [anchor, tail], activityByEpoch: [span.analysisEpochID: activity]
+ )
+ XCTAssertEqual(assignments[tail.id], .unassigned, "a competing onset must veto inherited identity")
+ XCTAssertFalse(MeetingSpeakerActivityCoverage.isOutside(
+ tail, activityByEpoch: [span.analysisEpochID: activity], boundaryAssignments: assignments
+ ), "timing uncertainty must not remove recognized text")
+ }
+ }
+
+ func testFloat32EndpointAllowanceIsBoundedForLongEpochs() throws {
+ let (_, _, span) = try self.onlineMicFixture(duration: 7210)
+ let token = MeetingBackendSpeakerToken(analysisEpochID: span.analysisEpochID, label: "slot-0")
+ for endpoint in [19.2, 7201.44] {
+ let roundedEndpoint = Double(Float(endpoint))
+ let start = endpoint + 0.08
+ let anchor = self.unit(id: "anchor", span: span, analysisStart: start - 0.32, analysisEnd: start)
+ let tail = self.unit(id: "tail", span: span, analysisStart: start, analysisEnd: start + 0.32, speaker: .unassigned)
+ let activity = [MeetingBackendSpeakerActivity(token: token, start: 0, end: roundedEndpoint)]
+ let recovered = MeetingWordBoundaryRecovery.assignments(
+ units: [anchor, tail], activityByEpoch: [span.analysisEpochID: activity]
+ )
+ XCTAssertEqual(recovered[tail.id], .assigned(token), "Float32 rounding must not exclude the last word")
+ let later = self.unit(id: "later", span: span, analysisStart: start + 0.003, analysisEnd: start + 0.32, speaker: .unassigned)
+ XCTAssertNil(MeetingWordBoundaryRecovery.assignments(
+ units: [anchor, later], activityByEpoch: [span.analysisEpochID: activity]
+ )[later.id], "precision allowance must not become an unbounded speech extension")
+ }
+ }
+
+ func testDirectlySupportedShortWordCanStillAnchorAnEnding() throws {
+ let (_, _, span) = try self.onlineMicFixture()
+ let token = MeetingBackendSpeakerToken(analysisEpochID: span.analysisEpochID, label: "slot-0")
+ let words = [
+ self.unit(id: "first", span: span, analysisStart: 0.6, analysisEnd: 1),
+ self.unit(id: "short", span: span, analysisStart: 1, analysisEnd: 1.16),
+ self.unit(id: "tail", span: span, analysisStart: 1.16, analysisEnd: 1.6, speaker: .unassigned),
+ ]
+ let result = MeetingWordBoundaryRecovery.assignments(
+ units: words,
+ activityByEpoch: [span.analysisEpochID: [
+ MeetingBackendSpeakerActivity(token: token, start: 0, end: 1.08),
+ ]]
+ )
+ XCTAssertNil(result["short"], "direct evidence needs no repair")
+ XCTAssertEqual(result["tail"], .assigned(token))
+ }
+
+ func testBoundaryRecoveryCannotChainThroughSilenceOrOverrideSpeakers() throws {
+ let (_, _, span) = try self.onlineMicFixture()
+ let token = MeetingBackendSpeakerToken(analysisEpochID: span.analysisEpochID, label: "slot-0")
+ let other = MeetingBackendSpeakerToken(analysisEpochID: span.analysisEpochID, label: "slot-1")
+ let activity = [MeetingBackendSpeakerActivity(token: token, start: 0, end: 1.439_999_938)]
+ let anchor = self.unit(id: "anchor", span: span, analysisStart: 1.2, analysisEnd: 1.52)
+ let tail = self.unit(id: "tail", span: span, analysisStart: 1.52, analysisEnd: 2.16)
+ let silence = self.unit(id: "silence", span: span, analysisStart: 2.16, analysisEnd: 2.4, speaker: .unassigned)
+ let recovered = MeetingWordBoundaryRecovery.assignments(
+ units: [anchor, tail, silence], activityByEpoch: [span.analysisEpochID: activity]
+ )
+ XCTAssertEqual(recovered[tail.id], .assigned(token))
+ XCTAssertNil(recovered[silence.id], "a recovered tail cannot anchor another recovery")
+ for candidate in [
+ self.unit(id: "gap", span: span, analysisStart: 1.61, analysisEnd: 2.16, speaker: .unassigned),
+ self.unit(id: "long", span: span, analysisStart: 1.52, analysisEnd: 3, speaker: .unassigned),
+ self.unit(id: "other", span: span, analysisStart: 1.52, analysisEnd: 2.16, speaker: .assigned(other)),
+ self.unit(id: "ambiguous", span: span, analysisStart: 1.52, analysisEnd: 2.16, speaker: .ambiguous([token, other])),
+ self.unit(id: "span", span: span, analysisStart: 1.52, analysisEnd: 2.16, spanIDs: ["another-span"]),
+ self.unit(id: "epoch", span: span, analysisStart: 1.52, analysisEnd: 2.16, epoch: .init(trackID: span.trackID, ordinal: 9)),
+ self.unit(id: "invalid", span: span, analysisStart: .nan, analysisEnd: 2.16),
+ ] {
+ XCTAssertNil(MeetingWordBoundaryRecovery.assignments(
+ units: [anchor, candidate], activityByEpoch: [span.analysisEpochID: activity]
+ )[candidate.id], "unsafe candidate \(candidate.id) must not recover")
+ }
+ }
+
func testSpeakerActivityMustStayInsideItsEpoch() throws {
let (plan, manifest, span) = try self.onlineMicFixture()
let unit = self.unit(id: "u-0", span: span, analysisStart: 1, analysisEnd: 2)
diff --git a/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift b/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
index e1782350..e1e64a60 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
@@ -169,13 +169,13 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
isHeadphonesDataSource: false,
terminalTypes: []
)
- XCTAssertEqual(MeetingRawMicrophoneProtection.classify(unknownBluetooth), .unprotected)
+ XCTAssertEqual(MeetingRawMicrophoneProtection.classify(unknownBluetooth), .unfiltered)
var externalWithoutDataSource = unknownBluetooth
externalWithoutDataSource.isHeadphonesDataSource = true
XCTAssertEqual(
MeetingRawMicrophoneProtection.classify(externalWithoutDataSource),
- .unprotected,
+ .unfiltered,
"AudioDevice reports missing data-source properties pessimistically; that alone is not closed-route proof"
)
diff --git a/Tests/FluidDictationIntegrationTests/NativeButtonStylesHitTargetTests.swift b/Tests/FluidDictationIntegrationTests/NativeButtonStylesHitTargetTests.swift
index a75c14ef..0fe833c3 100644
--- a/Tests/FluidDictationIntegrationTests/NativeButtonStylesHitTargetTests.swift
+++ b/Tests/FluidDictationIntegrationTests/NativeButtonStylesHitTargetTests.swift
@@ -141,6 +141,69 @@ final class NativeButtonStylesHitTargetTests: XCTestCase {
},
]
+ func testDisclosureHeadersAcceptClicksAcrossTheirWholeRow() throws {
+ for x: CGFloat in [7, 90, 200, 316] {
+ var expanded = false
+ var writes = 0
+ let view = DisclosureGroup("Previous versions", isExpanded: Binding(
+ get: { expanded },
+ set: { expanded = $0; writes += 1 }
+ )) { Text("Details") }
+ .disclosureGroupStyle(FluidDisclosureGroupStyle())
+ .appTheme(.dark)
+ .frame(width: 320)
+ let harness = self.makeHarness(view)
+ defer { harness.tearDown() }
+ try self.simulateClick(in: harness.window, at: harness.hostPoint(x, harness.targetSize.height / 2), in: harness.hostingView)
+ XCTAssertTrue(expanded, "The chevron, text, blank middle, and trailing edge must all toggle")
+ XCTAssertEqual(writes, 1, "One click must toggle exactly once")
+ }
+ }
+
+ func testDisabledDisclosureDoesNotToggle() throws {
+ var expanded = false
+ var writes = 0
+ let view = DisclosureGroup("Previous versions", isExpanded: Binding(
+ get: { expanded },
+ set: { expanded = $0; writes += 1 }
+ )) { Text("Details") }
+ .disclosureGroupStyle(FluidDisclosureGroupStyle())
+ .appTheme(.dark)
+ .disabled(true)
+ .frame(width: 320)
+ let harness = self.makeHarness(view)
+ defer { harness.tearDown() }
+ for x: CGFloat in [7, 90, 200, 316] {
+ try self.simulateClick(in: harness.window, at: harness.hostPoint(x, harness.targetSize.height / 2), in: harness.hostingView)
+ }
+ XCTAssertFalse(expanded)
+ XCTAssertEqual(writes, 0)
+ }
+
+ func testExpandedContentActionDoesNotToggleItsHeader() throws {
+ var expanded = true
+ var toggles = 0
+ var actions = 0
+ let view = DisclosureGroup("Previous versions", isExpanded: Binding(
+ get: { expanded },
+ set: { expanded = $0; toggles += 1 }
+ )) {
+ Button { actions += 1 } label: {
+ Text("Content action").frame(width: 300, height: 32).contentShape(Rectangle())
+ }
+ .buttonStyle(.plain)
+ }
+ .disclosureGroupStyle(FluidDisclosureGroupStyle())
+ .appTheme(.dark)
+ .frame(width: 320)
+ let harness = self.makeHarness(view)
+ defer { harness.tearDown() }
+ try self.simulateClick(in: harness.window, at: harness.hostPoint(160, harness.targetSize.height - 16), in: harness.hostingView)
+ XCTAssertTrue(expanded)
+ XCTAssertEqual(toggles, 0)
+ XCTAssertEqual(actions, 1)
+ }
+
func testButtonStylesAcceptClicksAcrossTheirShape() throws {
for style in self.styleCases {
var clicks = 0
diff --git a/Tests/FluidDictationIntegrationTests/PronunciationModelRevisionTests.swift b/Tests/FluidDictationIntegrationTests/PronunciationModelRevisionTests.swift
index 93a299c3..69e002a8 100644
--- a/Tests/FluidDictationIntegrationTests/PronunciationModelRevisionTests.swift
+++ b/Tests/FluidDictationIntegrationTests/PronunciationModelRevisionTests.swift
@@ -56,6 +56,42 @@ final class PronunciationModelRevisionTests: XCTestCase {
#endif
}
+ func testFamilySelectionPrefersExactEligibleProfileAndPreservesStorage() async throws {
+ let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+ defer { try? FileManager.default.removeItem(at: directory) }
+ let url = directory.appendingPathComponent("profiles.json")
+ let store = PronunciationDictionaryStore(fileURL: url)
+ let id = UUID()
+ let target = ParakeetSpeechModelCatalog.mini.pronunciationModelKey
+ let source = Self.profile(entryID: id, key: "parakeet-v2")
+ try await store.upsert(dictionaryEntryID: id, label: source.label, modelKey: source.modelKey, enrollments: source.enrollments)
+ let before = try Data(contentsOf: url)
+ let shared = await store.sharedProfiles(modelKey: target, hiddenSize: source.hiddenSize)
+ XCTAssertEqual(shared.map(\.modelKey), [source.modelKey])
+ XCTAssertEqual(shared.first?.enrollments, source.enrollments)
+ XCTAssertEqual(try Data(contentsOf: url), before, "Family reads never relabel or rewrite enrollment data")
+ let incompatible = await store.sharedProfiles(modelKey: target, hiddenSize: source.hiddenSize + 1)
+ XCTAssertTrue(incompatible.isEmpty)
+ let unrelated = await store.sharedProfiles(modelKey: "whisper", hiddenSize: source.hiddenSize)
+ XCTAssertTrue(unrelated.isEmpty)
+ let exact = Self.profile(entryID: id, key: target)
+ var incomplete = exact
+ incomplete.enrollments = Array(exact.enrollments.prefix(1))
+ XCTAssertEqual(PronunciationDictionaryStore.selectSharedProfiles([source, incomplete], modelKey: target, hiddenSize: 2), [source])
+ for ordered in [[source, exact], [exact, source]] {
+ XCTAssertEqual(PronunciationDictionaryStore.selectSharedProfiles(ordered, modelKey: target, hiddenSize: 2), [exact])
+ }
+ let keys = ParakeetSpeechModelCatalog.descriptors.map(\.pronunciationModelKey)
+ for sourceKey in keys {
+ for targetKey in keys {
+ XCTAssertTrue(ParakeetSpeechModelCatalog.sharesPronunciationFamily(sourceKey, with: targetKey))
+ }
+ }
+ for invalid in ["", "whisper", "parakeet-v4", "fluid-parakeet-mini:sha256:bad"] {
+ XCTAssertFalse(ParakeetSpeechModelCatalog.sharesPronunciationFamily(invalid, with: target))
+ }
+ }
+
func testOldNegativeFramesAreInactiveAndRemainStoredWithoutAReadMutation() async throws {
let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
defer { try? FileManager.default.removeItem(at: directory) }
@@ -94,9 +130,7 @@ final class PronunciationModelRevisionTests: XCTestCase {
let old = Self.profile(entryID: id, key: descriptor.modelID)
let current = Self.profile(entryID: id, key: descriptor.pronunciationModelKey)
let notice = DictionaryPronunciationRevisionPolicy.notice(profiles: [old], descriptor: descriptor, entryIDs: [id])
- XCTAssertTrue(notice?.contains("FluidVoice") == true)
- XCTAssertTrue(notice?.contains("need retraining") == true)
- XCTAssertTrue(notice?.contains("text corrections remain saved") == true)
+ XCTAssertNil(notice, "Parakeet family recordings remain available after a model update")
XCTAssertNil(DictionaryPronunciationRevisionPolicy.notice(profiles: [old, current], descriptor: descriptor, entryIDs: [id]))
XCTAssertNil(DictionaryPronunciationRevisionPolicy.notice(profiles: [old], descriptor: descriptor, entryIDs: []))
XCTAssertNil(DictionaryPronunciationRevisionPolicy.notice(profiles: [old], descriptor: ParakeetSpeechModelCatalog.pico, entryIDs: [id]))
@@ -136,23 +170,22 @@ final class PronunciationModelRevisionTests: XCTestCase {
#endif
#if arch(arm64)
- func testLoadedOldCompactRevisionMatchesOnlyItsOwnEnrollmentVectors() {
+ func testExplicitMatchingSharesParakeetFamilyWithoutMixingStoredEnrollments() {
let descriptor = ParakeetSpeechModelCatalog.mini
let oldKey = descriptor.modelID + ":sha256:" + String(repeating: "a", count: 64)
let old = Self.profile(entryID: UUID(), key: oldKey)
let latest = Self.profile(entryID: UUID(), key: descriptor.pronunciationModelKey)
XCTAssertTrue(DictionaryPronunciationReferences.make(profiles: [old]).isEmpty)
let oldReferences = DictionaryPronunciationReferences.make(profiles: [old, latest], compatibleModelKey: oldKey)
- XCTAssertEqual(oldReferences.count, 3)
- XCTAssertTrue(oldReferences.allSatisfy { $0.profile.modelKey == oldKey })
+ XCTAssertEqual(oldReferences.count, 6)
+ XCTAssertEqual(Set(oldReferences.map { $0.profile.modelKey }), [oldKey, descriptor.pronunciationModelKey])
let latestReferences = DictionaryPronunciationReferences.make(profiles: [old, latest], compatibleModelKey: descriptor.pronunciationModelKey)
- XCTAssertEqual(latestReferences.count, 3)
- XCTAssertTrue(latestReferences.allSatisfy { $0.profile.modelKey == descriptor.pronunciationModelKey })
+ XCTAssertEqual(latestReferences.count, 6)
var mixed = old
mixed.enrollments[0] = latest.enrollments[0]
XCTAssertTrue(DictionaryPronunciationReferences.make(profiles: [mixed], compatibleModelKey: oldKey).isEmpty)
let unversioned = Self.profile(entryID: UUID(), key: descriptor.modelID)
- XCTAssertTrue(DictionaryPronunciationReferences.make(profiles: [unversioned], compatibleModelKey: descriptor.modelID).isEmpty)
+ XCTAssertEqual(DictionaryPronunciationReferences.make(profiles: [unversioned], compatibleModelKey: descriptor.modelID).count, 3)
}
func testNormalProviderPreparationKeepsCompleteOldWeightsAndTheirActualPronunciationKey() async throws {
diff --git a/Tests/FluidDictationIntegrationTests/SettingsBackupRoundTripTests.swift b/Tests/FluidDictationIntegrationTests/SettingsBackupRoundTripTests.swift
index 72c0e9e7..e48dad68 100644
--- a/Tests/FluidDictationIntegrationTests/SettingsBackupRoundTripTests.swift
+++ b/Tests/FluidDictationIntegrationTests/SettingsBackupRoundTripTests.swift
@@ -155,3 +155,31 @@ final class SettingsBackupRoundTripTests: XCTestCase {
try body()
}
}
+
+extension SettingsBackupRoundTripTests {
+ func testMeetingShortcutRestoreAssignedUnassignedAndLegacyBackups() throws {
+ try self.withSavedDefaults {
+ let settings = SettingsStore.shared
+ let assigned = HotkeyShortcut(keyCode: 46, modifierFlags: [.control, .option, .command])
+ let replacement = HotkeyShortcut(keyCode: 46, modifierFlags: [.command, .shift])
+ let primary = settings.primaryDictationShortcuts
+ let meetingDefaults = settings.meetingRecordingDefaults
+ settings.meetingRecordingHotkeyShortcut = assigned
+ let assignedPayload = try JSONDecoder().decode(SettingsBackupPayload.self, from: JSONEncoder().encode(settings.makeBackupPayload()))
+ settings.meetingRecordingHotkeyShortcut = nil
+ let unassignedPayload = try JSONDecoder().decode(SettingsBackupPayload.self, from: JSONEncoder().encode(settings.makeBackupPayload()))
+ let legacyPayload = try self.payload(removing: ["meetingRecordingHotkeyShortcut", "meetingRecordingShortcutEnabled"])
+
+ settings.meetingRecordingHotkeyShortcut = replacement
+ settings.restore(from: assignedPayload)
+ XCTAssertEqual(settings.meetingRecordingHotkeyShortcut, assigned)
+ settings.restore(from: unassignedPayload)
+ XCTAssertNil(settings.meetingRecordingHotkeyShortcut)
+ settings.meetingRecordingHotkeyShortcut = replacement
+ settings.restore(from: legacyPayload)
+ XCTAssertEqual(settings.meetingRecordingHotkeyShortcut, replacement)
+ XCTAssertEqual(settings.primaryDictationShortcuts, primary)
+ XCTAssertEqual(settings.meetingRecordingDefaults, meetingDefaults)
+ }
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/SettingsNavigationStateTests.swift b/Tests/FluidDictationIntegrationTests/SettingsNavigationStateTests.swift
index 75ccf66d..3efeb37e 100644
--- a/Tests/FluidDictationIntegrationTests/SettingsNavigationStateTests.swift
+++ b/Tests/FluidDictationIntegrationTests/SettingsNavigationStateTests.swift
@@ -1,9 +1,48 @@
import AppKit
+import Combine
@testable import FluidVoice_Debug
import XCTest
@MainActor
final class SettingsNavigationStateTests: XCTestCase {
+ func testFluidMeetHasASettingsDestinationEvenWhenDisabled() {
+ var state = SettingsNavigationState()
+ state.present(.fluidMeet, returningTo: .history)
+ XCTAssertEqual(state.selectedSection?.title, "FluidMeet Settings")
+ XCTAssertEqual(SettingsSection.fluidMeet.searchTarget, .fluidMeet)
+ XCTAssertEqual(SettingsSearchTarget.fluidMeet.section, .fluidMeet)
+ XCTAssertEqual(SettingsSearchIndex.results(for: "disable meetings").first?.section, .fluidMeet)
+ XCTAssertEqual(state.dismiss(), .history)
+ }
+
+ func testFluidMeetTogglePreservesDetectionPreferencesAndOnlyNotifiesOnChanges() {
+ let defaults = UserDefaults.standard
+ let original = defaults.object(forKey: "FluidMeetEnabled")
+ defer { defaults.set(original, forKey: "FluidMeetEnabled") }
+ defaults.removeObject(forKey: "FluidMeetEnabled")
+ let settings = SettingsStore.shared
+ let nativeDetection = settings.meetingAutoDetectEnabled
+ let browserDetection = settings.meetingAutoDetectBrowserEnabled
+ let recordingDefaults = defaults.object(forKey: "MeetingRecordingDefaults") as? Data
+ var changes = 0
+ let observation = NotificationCenter.default.publisher(for: .fluidMeetEnabledChanged)
+ .sink { _ in changes += 1 }
+ defer { observation.cancel() }
+
+ XCTAssertTrue(settings.fluidMeetEnabled, "Existing users keep FluidMeet enabled")
+ for _ in 0..<3 {
+ settings.fluidMeetEnabled = false
+ XCTAssertFalse(settings.fluidMeetEnabled)
+ settings.fluidMeetEnabled = false
+ settings.fluidMeetEnabled = true
+ XCTAssertTrue(settings.fluidMeetEnabled)
+ }
+ XCTAssertEqual(changes, 6, "Repeated assignments must not restart detection")
+ XCTAssertEqual(settings.meetingAutoDetectEnabled, nativeDetection)
+ XCTAssertEqual(settings.meetingAutoDetectBrowserEnabled, browserDetection)
+ XCTAssertEqual(defaults.object(forKey: "MeetingRecordingDefaults") as? Data, recordingDefaults)
+ }
+
func testPresentAndDismissRestoresPreviousAppDestination() {
var state = SettingsNavigationState()
@@ -114,7 +153,7 @@ final class SettingsNavigationStateTests: XCTestCase {
func testSettingsSectionsHaveStableTitlesAndIcons() {
XCTAssertEqual(
SettingsSection.allCases.map(\.title),
- ["General", "Dictation", "Dictation Formatting", "Shortcuts", "Notifications", "Audio", "Overlay", "Data & Diagnostics", "Experimental"]
+ ["General", "FluidMeet Settings", "Dictation", "Dictation Formatting", "Shortcuts", "Notifications", "Audio", "Overlay", "Data & Diagnostics", "Experimental"]
)
XCTAssertTrue(SettingsSection.allCases.allSatisfy { !$0.systemImage.isEmpty })
XCTAssertEqual(SettingsSection.overlay.systemImage, "rectangle.on.rectangle")
@@ -154,6 +193,7 @@ final class SettingsNavigationStateTests: XCTestCase {
func testSettingsSearchMatchesPrefixesAliasesAndMultipleWords() {
XCTAssertEqual(SettingsSearchIndex.results(for: "start").first?.target, .launchAtStartup)
+ XCTAssertEqual(SettingsSearchIndex.results(for: "meeting shortcut").first?.target, .meetingRecordingShortcut)
XCTAssertTrue(SettingsSearchIndex.results(for: "mic").contains { $0.target == .inputDevicePriority })
XCTAssertEqual(SettingsSearchIndex.results(for: "audio device").first?.section, .audio)
}
diff --git a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift
index 1817d00a..a494c09b 100644
--- a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift
+++ b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift
@@ -594,6 +594,68 @@ extension MeetingSessionModelTests {
}
}
+ func testMarkdownExportIncludesTitleAndChronologicalSpeakerTurns() {
+ var session = MeetingModelFixture.makeSession()
+ session.title = "Planning"
+ var later = session.transcriptSegments[0]
+ later.id = UUID()
+ later.start = MeetingModelFixture.mediaTime(65)
+ later.end = MeetingModelFixture.mediaTime(70)
+ later.speakerID = nil
+ later.text = "Next steps"
+ session.transcriptSegments.insert(later, at: 0)
+
+ XCTAssertEqual(MeetingTranscriptExporter.markdown(for: session), """
+ # Planning
+
+ ## Transcript
+
+ ### [00:04] Speaker 1
+
+ Initial provisional text
+
+ ### [01:05] Unknown speaker
+
+ Next steps
+
+ """)
+ XCTAssertEqual(MeetingTranscriptExportFormat.markdown.fileExtension, "md")
+ XCTAssertEqual(MeetingTranscriptExportFormat.text.fileExtension, "txt")
+ XCTAssertEqual(MeetingTranscriptExportFormat.json.fileExtension, "json")
+ }
+
+ func testMarkdownExportEscapesContentAndKeepsOtherFormatsUnchanged() throws {
+ var session = MeetingModelFixture.makeSession()
+ session.title = "Planning *notes*\n# next"
+ try session.renameSpeaker(id: MeetingModelFixture.speakerID, to: "[Alex]")
+ session.transcriptSegments[0].text = "Use **bold** & <tags>\n# literal heading"
+ let plainText = MeetingTranscriptExporter.text(for: session)
+ let json = try MeetingTranscriptExporter.json(for: session)
+
+ let markdown = MeetingTranscriptExporter.markdown(for: session)
+
+ XCTAssertTrue(markdown.hasPrefix(##"# Planning \*notes\* \# next"## + "\n"))
+ XCTAssertTrue(markdown.contains(#"### [00:04] \[Alex\]"#))
+ XCTAssertTrue(markdown.contains(#"Use \*\*bold\*\* & \<tags\>"# + "\n" + ##"\# literal heading"##))
+ XCTAssertEqual(MeetingTranscriptExporter.text(for: session), plainText)
+ XCTAssertEqual(try MeetingTranscriptExporter.json(for: session), json)
+ }
+
+ func testMarkdownExportHonorsEchoVisibilityAndEmptyTranscript() {
+ var session = MeetingModelFixture.makeSession()
+ var echo = session.transcriptSegments[0]
+ echo.id = UUID()
+ echo.text = "Echoed remote text"
+ echo.isLikelyEcho = true
+ session.transcriptSegments.append(echo)
+
+ XCTAssertFalse(MeetingTranscriptExporter.markdown(for: session).contains(echo.text))
+ XCTAssertTrue(MeetingTranscriptExporter.markdown(for: session, includeEchoes: true).contains(echo.text))
+ session.title = "Empty meeting"
+ session.transcriptSegments = []
+ XCTAssertEqual(MeetingTranscriptExporter.markdown(for: session), "# Empty meeting\n\n## Transcript\n")
+ }
+
func testTextFormatMatchesTimestampFormatAndLegacyUnassignedSpeaker() {
var session = MeetingModelFixture.makeSession()
var unknownSpeakerSegment = session.transcriptSegments[0]
diff --git a/Tests/FluidDictationIntegrationTests/UpdatePromptIntegrationTests.swift b/Tests/FluidDictationIntegrationTests/UpdatePromptIntegrationTests.swift
index ccaadddc..31267f52 100644
--- a/Tests/FluidDictationIntegrationTests/UpdatePromptIntegrationTests.swift
+++ b/Tests/FluidDictationIntegrationTests/UpdatePromptIntegrationTests.swift
@@ -4,6 +4,79 @@ import XCTest
@MainActor
final class UpdatePromptIntegrationTests: XCTestCase {
+ func testRollbackListOnlyOffersOlderInstallableReleases() async throws {
+ let context = try UpdateTestContext()
+ defer { context.cleanup() }
+ for (version, downloadable, expected) in [
+ ("v0.0.1", true, ["v0.0.1"]),
+ ("v99.0.0", true, []),
+ ("v0.0.1", false, []),
+ ] {
+ context.fixture.setReply(version: version, downloadable: downloadable)
+ let options = try await context.updater.fetchRecentReleaseBuildOptions(
+ owner: "altic-dev", repo: "Fluid-oss", olderThanCurrent: true
+ )
+ XCTAssertEqual(options.map(\.version), expected)
+ XCTAssertFalse(context.updater.isUpdateInProgress, "Listing versions never starts installation")
+ }
+ }
+
+ func testSelectedRollbackInstallsOnlyTheExactOlderRelease() async throws {
+ let context = try UpdateTestContext()
+ defer { context.cleanup() }
+ context.fixture.setReply(version: "v0.0.1")
+ var installs = 0
+ context.updater.simulationInstallHandler = { installs += 1 }
+ try await context.updater.installPreviousRelease(version: "v0.0.1")
+ XCTAssertEqual(installs, 1)
+ XCTAssertFalse(context.updater.isUpdateInProgress)
+ XCTAssertNil(context.defaults.object(forKey: SettingsStore.UpdateKeys.betaReleasesEnabled))
+ for selected in ["v0.0.2", "v99.0.0"] {
+ context.fixture.setReply(version: "v99.0.0")
+ do {
+ try await context.updater.installPreviousRelease(version: selected)
+ XCTFail("A missing or newer release must never install")
+ } catch {}
+ }
+ XCTAssertEqual(installs, 1)
+ XCTAssertFalse(context.updater.isUpdateInProgress)
+ }
+
+ func testFailedRollbackReleasesGateForRetry() async throws {
+ let context = try UpdateTestContext()
+ defer { context.cleanup() }
+ context.fixture.setReply(version: "v0.0.1")
+ context.updater.simulationInstallHandler = { throw URLError(.cannotConnectToHost) }
+ do {
+ try await context.updater.installPreviousRelease(version: "v0.0.1")
+ XCTFail("Expected installation failure")
+ } catch {}
+ XCTAssertFalse(context.updater.isUpdateInProgress)
+ var installs = 0
+ context.updater.simulationInstallHandler = { installs += 1 }
+ try await context.updater.installPreviousRelease(version: "v0.0.1")
+ XCTAssertEqual(installs, 1)
+ }
+
+ func testRollbackRejectsConcurrentInstallAndCancelledDiscovery() async throws {
+ let context = try UpdateTestContext()
+ defer { context.cleanup() }
+ context.fixture.setReply(version: "v0.0.1", held: true)
+ var installs = 0
+ context.updater.simulationInstallHandler = { installs += 1 }
+ let pending = Task { try await context.updater.installPreviousRelease(version: "v0.0.1") }
+ try await self.waitFor { context.fixture.requestCount == 1 }
+ do {
+ try await context.updater.installPreviousRelease(version: "v0.0.1")
+ XCTFail("Concurrent operations must be rejected")
+ } catch {}
+ pending.cancel()
+ context.fixture.release()
+ do { try await pending.value; XCTFail("Cancelled request must not install") } catch {}
+ XCTAssertEqual(installs, 0)
+ XCTAssertFalse(context.updater.isUpdateInProgress)
+ }
+
func testPopupPreferenceBackupDecodesFalseAndMissingLegacyField() throws {
var payload = SettingsStore.shared.makeBackupPayload()
payload.showUpdatePopups = false
@@ -559,13 +632,19 @@ private final nonisolated class UpdateFixture: @unchecked Sendable {
var requestCount: Int { self.lock.withLock { self.count } }
- func setReply(version: String?, status: Int = 200, held: Bool = false) {
+ func setReply(version: String?, status: Int = 200, held: Bool = false, downloadable: Bool = false) {
self.lock.withLock {
self.status = status
self.held = held
self.body = version.map {
- """
- [{"tag_name":"\($0)","prerelease":false,"assets":[]}]
+ let raw = $0.hasPrefix("v") ? String($0.dropFirst()) : $0
+ let assets = downloadable
+ ? """
+ [{"name":"fluid-oss-\(raw).zip","content_type":"application/zip","browser_download_url":"https://example.com/build.zip"}]
+ """
+ : "[]"
+ return """
+ [{"tag_name":"\($0)","prerelease":false,"assets":\(assets)}]
"""
} ?? "[{\"tag_name\":\"v0.0.1\",\"prerelease\":false,\"assets\":[]}]"
}
diff --git a/Tests/FluidDictationIntegrationTests/VoiceModelDeletionTests.swift b/Tests/FluidDictationIntegrationTests/VoiceModelDeletionTests.swift
index 08f05ac3..5ce40d4c 100644
--- a/Tests/FluidDictationIntegrationTests/VoiceModelDeletionTests.swift
+++ b/Tests/FluidDictationIntegrationTests/VoiceModelDeletionTests.swift
@@ -91,9 +91,9 @@ final class VoiceModelDeletionTests: XCTestCase {
try await body(XCTUnwrap(ready, "SwiftUI must install the real ContentView state"))
}
- private func replayPrimaryTap(_ manager: GlobalHotkeyManager) throws {
+ private func replayPrimaryTap(_ manager: GlobalHotkeyManager, keyCode: CGKeyCode = 2) throws {
for type in [CGEventType.keyDown, .keyUp] {
- let event = try XCTUnwrap(CGEvent(keyboardEventSource: nil, virtualKey: 2, keyDown: type == .keyDown))
+ let event = try XCTUnwrap(CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: type == .keyDown))
event.type = type
event.flags = [.maskControl, .maskCommand]
_ = manager.handleKeyEvent(type: type, event: event)
@@ -110,17 +110,27 @@ final class VoiceModelDeletionTests: XCTestCase {
asrService: asr,
primaryShortcuts: [HotkeyShortcut(keyCode: 2, modifierFlags: [.control, .command])],
promptModeShortcut: HotkeyShortcut(keyCode: 60, modifierFlags: []),
- commandModeShortcut: nil,
- rewriteModeShortcut: HotkeyShortcut(keyCode: 58, modifierFlags: []),
+ commandModeShortcut: HotkeyShortcut(keyCode: 14, modifierFlags: [.control, .command]),
+ rewriteModeShortcut: HotkeyShortcut(keyCode: 15, modifierFlags: [.control, .command]),
promptModeShortcutEnabled: false,
- commandModeShortcutEnabled: false,
- rewriteModeShortcutEnabled: false,
+ commandModeShortcutEnabled: true,
+ rewriteModeShortcutEnabled: true,
dictationModeCallback: {
onSwitch()
await view.beginDictationModeSwitchForTesting()?.value
},
stopAndProcessCallback: { _ in onStop() },
+ commandModeCallback: {
+ onSwitch()
+ await view.beginCommandModeSwitchForTesting()
+ },
+ rewriteModeCallback: {
+ onSwitch()
+ await view.beginRewriteModeSwitchForTesting()
+ },
isDictateRecordingProvider: { view.dictationModeSwitchStateForTesting.mode == "dictate" },
+ isCommandRecordingProvider: { view.dictationModeSwitchStateForTesting.mode == "command" },
+ isRewriteRecordingProvider: { view.dictationModeSwitchStateForTesting.mode == "edit" },
isSessionLockedProvider: { false }
)
manager.setHotkeyMode(.toggle)
@@ -192,6 +202,84 @@ final class VoiceModelDeletionTests: XCTestCase {
asr.releaseExclusiveActivity(lease)
}
+ func testDictationEditAndCommandSwitchesKeepTheSameCapture() async throws {
+ let asr = ASRService()
+ let provider = DeletionFixtureProvider()
+ asr.modelProvidersForTesting[SettingsStore.shared.selectedSpeechModel] = provider
+ asr.isAsrReady = true
+ asr.finalText = "retained transcript"
+ let samples: [Float] = [0.12, -0.24, 0.31]
+ asr.configureAudioRouteRecoveryForTesting(controller: self.makeNoHardwareController(for: asr), devices: [], initialSamples: samples)
+ let lease = try asr.acquireExclusiveActivity(.dictation)
+ asr.configureDictationModeSwitchForTesting(ownedLease: lease)
+ try await self.withDictationModeSwitchFixture(asr: asr) { view in
+ let generation = view.dictationModeSwitchStateForTesting.generation
+ for target in ["dictate", "command", "edit", "command", "dictate", "edit", "edit"] {
+ asr.errorTitle = "Dictation Unavailable"
+ asr.errorMessage = "Wait for the active dictation to finish."
+ asr.showError = true
+ switch target {
+ case "command": await view.beginCommandModeSwitchForTesting()
+ case "edit": await view.beginRewriteModeSwitchForTesting()
+ default: XCTAssertNil(view.beginDictationModeSwitchForTesting())
+ }
+ XCTAssertEqual(view.dictationModeSwitchStateForTesting.mode, target)
+ XCTAssertEqual(NotchContentState.shared.mode, target == "dictate" ? .dictation : target == "command" ? .command : .edit)
+ XCTAssertEqual(view.dictationModeSwitchStateForTesting.generation, generation)
+ XCTAssertTrue(asr.isRunning)
+ XCTAssertTrue(asr.canSwitchOwnedDictationCaptureMode)
+ XCTAssertFalse(asr.showError)
+ XCTAssertEqual(asr.activeExclusiveActivity, .dictation)
+ XCTAssertEqual(asr.audioRouteRecoveryStateForTesting.samples, samples)
+ XCTAssertTrue(asr.audioRouteRecoveryStateForTesting.acceptingPCM)
+ XCTAssertEqual(asr.finalText, "retained transcript")
+ XCTAssertTrue((asr.fileTranscriptionProvider as? DeletionFixtureProvider) === provider)
+ XCTAssertEqual(provider.prepareCalls + provider.clearCalls, 0)
+ }
+ asr.errorTitle = "Different error"
+ asr.errorMessage = "Keep this error"
+ asr.showError = true
+ await view.beginCommandModeSwitchForTesting()
+ XCTAssertTrue(asr.showError, "Mode switching must preserve unrelated errors")
+ XCTAssertEqual(asr.errorMessage, "Keep this error")
+ }
+ asr.configureDictationModeSwitchForTesting(ownedLease: nil)
+ await asr.finishAudioRouteRecoveryTest()
+ asr.releaseExclusiveActivity(lease)
+ }
+
+ func testCommandAndEditShortcutSwitchThenRapidSecondTapStops() async throws {
+ for (keyCode, mode) in [(CGKeyCode(14), "command"), (CGKeyCode(15), "edit")] {
+ let asr = ASRService()
+ asr.configureAudioRouteRecoveryForTesting(controller: self.makeNoHardwareController(for: asr), devices: [], initialSamples: [0.5])
+ let lease = try asr.acquireExclusiveActivity(.dictation)
+ asr.configureDictationModeSwitchForTesting(ownedLease: lease)
+ try await self.withDictationModeSwitchFixture(asr: asr) { view in
+ XCTAssertNil(view.beginDictationModeSwitchForTesting())
+ var switches = 0
+ var stops = 0
+ let manager = self.makeModeSwitchHotkeyManager(asr: asr, view: view, onSwitch: { switches += 1 }, onStop: {
+ stops += 1
+ asr.isRunning = false
+ asr.releaseExclusiveActivity(lease)
+ })
+ try self.replayPrimaryTap(manager, keyCode: keyCode)
+ try self.replayPrimaryTap(manager, keyCode: keyCode)
+ for _ in 0..<30 {
+ await Task.yield()
+ }
+ XCTAssertEqual(switches, 1, mode)
+ XCTAssertEqual(stops, 1, mode)
+ XCTAssertEqual(view.dictationModeSwitchStateForTesting.mode, mode)
+ XCTAssertFalse(asr.showError, mode)
+ XCTAssertEqual(asr.audioRouteRecoveryStateForTesting.samples, [0.5])
+ }
+ asr.configureDictationModeSwitchForTesting(ownedLease: nil)
+ await asr.finishAudioRouteRecoveryTest()
+ asr.releaseExclusiveActivity(lease)
+ }
+ }
+
func testRapidRewriteToDictateDoubleTapSwitchesThenStopsOnce() async throws {
let asr = ASRService()
asr.configureAudioRouteRecoveryForTesting(controller: self.makeNoHardwareController(for: asr), devices: [], initialSamples: [0.5])
@@ -248,6 +336,8 @@ final class VoiceModelDeletionTests: XCTestCase {
view.blockDictationModeSwitchForTesting(savingCancellation: name == "cancel save", processing: name == "processing")
let before = view.dictationModeSwitchStateForTesting
XCTAssertNil(view.beginDictationModeSwitchForTesting(selection: attempted), name)
+ await view.beginCommandModeSwitchForTesting()
+ await view.beginRewriteModeSwitchForTesting()
XCTAssertEqual(view.dictationModeSwitchStateForTesting.mode, before.mode, name)
XCTAssertEqual(view.dictationModeSwitchStateForTesting.slot, before.slot, name)
XCTAssertEqual(view.dictationModeSwitchStateForTesting.generation, before.generation, name)
diff --git a/Tests/VoiceTrainingAliasFilterTests.swift b/Tests/VoiceTrainingAliasFilterTests.swift
index 89dfca09..67c3c95d 100644
--- a/Tests/VoiceTrainingAliasFilterTests.swift
+++ b/Tests/VoiceTrainingAliasFilterTests.swift
@@ -1,52 +1,35 @@
-import AppKit
+import Foundation
@main
struct VoiceTrainingAliasFilterTests {
@MainActor
static func main() async {
- let ordinary = ["but", "NOW", "okay", "hello", "right now", "let’s do that"]
- let native = await VoiceTrainingAliasFilter.filter(ordinary + ["fluwidvois"])
- FileHandle.standardOutput.write(Data("native accepted=\(native.accepted) rejected=\(native.rejected) available=\(native.lookupAvailable)\n".utf8))
- precondition(native.lookupAvailable, "Native English word lookup must be available for this smoke test")
- precondition(native.accepted == ["fluwidvois"])
- precondition(native.rejected == ordinary)
+ let ordinary = ["NOW", "okay", "hello", "right now", "let’s do that", "fluid voice", "Apple", "May", "résumé", "こんにちは", "but", "you", "is"]
+ let result = await VoiceTrainingAliasFilter.filter(ordinary + ["fluwidvois"])
+ precondition(result.accepted == ordinary + ["fluwidvois"], "Normal words and phrases must remain teachable")
+ precondition(result.rejected.isEmpty)
- // Deterministic policy coverage, independent of the installed dictionary version.
- let aliases = ["now", "right now", "fluid zqxword", "🙂 zqxword", "123", "", String(repeating: "x", count: 161)]
- let filtered = await VoiceTrainingAliasFilter.filter(aliases) { text in
- guard let regex = try? NSRegularExpression(pattern: "zqxword") else { preconditionFailure("Invalid fixture pattern") }
- return .checked(regex.matches(in: text, range: NSRange(text.startIndex..., in: text)).map(\.range))
- }
- precondition(filtered.accepted == ["fluid zqxword", "🙂 zqxword"])
- precondition(filtered.rejected == ["now", "right now", "123", "", String(repeating: "x", count: 161)])
-
- let unavailable = await VoiceTrainingAliasFilter.filter(["now", "zqxword"]) { _ in .unavailable }
- precondition(!unavailable.lookupAvailable && unavailable.accepted.isEmpty)
+ let basic = ["the", "AND", " a ", "an.", "there!", "or"]
+ let guarded = await VoiceTrainingAliasFilter.filter(basic + ["the voice", "and now", "now"])
+ precondition(guarded.accepted == ["the voice", "and now", "now"])
+ precondition(guarded.rejected == basic, "Only standalone basic words need a manual rule")
+ precondition(guarded.rejectionMessage.contains("Replacement found these words:"))
+ precondition(guarded.rejectionMessage.contains("“there”"))
+ precondition(guarded.rejectionMessage.contains("single words"))
+ let phrase = await VoiceTrainingAliasFilter.filter(["a an the", "the and", "there or"])
+ precondition(phrase.accepted == ["a an the", "the and", "there or"], "Multiple basic words form a valid phrase")
- let cancelled = Task { @MainActor in
- await VoiceTrainingAliasFilter.filter(["zqxword"]) { text in
- try? await Task.sleep(for: .milliseconds(30))
- return .checked([NSRange(text.startIndex..., in: text)])
- }
- }
- cancelled.cancel()
- let cancelledResult = await cancelled.value
- precondition(cancelledResult.accepted.isEmpty)
+ let invalid = ["123", "", "🙂", String(repeating: "x", count: 161)]
+ let bounded = await VoiceTrainingAliasFilter.filter(invalid + Array(repeating: "hello", count: 25))
+ precondition(bounded.accepted.count == 16, "Inspect at most twenty input samples")
+ precondition(bounded.rejected == invalid)
- let results = await withTaskGroup(of: Bool.self) { group in
- for _ in 0..<3 {
- group.addTask {
- let result = await VoiceTrainingAliasFilter.filter(["now", "fluwidvois"])
- return result.accepted == ["fluwidvois"]
- }
- }
- var values: [Bool] = []
- for await result in group {
- values.append(result)
- }
- return values
+ let task = Task { @MainActor in
+ await VoiceTrainingAliasFilter.filter(["hello"])
}
- precondition(results.count == 3 && results.allSatisfy { $0 })
- print("PASS: native vocabulary, common phrases, rare aliases, UTF-16 ranges, invalid input, unavailable lookup, cancellation, concurrent requests")
+ task.cancel()
+ let cancelled = await task.value
+ precondition(cancelled.accepted.isEmpty)
+ print("PASS: ordinary words, phrases, names, multilingual terms, basic-word guard, punctuation, invalid input, bounds, cancellation")
}
}
diff --git a/Tests/run_dictionary_training_save_tests.py b/Tests/run_dictionary_training_save_tests.py
index c5c38d3e..df6224ce 100644
--- a/Tests/run_dictionary_training_save_tests.py
+++ b/Tests/run_dictionary_training_save_tests.py
@@ -42,10 +42,8 @@ struct VoiceTrainingAliasFilter {
struct Result {
let accepted: [String]
let rejected: [String] = []
- let lookupAvailable: Bool
- init(accepted: [String], lookupAvailable: Bool = true) {
- self.accepted = accepted; self.lookupAvailable = lookupAvailable
- }
+ var rejectionMessage: String { "Replacement found these words: “the”. These single words are too broad to replace automatically. Add a manual replacement if intended." }
+ init(accepted: [String]) { self.accepted = accepted }
}
static var response: Result?
static func filter(_ values: [String]) async -> Result { response ?? Result(accepted: values) }
@@ -242,18 +240,18 @@ func check(_ condition: @autoclosure () -> Bool, _ message: String) {
passes += 1
}
// Filtering cannot create a text rule when no safe aliases and no ready voice exist.
- for available in [true, false] {
+ do {
let original = baseline()
let view = reset(original)
view.trainingPronunciationEnrollments = []
- VoiceTrainingAliasFilter.response = .init(accepted: [], lookupAvailable: available)
+ VoiceTrainingAliasFilter.response = .init(accepted: [])
await PronunciationDictionaryStore.shared.configure(gate: nil)
await view.addTrainedReplacement()
let calls = await PronunciationDictionaryStore.shared.calls
check(view.writes == 0 && calls == 0, "Rejected aliases must not write text rules or profiles")
check(SettingsStore.shared.customDictionaryEntries == original, "Rejected aliases preserve the dictionary")
- check(view.trainingHasError && view.wizardStep == .recording, "Rejected or unavailable aliases leave a visible retry")
- check(view.trainingStatusMessage.contains(available ? "everyday words" : "Couldn't check"), "Alias failure explains its actual cause")
+ check(view.trainingHasError && view.wizardStep == .recording, "Rejected aliases leave a visible manual fallback")
+ check(view.trainingStatusMessage.contains("manual replacement"), "Alias failure explains its actual cause")
check(!view.trainingVariants.isEmpty && !view.isTrainingProcessing && view.trainingSaveID == nil, "Alias failure retains recordings and releases busy state")
passes += 1
}
diff --git a/Vendor/WebRTCAudioProcessing/Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a b/Vendor/WebRTCAudioProcessing/Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a
index 48bd8bbe..4fff0b4e 100644
Binary files a/Vendor/WebRTCAudioProcessing/Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a and b/Vendor/WebRTCAudioProcessing/Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a differ
diff --git a/Vendor/WebRTCAudioProcessing/BridgeSources/fluid_aec3_bridge.cc b/Vendor/WebRTCAudioProcessing/BridgeSources/fluid_aec3_bridge.cc
index f8c8bed8..852e609e 100644
--- a/Vendor/WebRTCAudioProcessing/BridgeSources/fluid_aec3_bridge.cc
+++ b/Vendor/WebRTCAudioProcessing/BridgeSources/fluid_aec3_bridge.cc
@@ -17,7 +17,7 @@ namespace {
constexpr char kUpstreamRevision[] =
"d9bd07ba5f614156021df666ba4052ac73cf7953";
constexpr char kConfigurationID[] =
- "webrtc-aec3-48k-mono-cxx20-no-protobuf-no-log-v1";
+ "webrtc-aec3-48k-mono-speech-protection-v2";
webrtc::scoped_refptr<webrtc::AudioProcessing> BuildAudioProcessing() {
webrtc::AudioProcessing::Config config;
@@ -36,6 +36,19 @@ webrtc::scoped_refptr<webrtc::AudioProcessing> BuildAudioProcessing() {
config.pipeline.multi_channel_capture = false;
webrtc::BuiltinAudioProcessingBuilder builder(config);
+ webrtc::EchoCanceller3Config echo;
+ // Meetings can contain quiet local speech beneath loud speaker playback. The
+ // default call profile waits for a dominant near-end voice and aggressively
+ // masks speech bands in that overlap. Enter near-end protection sooner and
+ // tolerate more estimated residual echo only while that voice is detected.
+ // Keep normal (echo-only) suppression, adaptive timing, and gain unchanged.
+ echo.suppressor.dominant_nearend_detection.enr_threshold = 1.f;
+ echo.suppressor.dominant_nearend_detection.snr_threshold = 10.f;
+ echo.suppressor.dominant_nearend_detection.trigger_threshold = 3;
+ echo.suppressor.nearend_tuning.mask_lf = {4.f, 8.f, .3f};
+ echo.suppressor.nearend_tuning.mask_hf = {4.f, 8.f, .3f};
+ if (!webrtc::EchoCanceller3Config::Validate(&echo)) return nullptr;
+ builder.SetEchoCancellerConfig(echo, std::nullopt);
return builder.Build(webrtc::CreateEnvironment());
}
diff --git a/Vendor/WebRTCAudioProcessing/BridgeSources/phase1_bridge.cc b/Vendor/WebRTCAudioProcessing/BridgeSources/phase1_bridge.cc
index 97b8904d..2a6744fc 100644
--- a/Vendor/WebRTCAudioProcessing/BridgeSources/phase1_bridge.cc
+++ b/Vendor/WebRTCAudioProcessing/BridgeSources/phase1_bridge.cc
@@ -31,7 +31,7 @@ int BasicContractTest() {
if (std::strcmp(fv_aec3_upstream_revision(),
"d9bd07ba5f614156021df666ba4052ac73cf7953") != 0 ||
std::strcmp(fv_aec3_configuration_id(),
- "webrtc-aec3-48k-mono-cxx20-no-protobuf-no-log-v1") != 0) {
+ "webrtc-aec3-48k-mono-speech-protection-v2") != 0) {
return 10;
}
if (fv_aec3_reset(nullptr) != FV_AEC3_INVALID_ARGUMENT) return 11;
@@ -94,10 +94,10 @@ int BasicContractTest() {
return 0;
}
-int SyntheticEchoAndDoubleTalkTest() {
+int SyntheticEchoAndDoubleTalkTest(bool weak_voice = false) {
constexpr int kFrames = 1200;
constexpr int kSamples = kFrames * FV_AEC3_FRAME_SAMPLES;
- constexpr int kEchoDelaySamples = 240;
+ const int kEchoDelaySamples = weak_voice ? 2346 : 240;
std::vector<float> render(kSamples), near_end(kSamples), capture(kSamples),
output(kSamples);
uint32_t render_state = 0x12345678u;
@@ -116,9 +116,19 @@ int SyntheticEchoAndDoubleTalkTest() {
near_state = near_state * 1103515245u + 12345u;
const float near_white =
(static_cast<float>(static_cast<int32_t>(near_state)) /
- 2147483648.0f) * 0.6f;
+ 2147483648.0f) * (weak_voice ? 0.04f : 0.6f);
filtered_near = 0.82f * filtered_near + 0.18f * near_white;
near_end[index] = filtered_near;
+ if (weak_voice) {
+ // Deterministic voiced harmonics plus breath noise, independent of render.
+ // The original strong-noise fixture missed quiet speech being suppressed.
+ const double t = static_cast<double>(index) / 48000;
+ const double phase = 2 * M_PI * (170 * t + 4 * std::sin(2 * M_PI * 1.3 * t));
+ const float envelope = .4f + .6f * std::pow(std::sin(2 * M_PI * 2.1 * t), 2);
+ near_end[index] = envelope *
+ (.008f * (std::sin(phase) + .5f * std::sin(3 * phase) +
+ .25f * std::sin(5 * phase)) + .2f * filtered_near);
+ }
}
const float echo = index >= kEchoDelaySamples
? 0.62f * render[index - kEchoDelaySamples]
@@ -155,7 +165,19 @@ int SyntheticEchoAndDoubleTalkTest() {
const double near_attenuation_db = 20.0 * std::log10(
Rms(output, double_begin, double_end) /
Rms(near_end, double_begin, double_end));
- if (!(echo_reduction_db >= 6.0) || !(near_attenuation_db >= -3.0)) {
+ // Measure the component correlated with the known voice, not the total mix:
+ // residual video echo must never count as preserved microphone speech.
+ double voice_dot = 0.0, voice_power = 0.0;
+ constexpr int kProcessingDelaySamples = 433;
+ for (int index = double_begin + 4800; index < double_end - 4800; ++index) {
+ voice_dot += output[index + kProcessingDelaySamples] * near_end[index];
+ voice_power += near_end[index] * near_end[index];
+ }
+ const double voice_gain_db = 20 * std::log10(std::abs(voice_dot / voice_power));
+ std::cout << "double_talk weak=" << weak_voice << " voice_gain_db=" << voice_gain_db
+ << " echo_reduction_db=" << echo_reduction_db << '\n';
+ if (!(echo_reduction_db >= (weak_voice ? 20.0 : 6.0)) ||
+ !(weak_voice ? voice_gain_db >= -8.0 : near_attenuation_db >= -3.0)) {
fv_aec3_destroy(engine);
return 23;
}
@@ -199,6 +221,8 @@ int main() {
if (const int result = BasicContractTest(); result != 0) return result;
if (const int result = SyntheticEchoAndDoubleTalkTest(); result != 0)
return result;
+ if (const int result = SyntheticEchoAndDoubleTalkTest(true); result != 0)
+ return result;
std::cout << "phase1_bridge_ok revision=" << fv_aec3_upstream_revision()
<< " configuration=" << fv_aec3_configuration_id() << '\n';
return 0;
diff --git a/Vendor/WebRTCAudioProcessing/BridgeSources/rebuild_bridge.sh b/Vendor/WebRTCAudioProcessing/BridgeSources/rebuild_bridge.sh
new file mode 100755
index 00000000..2e6c0400
--- /dev/null
+++ b/Vendor/WebRTCAudioProcessing/BridgeSources/rebuild_bridge.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# Rebuild only FluidVoice's bridge; preserve every pinned upstream archive member.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+selected="${DEVELOPER_DIR:-$(xcode-select -p)}"
+if [[ ! "$selected" == *.app/Contents/Developer ]]; then
+ export DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer
+fi
+library=Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a
+shasum -a 256 -c Metadata/artifact-files.sha256
+python3 - <<'VERIFY'
+import hashlib, pathlib
+for line in pathlib.Path('Metadata/source-files.sha256').read_text().splitlines():
+ expected, path = line.split(' ', 1)
+ if path not in ('BridgeSources/fluid_aec3_bridge.cc', 'BridgeSources/phase1_bridge.cc'):
+ actual = hashlib.sha256(pathlib.Path(path).read_bytes()).hexdigest()
+ if actual != expected:
+ raise SystemExit(f'Pinned source changed: {path}')
+VERIFY
+scratch=$(mktemp -d "${TMPDIR:-/tmp}/fluid-aec-bridge.XXXXXX")
+trap 'rm -rf "$scratch"' EXIT
+for arch in arm64 x86_64; do
+ xcrun lipo "$library" -thin "$arch" -output "$scratch/upstream-$arch.a"
+ xcrun ar -d "$scratch/upstream-$arch.a" fluid_aec3_bridge.o
+ mkdir -p "$scratch/$arch"
+ xcrun clang++ -arch "$arch" -O2 -std=c++20 -fno-exceptions -fno-rtti \
+ -fvisibility=hidden -Wno-nullability-completeness -mmacosx-version-min=15.0 \
+ -DWEBRTC_POSIX -DWEBRTC_MAC -DWEBRTC_LIBRARY_IMPL -DNDEBUG \
+ -I BridgeSources -I UpstreamSources/src \
+ -I UpstreamSources/src/third_party/abseil-cpp -I "UpstreamSources/generated/$arch" \
+ -c BridgeSources/fluid_aec3_bridge.cc -o "$scratch/$arch/fluid_aec3_bridge.o"
+ xcrun libtool -static -o "$scratch/bridge-$arch.a" \
+ "$scratch/upstream-$arch.a" "$scratch/$arch/fluid_aec3_bridge.o"
+done
+xcrun lipo -create "$scratch/bridge-arm64.a" "$scratch/bridge-x86_64.a" -output "$scratch/universal.a"
+# Test the actual replacement archive before installing it in the package.
+xcrun clang++ -O2 -std=c++20 -mmacosx-version-min=15.0 -I BridgeSources \
+ BridgeSources/phase1_bridge.cc "$scratch/universal.a" -framework Foundation -o "$scratch/probe"
+"$scratch/probe"
+cp "$scratch/universal.a" "$library"
+python3 - <<'PY'
+import hashlib, pathlib
+for name in ('source-files.sha256', 'artifact-files.sha256'):
+ manifest = pathlib.Path('Metadata') / name
+ lines = []
+ for line in manifest.read_text().splitlines():
+ old, path = line.split(' ', 1)
+ # Only these bridge inputs and its archive changed; never bless upstream drift.
+ if path in ('BridgeSources/fluid_aec3_bridge.cc', 'BridgeSources/phase1_bridge.cc',
+ 'Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a'):
+ old = hashlib.sha256(pathlib.Path(path).read_bytes()).hexdigest()
+ lines.append(f'{old} {path}')
+ manifest.write_text('\n'.join(lines) + '\n')
+PY
+shasum -a 256 -c Metadata/source-files.sha256 > "$scratch/source-check.log"
+shasum -a 256 -c Metadata/artifact-files.sha256
+xcrun lipo -archs "$library"
diff --git a/Vendor/WebRTCAudioProcessing/Metadata/artifact-files.sha256 b/Vendor/WebRTCAudioProcessing/Metadata/artifact-files.sha256
index 098e8174..3745e48e 100644
--- a/Vendor/WebRTCAudioProcessing/Metadata/artifact-files.sha256
+++ b/Vendor/WebRTCAudioProcessing/Metadata/artifact-files.sha256
@@ -1,3 +1,3 @@
bcbe80ca2b0f0201725bc288efe3d54bb2bdc9ec4af8a6726dc31095093a1c6a Artifacts/FluidAEC3.xcframework/Info.plist
96c88f253d485533a1325c693f1a86f8199285fd25465907af2b5c16a3457ed2 Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/Headers/FluidAEC3Binary.h
-7c06ab158e3c48f109e4e9bd5cf6b643049f61fc04177d13026a3b878de9654f Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a
+295a3b4280f0c020bcf14cf7755596664c27e590cb547315dd282cc7e976b851 Artifacts/FluidAEC3.xcframework/macos-arm64_x86_64/libFluidAEC3-system-exact.a
diff --git a/Vendor/WebRTCAudioProcessing/Metadata/manifest.json b/Vendor/WebRTCAudioProcessing/Metadata/manifest.json
index bfc385eb..e3518d73 100644
--- a/Vendor/WebRTCAudioProcessing/Metadata/manifest.json
+++ b/Vendor/WebRTCAudioProcessing/Metadata/manifest.json
@@ -16,7 +16,7 @@
"third_party/protobuf": "4c56f21bb6d7e9cb8856ba0f6ca26f7e95e546f5"
},
"bridge": {
- "configurationID": "webrtc-aec3-48k-mono-cxx20-no-protobuf-no-log-v1",
+ "configurationID": "webrtc-aec3-48k-mono-speech-protection-v2",
"sampleRateHz": 48000,
"frameSamples": 480,
"languageStandard": "c++20",
@@ -28,7 +28,31 @@
"usesAppleSystemLibcxx": true,
"aecDumpFactory": "null",
"protobufRuntimeLinked": false,
- "perfettoRuntimeLinked": false
+ "perfettoRuntimeLinked": false,
+ "speechProtection": {
+ "dominantNearendEchoToSpeechThreshold": 1,
+ "dominantNearendSignalToNoiseThreshold": 10,
+ "dominantNearendTriggerBlocks": 3,
+ "nearendMaskingLowAndHigh": [
+ 4,
+ 8,
+ 0.3
+ ],
+ "normalSuppressionUnchanged": true
+ },
+ "bridgeOnlyRebuild": {
+ "script": "BridgeSources/rebuild_bridge.sh",
+ "compiler": "Apple clang via xcrun",
+ "flags": [
+ "-O2",
+ "-std=c++20",
+ "-fno-exceptions",
+ "-fno-rtti",
+ "-fvisibility=hidden",
+ "-mmacosx-version-min=15.0"
+ ],
+ "preservesPinnedUpstreamArchiveMembers": true
+ }
},
"gnArgs": {
"common": {
diff --git a/Vendor/WebRTCAudioProcessing/Metadata/source-files.sha256 b/Vendor/WebRTCAudioProcessing/Metadata/source-files.sha256
index edca47b8..039de217 100644
--- a/Vendor/WebRTCAudioProcessing/Metadata/source-files.sha256
+++ b/Vendor/WebRTCAudioProcessing/Metadata/source-files.sha256
@@ -527,8 +527,8 @@ ba857529494ceb20a89d01fe7eb4f846d2fbf2e8ff42bb389413d3a59371db92 UpstreamSource
c3609fd6c86efd2f8fb10c2bab5a1b5faeba4ed571818a8e73369be7ab14cfee UpstreamSources/src/third_party/rnnoise/src/rnn_vad_weights.cc
3a2c9115381b1d0363ce73927cee9ce99dccff3405857e46e710bbd528b86f85 UpstreamSources/src/third_party/rnnoise/src/rnn_vad_weights.h
d13c6ff58f039d670b95234966dfd2abc35be2812507bb458097ca11c2dc7c35 BridgeSources/BUILD.gn
-55dcc69fc169205eac4e3a4c665f089808110020bc12aef6f7dd522743533502 BridgeSources/fluid_aec3_bridge.cc
+5ff972adbba3caeb3d306dcbcdb0cae58edf9a6b770846c004e57c0a104c734e BridgeSources/fluid_aec3_bridge.cc
96c88f253d485533a1325c693f1a86f8199285fd25465907af2b5c16a3457ed2 BridgeSources/fluid_aec3_bridge.h
bc461ce317f7137a3e7a6c8ffd51a29d3db814784b8131af040b62e675744e5b BridgeSources/module.modulemap
-40ecfc5989a862135537a8cb3c441d25bd10bfa070420f5f619adaa4e0bdb2da BridgeSources/phase1_bridge.cc
+9c6e23f5f9a5f4284e250319413dd4098da03c5d0e0bb5eaaf2cd1c31ab4efb7 BridgeSources/phase1_bridge.cc
18e9a754c5ad8357f3cb45424e83079e65dd4af2d3b0a58f9bed32ee6fd0773f BridgeSources/realtime_benchmark.cc
diff --git a/Vendor/WebRTCAudioProcessing/README.md b/Vendor/WebRTCAudioProcessing/README.md
index da155ac2..73be6b81 100644
--- a/Vendor/WebRTCAudioProcessing/README.md
+++ b/Vendor/WebRTCAudioProcessing/README.md
@@ -31,3 +31,21 @@ an exact gclient checkout plus depot_tools. The refresh script rejects any revis
below 10 ms, bounded RSS variation, and no monotonic RSS growth.
Licenses and required attribution for WebRTC and linked third-party source are in `Notices`.
+
+## Meeting speech protection
+
+The v2 bridge keeps the pinned WebRTC revision and default echo-only suppression.
+It enters local-speech protection sooner and relaxes the low/high-band residual-echo
+mask only during detected local speech. It adds no capture delay, gain, noise
+suppression, extra model, or realtime statistics lock. This improves quiet speech
+under loud speaker playback; some residual echo during overlapping speech remains
+possible. It does not guarantee separation of every speaker mixture.
+
+`BridgeSources/rebuild_bridge.sh` recompiles the bridge for both architectures using the
+vendored headers and system libc++, replacing only `fluid_aec3_bridge.o` in each
+verified archive slice. It preserves all pinned upstream objects, runs the contract,
+strong-voice and weak-voice overlap probes, and updates the two integrity manifests.
+It refuses modified upstream inputs. This is a bridge-only rebuild, not a new GN
+build of upstream WebRTC; the manifest records that distinction. The weak-voice
+regression correlates output with an independent known voice so leaked playback
+cannot count as preserved speech. XCTest repeats that check through the app wrapper.