Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

feat(meeting): harden audio capture and processing

argus-vault02bb636c173b
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj
index f3f57f2e..e005dc93 100644
--- a/Fluid.xcodeproj/project.pbxproj
+++ b/Fluid.xcodeproj/project.pbxproj
@@ -26,6 +26,7 @@
2E9B0000000000000000000B /* AppSearchServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2E9B0000000000000000000A /* AppSearchServiceTests.swift */; };
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; };
D1A600000000000000000202 /* SpeakerTurnMergingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A600000000000000000201 /* SpeakerTurnMergingTests.swift */; };
+ D1A700000000000000000002 /* MeetingExistingAudioReplayTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A700000000000000000001 /* MeetingExistingAudioReplayTests.swift */; };
F1BD00000000000000000002 /* StripThinkingTagsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F1BD00000000000000000001 /* StripThinkingTagsTests.swift */; };
D1A600000000000000000204 /* MeetingWindowSelectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */; };
D1A600000000000000000206 /* MeetingRecoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A600000000000000000205 /* MeetingRecoveryTests.swift */; };
@@ -99,6 +100,7 @@
2E9B0000000000000000000A /* AppSearchServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppSearchServiceTests.swift; sourceTree = "<group>"; };
A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = "<group>"; };
D1A600000000000000000201 /* SpeakerTurnMergingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpeakerTurnMergingTests.swift; sourceTree = "<group>"; };
+ D1A700000000000000000001 /* MeetingExistingAudioReplayTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingExistingAudioReplayTests.swift; sourceTree = "<group>"; };
F1BD00000000000000000001 /* StripThinkingTagsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StripThinkingTagsTests.swift; sourceTree = "<group>"; };
D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingWindowSelectorTests.swift; sourceTree = "<group>"; };
D1A600000000000000000205 /* MeetingRecoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingRecoveryTests.swift; sourceTree = "<group>"; };
@@ -220,6 +222,7 @@
2E9B0000000000000000000A /* AppSearchServiceTests.swift */,
A62300000000000000000001 /* AudioBufferConverterTests.swift */,
D1A600000000000000000201 /* SpeakerTurnMergingTests.swift */,
+ D1A700000000000000000001 /* MeetingExistingAudioReplayTests.swift */,
F1BD00000000000000000001 /* StripThinkingTagsTests.swift */,
D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */,
D1A600000000000000000205 /* MeetingRecoveryTests.swift */,
@@ -426,6 +429,7 @@
2E9B0000000000000000000B /* AppSearchServiceTests.swift in Sources */,
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */,
D1A600000000000000000202 /* SpeakerTurnMergingTests.swift in Sources */,
+ D1A700000000000000000002 /* MeetingExistingAudioReplayTests.swift in Sources */,
F1BD00000000000000000002 /* StripThinkingTagsTests.swift in Sources */,
D1A600000000000000000204 /* MeetingWindowSelectorTests.swift in Sources */,
D1A600000000000000000206 /* MeetingRecoveryTests.swift in Sources */,
@@ -794,8 +798,8 @@
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/altic-dev/FluidAudio.git";
requirement = {
- branch = main;
- kind = branch;
+ kind = revision;
+ revision = 3fd63887eef1dc25edea8263ce4b44aa854d898b;
};
};
7C3697872ED70F9C005874CE /* XCRemoteSwiftPackageReference "DynamicNotchKit" */ = {
diff --git a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
index 86f7739e..5a9aa57b 100644
--- a/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
+++ b/Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -33,7 +33,6 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/altic-dev/FluidAudio.git",
"state" : {
- "branch" : "main",
"revision" : "3fd63887eef1dc25edea8263ce4b44aa854d898b"
}
},
diff --git a/Package.resolved b/Package.resolved
index 5ccb7bb2..d267136b 100644
--- a/Package.resolved
+++ b/Package.resolved
@@ -32,7 +32,6 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/altic-dev/FluidAudio.git",
"state" : {
- "branch" : "main",
"revision" : "3fd63887eef1dc25edea8263ce4b44aa854d898b"
}
},
diff --git a/Package.swift b/Package.swift
index 7d0e9f85..86f85087 100644
--- a/Package.swift
+++ b/Package.swift
@@ -10,7 +10,12 @@ let package = Package(
],
dependencies: [
.package(url: "https://github.com/mxcl/AppUpdater.git", from: "1.0.0"),
- .package(url: "https://github.com/altic-dev/FluidAudio.git", branch: "main"),
+ // Diarization evaluation and checkpoints depend on exact clustering behavior.
+ // Keep this immutable and update it only with a reviewed before/after evaluation.
+ .package(
+ url: "https://github.com/altic-dev/FluidAudio.git",
+ revision: "3fd63887eef1dc25edea8263ce4b44aa854d898b"
+ ),
.package(url: "https://github.com/mxcl/PromiseKit", from: "6.0.0"),
.package(url: "https://github.com/altic-dev/DynamicNotchKit.git", branch: "main"),
.package(url: "https://github.com/altic-dev/transcribe-cpp-swift.git", exact: "0.1.2"),
diff --git a/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c b/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c
index 4d7ac7a6..4d1c4e8d 100644
--- a/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c
+++ b/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c
@@ -3,6 +3,7 @@
#include <dispatch/dispatch.h>
#include <limits.h>
#include <mach/mach_time.h>
+#include <pthread.h>
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
@@ -11,7 +12,7 @@
// realtime producer strictly allocation-free.
#define FV_RING_CAPACITY 64u
#define FV_MAX_FRAMES_PER_PACKET 8192u
-#define FV_AUDIO_TOPOLOGY_TRACE_CAPACITY 2048u
+#define FV_AUDIO_TOPOLOGY_TRACE_CAPACITY 8192u
typedef struct {
float samples[FV_MAX_FRAMES_PER_PACKET];
@@ -52,6 +53,7 @@ typedef struct {
_Atomic uint32_t queueRole;
_Atomic uint32_t phase;
_Atomic uint32_t transport;
+ _Atomic uint32_t isMainThread;
_Atomic int32_t status;
} FVAudioTopologyTraceSlot;
@@ -119,6 +121,7 @@ uint64_t fv_audio_topology_trace_record(
atomic_store_explicit(&slot->queueRole, queueRole, memory_order_relaxed);
atomic_store_explicit(&slot->phase, phase, memory_order_relaxed);
atomic_store_explicit(&slot->transport, transport, memory_order_relaxed);
+ atomic_store_explicit(&slot->isMainThread, pthread_main_np() != 0, memory_order_relaxed);
atomic_store_explicit(&slot->status, status, memory_order_relaxed);
atomic_store_explicit(
&slot->publishedSequence,
@@ -178,6 +181,7 @@ uint32_t fv_audio_topology_trace_snapshot(
.queueRole = atomic_load_explicit(&slot->queueRole, memory_order_relaxed),
.phase = atomic_load_explicit(&slot->phase, memory_order_relaxed),
.transport = atomic_load_explicit(&slot->transport, memory_order_relaxed),
+ .isMainThread = atomic_load_explicit(&slot->isMainThread, memory_order_relaxed),
.status = atomic_load_explicit(&slot->status, memory_order_relaxed),
};
atomic_thread_fence(memory_order_acquire);
@@ -248,6 +252,7 @@ void fv_audio_topology_trace_reset(void) {
atomic_store_explicit(&slot->queueRole, 0, memory_order_relaxed);
atomic_store_explicit(&slot->phase, 0, memory_order_relaxed);
atomic_store_explicit(&slot->transport, 0, memory_order_relaxed);
+ atomic_store_explicit(&slot->isMainThread, 0, memory_order_relaxed);
atomic_store_explicit(&slot->status, 0, memory_order_relaxed);
}
}
diff --git a/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h b/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h
index 066d0b5c..911faf1b 100644
--- a/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h
+++ b/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h
@@ -83,6 +83,7 @@ typedef struct {
uint32_t queueRole;
uint32_t phase;
uint32_t transport;
+ uint32_t isMainThread;
int32_t status;
} FVAudioTopologyTraceEvent;
diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift
index 32ca4fb0..b100569b 100644
--- a/Sources/Fluid/ContentView.swift
+++ b/Sources/Fluid/ContentView.swift
@@ -1072,7 +1072,7 @@ struct ContentView: View {
case .settings:
self.openSettings(.general)
case .meetingTranscription:
- self.navigateToApp(.meetingTools)
+ self.navigateToApp(.meetingTranscription)
case .preferences:
self.openSettings(.general)
}
@@ -1087,7 +1087,7 @@ struct ContentView: View {
case .history:
self.navigateToApp(.history)
case .meetingTranscription:
- self.navigateToApp(.meetingTools)
+ self.navigateToApp(.meetingTranscription)
}
}
@@ -1751,8 +1751,10 @@ struct ContentView: View {
shortcutRecordingMessage: self.$shortcutRecordingMessage,
revealTarget: self.$appSearchRevealTarget
))
- case .meetingTools:
- return AnyView(self.meetingToolsView)
+ case .fileTranscription:
+ return AnyView(self.fileTranscriptionView)
+ case .meetingTranscription:
+ return AnyView(self.meetingTranscriptionView)
case .customDictionary:
return AnyView(CustomDictionaryView(revealTarget: self.$appSearchRevealTarget))
case .stats:
diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift
index 4e42f73d..b2ccd615 100644
--- a/Sources/Fluid/Services/ASRService.swift
+++ b/Sources/Fluid/Services/ASRService.swift
@@ -396,6 +396,54 @@ enum ASRActivityError: LocalizedError {
}
}
+nonisolated enum ASRHardwareListenerEventDisposition: Equatable {
+ case ignore
+ case latchUntilInstalled
+ case handle
+}
+
+nonisolated enum ASRHardwareListenerPolicy {
+ static func disposition(
+ isTerminating: Bool,
+ lifecycleMatches: Bool,
+ isInstalled: Bool,
+ pendingIdentityMatches: Bool
+ ) -> ASRHardwareListenerEventDisposition {
+ guard !isTerminating, lifecycleMatches else { return .ignore }
+ if isInstalled { return .handle }
+ return pendingIdentityMatches ? .latchUntilInstalled : .ignore
+ }
+
+ static func selectedDeviceQueryCanApply(
+ isTerminating: Bool,
+ capturedLifecycle: UInt64,
+ currentLifecycle: UInt64,
+ capturedEpoch: UInt64,
+ currentEpoch: UInt64,
+ capturedDeviceID: AudioObjectID,
+ currentDeviceID: AudioObjectID?,
+ listenerIsInstalled: Bool
+ ) -> Bool {
+ !isTerminating
+ && capturedLifecycle == currentLifecycle
+ && capturedEpoch == currentEpoch
+ && currentDeviceID == capturedDeviceID
+ && listenerIsInstalled
+ }
+
+ static func deviceListQueryCanApply(
+ isTerminating: Bool,
+ capturedLifecycle: UInt64,
+ currentLifecycle: UInt64,
+ capturedQueryGeneration: UInt64,
+ currentQueryGeneration: UInt64
+ ) -> Bool {
+ !isTerminating
+ && capturedLifecycle == currentLifecycle
+ && capturedQueryGeneration == currentQueryGeneration
+ }
+}
+
// swiftlint:disable file_length type_body_length
/// A comprehensive speech recognition service that handles real-time audio transcription.
///
@@ -564,6 +612,15 @@ final class ASRService: ObservableObject {
private var activeActivityLease: ASRActivityLease?
private var dictationActivityLease: ASRActivityLease?
private var providerResetPending = false
+ private var meetingAudioHandoffGeneration: UInt64 = 0
+ private var meetingTopologyObserverSuspension: UInt64?
+ private var meetingAudioHandbackLease: ASRActivityLease?
+ private var meetingAudioHandbackTask: Task<Void, Never>?
+ private var meetingAudioHandbackTaskID: UUID?
+ private var meetingHandoffShouldRestorePreparedCapture = false
+ private var deferredMeetingRouteRecoveryReason: String?
+ private var deferredMeetingRouteRecoveryRequiresPrewarm = false
+ private var deferredMeetingRouteRecoveryReconcilesInput = false
var isRunningOrStarting: Bool {
self.isRunning || self.isStarting
}
@@ -580,6 +637,10 @@ final class ASRService: ObservableObject {
func releaseExclusiveActivity(_ lease: ASRActivityLease) {
guard self.activeActivityLease == lease else { return }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .asrActivityLease, queueRole: .mainControl, phase: .activityLease)
+ defer { AudioTopologyDiagnostics.record(.phaseEnd, owner: .asrActivityLease, queueRole: .mainControl, phase: .activityLease) }
+ #endif
self.activeActivityLease = nil
self.activeExclusiveActivity = nil
@@ -588,6 +649,152 @@ final class ASRService: ObservableObject {
self.resetTranscriptionProvider()
}
+ func prepareMeetingAudioHandoff(_ lease: ASRActivityLease) async throws {
+ guard self.activeActivityLease == lease, lease.activity == .meeting,
+ self.isRunning == false, self.isStarting == false,
+ self.isTerminating == false
+ else { throw ASRActivityError.activityInProgress(self.activeExclusiveActivity ?? .meeting) }
+
+ self.meetingAudioHandoffGeneration &+= 1
+ let generation = self.meetingAudioHandoffGeneration
+ self.meetingHandoffShouldRestorePreparedCapture = self.hasPreparedAudioCapture
+ self.audioEngineStandbyTask?.cancel()
+ self.audioEngineStandbyTask = nil
+
+ let observerSuspension = await AppServices.shared.audioObserver.suspendTopologyReconciliation()
+ guard self.activeActivityLease == lease,
+ self.meetingAudioHandoffGeneration == generation
+ else {
+ await AppServices.shared.audioObserver.resumeTopologyReconciliation(observerSuspension)
+ throw CancellationError()
+ }
+ self.meetingTopologyObserverSuspension = observerSuspension
+
+ // Preview owns DirectCoreAudio without an exclusive activity lease. Stopping it
+ // advances its operation generation, clears requested/active state and monitoring,
+ // and drains its direct-capture stop before the meeting backend may materialize.
+ await self.stopMicrophonePreview(retainPreparedCapture: false)
+ await self.cancelAudioRouteRecoveryAndWait()
+ await self.directAudioLifecycleController.invalidate(reason: "meeting_audio_handoff")
+ self.activeAudioCaptureBackend = .none
+ await self.retireAudioEngineAndWait(reason: "meeting_audio_handoff")
+
+ guard self.activeActivityLease == lease,
+ self.meetingAudioHandoffGeneration == generation,
+ self.isRunning == false, self.isStarting == false,
+ self.isTerminating == false,
+ self.hasPreparedAudioCapture == false
+ else { throw CancellationError() }
+ try Task.checkCancellation()
+ }
+
+ func completeMeetingAudioHandback(_ lease: ASRActivityLease) async {
+ if self.meetingAudioHandbackLease == lease,
+ let handbackTask = self.meetingAudioHandbackTask
+ {
+ await handbackTask.value
+ return
+ }
+ guard self.activeActivityLease == lease, lease.activity == .meeting else { return }
+
+ let taskID = UUID()
+ let handbackTask = Task { @MainActor [weak self] () -> Void in
+ guard let self else { return }
+ await self.performMeetingAudioHandback(lease)
+ }
+ self.meetingAudioHandbackLease = lease
+ self.meetingAudioHandbackTask = handbackTask
+ self.meetingAudioHandbackTaskID = taskID
+ await handbackTask.value
+ if self.meetingAudioHandbackTaskID == taskID {
+ self.meetingAudioHandbackLease = nil
+ self.meetingAudioHandbackTask = nil
+ self.meetingAudioHandbackTaskID = nil
+ }
+ }
+
+ private func performMeetingAudioHandback(_ lease: ASRActivityLease) async {
+ guard self.activeActivityLease == lease, lease.activity == .meeting else { return }
+ self.meetingAudioHandoffGeneration &+= 1
+ let generation = self.meetingAudioHandoffGeneration
+
+ await self.cancelAudioRouteRecoveryAndWait()
+ await self.directAudioLifecycleController.invalidate(reason: "meeting_audio_handback")
+ self.activeAudioCaptureBackend = .none
+ await self.retireAudioEngineAndWait(reason: "meeting_audio_handback")
+ let snapshot = await AudioTopologyListenerExecution.perform {
+ (
+ AudioDevice.listInputDevicesRefreshingLiveness(diagnosticsOwner: .asrService),
+ AudioDevice.getDefaultInputDevice()?.uid
+ )
+ }
+
+ guard self.activeActivityLease == lease,
+ self.meetingAudioHandoffGeneration == generation
+ else { return }
+
+ guard self.isTerminating == false else {
+ if let observerSuspension = self.meetingTopologyObserverSuspension {
+ await AppServices.shared.audioObserver.resumeTopologyReconciliation(observerSuspension)
+ guard self.activeActivityLease == lease,
+ self.meetingAudioHandoffGeneration == generation
+ else { return }
+ if self.meetingTopologyObserverSuspension == observerSuspension {
+ self.meetingTopologyObserverSuspension = nil
+ }
+ }
+ guard self.activeActivityLease == lease,
+ self.meetingAudioHandoffGeneration == generation
+ else { return }
+ self.meetingHandoffShouldRestorePreparedCapture = false
+ self.deferredMeetingRouteRecoveryReason = nil
+ self.deferredMeetingRouteRecoveryRequiresPrewarm = false
+ self.deferredMeetingRouteRecoveryReconcilesInput = false
+ self.releaseExclusiveActivity(lease)
+ return
+ }
+
+ AppServices.shared.microphonePreferenceCoordinator.reconcileMicrophoneSelection(
+ availableInputs: snapshot.0,
+ defaultInputUID: snapshot.1
+ )
+ self.cacheCurrentDeviceList(snapshot.0)
+
+ if let observerSuspension = self.meetingTopologyObserverSuspension {
+ await AppServices.shared.audioObserver.resumeTopologyReconciliation(observerSuspension)
+ guard self.activeActivityLease == lease,
+ self.meetingAudioHandoffGeneration == generation
+ else { return }
+ if self.meetingTopologyObserverSuspension == observerSuspension {
+ self.meetingTopologyObserverSuspension = nil
+ }
+ }
+
+ // Observer resume can deliver coalesced topology events while this meeting
+ // lease is still held. Read and clear the deferred aggregate only after that
+ // drain so no handback event is stranded behind the lease release.
+ guard self.activeActivityLease == lease,
+ self.meetingAudioHandoffGeneration == generation
+ else { return }
+ let deferredReason = self.deferredMeetingRouteRecoveryReason
+ let shouldPrewarm = self.meetingHandoffShouldRestorePreparedCapture ||
+ self.deferredMeetingRouteRecoveryRequiresPrewarm
+ let shouldReconcile = self.deferredMeetingRouteRecoveryReconcilesInput
+ self.meetingHandoffShouldRestorePreparedCapture = false
+ self.deferredMeetingRouteRecoveryReason = nil
+ self.deferredMeetingRouteRecoveryRequiresPrewarm = false
+ self.deferredMeetingRouteRecoveryReconcilesInput = false
+ self.releaseExclusiveActivity(lease)
+
+ if shouldPrewarm || deferredReason != nil {
+ self.scheduleAudioRouteRecovery(
+ reason: deferredReason ?? "meeting audio handback",
+ requiresIdlePrewarm: shouldPrewarm,
+ reconcilesInputSelection: shouldReconcile
+ )
+ }
+ }
+
private func releaseDictationActivityIfNeeded() {
guard let lease = self.dictationActivityLease else { return }
self.dictationActivityLease = nil
@@ -1304,6 +1511,12 @@ final class ASRService: ObservableObject {
DebugLogger.shared.debug("Audio capture prewarm skipped - app is terminating", source: "ASRService")
return
}
+ guard self.activeExclusiveActivity != .meeting else {
+ DebugLogger.shared.debug("Audio capture prewarm deferred - meeting owns audio topology", source: "ASRService")
+ self.meetingHandoffShouldRestorePreparedCapture = true
+ return
+ }
+ let handoffGeneration = self.meetingAudioHandoffGeneration
guard self.micStatus == .authorized else {
DebugLogger.shared.debug("Audio engine prewarm skipped - mic not authorized", source: "ASRService")
return
@@ -1343,6 +1556,8 @@ final class ASRService: ObservableObject {
// completed, then recheck state because this await yields the main actor.
await self.audioEngineRetirementDrain.waitForScheduledReleases()
guard self.isTerminating == false,
+ self.activeExclusiveActivity != .meeting,
+ self.meetingAudioHandoffGeneration == handoffGeneration,
self.isRunning == false,
self.isStarting == false || allowDuringRouteRecovery,
self.hasPreparedAudioCapture == false
@@ -1357,6 +1572,14 @@ final class ASRService: ObservableObject {
let startedAt = Date().timeIntervalSince1970
do {
_ = try await self.prepareDirectAudioInput(reason: reason)
+ guard self.activeExclusiveActivity != .meeting,
+ self.meetingAudioHandoffGeneration == handoffGeneration,
+ self.isTerminating == false
+ else {
+ await self.directAudioLifecycleController.invalidate(reason: "stale_prewarm_after_meeting_handoff")
+ self.activeAudioCaptureBackend = .none
+ return
+ }
self.benchmarkLog("direct_audio_prewarm reason=\(reason) elapsedMs=\(self.elapsedMilliseconds(since: startedAt))")
} catch {
DebugLogger.shared.warning(
@@ -2070,6 +2293,7 @@ final class ASRService: ObservableObject {
guard self.micStatus == .authorized,
self.isRunning == false,
self.isStarting == false,
+ self.activeExclusiveActivity != .meeting,
self.isTerminating == false
else { return }
@@ -2087,6 +2311,7 @@ final class ASRService: ObservableObject {
self.isMicrophonePreviewRequested,
self.isRunning == false,
self.isStarting == false,
+ self.activeExclusiveActivity != .meeting,
self.isTerminating == false,
Task.isCancelled == false
else {
@@ -2105,6 +2330,7 @@ final class ASRService: ObservableObject {
self.isMicrophonePreviewRequested,
self.isRunning == false,
self.isStarting == false,
+ self.activeExclusiveActivity != .meeting,
self.isTerminating == false,
Task.isCancelled == false
else {
@@ -2130,6 +2356,7 @@ final class ASRService: ObservableObject {
guard operationGeneration == self.microphonePreviewOperationGeneration,
self.isRunning == false,
self.isStarting == false,
+ self.activeExclusiveActivity != .meeting,
self.isTerminating == false
else { throw CancellationError() }
_ = try await self.directAudioLifecycleController.start(
@@ -2141,6 +2368,7 @@ final class ASRService: ObservableObject {
guard operationGeneration == self.microphonePreviewOperationGeneration,
self.isRunning == false,
self.isStarting == false,
+ self.activeExclusiveActivity != .meeting,
self.isTerminating == false
else { throw CancellationError() }
self.activeAudioCaptureBackend = .directCoreAudio
@@ -4062,6 +4290,15 @@ final class ASRService: ObservableObject {
self.benchmarkLog("route_recovery_ignored reason=app_terminating event=\(reason)")
return
}
+ if self.activeExclusiveActivity == .meeting {
+ self.deferredMeetingRouteRecoveryReason = reason
+ self.deferredMeetingRouteRecoveryRequiresPrewarm =
+ self.deferredMeetingRouteRecoveryRequiresPrewarm || requiresIdlePrewarm
+ self.deferredMeetingRouteRecoveryReconcilesInput =
+ self.deferredMeetingRouteRecoveryReconcilesInput || reconcilesInputSelection
+ self.benchmarkLog("route_recovery_deferred reason=meeting_owns_topology event=\(reason)")
+ return
+ }
if AudioCaptureIdlePolicy.shouldDeferRouteRecoveryToBluetoothStart(
directCaptureEnabled: SettingsStore.shared.experimentalDirectAudioCaptureEnabled,
isStarting: self.isStarting,
@@ -4636,9 +4873,17 @@ final class ASRService: ObservableObject {
object: nil,
queue: nil
) { [weak self] notification in
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .asrEngineConfiguration, queueRole: .callbackCurrent, phase: .engine)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .asrEngineConfiguration, queueRole: .callbackCurrent, phase: .engine) }
+ #endif
guard let changedEngine = notification.object as? AVAudioEngine else { return }
let changedEngineIdentifier = ObjectIdentifier(changedEngine)
Task { @MainActor [weak self] in
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.mainHopBegin, owner: .asrEngineConfiguration, queueRole: .mainControl, phase: .engine)
+ defer { AudioTopologyDiagnostics.record(.mainHopEnd, owner: .asrEngineConfiguration, queueRole: .mainControl, phase: .engine) }
+ #endif
self?.handleEngineConfigurationChanged(changedEngineIdentifier)
}
}
@@ -4647,10 +4892,24 @@ final class ASRService: ObservableObject {
private var defaultInputListenerInstalled = false
private var defaultInputListenerToken: AudioObjectPropertyListenerBlock?
private var defaultOutputListenerToken: AudioObjectPropertyListenerBlock?
+ private var audioHardwareListenerLifecycleGeneration: UInt64 = 0
+ private var defaultInputListenerPendingGeneration: UInt64?
+ private var defaultOutputListenerPendingGeneration: UInt64?
+ private var defaultInputListenerDirtyGeneration: UInt64?
+ private var defaultOutputListenerDirtyGeneration: UInt64?
private func reestablishAudioHardwareListenersAfterServiceReset() {
// The reset invalidates these registrations in HAL; do not attempt to
// remove the stale tokens before installing replacements.
+ self.audioHardwareListenerLifecycleGeneration &+= 1
+ self.defaultInputListenerPendingGeneration = nil
+ self.defaultOutputListenerPendingGeneration = nil
+ self.deviceListListenerPendingGeneration = nil
+ self.defaultInputListenerDirtyGeneration = nil
+ self.defaultOutputListenerDirtyGeneration = nil
+ self.deviceListListenerDirtyGeneration = nil
+ self.monitoredDeviceListenerPendingEpoch = nil
+ self.monitoredDeviceAvailabilityDirtyEpoch = nil
self.defaultInputListenerInstalled = false
self.defaultInputListenerToken = nil
self.defaultOutputListenerToken = nil
@@ -4660,101 +4919,165 @@ final class ASRService: ObservableObject {
self.monitoredDeviceIsAliveListenerToken = nil
self.registerDefaultDeviceChangeListener()
self.registerDeviceListChangeListener()
- let defaultInputReady = self.defaultInputListenerInstalled
- let defaultOutputReady = self.defaultOutputListenerToken != nil
- let deviceListReady = self.deviceListListenerInstalled
self.benchmarkLog(
- "audio_service_restart listeners_reestablished " +
- "defaultInput=\(defaultInputReady) defaultOutput=\(defaultOutputReady) " +
- "deviceList=\(deviceListReady)"
+ "audio_service_restart listener_reregistration_scheduled generation=" +
+ "\(self.audioHardwareListenerLifecycleGeneration)"
+ )
+ DebugLogger.shared.warning(
+ "Scheduled ASR hardware listener re-registration after Core Audio service reset",
+ source: "ASRService"
)
- let logMessage =
- "ASR hardware listeners after Core Audio service reset: " +
- "defaultInput=\(defaultInputReady), defaultOutput=\(defaultOutputReady), " +
- "deviceList=\(deviceListReady)"
- if defaultInputReady, defaultOutputReady, deviceListReady {
- DebugLogger.shared.warning(
- "Re-registered \(logMessage)",
- source: "ASRService"
- )
- } else {
- DebugLogger.shared.error(
- "Failed to fully re-register \(logMessage)",
- source: "ASRService"
- )
- }
}
private func registerDefaultDeviceChangeListener() {
guard self.defaultInputListenerInstalled == false || self.defaultOutputListenerToken == nil else { return }
- var inputAddress = AudioObjectPropertyAddress(
+ let inputAddress = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultInputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- var outputAddress = AudioObjectPropertyAddress(
+ let outputAddress = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- if self.defaultInputListenerInstalled == false {
+ let generation = self.audioHardwareListenerLifecycleGeneration
+ if self.defaultInputListenerInstalled == false,
+ self.defaultInputListenerPendingGeneration == nil
+ {
let inputToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
// Defer to next runloop pass — CoreAudio may hold an internal lock during
// this callback, and our handler makes synchronous CoreAudio queries that
// would deadlock waiting for the same lock.
- DispatchQueue.main.async { self?.handleDefaultInputChanged() }
+ DispatchQueue.main.async {
+ guard let self,
+ self.audioHardwareListenerLifecycleGeneration == generation,
+ self.isTerminating == false
+ else { return }
+ guard self.defaultInputListenerToken != nil else {
+ if self.defaultInputListenerPendingGeneration == generation {
+ self.defaultInputListenerDirtyGeneration = generation
+ }
+ return
+ }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.mainHopBegin, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl)
+ defer { AudioTopologyDiagnostics.record(.mainHopEnd, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl) }
+ #endif
+ self.handleDefaultInputChanged()
+ }
}
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: inputAddress.mSelector, scope: inputAddress.mScope, element: inputAddress.mElement, queueRole: .mainControl, phase: .listener)
#endif
- let inputStatus = AudioObjectAddPropertyListenerBlock(
- AudioObjectID(kAudioObjectSystemObject),
- &inputAddress,
- DispatchQueue.main,
- inputToken
- )
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: inputAddress.mSelector, scope: inputAddress.mScope, element: inputAddress.mElement, queueRole: .mainControl, phase: .listener, status: inputStatus)
- #endif
-
- if inputStatus == noErr {
+ self.defaultInputListenerPendingGeneration = generation
+ Task { @MainActor [weak self] in
+ let inputStatus = await AudioTopologyListenerExecution.add(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: inputAddress,
+ token: inputToken
+ )
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrDefaultInput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: inputAddress.mSelector, scope: inputAddress.mScope, element: inputAddress.mElement, queueRole: .dedicatedControl, phase: .listener, status: inputStatus, generation: generation)
+ #endif
+ guard let self else {
+ if inputStatus == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: AudioObjectID(kAudioObjectSystemObject), address: inputAddress, token: inputToken)
+ }
+ return
+ }
+ let ownsPending = self.defaultInputListenerPendingGeneration == generation
+ if ownsPending { self.defaultInputListenerPendingGeneration = nil }
+ guard inputStatus == noErr, ownsPending,
+ self.audioHardwareListenerLifecycleGeneration == generation,
+ self.isTerminating == false
+ else {
+ if ownsPending, self.defaultInputListenerDirtyGeneration == generation {
+ self.defaultInputListenerDirtyGeneration = nil
+ }
+ if inputStatus == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: AudioObjectID(kAudioObjectSystemObject), address: inputAddress, token: inputToken)
+ }
+ return
+ }
self.defaultInputListenerInstalled = true
self.defaultInputListenerToken = inputToken
- } else {
- self.defaultInputListenerToken = nil
- DebugLogger.shared.error("Failed to register default input listener: \(inputStatus)", source: "ASRService")
+ if self.defaultInputListenerDirtyGeneration == generation {
+ self.defaultInputListenerDirtyGeneration = nil
+ self.handleDefaultInputChanged()
+ }
}
}
- if self.defaultOutputListenerToken == nil {
+ if self.defaultOutputListenerToken == nil,
+ self.defaultOutputListenerPendingGeneration == nil
+ {
let outputToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
- DispatchQueue.main.async { self?.handleDefaultOutputChanged() }
+ DispatchQueue.main.async {
+ guard let self,
+ self.audioHardwareListenerLifecycleGeneration == generation,
+ self.isTerminating == false
+ else { return }
+ guard self.defaultOutputListenerToken != nil else {
+ if self.defaultOutputListenerPendingGeneration == generation {
+ self.defaultOutputListenerDirtyGeneration = generation
+ }
+ return
+ }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.mainHopBegin, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl)
+ defer { AudioTopologyDiagnostics.record(.mainHopEnd, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl) }
+ #endif
+ self.handleDefaultOutputChanged()
+ }
}
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: outputAddress.mSelector, scope: outputAddress.mScope, element: outputAddress.mElement, queueRole: .mainControl, phase: .listener)
#endif
- let outputStatus = AudioObjectAddPropertyListenerBlock(
- AudioObjectID(kAudioObjectSystemObject),
- &outputAddress,
- DispatchQueue.main,
- outputToken
- )
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: outputAddress.mSelector, scope: outputAddress.mScope, element: outputAddress.mElement, queueRole: .mainControl, phase: .listener, status: outputStatus)
- #endif
-
- if outputStatus == noErr {
+ self.defaultOutputListenerPendingGeneration = generation
+ Task { @MainActor [weak self] in
+ let outputStatus = await AudioTopologyListenerExecution.add(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: outputAddress,
+ token: outputToken
+ )
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrDefaultOutput, objectID: AudioObjectID(kAudioObjectSystemObject), selector: outputAddress.mSelector, scope: outputAddress.mScope, element: outputAddress.mElement, queueRole: .dedicatedControl, phase: .listener, status: outputStatus, generation: generation)
+ #endif
+ guard let self else {
+ if outputStatus == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: AudioObjectID(kAudioObjectSystemObject), address: outputAddress, token: outputToken)
+ }
+ return
+ }
+ let ownsPending = self.defaultOutputListenerPendingGeneration == generation
+ if ownsPending { self.defaultOutputListenerPendingGeneration = nil }
+ guard outputStatus == noErr, ownsPending,
+ self.audioHardwareListenerLifecycleGeneration == generation,
+ self.isTerminating == false
+ else {
+ if ownsPending, self.defaultOutputListenerDirtyGeneration == generation {
+ self.defaultOutputListenerDirtyGeneration = nil
+ }
+ if outputStatus == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: AudioObjectID(kAudioObjectSystemObject), address: outputAddress, token: outputToken)
+ }
+ return
+ }
self.defaultOutputListenerToken = outputToken
- } else {
- self.defaultOutputListenerToken = nil
- DebugLogger.shared.warning("Failed to register default output listener: \(outputStatus)", source: "ASRService")
+ if self.defaultOutputListenerDirtyGeneration == generation {
+ self.defaultOutputListenerDirtyGeneration = nil
+ self.handleDefaultOutputChanged()
+ }
}
}
}
@@ -4763,51 +5086,103 @@ final class ASRService: ObservableObject {
private var deviceListListenerInstalled = false
private var deviceListListenerToken: AudioObjectPropertyListenerBlock?
- private let deviceListListenerQueue = DispatchQueue(label: "com.fluidvoice.audio.topology-notifications", qos: .userInitiated)
+ private var deviceListListenerPendingGeneration: UInt64?
+ private var deviceListListenerDirtyGeneration: UInt64?
+ private var deviceListQueryGeneration: UInt64 = 0
private var monitoredDeviceID: AudioObjectID?
private var monitoredDeviceIsAliveListenerToken: AudioObjectPropertyListenerBlock?
+ private var monitoredDeviceListenerEpoch: UInt64 = 0
+ private var monitoredDeviceListenerPendingEpoch: UInt64?
+ private var monitoredDeviceAvailabilityDirtyEpoch: UInt64?
/// Registers a listener for device list changes (additions/removals)
/// This enables auto-switching to newly connected devices (especially Bluetooth)
private func registerDeviceListChangeListener() {
- guard self.deviceListListenerInstalled == false else { return }
+ guard self.deviceListListenerInstalled == false,
+ self.deviceListListenerPendingGeneration == nil
+ else { return }
- var address = AudioObjectPropertyAddress(
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDevices,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
let controller = self.directAudioLifecycleController
+ let generation = self.audioHardwareListenerLifecycleGeneration
let token: AudioObjectPropertyListenerBlock = { [weak self, weak controller] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .dedicatedDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
- // Mark freshness immediately, even if the UI is busy. This only
- // updates an in-memory revision; HAL queries remain outside callbacks.
+ // Invalidate cached topology immediately without querying HAL from its callback.
controller?.noteHardwareTopologyChanged()
- DispatchQueue.main.async { self?.handleDeviceListChanged() }
+ // Defer to next runloop pass — CoreAudio may hold an internal lock during
+ // this callback, and our handler makes synchronous CoreAudio queries that
+ // would deadlock waiting for the same lock.
+ DispatchQueue.main.async {
+ guard let self else { return }
+ switch ASRHardwareListenerPolicy.disposition(
+ isTerminating: self.isTerminating,
+ lifecycleMatches: self.audioHardwareListenerLifecycleGeneration == generation,
+ isInstalled: self.deviceListListenerToken != nil,
+ pendingIdentityMatches: self.deviceListListenerPendingGeneration == generation
+ ) {
+ case .ignore:
+ return
+ case .latchUntilInstalled:
+ self.deviceListListenerDirtyGeneration = generation
+ return
+ case .handle:
+ break
+ }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.mainHopBegin, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl)
+ defer { AudioTopologyDiagnostics.record(.mainHopEnd, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl) }
+ #endif
+ self.handleDeviceListChanged()
+ }
}
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
#endif
- let status = AudioObjectAddPropertyListenerBlock(
- AudioObjectID(kAudioObjectSystemObject),
- &address,
- self.deviceListListenerQueue,
- token
- )
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
-
- if status == noErr {
+ self.deviceListListenerPendingGeneration = generation
+ Task { @MainActor [weak self] in
+ let status = await AudioTopologyListenerExecution.add(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: address,
+ token: token
+ )
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrDeviceList, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status, generation: generation)
+ #endif
+ guard let self else {
+ if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: AudioObjectID(kAudioObjectSystemObject), address: address, token: token)
+ }
+ return
+ }
+ let ownsPending = self.deviceListListenerPendingGeneration == generation
+ if ownsPending { self.deviceListListenerPendingGeneration = nil }
+ guard status == noErr, ownsPending,
+ self.audioHardwareListenerLifecycleGeneration == generation,
+ self.isTerminating == false
+ else {
+ if ownsPending, self.deviceListListenerDirtyGeneration == generation {
+ self.deviceListListenerDirtyGeneration = nil
+ }
+ if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: AudioObjectID(kAudioObjectSystemObject), address: address, token: token)
+ }
+ return
+ }
self.deviceListListenerInstalled = true
self.deviceListListenerToken = token
+ if self.deviceListListenerDirtyGeneration == generation {
+ self.deviceListListenerDirtyGeneration = nil
+ self.handleDeviceListChanged()
+ }
DebugLogger.shared.debug("Device list change listener registered", source: "ASRService")
- } else {
- self.deviceListListenerToken = nil
- DebugLogger.shared.error("Failed to register device list listener: \(status)", source: "ASRService")
}
}
@@ -4817,39 +5192,102 @@ final class ASRService: ObservableObject {
// Unregister previous device if any
self.stopMonitoringDevice()
- var address = AudioObjectPropertyAddress(
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyDeviceIsAlive,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
+ self.monitoredDeviceListenerEpoch &+= 1
+ let epoch = self.monitoredDeviceListenerEpoch
+ let generation = self.audioHardwareListenerLifecycleGeneration
+ self.monitoredDeviceID = deviceID
+ self.monitoredDeviceListenerPendingEpoch = epoch
let token: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .asrMonitoredInput, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .asrMonitoredInput, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: epoch)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .asrMonitoredInput, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: epoch) }
#endif
- DispatchQueue.main.async { self?.handleDeviceAvailabilityChanged(deviceID: deviceID) }
+ DispatchQueue.main.async {
+ guard let self else { return }
+ let identityMatches = self.audioHardwareListenerLifecycleGeneration == generation
+ && self.monitoredDeviceListenerEpoch == epoch
+ && self.monitoredDeviceID == deviceID
+ switch ASRHardwareListenerPolicy.disposition(
+ isTerminating: self.isTerminating,
+ lifecycleMatches: identityMatches,
+ isInstalled: self.monitoredDeviceIsAliveListenerToken != nil,
+ pendingIdentityMatches: self.monitoredDeviceListenerPendingEpoch == epoch
+ ) {
+ case .ignore:
+ return
+ case .latchUntilInstalled:
+ self.monitoredDeviceAvailabilityDirtyEpoch = epoch
+ return
+ case .handle:
+ break
+ }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.mainHopBegin, owner: .asrMonitoredInput, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl)
+ defer { AudioTopologyDiagnostics.record(.mainHopEnd, owner: .asrMonitoredInput, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl) }
+ #endif
+ self.handleDeviceAvailabilityChanged(
+ deviceID: deviceID,
+ lifecycleGeneration: generation,
+ listenerEpoch: epoch
+ )
+ }
}
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .asrMonitoredInput, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
#endif
- let status = AudioObjectAddPropertyListenerBlock(
- deviceID,
- &address,
- DispatchQueue.main,
- token
- )
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrMonitoredInput, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
-
- if status == noErr {
- self.monitoredDeviceID = deviceID
+ Task { @MainActor [weak self] in
+ let status = await AudioTopologyListenerExecution.add(
+ objectID: deviceID,
+ address: address,
+ token: token
+ )
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .asrMonitoredInput, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status, generation: epoch)
+ #endif
+ guard let self else {
+ if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: deviceID, address: address, token: token)
+ }
+ return
+ }
+ let ownsPending = self.monitoredDeviceListenerPendingEpoch == epoch
+ if ownsPending { self.monitoredDeviceListenerPendingEpoch = nil }
+ guard status == noErr,
+ ownsPending,
+ self.audioHardwareListenerLifecycleGeneration == generation,
+ self.monitoredDeviceListenerEpoch == epoch,
+ self.monitoredDeviceID == deviceID,
+ self.isTerminating == false
+ else {
+ if ownsPending,
+ self.monitoredDeviceListenerEpoch == epoch,
+ self.monitoredDeviceID == deviceID
+ {
+ self.monitoredDeviceAvailabilityDirtyEpoch = nil
+ self.monitoredDeviceID = nil
+ self.monitoredDeviceListenerEpoch &+= 1
+ }
+ if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: deviceID, address: address, token: token)
+ }
+ return
+ }
self.monitoredDeviceIsAliveListenerToken = token
+ if self.monitoredDeviceAvailabilityDirtyEpoch == epoch {
+ self.monitoredDeviceAvailabilityDirtyEpoch = nil
+ self.handleDeviceAvailabilityChanged(
+ deviceID: deviceID,
+ lifecycleGeneration: generation,
+ listenerEpoch: epoch
+ )
+ }
DebugLogger.shared.debug("Started monitoring device ID: \(deviceID)", source: "ASRService")
- } else {
- self.monitoredDeviceID = nil
- self.monitoredDeviceIsAliveListenerToken = nil
- DebugLogger.shared.error("Failed to monitor device \(deviceID): \(status)", source: "ASRService")
}
}
@@ -4857,23 +5295,33 @@ final class ASRService: ObservableObject {
private func stopMonitoringDevice() {
guard let deviceID = self.monitoredDeviceID else { return }
- var address = AudioObjectPropertyAddress(
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyDeviceIsAlive,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- if let token = self.monitoredDeviceIsAliveListenerToken {
+ let token = self.monitoredDeviceIsAliveListenerToken
+ self.monitoredDeviceListenerEpoch &+= 1
+ self.monitoredDeviceID = nil
+ self.monitoredDeviceIsAliveListenerToken = nil
+ self.monitoredDeviceListenerPendingEpoch = nil
+ self.monitoredDeviceAvailabilityDirtyEpoch = nil
+ if let token {
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .asrMonitoredInput, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
#endif
- let status = AudioObjectRemovePropertyListenerBlock(deviceID, &address, DispatchQueue.main, token)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .asrMonitoredInput, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
+ Task {
+ let status = await AudioTopologyListenerExecution.remove(
+ objectID: deviceID,
+ address: address,
+ token: token
+ )
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .asrMonitoredInput, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
+ #endif
+ }
}
- self.monitoredDeviceID = nil
- self.monitoredDeviceIsAliveListenerToken = nil
DebugLogger.shared.debug("Stopped monitoring device ID: \(deviceID)", source: "ASRService")
}
@@ -4881,14 +5329,31 @@ final class ASRService: ObservableObject {
private func handleDeviceListChanged() {
DebugLogger.shared.info("🔄 Device list changed - checking for new/removed devices", source: "ASRService")
- // Perform CoreAudio queries off the main thread — during a device topology change
- // the HAL may still be settling, and synchronous queries on main can deadlock.
- DispatchQueue.global(qos: .userInitiated).async { [weak self] in
- let currentDevices = AudioDevice.listInputDevicesRefreshingLiveness(diagnosticsOwner: .asrService)
- let defaultInputUID = AudioDevice.getDefaultInputDevice()?.uid
+ // Every request gets an identity before leaving MainActor. Results are last-request-wins:
+ // an older HAL snapshot must not mutate dictation state after a newer topology event,
+ // a Core Audio service restart, or termination.
+ self.deviceListQueryGeneration &+= 1
+ let queryGeneration = self.deviceListQueryGeneration
+ let lifecycleGeneration = self.audioHardwareListenerLifecycleGeneration
+ Task { @MainActor [weak self] in
+ let snapshot = await AudioTopologyListenerExecution.perform {
+ (
+ AudioDevice.listInputDevicesRefreshingLiveness(diagnosticsOwner: .asrService),
+ AudioDevice.getDefaultInputDevice()?.uid
+ )
+ }
- DispatchQueue.main.async { [weak self] in
- guard let self else { return }
+ guard let self,
+ ASRHardwareListenerPolicy.deviceListQueryCanApply(
+ isTerminating: self.isTerminating,
+ capturedLifecycle: lifecycleGeneration,
+ currentLifecycle: self.audioHardwareListenerLifecycleGeneration,
+ capturedQueryGeneration: queryGeneration,
+ currentQueryGeneration: self.deviceListQueryGeneration
+ )
+ else { return }
+
+ let (currentDevices, defaultInputUID) = snapshot
let cachedUIDs = self.cachedDeviceUIDs
let cachedDeviceIDsByUID = self.cachedInputDeviceIDsByUID
let cachedLivenessByUID = self.cachedInputLivenessByUID
@@ -4941,17 +5406,32 @@ final class ASRService: ObservableObject {
}
self.cacheCurrentDeviceList(currentDevices)
- }
}
}
/// Handles device availability changes (device disconnected or reconnected)
- private func handleDeviceAvailabilityChanged(deviceID: AudioObjectID) {
+ private func handleDeviceAvailabilityChanged(
+ deviceID: AudioObjectID,
+ lifecycleGeneration: UInt64,
+ listenerEpoch: UInt64
+ ) {
DebugLogger.shared.info("⚠️ Device availability changed for ID: \(deviceID)", source: "ASRService")
DispatchQueue.global(qos: .userInitiated).async { [weak self] in
let isAlive = AudioDevice.refreshInputDeviceLiveness(deviceID: deviceID)
DispatchQueue.main.async { [weak self] in
- self?.applyDeviceAvailability(isAlive: isAlive, deviceID: deviceID)
+ guard let self,
+ ASRHardwareListenerPolicy.selectedDeviceQueryCanApply(
+ isTerminating: self.isTerminating,
+ capturedLifecycle: lifecycleGeneration,
+ currentLifecycle: self.audioHardwareListenerLifecycleGeneration,
+ capturedEpoch: listenerEpoch,
+ currentEpoch: self.monitoredDeviceListenerEpoch,
+ capturedDeviceID: deviceID,
+ currentDeviceID: self.monitoredDeviceID,
+ listenerIsInstalled: self.monitoredDeviceIsAliveListenerToken != nil
+ )
+ else { return }
+ self.applyDeviceAvailability(isAlive: isAlive, deviceID: deviceID)
}
}
}
diff --git a/Sources/Fluid/Services/AudioDeviceService.swift b/Sources/Fluid/Services/AudioDeviceService.swift
index 6147d665..9fae3ed9 100644
--- a/Sources/Fluid/Services/AudioDeviceService.swift
+++ b/Sources/Fluid/Services/AudioDeviceService.swift
@@ -522,19 +522,171 @@ nonisolated enum AggregateInputLivenessLedgerPolicy {
}
#endif
+/// Process-wide execution boundary for app-owned Core Audio property listeners.
+///
+/// Core Audio may synchronously wait for the listener delivery queue while it tears a device down.
+/// Keeping delivery off main, and serializing add/remove calls on a different queue, prevents main/HAL
+/// circular waits. Callers always await asynchronously; MainActor must never `sync` onto either queue.
+nonisolated enum AudioTopologyListenerExecution {
+ static let deliveryQueue = DispatchQueue(
+ label: "com.fluidvoice.audio.topology-listener-delivery",
+ qos: .userInitiated
+ )
+
+ private static let controlQueue = DispatchQueue(
+ label: "com.fluidvoice.audio.topology-listener-control",
+ qos: .userInitiated
+ )
+
+ /// Runs synchronous HAL catalog/property work away from MainActor and listener delivery.
+ static func perform<T: Sendable>(_ work: @escaping @Sendable () -> T) async -> T {
+ await withCheckedContinuation { continuation in
+ Self.controlQueue.async {
+ continuation.resume(returning: work())
+ }
+ }
+ }
+
+ static func add(
+ objectID: AudioObjectID,
+ address: AudioObjectPropertyAddress,
+ token: @escaping AudioObjectPropertyListenerBlock
+ ) async -> OSStatus {
+ await withCheckedContinuation { continuation in
+ Self.controlQueue.async {
+ var mutableAddress = address
+ continuation.resume(returning: AudioObjectAddPropertyListenerBlock(
+ objectID,
+ &mutableAddress,
+ Self.deliveryQueue,
+ token
+ ))
+ }
+ }
+ }
+
+ static func remove(
+ objectID: AudioObjectID,
+ address: AudioObjectPropertyAddress,
+ token: @escaping AudioObjectPropertyListenerBlock
+ ) async -> OSStatus {
+ await withCheckedContinuation { continuation in
+ Self.controlQueue.async {
+ var mutableAddress = address
+ continuation.resume(returning: AudioObjectRemovePropertyListenerBlock(
+ objectID,
+ &mutableAddress,
+ Self.deliveryQueue,
+ token
+ ))
+ }
+ }
+ }
+
+ static func isBenignRemovalStatus(_ status: OSStatus) -> Bool {
+ status == noErr || status == kAudioHardwareBadObjectError
+ }
+}
+
+nonisolated struct AudioInputAvailabilityListenerIdentity: Equatable {
+ let deviceID: AudioObjectID
+ let uid: String
+ let epoch: UInt64
+ let lifecycleGeneration: UInt64
+}
+
+/// Pure lifecycle rules for the asynchronous CoreAudio liveness-listener ledger.
+/// Keeping these decisions independent of HAL makes the race cases deterministic in tests.
+nonisolated enum AudioInputAvailabilityListenerPolicy {
+ static func callbackIsCurrent(
+ captured: AudioInputAvailabilityListenerIdentity,
+ registered: AudioInputAvailabilityListenerIdentity?,
+ installed: Bool
+ ) -> Bool {
+ installed && registered == captured
+ }
+
+ static func registrationNeedsReplacement(registeredUID: String?, desiredUID: String?) -> Bool {
+ registeredUID != desiredUID
+ }
+
+ static func completionCanInstall(
+ statusSucceeded: Bool,
+ ownsPendingMarker: Bool,
+ installed: Bool,
+ capturedLifecycleGeneration: UInt64,
+ currentLifecycleGeneration: UInt64,
+ capturedUID: String,
+ desiredUID: String?,
+ reconciliationIsCurrent: Bool
+ ) -> Bool {
+ statusSucceeded
+ && ownsPendingMarker
+ && installed
+ && capturedLifecycleGeneration == currentLifecycleGeneration
+ && desiredUID == capturedUID
+ && reconciliationIsCurrent
+ }
+
+ static func shouldRetryStaleCompletion(
+ ownsPendingMarker: Bool,
+ installed: Bool,
+ capturedLifecycleGeneration: UInt64,
+ currentLifecycleGeneration: UInt64,
+ capturedUID: String,
+ desiredUID: String?,
+ hasRegistration: Bool,
+ reconciliationIsStale: Bool
+ ) -> Bool {
+ ownsPendingMarker
+ && installed
+ && capturedLifecycleGeneration == currentLifecycleGeneration
+ && desiredUID == capturedUID
+ && !hasRegistration
+ && reconciliationIsStale
+ }
+}
+
+nonisolated enum AudioInputAvailabilityPolicy {
+ static func hasAvailableInput(liveness: [Bool]) -> Bool {
+ liveness.contains(true)
+ }
+}
+
final class AudioHardwareObserver: ObservableObject {
+ private struct InputAvailabilityRegistration {
+ let identity: AudioInputAvailabilityListenerIdentity
+ let token: AudioObjectPropertyListenerBlock
+ }
+
+ private struct PendingInputAvailabilityRegistration: Equatable {
+ let identity: AudioInputAvailabilityListenerIdentity
+ }
+
/// Incremented every time CoreAudio reports a hardware/default-device change.
/// Using a simple `@Published` value avoids putting `AnyPublisher`/`SubscriptionView` generics into
/// SwiftUI's root view type, which can trigger AttributeGraph metadata-instantiation crashes at launch.
@Published private(set) var changeTick: UInt64 = 0
@Published private(set) var inputAvailabilityTick: UInt64 = 0
+ @Published private(set) var hasAvailableInputDevice = false
private var installed: Bool = false
private var devicesListenerToken: AudioObjectPropertyListenerBlock?
private var defaultInputListenerToken: AudioObjectPropertyListenerBlock?
private var defaultOutputListenerToken: AudioObjectPropertyListenerBlock?
- private var inputAvailabilityListenerTokens: [AudioObjectID: AudioObjectPropertyListenerBlock] = [:]
+ private var inputAvailabilityRegistrations: [AudioObjectID: InputAvailabilityRegistration] = [:]
+ private var pendingInputAvailabilityRegistrations: [AudioObjectID: PendingInputAvailabilityRegistration] = [:]
+ private var desiredInputAvailabilityUIDs: [AudioObjectID: String] = [:]
+ private var nextInputAvailabilityEpoch: UInt64 = 0
private var inputAvailabilityRefreshGeneration: UInt64 = 0
+ private var listenerLifecycleGeneration: UInt64 = 0
+ private var registrationTask: Task<Void, Never>?
+ private var trackedTopologyTasks: [UUID: Task<Void, Never>] = [:]
+ private var topologyReconciliationSuspended = false
+ private var topologyReconciliationDirty = false
+ private var topologySuspensionGeneration: UInt64 = 0
+ private var activeTopologySuspension: UInt64?
+ private var registrationDeferredByTopologySuspension = false
private var clamshellRootDomain: io_service_t = IO_OBJECT_NULL
private var clamshellNotificationPort: IONotificationPortRef?
private var clamshellNotification: io_object_t = IO_OBJECT_NULL
@@ -553,6 +705,72 @@ final class AudioHardwareObserver: ObservableObject {
self.register()
}
+ /// Prevents this observer's catalog queries and per-device listener maintenance from
+ /// overlapping a capture-backend transition. The returned generation owns the resume.
+ func suspendTopologyReconciliation() async -> UInt64 {
+ if let activeTopologySuspension {
+ await self.drainTrackedTopologyWork()
+ return activeTopologySuspension
+ }
+ self.topologySuspensionGeneration &+= 1
+ let generation = self.topologySuspensionGeneration
+ self.activeTopologySuspension = generation
+ self.topologyReconciliationSuspended = true
+ self.topologyReconciliationDirty = true
+ self.inputAvailabilityRefreshGeneration &+= 1
+ await self.drainTrackedTopologyWork()
+ return generation
+ }
+
+ /// Resumes only the exact suspension owner and does not return until the single
+ /// coalesced catalog/listener reconciliation has reached stable quiescence.
+ func resumeTopologyReconciliation(_ generation: UInt64) async {
+ guard self.activeTopologySuspension == generation else { return }
+ self.topologyReconciliationSuspended = false
+ self.topologyReconciliationDirty = false
+
+ if self.registrationDeferredByTopologySuspension {
+ self.registrationDeferredByTopologySuspension = false
+ self.register()
+ } else {
+ self.refreshInputAvailabilityListeners()
+ }
+ await self.drainTrackedTopologyWork()
+ guard self.activeTopologySuspension == generation else { return }
+ self.activeTopologySuspension = nil
+ }
+
+ private func launchTrackedTopologyTask(
+ _ operation: @escaping @MainActor () async -> Void
+ ) {
+ let id = UUID()
+ let task = Task { @MainActor [weak self] in
+ await operation()
+ self?.trackedTopologyTasks.removeValue(forKey: id)
+ }
+ self.trackedTopologyTasks[id] = task
+ }
+
+ private func retainRegistrationTaskForDrain(_ task: Task<Void, Never>) {
+ self.launchTrackedTopologyTask {
+ await task.value
+ }
+ }
+
+ private func drainTrackedTopologyWork() async {
+ while true {
+ if let registrationTask = self.registrationTask {
+ await registrationTask.value
+ }
+ let tasks = Array(self.trackedTopologyTasks.values)
+ for task in tasks { await task.value }
+ // Drains any exact-token cleanup queued by a completion before it returned.
+ await AudioTopologyListenerExecution.perform {}
+ await Task.yield()
+ if self.registrationTask == nil, self.trackedTopologyTasks.isEmpty { return }
+ }
+ }
+
@MainActor
func signalInputAvailabilityChanged() {
self.inputAvailabilityTick &+= 1
@@ -561,12 +779,26 @@ final class AudioHardwareObserver: ObservableObject {
func restartObservingAfterAudioServiceReset() {
// Core Audio discards every previously registered listener when its
// service resets, so the old tokens must not be removed or reused.
+ self.listenerLifecycleGeneration &+= 1
+ if let registrationTask = self.registrationTask {
+ registrationTask.cancel()
+ self.retainRegistrationTaskForDrain(registrationTask)
+ }
+ self.registrationTask = nil
self.devicesListenerToken = nil
self.defaultInputListenerToken = nil
self.defaultOutputListenerToken = nil
- self.inputAvailabilityListenerTokens.removeAll()
+ self.inputAvailabilityRegistrations.removeAll()
+ self.pendingInputAvailabilityRegistrations.removeAll()
+ self.desiredInputAvailabilityUIDs.removeAll()
self.inputAvailabilityRefreshGeneration &+= 1
self.installed = false
+ if self.topologyReconciliationSuspended {
+ self.registrationDeferredByTopologySuspension = true
+ self.topologyReconciliationDirty = true
+ self.changeTick &+= 1
+ return
+ }
self.register()
self.changeTick &+= 1
if self.installed {
@@ -587,199 +819,206 @@ final class AudioHardwareObserver: ObservableObject {
}
private func register() {
- guard self.installed == false else { return }
- var addrDevices = AudioObjectPropertyAddress(
+ guard self.topologyReconciliationSuspended == false else {
+ self.registrationDeferredByTopologySuspension = true
+ return
+ }
+ guard self.installed == false, self.registrationTask == nil else { return }
+ self.listenerLifecycleGeneration &+= 1
+ let generation = self.listenerLifecycleGeneration
+ let addrDevices = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDevices,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- var addrDefaultIn = AudioObjectPropertyAddress(
+ let addrDefaultIn = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultInputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- var addrDefaultOut = AudioObjectPropertyAddress(
+ let addrDefaultOut = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- let queue = DispatchQueue.main
let sys = AudioObjectID(kAudioObjectSystemObject)
let devicesToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
AudioTopologyDiagnostics.record(
- .callback,
+ .callbackBegin,
owner: .audioHardwareObserver,
objectID: sys,
selector: kAudioHardwarePropertyDevices,
scope: kAudioObjectPropertyScopeGlobal,
element: kAudioObjectPropertyElementMain,
- queueRole: .mainDelivery,
- generation: self?.inputAvailabilityRefreshGeneration ?? 0
+ queueRole: .callbackCurrent,
+ generation: generation
)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .audioHardwareObserver, objectID: sys, selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
- self?.changeTick &+= 1
- self?.refreshInputAvailabilityListeners()
+ DispatchQueue.main.async { [weak self] in
+ guard let self, self.installed, self.listenerLifecycleGeneration == generation else { return }
+ self.changeTick &+= 1
+ self.refreshInputAvailabilityListeners()
+ }
}
let defaultInToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
AudioTopologyDiagnostics.record(
- .callback,
+ .callbackBegin,
owner: .audioHardwareObserver,
objectID: sys,
selector: kAudioHardwarePropertyDefaultInputDevice,
scope: kAudioObjectPropertyScopeGlobal,
element: kAudioObjectPropertyElementMain,
- queueRole: .mainDelivery,
- generation: self?.inputAvailabilityRefreshGeneration ?? 0
+ queueRole: .callbackCurrent,
+ generation: generation
)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .audioHardwareObserver, objectID: sys, selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
- self?.inputAvailabilityTick &+= 1
+ DispatchQueue.main.async { [weak self] in
+ guard let self, self.installed, self.listenerLifecycleGeneration == generation else { return }
+ self.inputAvailabilityTick &+= 1
+ }
}
let defaultOutToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
AudioTopologyDiagnostics.record(
- .callback,
+ .callbackBegin,
owner: .audioHardwareObserver,
objectID: sys,
selector: kAudioHardwarePropertyDefaultOutputDevice,
scope: kAudioObjectPropertyScopeGlobal,
element: kAudioObjectPropertyElementMain,
- queueRole: .mainDelivery,
- generation: self?.inputAvailabilityRefreshGeneration ?? 0
+ queueRole: .callbackCurrent,
+ generation: generation
)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .audioHardwareObserver, objectID: sys, selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
- self?.changeTick &+= 1
+ DispatchQueue.main.async { [weak self] in
+ guard let self, self.installed, self.listenerLifecycleGeneration == generation else { return }
+ self.changeTick &+= 1
+ }
}
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let devicesStatus = AudioObjectAddPropertyListenerBlock(sys, &addrDevices, queue, devicesToken)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .mainControl, phase: .listener, status: devicesStatus)
- AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultIn.mSelector, scope: addrDefaultIn.mScope, element: addrDefaultIn.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let defaultInStatus = AudioObjectAddPropertyListenerBlock(sys, &addrDefaultIn, queue, defaultInToken)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultIn.mSelector, scope: addrDefaultIn.mScope, element: addrDefaultIn.mElement, queueRole: .mainControl, phase: .listener, status: defaultInStatus)
- AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultOut.mSelector, scope: addrDefaultOut.mScope, element: addrDefaultOut.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let defaultOutStatus = AudioObjectAddPropertyListenerBlock(sys, &addrDefaultOut, queue, defaultOutToken)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultOut.mSelector, scope: addrDefaultOut.mScope, element: addrDefaultOut.mElement, queueRole: .mainControl, phase: .listener, status: defaultOutStatus)
- AudioTopologyDiagnostics.record(.readiness, owner: .audioHardwareObserver, queueRole: .mainControl, status: (devicesStatus == noErr && defaultInStatus == noErr && defaultOutStatus == noErr) ? noErr : -1)
- #endif
-
- guard devicesStatus == noErr, defaultInStatus == noErr, defaultOutStatus == noErr else {
- // Best-effort cleanup for any partial installs.
- if devicesStatus == noErr {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &addrDevices, queue, devicesToken)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
- }
- if defaultInStatus == noErr {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultIn.mSelector, scope: addrDefaultIn.mScope, element: addrDefaultIn.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &addrDefaultIn, queue, defaultInToken)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultIn.mSelector, scope: addrDefaultIn.mScope, element: addrDefaultIn.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
+ self.registrationTask = Task { @MainActor [weak self] in
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .dedicatedControl, phase: .listener, generation: generation)
+ #endif
+ let devicesStatus = await AudioTopologyListenerExecution.add(objectID: sys, address: addrDevices, token: devicesToken)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .dedicatedControl, phase: .listener, status: devicesStatus, generation: generation)
+ #endif
+ let defaultInStatus = Task.isCancelled
+ ? kAudioHardwareUnspecifiedError
+ : await AudioTopologyListenerExecution.add(objectID: sys, address: addrDefaultIn, token: defaultInToken)
+ let defaultOutStatus = Task.isCancelled
+ ? kAudioHardwareUnspecifiedError
+ : await AudioTopologyListenerExecution.add(objectID: sys, address: addrDefaultOut, token: defaultOutToken)
+
+ guard let self, self.listenerLifecycleGeneration == generation, !Task.isCancelled else {
+ await Self.removeSuccessfulSystemListeners(
+ sys: sys,
+ registrations: [
+ (addrDevices, devicesToken, devicesStatus),
+ (addrDefaultIn, defaultInToken, defaultInStatus),
+ (addrDefaultOut, defaultOutToken, defaultOutStatus),
+ ],
+ generation: generation
+ )
+ return
}
- if defaultOutStatus == noErr {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultOut.mSelector, scope: addrDefaultOut.mScope, element: addrDefaultOut.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &addrDefaultOut, queue, defaultOutToken)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultOut.mSelector, scope: addrDefaultOut.mScope, element: addrDefaultOut.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
+ self.registrationTask = nil
+ guard devicesStatus == noErr, defaultInStatus == noErr, defaultOutStatus == noErr else {
+ await Self.removeSuccessfulSystemListeners(
+ sys: sys,
+ registrations: [
+ (addrDevices, devicesToken, devicesStatus),
+ (addrDefaultIn, defaultInToken, defaultInStatus),
+ (addrDefaultOut, defaultOutToken, defaultOutStatus),
+ ],
+ generation: generation
+ )
+ return
}
- self.devicesListenerToken = nil
- self.defaultInputListenerToken = nil
- self.defaultOutputListenerToken = nil
- self.installed = false
- return
+ self.devicesListenerToken = devicesToken
+ self.defaultInputListenerToken = defaultInToken
+ self.defaultOutputListenerToken = defaultOutToken
+ self.installed = true
+ self.refreshInputAvailabilityListeners()
+ self.registerClamshellStateListener()
}
-
- self.devicesListenerToken = devicesToken
- self.defaultInputListenerToken = defaultInToken
- self.defaultOutputListenerToken = defaultOutToken
- self.installed = true
- self.refreshInputAvailabilityListeners()
- self.registerClamshellStateListener()
}
private func unregister() {
- var addrDevices = AudioObjectPropertyAddress(
+ self.listenerLifecycleGeneration &+= 1
+ if let registrationTask = self.registrationTask {
+ registrationTask.cancel()
+ self.retainRegistrationTaskForDrain(registrationTask)
+ }
+ self.registrationTask = nil
+ let addrDevices = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDevices,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- var addrDefaultIn = AudioObjectPropertyAddress(
+ let addrDefaultIn = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultInputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- var addrDefaultOut = AudioObjectPropertyAddress(
+ let addrDefaultOut = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- let queue = DispatchQueue.main
let sys = AudioObjectID(kAudioObjectSystemObject)
-
- if self.installed {
- if let token = self.devicesListenerToken {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &addrDevices, queue, token)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDevices.mSelector, scope: addrDevices.mScope, element: addrDevices.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
- }
- if let token = self.defaultInputListenerToken {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultIn.mSelector, scope: addrDefaultIn.mScope, element: addrDefaultIn.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &addrDefaultIn, queue, token)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultIn.mSelector, scope: addrDefaultIn.mScope, element: addrDefaultIn.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
- }
- if let token = self.defaultOutputListenerToken {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultOut.mSelector, scope: addrDefaultOut.mScope, element: addrDefaultOut.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &addrDefaultOut, queue, token)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: sys, selector: addrDefaultOut.mSelector, scope: addrDefaultOut.mScope, element: addrDefaultOut.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
- }
- self.removeInputAvailabilityListeners()
- }
-
+ let registrations: [(AudioObjectPropertyAddress, AudioObjectPropertyListenerBlock)] = [
+ self.devicesListenerToken.map { (addrDevices, $0) },
+ self.defaultInputListenerToken.map { (addrDefaultIn, $0) },
+ self.defaultOutputListenerToken.map { (addrDefaultOut, $0) },
+ ].compactMap { $0 }
self.devicesListenerToken = nil
self.defaultInputListenerToken = nil
self.defaultOutputListenerToken = nil
self.installed = false
self.inputAvailabilityRefreshGeneration &+= 1
+ self.removeInputAvailabilityListeners()
self.unregisterClamshellStateListener()
+ Task {
+ for (address, token) in registrations {
+ _ = await AudioTopologyListenerExecution.remove(objectID: sys, address: address, token: token)
+ }
+ }
+ }
+
+ private nonisolated static func removeSuccessfulSystemListeners(
+ sys: AudioObjectID,
+ registrations: [(AudioObjectPropertyAddress, AudioObjectPropertyListenerBlock, OSStatus)],
+ generation: UInt64
+ ) async {
+ for (address, token, status) in registrations where status == noErr {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: sys, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, generation: generation)
+ #endif
+ let removalStatus = await AudioTopologyListenerExecution.remove(objectID: sys, address: address, token: token)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: sys, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: removalStatus, generation: generation)
+ #endif
+ }
}
private func refreshInputAvailabilityListeners() {
+ guard self.topologyReconciliationSuspended == false else {
+ self.topologyReconciliationDirty = true
+ return
+ }
self.inputAvailabilityRefreshGeneration &+= 1
let generation = self.inputAvailabilityRefreshGeneration
- DispatchQueue.global(qos: .utility).async { [weak self] in
+ self.launchTrackedTopologyTask { [weak self] in
#if DEBUG
AudioTopologyDiagnostics.record(
.enumerationBegin,
@@ -789,7 +1028,9 @@ final class AudioHardwareObserver: ObservableObject {
generation: generation
)
#endif
- let devices = AudioDevice.listInputDevicesRefreshingLiveness()
+ let devices = await AudioTopologyListenerExecution.perform {
+ AudioDevice.listInputDevicesRefreshingLiveness()
+ }
#if DEBUG
AudioTopologyDiagnostics.record(
.enumerationEnd,
@@ -800,17 +1041,25 @@ final class AudioHardwareObserver: ObservableObject {
generation: generation
)
#endif
- DispatchQueue.main.async { [weak self] in
- guard let self,
- self.inputAvailabilityRefreshGeneration == generation
- else { return }
- self.replaceInputAvailabilityListeners(with: devices)
+ guard let self,
+ self.topologyReconciliationSuspended == false,
+ self.inputAvailabilityRefreshGeneration == generation
+ else {
+ self?.topologyReconciliationDirty = true
+ return
}
+ self.replaceInputAvailabilityListeners(with: devices)
}
}
private func replaceInputAvailabilityListeners(with devices: [AudioDevice.Device]) {
- guard self.installed else { return }
+ guard self.installed, self.topologyReconciliationSuspended == false else {
+ self.topologyReconciliationDirty = true
+ return
+ }
+ self.hasAvailableInputDevice = AudioInputAvailabilityPolicy.hasAvailableInput(
+ liveness: devices.map(\.isAlive)
+ )
#if DEBUG
// Phase-0 isolation for the rebase-introduced all-input liveness ledger. The flag is
// deliberately scoped to this observer: default-device monitoring, ASR's existing
@@ -898,41 +1147,108 @@ final class AudioHardwareObserver: ObservableObject {
#else
let listenerDevices = productionScopedDevices
#endif
- let currentIDs = Set(listenerDevices.map(\.id))
- for (deviceID, token) in self.inputAvailabilityListenerTokens where currentIDs.contains(deviceID) == false {
- var address = Self.inputAvailabilityAddress
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, generation: self.inputAvailabilityRefreshGeneration)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(deviceID, &address, DispatchQueue.main, token)
+ let desiredUIDs = Dictionary(
+ listenerDevices.map { ($0.id, $0.uid) },
+ uniquingKeysWith: { _, latest in latest }
+ )
+ self.desiredInputAvailabilityUIDs = desiredUIDs
+ for (deviceID, registration) in self.inputAvailabilityRegistrations
+ where AudioInputAvailabilityListenerPolicy.registrationNeedsReplacement(
+ registeredUID: registration.identity.uid,
+ desiredUID: desiredUIDs[deviceID]
+ )
+ {
+ let address = Self.inputAvailabilityAddress
+ self.inputAvailabilityRegistrations.removeValue(forKey: deviceID)
#if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status, generation: self.inputAvailabilityRefreshGeneration)
+ AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, generation: self.inputAvailabilityRefreshGeneration)
#endif
- self.inputAvailabilityListenerTokens.removeValue(forKey: deviceID)
+ let generation = self.inputAvailabilityRefreshGeneration
+ self.launchTrackedTopologyTask {
+ let status = await AudioTopologyListenerExecution.remove(
+ objectID: deviceID,
+ address: address,
+ token: registration.token
+ )
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status, generation: generation)
+ #endif
+ }
}
- for device in listenerDevices where self.inputAvailabilityListenerTokens[device.id] == nil {
+ for device in listenerDevices {
let deviceID = device.id
- var address = Self.inputAvailabilityAddress
+ let uid = device.uid
+ if self.inputAvailabilityRegistrations[deviceID]?.identity.uid == uid
+ || self.pendingInputAvailabilityRegistrations[deviceID]?.identity.uid == uid
+ {
+ continue
+ }
+
+ let address = Self.inputAvailabilityAddress
+ let reconciliationGeneration = self.inputAvailabilityRefreshGeneration
+ let lifecycleGeneration = self.listenerLifecycleGeneration
+ self.nextInputAvailabilityEpoch &+= 1
+ let epoch = self.nextInputAvailabilityEpoch
+ let identity = AudioInputAvailabilityListenerIdentity(
+ deviceID: deviceID,
+ uid: uid,
+ epoch: epoch,
+ lifecycleGeneration: lifecycleGeneration
+ )
+ let pending = PendingInputAvailabilityRegistration(identity: identity)
let token: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
AudioTopologyDiagnostics.record(
- .callback,
+ .callbackBegin,
owner: .audioHardwareObserver,
objectID: deviceID,
selector: kAudioDevicePropertyDeviceIsAlive,
scope: kAudioObjectPropertyScopeGlobal,
element: kAudioObjectPropertyElementMain,
- queueRole: .mainDelivery,
- generation: self?.inputAvailabilityRefreshGeneration ?? 0
+ queueRole: .callbackCurrent,
+ generation: reconciliationGeneration
)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: reconciliationGeneration) }
#endif
- // Never query Core Audio synchronously from its property callback.
- // Refresh the cached snapshot off-main before observers resolve priority.
- DispatchQueue.global(qos: .userInitiated).async { [weak self] in
- _ = AudioDevice.listInputDevicesRefreshingLiveness()
- DispatchQueue.main.async { [weak self] in
- guard let self else { return }
+ // First reject callbacks from replaced registrations on main, then query Core Audio
+ // off-main. Validate the same registration again before publishing the result.
+ DispatchQueue.main.async { [weak self] in
+ guard let self,
+ self.listenerLifecycleGeneration == lifecycleGeneration,
+ AudioInputAvailabilityListenerPolicy.callbackIsCurrent(
+ captured: identity,
+ registered: self.inputAvailabilityRegistrations[deviceID]?.identity,
+ installed: self.installed
+ )
+ else { return }
+ guard self.topologyReconciliationSuspended == false else {
+ self.topologyReconciliationDirty = true
+ return
+ }
+ self.launchTrackedTopologyTask { [weak self] in
+ let devices = await AudioTopologyListenerExecution.perform {
+ AudioDevice.listInputDevicesRefreshingLiveness()
+ }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.mainHopBegin, owner: .audioHardwareObserver, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl, generation: reconciliationGeneration)
+ defer { AudioTopologyDiagnostics.record(.mainHopEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: kAudioDevicePropertyDeviceIsAlive, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainControl, generation: reconciliationGeneration) }
+ #endif
+ guard let self,
+ self.topologyReconciliationSuspended == false,
+ self.listenerLifecycleGeneration == lifecycleGeneration,
+ AudioInputAvailabilityListenerPolicy.callbackIsCurrent(
+ captured: identity,
+ registered: self.inputAvailabilityRegistrations[deviceID]?.identity,
+ installed: self.installed
+ )
+ else {
+ self?.topologyReconciliationDirty = true
+ return
+ }
+ self.hasAvailableInputDevice = AudioInputAvailabilityPolicy.hasAvailableInput(
+ liveness: devices.map(\.isAlive)
+ )
self.inputAvailabilityTick &+= 1
NotificationCenter.default.post(
name: .inputDeviceAvailabilityDidChange,
@@ -942,20 +1258,68 @@ final class AudioHardwareObserver: ObservableObject {
}
}
}
+ self.pendingInputAvailabilityRegistrations[deviceID] = pending
#if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, transport: AudioTopologyDiagnostics.transportClassification(device.transportType), generation: self.inputAvailabilityRefreshGeneration)
- #endif
- let status = AudioObjectAddPropertyListenerBlock(
- deviceID,
- &address,
- DispatchQueue.main,
- token
- )
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, transport: AudioTopologyDiagnostics.transportClassification(device.transportType), status: status, generation: self.inputAvailabilityRefreshGeneration)
+ AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, transport: AudioTopologyDiagnostics.transportClassification(device.transportType), generation: reconciliationGeneration)
#endif
- if status == noErr {
- self.inputAvailabilityListenerTokens[deviceID] = token
+ self.launchTrackedTopologyTask { [weak self] in
+ let status = await AudioTopologyListenerExecution.add(objectID: deviceID, address: address, token: token)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, transport: AudioTopologyDiagnostics.transportClassification(device.transportType), status: status, generation: reconciliationGeneration)
+ #endif
+ guard let self else {
+ if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: deviceID, address: address, token: token)
+ }
+ return
+ }
+ let ownsPendingMarker = self.pendingInputAvailabilityRegistrations[deviceID] == pending
+ if ownsPendingMarker {
+ self.pendingInputAvailabilityRegistrations.removeValue(forKey: deviceID)
+ }
+ let completionCanInstall = AudioInputAvailabilityListenerPolicy.completionCanInstall(
+ statusSucceeded: status == noErr,
+ ownsPendingMarker: ownsPendingMarker,
+ installed: self.installed,
+ capturedLifecycleGeneration: lifecycleGeneration,
+ currentLifecycleGeneration: self.listenerLifecycleGeneration,
+ capturedUID: uid,
+ desiredUID: self.desiredInputAvailabilityUIDs[deviceID],
+ reconciliationIsCurrent: self.inputAvailabilityRefreshGeneration
+ == reconciliationGeneration
+ )
+ guard self.topologyReconciliationSuspended == false,
+ completionCanInstall
+ else {
+ if self.topologyReconciliationSuspended {
+ self.topologyReconciliationDirty = true
+ }
+ if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: deviceID, address: address, token: token)
+ }
+ let staleReconciliationNeedsRetry = AudioInputAvailabilityListenerPolicy
+ .shouldRetryStaleCompletion(
+ ownsPendingMarker: ownsPendingMarker,
+ installed: self.installed,
+ capturedLifecycleGeneration: lifecycleGeneration,
+ currentLifecycleGeneration: self.listenerLifecycleGeneration,
+ capturedUID: uid,
+ desiredUID: self.desiredInputAvailabilityUIDs[deviceID],
+ hasRegistration: self.inputAvailabilityRegistrations[deviceID] != nil,
+ reconciliationIsStale: self.inputAvailabilityRefreshGeneration
+ != reconciliationGeneration
+ )
+ if staleReconciliationNeedsRetry,
+ self.topologyReconciliationSuspended == false
+ {
+ self.refreshInputAvailabilityListeners()
+ }
+ return
+ }
+ self.inputAvailabilityRegistrations[deviceID] = InputAvailabilityRegistration(
+ identity: identity,
+ token: token
+ )
}
}
#if DEBUG
@@ -971,17 +1335,27 @@ final class AudioHardwareObserver: ObservableObject {
}
private func removeInputAvailabilityListeners() {
- for (deviceID, token) in self.inputAvailabilityListenerTokens {
- var address = Self.inputAvailabilityAddress
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, generation: self.inputAvailabilityRefreshGeneration)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(deviceID, &address, DispatchQueue.main, token)
+ let registrations = self.inputAvailabilityRegistrations
+ let generation = self.inputAvailabilityRefreshGeneration
+ self.inputAvailabilityRegistrations.removeAll()
+ self.pendingInputAvailabilityRegistrations.removeAll()
+ self.desiredInputAvailabilityUIDs.removeAll()
+ for (deviceID, registration) in registrations {
+ let address = Self.inputAvailabilityAddress
#if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status, generation: self.inputAvailabilityRefreshGeneration)
+ AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, generation: generation)
#endif
+ self.launchTrackedTopologyTask {
+ let status = await AudioTopologyListenerExecution.remove(
+ objectID: deviceID,
+ address: address,
+ token: registration.token
+ )
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .audioHardwareObserver, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status, generation: generation)
+ #endif
+ }
}
- self.inputAvailabilityListenerTokens.removeAll()
}
private static var inputAvailabilityAddress: AudioObjectPropertyAddress {
diff --git a/Sources/Fluid/Services/AudioTopologyDiagnostics.swift b/Sources/Fluid/Services/AudioTopologyDiagnostics.swift
index 8cbb9666..5d1c6ae6 100644
--- a/Sources/Fluid/Services/AudioTopologyDiagnostics.swift
+++ b/Sources/Fluid/Services/AudioTopologyDiagnostics.swift
@@ -48,6 +48,11 @@ nonisolated enum AudioTopologyTraceEvent: UInt32, CaseIterable, Sendable {
case readiness
case isolationActive
case policyExcluded
+ case callbackBegin
+ case callbackEnd
+ case mainHopBegin
+ case mainHopEnd
+ case ownerWillDeinit
var label: String {
String(describing: self)
@@ -69,6 +74,11 @@ nonisolated enum AudioTopologyTraceOwner: UInt32, CaseIterable, Sendable {
case meetingCatalog
case meetingCoordinator
case phaseZeroProbe
+ case meetingCaptureEngine
+ case screenCapture
+ case avCapture
+ case asrActivityLease
+ case asrEngineConfiguration
var label: String {
String(describing: self)
@@ -98,6 +108,16 @@ nonisolated enum AudioTopologyTracePhase: UInt32, CaseIterable, Sendable {
case recovery
case handoff
case quiescing
+ case coordinatorStop
+ case captureStop
+ case runtimeStop
+ case writerStop
+ case outputRouteRelease
+ case avCaptureStop
+ case screenCaptureStop
+ case processing
+ case activityLease
+ case microphoneStop
var label: String {
String(describing: self)
@@ -173,7 +193,7 @@ final nonisolated class AudioTopologyDiagnostics: @unchecked Sendable {
"kind", "sequence", "continuous_time", "wall_unix_ns", "event", "owner",
"object_id", "selector", "scope", "element", "queue_role", "phase",
"transport", "status", "generation", "pid", "heartbeat_age_ms",
- "stall_classification", "trace_path", "stall_path",
+ "stall_classification", "trace_path", "stall_path", "is_main_thread",
]
private let drainQueue = DispatchQueue(
@@ -500,13 +520,26 @@ final nonisolated class AudioTopologyDiagnostics: @unchecked Sendable {
var category: UInt8
var owner: UInt32
var objectID: UInt32
+ var selector: UInt32
+ var scope: UInt32
+ var element: UInt32
+ var phase: UInt32
var generation: UInt64
}
var openCounts: [Key: Int] = [:]
for event in events {
guard let kind = AudioTopologyTraceEvent(rawValue: event.event) else { continue }
guard let pair = Self.phasePair(for: kind) else { continue }
- let key = Key(category: pair.category, owner: event.owner, objectID: event.objectID, generation: event.generation)
+ let key = Key(
+ category: pair.category,
+ owner: event.owner,
+ objectID: event.objectID,
+ selector: event.selector,
+ scope: event.scope,
+ element: event.element,
+ phase: event.phase,
+ generation: event.generation
+ )
if pair.isBegin {
openCounts[key, default: 0] += 1
} else if let count = openCounts[key], count > 0 {
@@ -550,6 +583,10 @@ final nonisolated class AudioTopologyDiagnostics: @unchecked Sendable {
case .phaseEnd: (15, false)
case .probeCycleBegin: (16, true)
case .probeCycleEnd: (16, false)
+ case .callbackBegin: (17, true)
+ case .callbackEnd: (17, false)
+ case .mainHopBegin: (18, true)
+ case .mainHopEnd: (18, false)
default: nil
}
}
@@ -566,6 +603,7 @@ final nonisolated class AudioTopologyDiagnostics: @unchecked Sendable {
"\"selector\":\(event.selector),\"scope\":\(event.scope)," +
"\"element\":\(event.element),\"queue_role\":\"\(queueLabel)\"," +
"\"phase\":\"\(phaseLabel)\",\"transport\":\"\(transportLabel)\"," +
+ "\"is_main_thread\":\(event.isMainThread != 0)," +
"\"status\":\(event.status),\"generation\":\(event.generation)}\n"
return Array(line.utf8)
}
diff --git a/Sources/Fluid/Services/DirectCoreAudioInput.swift b/Sources/Fluid/Services/DirectCoreAudioInput.swift
index 291fc4bb..ab7b0a0d 100644
--- a/Sources/Fluid/Services/DirectCoreAudioInput.swift
+++ b/Sources/Fluid/Services/DirectCoreAudioInput.swift
@@ -1491,7 +1491,7 @@ final nonisolated class DirectCoreAudioLifecycleController: @unchecked Sendable
let block = Self.makeDevicePropertyListener(objectID: objectID) { [weak self, weak input] objectID in
#if DEBUG
AudioTopologyDiagnostics.record(
- .callback,
+ .callbackBegin,
owner: .directCoreAudio,
objectID: objectID,
selector: selector,
@@ -1500,6 +1500,7 @@ final nonisolated class DirectCoreAudioLifecycleController: @unchecked Sendable
queueRole: .dedicatedDelivery,
generation: generation
)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .directCoreAudio, objectID: listenerObjectID, selector: selector, scope: scope, element: kAudioObjectPropertyElementMain, queueRole: .dedicatedDelivery, generation: generation) }
#endif
let deviceIsAlive =
name == "device_is_alive"
diff --git a/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift b/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
index 5c31b879..77d4607d 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
@@ -51,7 +51,7 @@ protocol MicActivitySignalProviding: AnyObject {
@MainActor
protocol AudioProcessActivityProviding: AnyObject {
- func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) -> Bool
+ func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) async -> Bool
}
nonisolated struct WindowSnapshot: Sendable, Equatable {
@@ -169,49 +169,64 @@ final class WorkspaceEventsMonitor: WorkspaceEventsProviding {
/// (deadlock-avoidance precedent) rather than acting synchronously inside the HAL callback.
@MainActor
final class CoreAudioMicActivitySignal: MicActivitySignalProviding {
+ private struct ListenerIdentity: Equatable, Sendable {
+ let deviceID: AudioObjectID
+ let uid: String
+ let listenerEpoch: UInt64
+ let lifecycleGeneration: UInt64
+ }
+
+ private struct Registration: @unchecked Sendable {
+ let identity: ListenerIdentity
+ let token: AudioObjectPropertyListenerBlock
+ }
+
var onEdge: ((MicActivityEdge) -> Void)?
private var deviceListToken: AudioObjectPropertyListenerBlock?
- private var perDeviceTokens: [AudioObjectID: AudioObjectPropertyListenerBlock] = [:]
+ private var serviceRestartedToken: AudioObjectPropertyListenerBlock?
+ private var perDeviceTokens: [AudioObjectID: Registration] = [:]
private var lastKnownRunning: [AudioObjectID: Bool] = [:]
+ private var desiredUIDs: [AudioObjectID: String] = [:]
private var started = false
+ private var lifecycleGeneration: UInt64 = 0
+ private var reconciliationEpoch: UInt64 = 0
+ private var listenerEpoch: UInt64 = 0
func start() {
guard !self.started else { return }
self.started = true
- self.registerDeviceListListener()
- self.resyncMonitoredDevices()
+ self.lifecycleGeneration &+= 1
+ let generation = self.lifecycleGeneration
+ Task { @MainActor [weak self] in
+ guard let self else { return }
+ await self.bootstrap(generation: generation)
+ }
}
func stop() {
guard self.started else { return }
self.started = false
- var devicesAddress = Self.devicesAddress
- if let token = self.deviceListToken {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: devicesAddress.mSelector, scope: devicesAddress.mScope, element: devicesAddress.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(AudioObjectID(kAudioObjectSystemObject), &devicesAddress, DispatchQueue.main, token)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: devicesAddress.mSelector, scope: devicesAddress.mScope, element: devicesAddress.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
- }
+ self.lifecycleGeneration &+= 1
+ self.reconciliationEpoch &+= 1
+ let deviceListToken = self.deviceListToken
+ let serviceRestartedToken = self.serviceRestartedToken
+ let registrations = Array(self.perDeviceTokens.values)
self.deviceListToken = nil
- for (deviceID, token) in self.perDeviceTokens {
- var address = Self.runningSomewhereAddress
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingDetector, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(deviceID, &address, DispatchQueue.main, token)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingDetector, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
- }
+ self.serviceRestartedToken = nil
self.perDeviceTokens = [:]
self.lastKnownRunning = [:]
+ self.desiredUIDs = [:]
+ Task {
+ await Self.remove(
+ deviceListToken: deviceListToken,
+ serviceRestartedToken: serviceRestartedToken,
+ registrations: registrations
+ )
+ }
}
- private static var devicesAddress: AudioObjectPropertyAddress {
+ nonisolated private static var devicesAddress: AudioObjectPropertyAddress {
AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDevices,
mScope: kAudioObjectPropertyScopeGlobal,
@@ -219,7 +234,7 @@ final class CoreAudioMicActivitySignal: MicActivitySignalProviding {
)
}
- private static var runningSomewhereAddress: AudioObjectPropertyAddress {
+ nonisolated private static var runningSomewhereAddress: AudioObjectPropertyAddress {
AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyDeviceIsRunningSomewhere,
mScope: kAudioObjectPropertyScopeGlobal,
@@ -227,102 +242,263 @@ final class CoreAudioMicActivitySignal: MicActivitySignalProviding {
)
}
- private func registerDeviceListListener() {
- var address = Self.devicesAddress
+ nonisolated private static var serviceRestartedAddress: AudioObjectPropertyAddress {
+ AudioObjectPropertyAddress(
+ mSelector: kAudioHardwarePropertyServiceRestarted,
+ mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain
+ )
+ }
+
+ private func bootstrap(generation: UInt64) async {
+ while self.isCurrent(generation) {
+ if await self.registerDeviceListListener(generation: generation) {
+ await self.resyncMonitoredDevices(generation: generation)
+ return
+ }
+ try? await Task.sleep(nanoseconds: 250_000_000)
+ }
+ }
+
+ private func registerDeviceListListener(generation: UInt64) async -> Bool {
+ let address = Self.devicesAddress
let token: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDevices, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery) }
#endif
- DispatchQueue.main.async { self?.resyncMonitoredDevices() }
+ guard let signal = self else { return }
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ Task { @MainActor in
+ await signal.resyncMonitoredDevices(generation: generation, replaceAll: true)
+ }
+ }
+ }
+ let restartedToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
+ guard let signal = self else { return }
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ Task { @MainActor in
+ await signal.handleServiceRestart(generation: generation)
+ }
+ }
}
+ // Restart protection must exist before any other token can be invalidated by coreaudiod.
+ let restartedStatus = await AudioTopologyListenerExecution.add(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: Self.serviceRestartedAddress,
+ token: restartedToken
+ )
+ if restartedStatus == noErr, self.isCurrent(generation) {
+ self.serviceRestartedToken = restartedToken
+ } else {
+ if restartedStatus == noErr {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: Self.serviceRestartedAddress,
+ token: restartedToken
+ )
+ }
+ return false
+ }
+
#if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
+ AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let status = AudioObjectAddPropertyListenerBlock(AudioObjectID(kAudioObjectSystemObject), &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.add(
+ objectID: AudioObjectID(kAudioObjectSystemObject), address: address, token: token
+ )
#if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingDetector, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
- if status == noErr {
+ if status == noErr, self.isCurrent(generation) {
self.deviceListToken = token
+ return true
+ }
+ if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: AudioObjectID(kAudioObjectSystemObject), address: address, token: token
+ )
+ }
+ if self.isCurrent(generation), let restartToken = self.serviceRestartedToken {
+ self.serviceRestartedToken = nil
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: Self.serviceRestartedAddress,
+ token: restartToken
+ )
}
+ return false
}
- private func resyncMonitoredDevices() {
- let currentIDs = Set(AudioDevice.listInputDevices().map(\.id))
- let previousIDs = Set(self.perDeviceTokens.keys)
-
- for removedID in previousIDs.subtracting(currentIDs) {
- var address = Self.runningSomewhereAddress
- if let token = self.perDeviceTokens[removedID] {
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingDetector, objectID: removedID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
- #endif
- let status = AudioObjectRemovePropertyListenerBlock(removedID, &address, DispatchQueue.main, token)
- #if DEBUG
- AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingDetector, objectID: removedID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
- #endif
- }
- self.perDeviceTokens[removedID] = nil
- self.lastKnownRunning[removedID] = nil
+ private func resyncMonitoredDevices(generation: UInt64, replaceAll: Bool = false) async {
+ guard self.isCurrent(generation) else { return }
+ self.reconciliationEpoch &+= 1
+ let reconciliation = self.reconciliationEpoch
+ let devices = await AudioTopologyListenerExecution.perform { AudioDevice.listInputDevices() }
+ guard self.isCurrent(generation), self.reconciliationEpoch == reconciliation else { return }
+
+ let desired = Dictionary(uniqueKeysWithValues: devices.map { ($0.id, $0.uid) })
+ self.desiredUIDs = desired
+ // A topology event is an incarnation boundary. HAL may have discarded a listener during
+ // a disconnect/reconnect even when AudioObjectID and UID are both reused.
+ let obsolete = self.perDeviceTokens.values.filter {
+ replaceAll || desired[$0.identity.deviceID] != $0.identity.uid
+ }
+ for registration in obsolete {
+ self.perDeviceTokens[registration.identity.deviceID] = nil
+ self.lastKnownRunning[registration.identity.deviceID] = nil
}
+ for registration in obsolete {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: registration.identity.deviceID,
+ address: Self.runningSomewhereAddress,
+ token: registration.token
+ )
+ }
+ guard self.isCurrent(generation), self.reconciliationEpoch == reconciliation else { return }
- for addedID in currentIDs.subtracting(previousIDs) {
- self.lastKnownRunning[addedID] = self.queryIsRunningSomewhere(addedID)
- var address = Self.runningSomewhereAddress
+ for device in devices where self.perDeviceTokens[device.id] == nil {
+ let initialRunning = await AudioTopologyListenerExecution.perform {
+ Self.queryIsRunningSomewhere(device.id)
+ }
+ guard self.isCurrent(generation), self.reconciliationEpoch == reconciliation,
+ self.desiredUIDs[device.id] == device.uid, self.perDeviceTokens[device.id] == nil
+ else { return }
+ self.listenerEpoch &+= 1
+ let identity = ListenerIdentity(
+ deviceID: device.id, uid: device.uid,
+ listenerEpoch: self.listenerEpoch, lifecycleGeneration: generation
+ )
let token: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .meetingDetector, objectID: addedID, selector: kAudioDevicePropertyDeviceIsRunningSomewhere, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingDetector, objectID: device.id, selector: kAudioDevicePropertyDeviceIsRunningSomewhere, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingDetector, objectID: device.id, selector: kAudioDevicePropertyDeviceIsRunningSomewhere, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery) }
#endif
- DispatchQueue.main.async { self?.handleRunningSomewhereChanged(deviceID: addedID) }
+ guard let signal = self else { return }
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ Task { @MainActor in
+ await signal.handleRunningSomewhereChanged(identity: identity)
+ }
+ }
}
#if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingDetector, objectID: addedID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
+ AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingDetector, objectID: device.id, selector: Self.runningSomewhereAddress.mSelector, scope: Self.runningSomewhereAddress.mScope, element: Self.runningSomewhereAddress.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let status = AudioObjectAddPropertyListenerBlock(addedID, &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.add(
+ objectID: device.id, address: Self.runningSomewhereAddress, token: token
+ )
#if DEBUG
- AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingDetector, objectID: addedID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingDetector, objectID: device.id, selector: Self.runningSomewhereAddress.mSelector, scope: Self.runningSomewhereAddress.mScope, element: Self.runningSomewhereAddress.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
- if status == noErr {
- self.perDeviceTokens[addedID] = token
+ guard status == noErr else { continue }
+ guard self.isCurrent(generation), self.reconciliationEpoch == reconciliation,
+ self.desiredUIDs[device.id] == device.uid, self.perDeviceTokens[device.id] == nil
+ else {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: device.id, address: Self.runningSomewhereAddress, token: token
+ )
+ return
}
+ self.perDeviceTokens[device.id] = Registration(identity: identity, token: token)
+ self.lastKnownRunning[device.id] = initialRunning
+
+ // Closes the callback-before-token-commit window: re-read after ownership is visible.
+ await self.handleRunningSomewhereChanged(identity: identity)
}
}
- private func handleRunningSomewhereChanged(deviceID: AudioObjectID) {
- let isRunning = self.queryIsRunningSomewhere(deviceID)
- let wasRunning = self.lastKnownRunning[deviceID] ?? false
- self.lastKnownRunning[deviceID] = isRunning
+ private func handleRunningSomewhereChanged(identity: ListenerIdentity) async {
+ guard self.isCurrent(identity.lifecycleGeneration),
+ self.perDeviceTokens[identity.deviceID]?.identity == identity
+ else { return }
+ let isRunning = await AudioTopologyListenerExecution.perform {
+ Self.queryIsRunningSomewhere(identity.deviceID)
+ }
+ guard self.isCurrent(identity.lifecycleGeneration),
+ self.perDeviceTokens[identity.deviceID]?.identity == identity
+ else { return }
+ let wasRunning = self.lastKnownRunning[identity.deviceID] ?? false
+ self.lastKnownRunning[identity.deviceID] = isRunning
guard isRunning != wasRunning else { return }
self.onEdge?(MicActivityEdge(isActive: isRunning))
}
- private func queryIsRunningSomewhere(_ deviceID: AudioObjectID) -> Bool {
+ private nonisolated static func queryIsRunningSomewhere(_ deviceID: AudioObjectID) -> Bool {
var address = Self.runningSomewhereAddress
var value: UInt32 = 0
var size = UInt32(MemoryLayout<UInt32>.size)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
+ AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
let status = AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, &value)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
+ AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
return status == noErr && value != 0
}
+
+ private func isCurrent(_ generation: UInt64) -> Bool {
+ self.started && self.lifecycleGeneration == generation
+ }
+
+ private func handleServiceRestart(generation: UInt64) async {
+ guard self.isCurrent(generation) else { return }
+ // coreaudiod discarded these registrations. Drop ownership without attempting removal,
+ // invalidate every callback/query, then build one fresh listener set.
+ self.lifecycleGeneration &+= 1
+ self.reconciliationEpoch &+= 1
+ let replacementGeneration = self.lifecycleGeneration
+ self.deviceListToken = nil
+ self.serviceRestartedToken = nil
+ self.perDeviceTokens = [:]
+ self.lastKnownRunning = [:]
+ self.desiredUIDs = [:]
+ await self.bootstrap(generation: replacementGeneration)
+ }
+
+ private nonisolated static func remove(
+ deviceListToken: AudioObjectPropertyListenerBlock?,
+ serviceRestartedToken: AudioObjectPropertyListenerBlock?,
+ registrations: [Registration]
+ ) async {
+ if let deviceListToken {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: Self.devicesAddress,
+ token: deviceListToken
+ )
+ }
+ if let serviceRestartedToken {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: Self.serviceRestartedAddress,
+ token: serviceRestartedToken
+ )
+ }
+ for registration in registrations {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: registration.identity.deviceID,
+ address: Self.runningSomewhereAddress,
+ token: registration.token
+ )
+ }
+ }
}
@MainActor
final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
- func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) -> Bool {
- guard #available(macOS 14.4, *) else { return false }
- return Self.processObjectIDs().contains { processID in
- guard let bundleIdentifier = Self.bundleIdentifier(for: processID), bundleIdentifiers.contains(bundleIdentifier) else { return false }
- return Self.isRunning(processID, selector: kAudioProcessPropertyIsRunningInput)
- || Self.isRunning(processID, selector: kAudioProcessPropertyIsRunningOutput)
+ func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) async -> Bool {
+ await AudioTopologyListenerExecution.perform {
+ guard #available(macOS 14.4, *) else { return false }
+ return Self.processObjectIDs().contains { processID in
+ guard let bundleIdentifier = Self.bundleIdentifier(for: processID), bundleIdentifiers.contains(bundleIdentifier) else { return false }
+ return Self.isRunning(processID, selector: kAudioProcessPropertyIsRunningInput)
+ || Self.isRunning(processID, selector: kAudioProcessPropertyIsRunningOutput)
+ }
}
}
- private static func processObjectIDs() -> [AudioObjectID] {
+ nonisolated private static func processObjectIDs() -> [AudioObjectID] {
var address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyProcessObjectList,
mScope: kAudioObjectPropertyScopeGlobal,
@@ -331,26 +507,26 @@ final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
var size: UInt32 = 0
let systemObject = AudioObjectID(kAudioObjectSystemObject)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
+ AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
let sizeStatus = AudioObjectGetPropertyDataSize(systemObject, &address, 0, nil, &size)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: sizeStatus)
+ AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: sizeStatus)
#endif
guard sizeStatus == noErr else { return [] }
var processIDs = Array(repeating: AudioObjectID(), count: Int(size) / MemoryLayout<AudioObjectID>.size)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
+ AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
let dataStatus = AudioObjectGetPropertyData(systemObject, &address, 0, nil, &size, &processIDs)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: dataStatus)
+ AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: dataStatus)
#endif
guard dataStatus == noErr else { return [] }
return processIDs
}
- private static func bundleIdentifier(for processID: AudioObjectID) -> String? {
+ nonisolated private static func bundleIdentifier(for processID: AudioObjectID) -> String? {
var address = AudioObjectPropertyAddress(
mSelector: kAudioProcessPropertyBundleID,
mScope: kAudioObjectPropertyScopeGlobal,
@@ -359,17 +535,17 @@ final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
var value: CFString = "" as CFString
var size = UInt32(MemoryLayout<CFString>.size)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
+ AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
let status = AudioObjectGetPropertyData(processID, &address, 0, nil, &size, &value)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
+ AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
guard status == noErr else { return nil }
return value as String
}
- private static func isRunning(_ processID: AudioObjectID, selector: AudioObjectPropertySelector) -> Bool {
+ nonisolated private static func isRunning(_ processID: AudioObjectID, selector: AudioObjectPropertySelector) -> Bool {
var address = AudioObjectPropertyAddress(
mSelector: selector,
mScope: kAudioObjectPropertyScopeGlobal,
@@ -378,11 +554,11 @@ final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
var value: UInt32 = 0
var size = UInt32(MemoryLayout<UInt32>.size)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener)
+ AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
let status = AudioObjectGetPropertyData(processID, &address, 0, nil, &size, &value)
#if DEBUG
- AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .mainControl, phase: .listener, status: status)
+ AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
return status == noErr && value != 0
}
@@ -557,7 +733,7 @@ final class LiveDetectionActivityGate: DetectionActivityGate {
guard self.isIdle else { return .busy }
guard CGPreflightScreenCaptureAccess() else { return .needsSetup(.screenRecording) }
guard AVCaptureDeviceAuthorization.isMicrophoneAuthorized() else { return .needsSetup(.microphone) }
- guard !AudioDevice.listInputDevices().isEmpty else { return .needsSetup(.noInputDevice) }
+ guard AppServices.shared.audioObserver.hasAvailableInputDevice else { return .needsSetup(.noInputDevice) }
guard MeetingStorageReadiness.hasSufficientFreeSpace() else { return .needsSetup(.storage) }
return .ready
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift b/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
index d8c12a2c..dc886327 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
@@ -48,6 +48,7 @@ final class MeetingAutoDetector {
var onHealthChanged: ((Health) -> Void)?
private struct CandidateRecord {
+ var incarnation: UInt64
var pid: Int32
var bundleIdentifier: String
var tier: MeetingDetectionTier
@@ -98,6 +99,8 @@ final class MeetingAutoDetector {
private var dismissedBundleUntil: [String: Date] = [:]
private var dismissalTimestamps: [Date] = []
private var pollTask: Task<Void, Never>?
+ private var runGeneration: UInt64 = 0
+ private var candidateIncarnation: UInt64 = 0
private var lastBrowserPollAt: [Int32: Date] = [:]
private var titleEnrichmentGeneration: [Int32: UInt64] = [:]
private var titleEnrichmentInFlight: Set<Int32> = []
@@ -129,6 +132,8 @@ final class MeetingAutoDetector {
func start() {
guard self.pollTask == nil else { return }
+ self.runGeneration &+= 1
+ let runGeneration = self.runGeneration
self.workspaceEvents.onEvent = { [weak self] event in self?.handleWorkspaceEvent(event, at: self?.clock.now() ?? Date()) }
self.workspaceEvents.start(isRegistryApp: { MeetingAppRegistry.tier(forBundleIdentifier: $0) != nil }) { [weak self] backfill in
self?.handleBackfill(backfill)
@@ -141,12 +146,13 @@ final class MeetingAutoDetector {
while !Task.isCancelled {
try? await Task.sleep(nanoseconds: 2_000_000_000)
guard let self else { return }
- await self.pollTick()
+ await self.pollTick(runGeneration: runGeneration)
}
}
}
func stop() {
+ self.runGeneration &+= 1
self.pollTask?.cancel()
self.pollTask = nil
self.workspaceEvents.stop()
@@ -156,10 +162,12 @@ final class MeetingAutoDetector {
self.removeTransientDisarmObservers()
}
- private func pollTick() async {
+ private func pollTick(runGeneration: UInt64) async {
+ guard self.isCurrentRun(runGeneration) else { return }
let now = self.clock.now()
if self.isNativeDetectionEnabled() {
- self.pollAudioProcessActivity(at: now)
+ await self.pollAudioProcessActivity(at: now, expectedRunGeneration: runGeneration)
+ guard self.isCurrentRun(runGeneration) else { return }
}
if self.shouldCollectEvidence(at: now) {
if self.isNativeDetectionEnabled() {
@@ -169,12 +177,18 @@ final class MeetingAutoDetector {
}
}
if self.isBrowserDetectionEnabled() {
- await self.pollBrowserTabsIfDue(at: now)
+ await self.pollBrowserTabsIfDue(at: now, runGeneration: runGeneration)
+ guard self.isCurrentRun(runGeneration) else { return }
}
}
+ guard self.isCurrentRun(runGeneration) else { return }
self.tick(at: self.clock.now())
}
+ private func isCurrentRun(_ generation: UInt64) -> Bool {
+ !Task.isCancelled && self.runGeneration == generation && self.pollTask != nil
+ }
+
private func shouldCollectEvidence(at now: Date) -> Bool {
if !self.episodesByKey.isEmpty { return true }
return self.records.values.contains { record in
@@ -183,12 +197,16 @@ final class MeetingAutoDetector {
}
}
- private func pollBrowserTabsIfDue(at now: Date) async {
+ private func pollBrowserTabsIfDue(at now: Date, runGeneration: UInt64) async {
for (pid, record) in self.records where record.tier == .browserTier2 {
+ guard self.isCurrentRun(runGeneration), self.isBrowserDetectionEnabled() else { return }
let lastPoll = self.lastBrowserPollAt[pid]
guard lastPoll == nil || now.timeIntervalSince(lastPoll!) >= 2 else { continue }
- self.lastBrowserPollAt[pid] = now
let url = await self.browserTabReader.frontmostTabURL(bundleIdentifier: record.bundleIdentifier, processID: pid)
+ guard self.isCurrentRun(runGeneration), self.isBrowserDetectionEnabled(),
+ self.records[pid]?.incarnation == record.incarnation
+ else { continue }
+ self.lastBrowserPollAt[pid] = now
self.handleBrowserTabURL(url, pid: pid, bundleIdentifier: record.bundleIdentifier, at: self.clock.now())
}
}
@@ -225,7 +243,13 @@ final class MeetingAutoDetector {
private func armIfNeeded(pid: Int32, bundleIdentifier: String, tier: MeetingDetectionTier) {
guard self.records[pid] == nil else { return }
- self.records[pid] = CandidateRecord(pid: pid, bundleIdentifier: bundleIdentifier, tier: tier)
+ self.candidateIncarnation &+= 1
+ self.records[pid] = CandidateRecord(
+ incarnation: self.candidateIncarnation,
+ pid: pid,
+ bundleIdentifier: bundleIdentifier,
+ tier: tier
+ )
DebugLogger.shared.log("record-armed bundle=\(bundleIdentifier)", source: "MeetingAutoDetector")
}
@@ -255,9 +279,17 @@ final class MeetingAutoDetector {
}
}
- func pollAudioProcessActivity(at now: Date) {
+ func pollAudioProcessActivity(at now: Date) async {
+ await self.pollAudioProcessActivity(at: now, expectedRunGeneration: nil)
+ }
+
+ private func pollAudioProcessActivity(at now: Date, expectedRunGeneration: UInt64?) async {
for (pid, record) in self.records where record.tier == .nativeTier1 {
- let isActive = self.audioProcessActivity.isAudioActive(forBundleIdentifiers: MeetingAppRegistry.audioProcessBundleIdentifiers(for: record.bundleIdentifier))
+ let isActive = await self.audioProcessActivity.isAudioActive(forBundleIdentifiers: MeetingAppRegistry.audioProcessBundleIdentifiers(for: record.bundleIdentifier))
+ guard !Task.isCancelled,
+ expectedRunGeneration.map({ self.runGeneration == $0 && self.pollTask != nil && self.isNativeDetectionEnabled() }) ?? true,
+ self.records[pid]?.incarnation == record.incarnation
+ else { continue }
self.handleAudioProcessActivity(isActive, pid: pid, at: now)
}
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
index 65de887c..82f1b207 100644
--- a/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
@@ -7,6 +7,10 @@ import IOKit.pwr_mgt
import ScreenCaptureKit
nonisolated protocol MeetingCaptureControlling: Sendable {
+ /// Any user-facing permission request must complete before the coordinator reserves
+ /// the exclusive ASR meeting lease.
+ func preflightPermissions() async throws
+
func start(
session: MeetingSession,
configuration: MeetingCaptureConfiguration,
@@ -32,6 +36,10 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
private var displaySleepAssertionID: IOPMAssertionID?
private var terminating = false
+ func preflightPermissions() async throws {
+ try await Self.requestMicrophonePermissionIfNeeded()
+ }
+
func start(
session: MeetingSession,
configuration: MeetingCaptureConfiguration,
@@ -46,7 +54,7 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
self.startingSessionID = session.id
defer { self.startingSessionID = nil }
try Self.preflightStorage(at: sessionDirectory)
- try await Self.ensureMicrophonePermission()
+ try Self.verifyMicrophonePermission()
let tracks = Self.makeTracks(session: session, configuration: configuration)
let writers = try tracks.map { track in
@@ -158,14 +166,29 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
}
let task = Task<MeetingCaptureStopResult, Error> {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .runtimeStop)
+ #endif
let runtimeFailure: Error?
do {
try await activeCapture.runtime.stop()
runtimeFailure = nil
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .runtimeStop)
+ #endif
} catch {
runtimeFailure = error
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .runtimeStop, status: -1)
+ #endif
}
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .writerStop)
+ #endif
let tracks = await Self.stopWriters(activeCapture.writers)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .writerStop)
+ #endif
let result = MeetingCaptureStopResult(tracks: tracks, stoppedAt: Date())
if let runtimeFailure {
throw MeetingCaptureError.captureStopFailed(
@@ -178,12 +201,24 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
self.stopTask = task
do {
let result = try await task.value
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .captureStop)
+ #endif
self.activeCapture = nil
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .captureStop)
+ #endif
self.stopTask = nil
self.releaseDisplaySleepAssertion()
return result
} catch {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .captureStop)
+ #endif
self.activeCapture = nil
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCaptureEngine, queueRole: .actorControl, phase: .captureStop, status: -1)
+ #endif
self.stopTask = nil
self.releaseDisplaySleepAssertion()
throw error
@@ -261,7 +296,7 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
return tracks
}
- private nonisolated static func ensureMicrophonePermission() async throws {
+ private nonisolated static func requestMicrophonePermissionIfNeeded() async throws {
switch AVCaptureDevice.authorizationStatus(for: .audio) {
case .authorized:
return
@@ -277,6 +312,12 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
}
}
+ private nonisolated static func verifyMicrophonePermission() throws {
+ guard AVCaptureDevice.authorizationStatus(for: .audio) == .authorized else {
+ throw MeetingCaptureError.microphonePermissionDenied
+ }
+ }
+
static func currentOutputRouteSnapshot() -> MeetingOutputRouteSnapshot {
guard let device = AudioDevice.getDefaultOutputDevice() else {
return MeetingOutputRouteSnapshot(deviceExists: false, isBluetooth: false, isBuiltIn: false, isHeadphonesDataSource: false)
@@ -411,12 +452,19 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void,
liveAudioHandler: (@Sendable (MeetingAudioTrackKind, CMSampleBuffer) -> Void)?
) async throws -> ScreenCaptureMeetingRuntime {
- let built = try await Self.buildStream(application: application, microphone: microphone, includeMicrophone: includeMicrophone)
+ let resolvedMicrophone = includeMicrophone
+ ? try MeetingAVCaptureMicrophoneResolver.resolve(microphone)
+ : microphone
+ let built = try await Self.buildStream(
+ application: application,
+ microphone: resolvedMicrophone,
+ includeMicrophone: includeMicrophone
+ )
let runtime = ScreenCaptureMeetingRuntime(
stream: built.stream,
scope: built.scope,
application: application,
- microphone: microphone,
+ microphone: resolvedMicrophone,
includeMicrophone: includeMicrophone,
applicationWriter: applicationWriter,
microphoneWriter: microphoneWriter,
@@ -533,6 +581,10 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
}
func stop() async throws {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .screenCapture, queueRole: .actorControl, phase: .screenCaptureStop)
+ defer { AudioTopologyDiagnostics.record(.phaseEnd, owner: .screenCapture, queueRole: .actorControl, phase: .screenCaptureStop) }
+ #endif
self.markStopping()
let (currentStream, streamAlreadyDead) = self.stateLock.withLock { (self.stream, self.rebuilding) }
// Mid-rebuild the current stream already stopped on its own; stopCapture would only time out.
@@ -705,7 +757,10 @@ private final nonisolated class InRoomMicrophoneCaptureRuntime: NSObject, Meetin
self.eventHandler = eventHandler
self.liveAudioHandler = liveAudioHandler
super.init()
- guard let device = AVCaptureDevice(uniqueID: microphone.captureDeviceID) else {
+ let resolvedMicrophone = try MeetingAVCaptureMicrophoneResolver.resolve(microphone)
+ guard let captureDeviceID = resolvedMicrophone.avCaptureDeviceID,
+ let device = AVCaptureDevice(uniqueID: captureDeviceID)
+ else {
throw MeetingCaptureError.microphoneUnavailable
}
let input = try AVCaptureDeviceInput(device: device)
@@ -724,6 +779,9 @@ private final nonisolated class InRoomMicrophoneCaptureRuntime: NSObject, Meetin
}
deinit {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.ownerWillDeinit, owner: .avCapture, queueRole: .callbackCurrent, phase: .avCaptureStop)
+ #endif
for observer in self.notificationObservers {
NotificationCenter.default.removeObserver(observer)
}
@@ -740,6 +798,10 @@ private final nonisolated class InRoomMicrophoneCaptureRuntime: NSObject, Meetin
}
func stop() async throws {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .avCapture, queueRole: .dedicatedControl, phase: .avCaptureStop)
+ defer { AudioTopologyDiagnostics.record(.phaseEnd, owner: .avCapture, queueRole: .dedicatedControl, phase: .avCaptureStop) }
+ #endif
self.stateLock.withLock { self.stopping = true }
let result = await withCheckedContinuation { continuation in
let completion = MeetingOneShotCompletion(continuation)
@@ -818,27 +880,95 @@ private final nonisolated class MeetingOneShotValue<T: Sendable>: @unchecked Sen
}
}
-/// Reusable AVCaptureSession mic component: `wasInterrupted` restarts only if still not running; runtime errors get ONE bounded restart.
+/// Serializes an uninterruptible framework start with its retirement barrier.
+/// `stop()` cannot complete until an already-enqueued `start()` has returned and
+/// the stop operation has run behind it on the same queue.
+final nonisolated class MeetingSerializedCaptureLifecycle: @unchecked Sendable {
+ private let queue: DispatchQueue
+ private let admissionLock = NSLock()
+ private let startOperation: @Sendable () -> Bool
+ private let stopOperation: @Sendable () -> Void
+ private var nextOrdinal: UInt64 = 0
+ private var retirementOrdinal: UInt64?
+
+ init(
+ label: String = "com.fluidvoice.meeting.avcapture.control",
+ start: @escaping @Sendable () -> Bool,
+ stop: @escaping @Sendable () -> Void
+ ) {
+ self.queue = DispatchQueue(label: label)
+ self.startOperation = start
+ self.stopOperation = stop
+ }
+
+ func start() async -> Bool {
+ await withCheckedContinuation { continuation in
+ let enqueued = self.admissionLock.withLock { () -> Bool in
+ guard self.retirementOrdinal == nil else { return false }
+ self.nextOrdinal &+= 1
+ let startOrdinal = self.nextOrdinal
+ // Enqueue while admission is locked. A competing stop therefore
+ // either queues behind this admitted start or seals admission first.
+ self.queue.async {
+ let mayRun = self.admissionLock.withLock {
+ self.retirementOrdinal.map { startOrdinal < $0 } ?? true
+ }
+ continuation.resume(returning: mayRun ? self.startOperation() : false)
+ }
+ return true
+ }
+ if !enqueued { continuation.resume(returning: false) }
+ }
+ }
+
+ func stop() async {
+ await withCheckedContinuation { continuation in
+ self.admissionLock.withLock {
+ if self.retirementOrdinal == nil {
+ self.nextOrdinal &+= 1
+ self.retirementOrdinal = self.nextOrdinal
+ }
+ // The permanent seal and barrier enqueue are one atomic admission
+ // operation, so no start can slip behind stopRunning.
+ self.queue.async {
+ self.stopOperation()
+ continuation.resume()
+ }
+ }
+ }
+ }
+
+ func perform(_ work: @escaping @Sendable () -> Void) async {
+ await withCheckedContinuation { continuation in
+ self.queue.async {
+ work()
+ continuation.resume()
+ }
+ }
+ }
+}
+
+/// Reusable AVCaptureSession mic component: `wasInterrupted` restarts only if still not running; runtime errors get ONE serialized restart.
final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
AVCaptureAudioDataOutputSampleBufferDelegate, @unchecked Sendable
{
private static let interruptionEndedWaitSeconds: Double = 3
- private static let restartSupervisionSeconds: Double = 3
- private static let stopTimeoutSeconds: Double = 3
let deviceUID: String?
let deviceName: String
- private let session = AVCaptureSession()
- private let output = AVCaptureAudioDataOutput()
- private let controlQueue = DispatchQueue(label: "com.fluidvoice.meeting.avcapture.control")
+ private let session: AVCaptureSession
+ private let output: AVCaptureAudioDataOutput
+ private let lifecycle: MeetingSerializedCaptureLifecycle
private let stateLock = NSLock()
private let onSample: @Sendable (CMSampleBuffer) -> Void
private let onTerminalStop: @Sendable (String) -> Void
private var stopping = false
+ private var fullyRetired = false
private var emittedTerminalStop = false
private var runtimeErrorRestarted = false
private var didStopRunningRestarted = false
+ private var deliveredSampleCount: UInt64 = 0
private var interruptionEndWaiters: [MeetingOneShotValue<Void>] = []
init(
@@ -846,12 +976,36 @@ final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
onSample: @escaping @Sendable (CMSampleBuffer) -> Void,
onTerminalStop: @escaping @Sendable (String) -> Void
) throws {
+ let session = AVCaptureSession()
+ let output = AVCaptureAudioDataOutput()
+ self.session = session
+ self.output = output
+ self.lifecycle = MeetingSerializedCaptureLifecycle(
+ start: {
+ session.startRunning()
+ return session.isRunning
+ },
+ stop: {
+ if session.isRunning { session.stopRunning() }
+ // `stopRunning()` only stops delivery; the session still owns its
+ // input device until its graph is dismantled. VPIO must never be
+ // enabled while those AVFoundation objects retain the device.
+ output.setSampleBufferDelegate(nil, queue: nil)
+ session.beginConfiguration()
+ for sessionOutput in session.outputs { session.removeOutput(sessionOutput) }
+ for sessionInput in session.inputs { session.removeInput(sessionInput) }
+ session.commitConfiguration()
+ }
+ )
self.deviceUID = microphone.coreAudioUID
self.deviceName = microphone.displayName
self.onSample = onSample
self.onTerminalStop = onTerminalStop
super.init()
- guard let device = AVCaptureDevice(uniqueID: microphone.captureDeviceID) else {
+ let resolvedMicrophone = try MeetingAVCaptureMicrophoneResolver.resolve(microphone)
+ guard let captureDeviceID = resolvedMicrophone.avCaptureDeviceID,
+ let device = AVCaptureDevice(uniqueID: captureDeviceID)
+ else {
throw MeetingCaptureError.microphoneUnavailable
}
let input = try AVCaptureDeviceInput(device: device)
@@ -870,66 +1024,60 @@ final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
}
deinit {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.ownerWillDeinit, owner: .avCapture, queueRole: .callbackCurrent, phase: .avCaptureStop)
+ #endif
for observer in self.observers { NotificationCenter.default.removeObserver(observer) }
}
private var observers: [NSObjectProtocol] = []
- /// Bounded retry at 0.5/1/2s on `startRunning` failure plus a supervised overall timeout.
+ /// Runs `startRunning` on the serial control queue. This intentionally has no
+ /// detached timeout: a timed-out AVFoundation call could otherwise finish after
+ /// meeting ownership was released and collide with dictation.
func start() async throws {
for (attempt, delay) in ([0] + Self.startRetryDelays).enumerated() {
if attempt > 0 { try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) }
if self.stateLock.withLock({ self.stopping }) { throw CancellationError() }
- let started = await MeetingSupervisedTimeout.run(seconds: Self.startSupervisionSeconds) { [session] in
- await withCheckedContinuation { (continuation: CheckedContinuation<Bool, Never>) in
- let box = MeetingOneShotValue<Bool>(continuation)
- self.controlQueue.async {
- session.startRunning()
- box.resume(session.isRunning)
- }
- }
- }
- if started == true { return }
+ if await self.startRunningOnControlQueue() { return }
}
throw MeetingCaptureError.captureStartFailed("The microphone did not start.")
}
private static let startRetryDelays: [Double] = [0.5, 1.0, 2.0]
- private static let startSupervisionSeconds: Double = 4
+
+ private func startRunningOnControlQueue() async -> Bool {
+ await self.lifecycle.start()
+ }
func stop() async throws {
- self.stateLock.lock()
- self.stopping = true
- let waiters = self.interruptionEndWaiters
- self.interruptionEndWaiters = []
- self.stateLock.unlock()
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .avCapture, queueRole: .dedicatedControl, phase: .avCaptureStop)
+ defer { AudioTopologyDiagnostics.record(.phaseEnd, owner: .avCapture, queueRole: .dedicatedControl, phase: .avCaptureStop) }
+ #endif
+ let waiters = self.stateLock.withLock { () -> [MeetingOneShotValue<Void>] in
+ self.stopping = true
+ defer { self.interruptionEndWaiters = [] }
+ return self.interruptionEndWaiters
+ }
waiters.forEach { $0.resume(()) }
- let result = await withCheckedContinuation { continuation in
- let completion = MeetingOneShotCompletion(continuation)
- self.controlQueue.async { [session] in
- if session.isRunning { session.stopRunning() }
- completion.resume(.success)
- }
- DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + Self.stopTimeoutSeconds) {
- completion.resume(.failure("Timed out while stopping microphone capture."))
- }
+ await self.lifecycle.stop()
+ let observers = self.stateLock.withLock { () -> [NSObjectProtocol] in
+ defer { self.observers.removeAll() }
+ return self.observers
}
- if case let .failure(detail) = result {
- throw MeetingCaptureRuntimeError.stopFailed(detail)
+ for observer in observers { NotificationCenter.default.removeObserver(observer) }
+ await self.lifecycle.perform { [self] in
+ let proven = !self.session.isRunning
+ && self.session.inputs.isEmpty
+ && self.session.outputs.isEmpty
+ self.stateLock.withLock { self.fullyRetired = proven }
}
}
- /// `onComplete(false)` means the supervision window elapsed before `stop()` finished.
- func stopDetached(supervisedSeconds: Double, onComplete: @escaping @Sendable (Bool) -> Void) {
- Task { [weak self] in
- guard let self else { onComplete(false); return }
- let completed = await MeetingSupervisedTimeout.run(seconds: supervisedSeconds) {
- try? await self.stop()
- return true
- }
- onComplete(completed == true)
- }
+ func isFullyRetired() -> Bool {
+ self.stateLock.withLock { self.fullyRetired }
}
func captureOutput(
@@ -939,11 +1087,16 @@ final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
) {
self.stateLock.lock()
let shouldAccept = !self.stopping
+ if shouldAccept { self.deliveredSampleCount &+= 1 }
self.stateLock.unlock()
guard shouldAccept else { return }
self.onSample(sampleBuffer)
}
+ func sampleCount() -> UInt64 {
+ self.stateLock.withLock { self.deliveredSampleCount }
+ }
+
private func observeSessionLifecycle() {
let center = NotificationCenter.default
self.observers = [
@@ -1027,17 +1180,8 @@ final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
private func attemptSupervisedRestart(reason: String) async {
guard !(self.stateLock.withLock({ self.stopping })) else { return }
- let restarted = await MeetingSupervisedTimeout.run(seconds: Self.restartSupervisionSeconds) { [session] in
- await withCheckedContinuation { (continuation: CheckedContinuation<Bool, Never>) in
- let box = MeetingOneShotValue<Bool>(continuation)
- self.controlQueue.async {
- guard !session.isRunning else { box.resume(true); return }
- session.startRunning()
- box.resume(session.isRunning)
- }
- }
- }
- guard restarted != true, !(self.stateLock.withLock({ self.stopping })) else { return }
+ let restarted = await self.startRunningOnControlQueue()
+ guard !restarted, !(self.stateLock.withLock({ self.stopping })) else { return }
self.emitTerminalStopIfNeeded(detail: reason)
}
@@ -1052,6 +1196,24 @@ final nonisolated class MeetingAVCaptureMicrophoneComponent: NSObject,
}
}
+/// Identity fence for terminal callbacks. A retired candidate from an earlier
+/// retry can share the transition generation but must never fail its successor.
+final nonisolated class MeetingAVCaptureAttemptToken: @unchecked Sendable {}
+
+nonisolated enum MeetingAVCaptureFailureAdmission {
+ static func accepts(
+ currentGeneration: UInt64,
+ eventGeneration: UInt64,
+ activeToken: MeetingAVCaptureAttemptToken?,
+ eventToken: MeetingAVCaptureAttemptToken,
+ stopRequested: Bool
+ ) -> Bool {
+ !stopRequested
+ && currentGeneration == eventGeneration
+ && activeToken === eventToken
+ }
+}
+
/// Bounds wall-clock time even when `operation` ignores cancellation; a late operation finishes detached, result discarded.
nonisolated enum MeetingSupervisedTimeout {
static func run<T: Sendable>(seconds: Double, operation: @escaping @Sendable () async -> T) async -> T? {
@@ -1194,7 +1356,7 @@ nonisolated enum MeetingCaptureError: LocalizedError {
}
}
-enum MeetingCaptureSourceCatalog {
+nonisolated enum MeetingCaptureSourceCatalog {
static let preferredFallbackBundleIdentifiers = [
"us.zoom.xos",
"com.google.Chrome",
@@ -1254,77 +1416,113 @@ enum MeetingCaptureSourceCatalog {
.sorted { $0.displayName.localizedCaseInsensitiveCompare($1.displayName) == .orderedAscending }
}
- static func availableMicrophones() -> [MeetingMicrophoneIdentity] {
- let coreAudioDevices = AudioDevice.listInputDevices()
- #if DEBUG
- if Self.usesCoreAudioOnlyIsolationCatalog {
- return coreAudioDevices.compactMap { device in
- guard device.uid.isEmpty == false else { return nil }
- return MeetingMicrophoneIdentity(
- captureDeviceID: device.uid,
- coreAudioUID: device.uid,
- displayName: device.name
- )
- }
+ static func microphoneSnapshot() async -> MeetingMicrophoneCatalogSnapshot {
+ await AudioTopologyListenerExecution.perform {
+ let identities = AudioDevice.listInputDevicesRefreshingLiveness().compactMap { device -> MeetingMicrophoneIdentity? in
+ guard device.isAlive, device.uid.isEmpty == false else { return nil }
+ return MeetingMicrophoneIdentity(
+ captureDeviceID: device.uid,
+ coreAudioUID: device.uid,
+ displayName: device.name
+ )
}
- #endif
- #if DEBUG
- AudioTopologyDiagnostics.record(.avfDiscoveryBegin, owner: .meetingCatalog, queueRole: .actorControl, phase: .catalog)
- #endif
- let devices = AVCaptureDevice.DiscoverySession(
- deviceTypes: [.microphone, .external],
- mediaType: .audio,
- position: .unspecified
- ).devices
- #if DEBUG
- AudioTopologyDiagnostics.record(.avfDiscoveryEnd, owner: .meetingCatalog, queueRole: .actorControl, phase: .catalog, status: Int32(clamping: devices.count))
- #endif
- return devices.map { device in
- let sameName = coreAudioDevices.filter { $0.name == device.localizedName }
- return MeetingMicrophoneIdentity(
- captureDeviceID: device.uniqueID,
- coreAudioUID: sameName.count == 1 ? sameName[0].uid : nil,
- displayName: device.localizedName
+ return MeetingMicrophoneCatalogSnapshot(
+ identities: identities,
+ defaultCoreAudioUID: AudioDevice.getDefaultInputDevice()?.uid
)
}
}
- static func defaultMicrophone(preferredCoreAudioUID: String? = nil) throws -> MeetingMicrophoneIdentity {
- let microphones = self.availableMicrophones()
+ static func availableMicrophones() async -> [MeetingMicrophoneIdentity] {
+ await self.microphoneSnapshot().identities
+ }
+
+ static func defaultMicrophone(preferredCoreAudioUID: String? = nil) async throws -> MeetingMicrophoneIdentity {
+ let snapshot = await self.microphoneSnapshot()
+ let microphones = snapshot.identities
if let preferredCoreAudioUID,
let preferred = microphones.first(where: { $0.coreAudioUID == preferredCoreAudioUID })
{
return preferred
}
- #if DEBUG
- if Self.usesCoreAudioOnlyIsolationCatalog {
- guard let defaultUID = AudioDevice.getDefaultInputDevice()?.uid,
- let microphone = microphones.first(where: { $0.coreAudioUID == defaultUID })
- else { throw MeetingCaptureError.microphoneUnavailable }
- return microphone
- }
- #endif
- #if DEBUG
- AudioTopologyDiagnostics.record(.avfDefaultBegin, owner: .meetingCatalog, queueRole: .actorControl, phase: .catalog)
- #endif
- let defaultDevice = AVCaptureDevice.default(for: .audio)
- #if DEBUG
- AudioTopologyDiagnostics.record(.avfDefaultEnd, owner: .meetingCatalog, queueRole: .actorControl, phase: .catalog, status: defaultDevice == nil ? -1 : noErr)
- #endif
- guard let defaultDevice,
- let microphone = microphones.first(where: { $0.captureDeviceID == defaultDevice.uniqueID })
+ guard let defaultUID = snapshot.defaultCoreAudioUID,
+ let microphone = microphones.first(where: { $0.coreAudioUID == defaultUID })
else { throw MeetingCaptureError.microphoneUnavailable }
return microphone
}
+}
- #if DEBUG
- /// Phase-0 isolation only. This is intentionally meeting-catalog scoped and inert unless
- /// the signed Debug app is launched with the exact opt-in value. It does not alter the
- /// dictation catalog, preferred microphone, or Core Audio listener ownership.
- private static var usesCoreAudioOnlyIsolationCatalog: Bool {
- ProcessInfo.processInfo.environment["FLUIDVOICE_MEETING_COREAUDIO_CATALOG_ISOLATION"] == "1"
+nonisolated struct MeetingMicrophoneCatalogSnapshot: Sendable, Equatable {
+ var identities: [MeetingMicrophoneIdentity]
+ var defaultCoreAudioUID: String?
+}
+
+nonisolated struct MeetingAVCaptureDeviceDescriptor: Equatable, Sendable {
+ var uniqueID: String
+ var displayName: String
+}
+
+/// Pure, fail-closed bridge from FluidVoice's stable Core Audio selection identity
+/// to the AVCaptureDevice unique ID required by ScreenCaptureKit/AVCaptureSession.
+nonisolated enum MeetingAVCaptureDeviceSelection {
+ static func select(
+ microphone: MeetingMicrophoneIdentity,
+ candidates: [MeetingAVCaptureDeviceDescriptor]
+ ) -> String? {
+ func exact(_ identifier: String?) -> String? {
+ guard let identifier, !identifier.isEmpty else { return nil }
+ return candidates.first(where: { $0.uniqueID == identifier })?.uniqueID
}
- #endif
+
+ if let explicit = exact(microphone.avCaptureDeviceID) { return explicit }
+ if microphone.identitySchemaVersion == nil,
+ let legacy = exact(microphone.captureDeviceID)
+ {
+ return legacy
+ }
+ if let coreAudio = exact(microphone.coreAudioUID) { return coreAudio }
+
+ let normalizedName = Self.normalize(microphone.displayName)
+ guard !normalizedName.isEmpty else { return nil }
+ let nameMatches = candidates.filter { Self.normalize($0.displayName) == normalizedName }
+ guard nameMatches.count == 1 else { return nil }
+ return nameMatches[0].uniqueID
+ }
+
+ private static func normalize(_ value: String) -> String {
+ value.trimmingCharacters(in: .whitespacesAndNewlines)
+ .folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current)
+ }
+}
+
+/// AVFoundation discovery is intentionally isolated here. Callers are capture adapters
+/// that run only after the ASR meeting lease has drained dictation audio ownership.
+nonisolated enum MeetingAVCaptureMicrophoneResolver {
+ static func resolve(_ microphone: MeetingMicrophoneIdentity) throws -> MeetingMicrophoneIdentity {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.avfDiscoveryBegin, owner: .meetingCatalog, queueRole: .actorControl, phase: .catalog)
+ #endif
+ let devices = AVCaptureDevice.DiscoverySession(
+ deviceTypes: [.microphone, .external],
+ mediaType: .audio,
+ position: .unspecified
+ ).devices
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.avfDiscoveryEnd, owner: .meetingCatalog, queueRole: .actorControl, phase: .catalog, status: Int32(clamping: devices.count))
+ #endif
+ let descriptors = devices.map {
+ MeetingAVCaptureDeviceDescriptor(uniqueID: $0.uniqueID, displayName: $0.localizedName)
+ }
+ guard let uniqueID = MeetingAVCaptureDeviceSelection.select(
+ microphone: microphone,
+ candidates: descriptors
+ ) else {
+ throw MeetingCaptureError.microphoneUnavailable
+ }
+ var resolved = microphone
+ resolved.avCaptureDeviceID = uniqueID
+ return resolved
+ }
}
private extension CGRect {
@@ -1339,6 +1537,222 @@ nonisolated enum MeetingCapturePathDecision: Sendable, Equatable {
case screenCaptureKit(reason: String)
}
+nonisolated enum MeetingCaptureTransitionPolicy {
+ static let allowsWithinSessionVoiceProcessingUpgrade = true
+ static let maximumRouteRecoveryAttempts = 3
+ static let recoveryDwellSeconds: Double = 2
+
+ static func shouldAttemptVoiceProcessingRecovery(
+ routeDrivenFallback: Bool,
+ attempts: Int,
+ microphone: MeetingMicrophoneIdentity,
+ outputRoute: MeetingOutputRouteSnapshot
+ ) -> Bool {
+ MeetingVoiceProcessingUpgradeDwell.shouldArm(
+ phaseIsAVCapture: true,
+ dwellAlreadyArmed: false,
+ quarantined: false,
+ stopRequested: false,
+ routeDrivenFallback: routeDrivenFallback,
+ attempts: attempts,
+ microphone: microphone,
+ outputRoute: outputRoute
+ )
+ }
+}
+
+nonisolated enum MeetingVoiceProcessingUpgradeDwell {
+ static var seconds: Double { MeetingCaptureTransitionPolicy.recoveryDwellSeconds }
+
+ static func shouldArm(
+ phaseIsAVCapture: Bool,
+ dwellAlreadyArmed: Bool,
+ quarantined: Bool,
+ stopRequested: Bool,
+ routeDrivenFallback: Bool,
+ attempts: Int,
+ microphone: MeetingMicrophoneIdentity,
+ outputRoute: MeetingOutputRouteSnapshot
+ ) -> Bool {
+ guard MeetingCaptureTransitionPolicy.allowsWithinSessionVoiceProcessingUpgrade,
+ phaseIsAVCapture,
+ !dwellAlreadyArmed,
+ !quarantined,
+ !stopRequested,
+ routeDrivenFallback,
+ attempts < MeetingCaptureTransitionPolicy.maximumRouteRecoveryAttempts
+ else { return false }
+ return MeetingCapturePathDecider.decide(
+ mode: .onlineCall,
+ microphone: microphone,
+ outputRoute: outputRoute
+ ) == .voiceProcessing
+ }
+}
+
+nonisolated enum MeetingVoiceProcessingUpgradeOwnership {
+ enum Step: Equatable {
+ case cancelWatchdog
+ case fenceAndQuiesce
+ case stopRunning
+ case removeObservers
+ case removeDelegate
+ case removeSessionIO
+ case clearRuntimeRef
+ case clearLocalRef
+ case electAndDecide
+ case parkCandidate
+ case constructVoiceProcessing
+ case startVoiceProcessing
+ case awaitFirstBuffer
+ case commit
+ case startWatchdog
+ }
+
+ static let stopOldFirstSequence: [Step] = [
+ .cancelWatchdog,
+ .fenceAndQuiesce,
+ .stopRunning,
+ .removeObservers,
+ .removeDelegate,
+ .removeSessionIO,
+ .clearRuntimeRef,
+ .clearLocalRef,
+ .electAndDecide,
+ .constructVoiceProcessing,
+ .parkCandidate,
+ .startVoiceProcessing,
+ .awaitFirstBuffer,
+ .commit,
+ .startWatchdog,
+ ]
+
+ static func mayConstructVoiceProcessing(
+ avCaptureRuntimeRef: Bool,
+ avCaptureLocalRef: Bool,
+ avCaptureRunning: Bool,
+ teardownProven: Bool
+ ) -> Bool {
+ !avCaptureRuntimeRef && !avCaptureLocalRef && !avCaptureRunning && teardownProven
+ }
+
+ static func mayPhaseAVCapture(hasComponent: Bool, componentStopped: Bool) -> Bool {
+ hasComponent && !componentStopped
+ }
+
+ enum TeardownFailureAction: Equatable {
+ case quarantineAndFail
+ }
+
+ static let actionIfTeardownUnproven = TeardownFailureAction.quarantineAndFail
+
+ enum RollbackStep: Equatable {
+ case stopAndReleaseVoiceProcessing
+ case elect
+ case recreateAVCapture
+ case awaitFirstBuffer
+ case commitAVCapture
+ }
+
+ static let rollbackSequence: [RollbackStep] = [
+ .stopAndReleaseVoiceProcessing,
+ .elect,
+ .recreateAVCapture,
+ .awaitFirstBuffer,
+ .commitAVCapture,
+ ]
+
+ static let watchdogRunsDuringIntentionalGap = false
+}
+
+nonisolated enum MeetingVoiceProcessingUpgradeCommit {
+ static let publishedCaptureMethod: MeetingAudioTrackCaptureMethod = .voiceProcessing
+ static let discardsOverlappingSamples = false
+
+ enum Stage: Equatable {
+ case beginSplice
+ case appendVoiceProcessingEra
+ case setCaptureMethod
+ case publishMethodChanged
+ case awaitAck
+ case flushRing
+ case phaseVPIO
+ case startWatchdog
+ }
+
+ static let orderedStages: [Stage] = [
+ .beginSplice,
+ .appendVoiceProcessingEra,
+ .setCaptureMethod,
+ .publishMethodChanged,
+ .awaitAck,
+ .flushRing,
+ .phaseVPIO,
+ .startWatchdog,
+ ]
+}
+
+nonisolated enum MeetingCaptureRouteReplay {
+ enum Backend: Equatable {
+ case vpio
+ case avCapture
+ }
+
+ static func shouldReplay(dirty: Bool, stopRequested: Bool, backend: Backend?) -> Bool {
+ dirty && !stopRequested && backend != nil
+ }
+
+ enum Action: Equatable {
+ case downgradeFromVPIO
+ case rebindAVCapture
+ case none
+ }
+
+ static func action(backend: Backend, outputRouteDeclineReason: String?) -> Action {
+ switch backend {
+ case .vpio:
+ return outputRouteDeclineReason == nil ? .none : .downgradeFromVPIO
+ case .avCapture:
+ return .rebindAVCapture
+ }
+ }
+}
+
+nonisolated enum MeetingVoiceProcessingUpgradeAdmission {
+ static func isCurrent(
+ currentGeneration: UInt64,
+ eventGeneration: UInt64,
+ stopRequested: Bool,
+ phaseIsTransitioning: Bool
+ ) -> Bool {
+ !stopRequested && currentGeneration == eventGeneration && phaseIsTransitioning
+ }
+
+ static func canCommit(
+ currentGeneration: UInt64,
+ eventGeneration: UInt64,
+ stopRequested: Bool,
+ phaseIsTransitioning: Bool
+ ) -> Bool {
+ self.isCurrent(
+ currentGeneration: currentGeneration,
+ eventGeneration: eventGeneration,
+ stopRequested: stopRequested,
+ phaseIsTransitioning: phaseIsTransitioning
+ )
+ }
+}
+
+/// Sticky means the safe AVCapture backend remains selected; it does not mean a
+/// disconnected physical microphone remains selected.
+nonisolated enum MeetingAVCaptureRebindDecision {
+ static func shouldRebind(activeCoreAudioUID: String?, electedCoreAudioUID: String?, forced: Bool) -> Bool {
+ if forced { return true }
+ guard let electedCoreAudioUID, !electedCoreAudioUID.isEmpty else { return false }
+ return activeCoreAudioUID != electedCoreAudioUID
+ }
+}
+
/// Output-route facts the decision needs, as a plain value so `decide` is unit-testable headless.
nonisolated struct MeetingOutputRouteSnapshot: Sendable, Equatable {
var deviceExists: Bool
@@ -1392,7 +1806,7 @@ nonisolated enum MeetingScreenCaptureAudioSettings {
configuration.excludesCurrentProcessAudio = true
configuration.captureMicrophone = includeMicrophone
if includeMicrophone {
- configuration.microphoneCaptureDeviceID = microphone.captureDeviceID
+ configuration.microphoneCaptureDeviceID = microphone.avCaptureDeviceID
}
}
}
@@ -1501,121 +1915,216 @@ nonisolated enum MeetingVoiceProcessingWatchdog {
}
}
-/// Re-evaluates the output-route predicate on default-output-device or data-source change.
+/// Re-evaluates meeting capture on default-input, default-output, or output data-source change.
/// Registration failure at init means VPIO is not viable — fail closed. Re-registers everything on
/// `kAudioHardwarePropertyServiceRestarted`, since CoreAudio discards all prior listeners on reset.
-private final nonisolated class MeetingOutputRouteListener: @unchecked Sendable {
+private actor MeetingOutputRouteListener {
+ private enum Lifecycle: Equatable {
+ case idle
+ case starting
+ case ready
+ }
+
private let onChange: @Sendable () -> Void
- private let lock = NSLock()
+ private var generation: UInt64 = 0
+ private var dataSourceEpoch: UInt64 = 0
+ private var active = false
+ private var lifecycle: Lifecycle = .idle
+ private var defaultInputDirty = false
+ private var defaultOutputDirty = false
+ private var serviceRestartDirty = false
+ private var defaultInputToken: AudioObjectPropertyListenerBlock?
private var defaultOutputToken: AudioObjectPropertyListenerBlock?
private var serviceRestartedToken: AudioObjectPropertyListenerBlock?
private var dataSourceToken: AudioObjectPropertyListenerBlock?
private var dataSourceDeviceID: AudioObjectID?
- init?(onChange: @escaping @Sendable () -> Void) {
+ private init(onChange: @escaping @Sendable () -> Void) {
self.onChange = onChange
- guard self.register() else { return nil }
}
- deinit {
- self.unregister()
+ static func make(onChange: @escaping @Sendable () -> Void) async -> MeetingOutputRouteListener? {
+ let listener = MeetingOutputRouteListener(onChange: onChange)
+ guard await listener.start() else { return nil }
+ return listener
}
- private func reevaluate() {
- self.onChange()
+ private func start() async -> Bool {
+ guard !self.active else { return true }
+ self.generation &+= 1
+ let generation = self.generation
+ self.active = true
+ self.lifecycle = .starting
+ self.defaultInputDirty = false
+ self.defaultOutputDirty = false
+ self.serviceRestartDirty = false
+ guard await self.registerSystemListeners(generation: generation),
+ await self.attachDataSourceListener(generation: generation)
+ else {
+ if self.isCurrent(generation) {
+ await self.stop()
+ }
+ return false
+ }
+ return await self.finishStarting(generation: generation)
}
- @discardableResult
- private func register() -> Bool {
+ private func registerSystemListeners(generation: UInt64) async -> Bool {
let sys = AudioObjectID(kAudioObjectSystemObject)
- var defaultOutAddr = AudioObjectPropertyAddress(
+ let defaultInAddr = AudioObjectPropertyAddress(
+ mSelector: kAudioHardwarePropertyDefaultInputDevice,
+ mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain
+ )
+ let defaultOutAddr = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- var restartedAddr = AudioObjectPropertyAddress(
+ let restartedAddr = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyServiceRestarted,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
+ let defaultInToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery) }
+ #endif
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ Task { await self?.handleDefaultInputChanged(generation: generation) }
+ }
+ }
let defaultOutToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyDefaultOutputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery) }
#endif
- self?.reattachDataSourceListener()
- self?.reevaluate()
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ Task { await self?.handleDefaultOutputChanged(generation: generation) }
+ }
}
let restartedToken: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyServiceRestarted, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyServiceRestarted, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingOutputRoute, objectID: sys, selector: kAudioHardwarePropertyServiceRestarted, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery) }
#endif
- self?.reregisterAfterServiceRestart()
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ Task { await self?.handleServiceRestart(generation: generation) }
+ }
+ }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingOutputRoute, objectID: sys, selector: defaultInAddr.mSelector, scope: defaultInAddr.mScope, element: defaultInAddr.mElement, queueRole: .dedicatedControl, phase: .listener)
+ #endif
+ let defaultInputStatus = await AudioTopologyListenerExecution.add(objectID: sys, address: defaultInAddr, token: defaultInToken)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingOutputRoute, objectID: sys, selector: defaultInAddr.mSelector, scope: defaultInAddr.mScope, element: defaultInAddr.mElement, queueRole: .dedicatedControl, phase: .listener, status: defaultInputStatus)
+ #endif
+ guard defaultInputStatus == noErr, self.isCurrent(generation) else {
+ if defaultInputStatus == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultInAddr, token: defaultInToken)
+ }
+ return false
}
+ self.defaultInputToken = defaultInToken
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingOutputRoute, objectID: sys, selector: defaultOutAddr.mSelector, scope: defaultOutAddr.mScope, element: defaultOutAddr.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let defaultStatus = AudioObjectAddPropertyListenerBlock(sys, &defaultOutAddr, DispatchQueue.main, defaultOutToken)
+ let defaultStatus = await AudioTopologyListenerExecution.add(objectID: sys, address: defaultOutAddr, token: defaultOutToken)
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingOutputRoute, objectID: sys, selector: defaultOutAddr.mSelector, scope: defaultOutAddr.mScope, element: defaultOutAddr.mElement, queueRole: .dedicatedControl, phase: .listener, status: defaultStatus)
#endif
guard defaultStatus == noErr else {
+ _ = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultInAddr, token: defaultInToken)
+ if self.generation == generation { self.defaultInputToken = nil }
return false
}
+ guard self.isCurrent(generation) else {
+ _ = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultInAddr, token: defaultInToken)
+ _ = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultOutAddr, token: defaultOutToken)
+ return false
+ }
+ self.defaultOutputToken = defaultOutToken
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingOutputRoute, objectID: sys, selector: restartedAddr.mSelector, scope: restartedAddr.mScope, element: restartedAddr.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let restartedStatus = AudioObjectAddPropertyListenerBlock(sys, &restartedAddr, DispatchQueue.main, restartedToken)
+ let restartedStatus = await AudioTopologyListenerExecution.add(objectID: sys, address: restartedAddr, token: restartedToken)
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingOutputRoute, objectID: sys, selector: restartedAddr.mSelector, scope: restartedAddr.mScope, element: restartedAddr.mElement, queueRole: .dedicatedControl, phase: .listener, status: restartedStatus)
#endif
- guard restartedStatus == noErr else {
+ guard restartedStatus == noErr, self.isCurrent(generation) else {
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingOutputRoute, objectID: sys, selector: defaultOutAddr.mSelector, scope: defaultOutAddr.mScope, element: defaultOutAddr.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let cleanupStatus = AudioObjectRemovePropertyListenerBlock(sys, &defaultOutAddr, DispatchQueue.main, defaultOutToken)
+ let cleanupStatus = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultOutAddr, token: defaultOutToken)
+ _ = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultInAddr, token: defaultInToken)
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingOutputRoute, objectID: sys, selector: defaultOutAddr.mSelector, scope: defaultOutAddr.mScope, element: defaultOutAddr.mElement, queueRole: .dedicatedControl, phase: .listener, status: cleanupStatus)
#endif
+ if self.generation == generation {
+ self.defaultInputToken = nil
+ self.defaultOutputToken = nil
+ }
+ if restartedStatus == noErr {
+ _ = await AudioTopologyListenerExecution.remove(objectID: sys, address: restartedAddr, token: restartedToken)
+ }
return false
}
- self.defaultOutputToken = defaultOutToken
self.serviceRestartedToken = restartedToken
- guard self.attachDataSourceListener() else {
- self.unregister()
- return false
- }
return true
}
- private func attachDataSourceListener() -> Bool {
- guard let device = AudioDevice.getDefaultOutputDevice() else { return false }
- var address = AudioObjectPropertyAddress(
- mSelector: kAudioDevicePropertyDataSource,
- mScope: kAudioObjectPropertyScopeOutput,
- mElement: kAudioObjectPropertyElementMain
+ private func handleDefaultInputChanged(generation: UInt64) {
+ guard self.isCurrent(generation) else { return }
+ guard self.lifecycle == .ready else {
+ self.defaultInputDirty = true
+ return
+ }
+ self.onChange()
+ }
+
+ private func attachDataSourceListener(generation: UInt64) async -> Bool {
+ guard self.isCurrent(generation) else { return false }
+ guard let device = AudioDevice.getDefaultOutputDevice() else { return false }
+ self.dataSourceEpoch &+= 1
+ let epoch = self.dataSourceEpoch
+ let address = AudioObjectPropertyAddress(
+ mSelector: kAudioDevicePropertyDataSource,
+ mScope: kAudioObjectPropertyScopeOutput,
+ mElement: kAudioObjectPropertyElementMain
)
let token: AudioObjectPropertyListenerBlock = { [weak self] _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .meetingOutputRoute, objectID: device.id, selector: kAudioDevicePropertyDataSource, scope: kAudioObjectPropertyScopeOutput, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingOutputRoute, objectID: device.id, selector: kAudioDevicePropertyDataSource, scope: kAudioObjectPropertyScopeOutput, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingOutputRoute, objectID: device.id, selector: kAudioDevicePropertyDataSource, scope: kAudioObjectPropertyScopeOutput, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery) }
#endif
- self?.reevaluate()
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ Task { await self?.handleDataSourceChanged(generation: generation, epoch: epoch) }
+ }
}
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingOutputRoute, objectID: device.id, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let status = AudioObjectAddPropertyListenerBlock(device.id, &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.add(objectID: device.id, address: address, token: token)
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingOutputRoute, objectID: device.id, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
guard status == noErr else { return false }
+ guard self.isCurrent(generation), self.dataSourceEpoch == epoch else {
+ _ = await AudioTopologyListenerExecution.remove(objectID: device.id, address: address, token: token)
+ return false
+ }
self.dataSourceToken = token
self.dataSourceDeviceID = device.id
return true
}
- private func detachDataSourceListener() {
+ private func detachDataSourceListener() async {
+ self.dataSourceEpoch &+= 1
guard let deviceID = self.dataSourceDeviceID, let token = self.dataSourceToken else { return }
- var address = AudioObjectPropertyAddress(
+ self.dataSourceToken = nil
+ self.dataSourceDeviceID = nil
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyDataSource,
mScope: kAudioObjectPropertyScopeOutput,
mElement: kAudioObjectPropertyElementMain
@@ -1623,61 +2132,178 @@ private final nonisolated class MeetingOutputRouteListener: @unchecked Sendable
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingOutputRoute, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let status = AudioObjectRemovePropertyListenerBlock(deviceID, &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.remove(objectID: deviceID, address: address, token: token)
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingOutputRoute, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
- self.dataSourceToken = nil
- self.dataSourceDeviceID = nil
}
- private func reattachDataSourceListener() {
- self.detachDataSourceListener()
- _ = self.attachDataSourceListener()
+ private func handleDefaultOutputChanged(generation: UInt64) async {
+ guard self.isCurrent(generation) else { return }
+ guard self.lifecycle == .ready else {
+ self.defaultOutputDirty = true
+ return
+ }
+ await self.detachDataSourceListener()
+ guard self.isCurrent(generation) else { return }
+ _ = await self.attachDataSourceListener(generation: generation)
+ guard self.isCurrent(generation) else { return }
+ self.onChange()
+ }
+
+ private func handleDataSourceChanged(generation: UInt64, epoch: UInt64) {
+ guard self.isCurrent(generation), self.dataSourceEpoch == epoch else { return }
+ self.onChange()
}
- private func reregisterAfterServiceRestart() {
- self.unregister()
- guard self.register() else {
+ private func handleServiceRestart(generation: UInt64) async {
+ guard self.isCurrent(generation) else { return }
+ guard self.lifecycle == .ready else {
+ self.serviceRestartDirty = true
+ return
+ }
+ let oldTokens = self.detachAll(invalidateLifecycle: true)
+ let replacementGeneration = self.generation
+ self.active = true
+ self.lifecycle = .starting
+ self.defaultInputDirty = false
+ self.defaultOutputDirty = false
+ self.serviceRestartDirty = false
+ await Self.remove(tokens: oldTokens)
+ guard self.isCurrent(replacementGeneration) else { return }
+ guard await self.registerSystemListeners(generation: replacementGeneration),
+ await self.attachDataSourceListener(generation: replacementGeneration)
+ else {
+ if self.isCurrent(replacementGeneration) {
+ await self.stop()
+ }
self.onChange()
return
}
- self.reevaluate()
+ if await self.finishStarting(generation: replacementGeneration),
+ self.generation == replacementGeneration
+ {
+ self.onChange()
+ }
+ }
+
+ /// Listener callbacks may arrive after HAL installs a token but before the async add
+ /// continuation commits ownership. Latch those callbacks while starting, then replay only
+ /// after all three tokens are owned. A service restart takes precedence over a route refresh.
+ private func finishStarting(generation: UInt64) async -> Bool {
+ guard self.isCurrent(generation), self.lifecycle == .starting else { return false }
+ self.lifecycle = .ready
+ let restartWasDirty = self.serviceRestartDirty
+ let inputWasDirty = self.defaultInputDirty
+ let defaultWasDirty = self.defaultOutputDirty
+ self.serviceRestartDirty = false
+ self.defaultInputDirty = false
+ self.defaultOutputDirty = false
+ if restartWasDirty {
+ await self.handleServiceRestart(generation: generation)
+ return self.active && self.lifecycle == .ready
+ }
+ if inputWasDirty {
+ self.handleDefaultInputChanged(generation: generation)
+ }
+ if defaultWasDirty {
+ await self.handleDefaultOutputChanged(generation: generation)
+ }
+ return self.active && self.lifecycle == .ready
+ }
+
+ func stop() async {
+ guard self.active || self.defaultInputToken != nil || self.defaultOutputToken != nil || self.serviceRestartedToken != nil || self.dataSourceToken != nil else { return }
+ let tokens = self.detachAll(invalidateLifecycle: true)
+ self.lifecycle = .idle
+ self.defaultInputDirty = false
+ self.defaultOutputDirty = false
+ self.serviceRestartDirty = false
+ await Self.remove(tokens: tokens)
}
- private func unregister() {
+ private struct Tokens: @unchecked Sendable {
+ let defaultInput: AudioObjectPropertyListenerBlock?
+ let defaultOutput: AudioObjectPropertyListenerBlock?
+ let serviceRestarted: AudioObjectPropertyListenerBlock?
+ let dataSource: AudioObjectPropertyListenerBlock?
+ let dataSourceDeviceID: AudioObjectID?
+ }
+
+ private func detachAll(invalidateLifecycle: Bool) -> Tokens {
+ if invalidateLifecycle {
+ self.active = false
+ self.generation &+= 1
+ }
+ self.dataSourceEpoch &+= 1
+ let tokens = Tokens(
+ defaultInput: self.defaultInputToken,
+ defaultOutput: self.defaultOutputToken,
+ serviceRestarted: self.serviceRestartedToken,
+ dataSource: self.dataSourceToken,
+ dataSourceDeviceID: self.dataSourceDeviceID
+ )
+ self.defaultInputToken = nil
+ self.defaultOutputToken = nil
+ self.serviceRestartedToken = nil
+ self.dataSourceToken = nil
+ self.dataSourceDeviceID = nil
+ return tokens
+ }
+
+ private static func remove(tokens: Tokens) async {
let sys = AudioObjectID(kAudioObjectSystemObject)
- var defaultOutAddr = AudioObjectPropertyAddress(
+ let defaultInAddr = AudioObjectPropertyAddress(
+ mSelector: kAudioHardwarePropertyDefaultInputDevice,
+ mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain
+ )
+ let defaultOutAddr = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultOutputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- var restartedAddr = AudioObjectPropertyAddress(
+ let restartedAddr = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyServiceRestarted,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
- if let token = self.defaultOutputToken {
+ if let token = tokens.defaultInput {
+ let status = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultInAddr, token: token)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingOutputRoute, objectID: sys, selector: defaultInAddr.mSelector, scope: defaultInAddr.mScope, element: defaultInAddr.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
+ #endif
+ }
+ if let token = tokens.defaultOutput {
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingOutputRoute, objectID: sys, selector: defaultOutAddr.mSelector, scope: defaultOutAddr.mScope, element: defaultOutAddr.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &defaultOutAddr, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.remove(objectID: sys, address: defaultOutAddr, token: token)
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingOutputRoute, objectID: sys, selector: defaultOutAddr.mSelector, scope: defaultOutAddr.mScope, element: defaultOutAddr.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
}
- if let token = self.serviceRestartedToken {
+ if let token = tokens.serviceRestarted {
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingOutputRoute, objectID: sys, selector: restartedAddr.mSelector, scope: restartedAddr.mScope, element: restartedAddr.mElement, queueRole: .dedicatedControl, phase: .listener)
#endif
- let status = AudioObjectRemovePropertyListenerBlock(sys, &restartedAddr, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.remove(objectID: sys, address: restartedAddr, token: token)
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingOutputRoute, objectID: sys, selector: restartedAddr.mSelector, scope: restartedAddr.mScope, element: restartedAddr.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
}
- self.defaultOutputToken = nil
- self.serviceRestartedToken = nil
- self.detachDataSourceListener()
+ if let token = tokens.dataSource, let deviceID = tokens.dataSourceDeviceID {
+ let address = AudioObjectPropertyAddress(
+ mSelector: kAudioDevicePropertyDataSource,
+ mScope: kAudioObjectPropertyScopeOutput,
+ mElement: kAudioObjectPropertyElementMain
+ )
+ _ = await AudioTopologyListenerExecution.remove(objectID: deviceID, address: address, token: token)
+ }
+ }
+
+ private func isCurrent(_ generation: UInt64) -> Bool {
+ self.active && self.generation == generation
}
}
@@ -1703,11 +2329,12 @@ nonisolated struct MeetingCaptureDeviceElection: Sendable, Equatable {
var identity: MeetingMicrophoneIdentity
var role: MeetingMicrophoneRole
- static func elect(original: MeetingMicrophoneIdentity) -> MeetingCaptureDeviceElection {
- Self.decide(
+ static func elect(original: MeetingMicrophoneIdentity) async -> MeetingCaptureDeviceElection {
+ let snapshot = await MeetingCaptureSourceCatalog.microphoneSnapshot()
+ return Self.decide(
original: original,
- catalog: MeetingCaptureSourceCatalog.availableMicrophones(),
- defaultInputUID: AudioDevice.getDefaultInputDevice()?.uid
+ catalog: snapshot.identities,
+ defaultInputUID: snapshot.defaultCoreAudioUID
)
}
@@ -1723,8 +2350,20 @@ nonisolated struct MeetingCaptureDeviceElection: Sendable, Equatable {
guard let matched = catalog.first(where: { $0.coreAudioUID == defaultInputUID }) else {
return MeetingCaptureDeviceElection(identity: original, role: original.role)
}
- if matched.captureDeviceID == original.captureDeviceID {
- return MeetingCaptureDeviceElection(identity: matched, role: original.role)
+ let isSamePhysicalDevice = {
+ if let originalUID = original.coreAudioUID, !originalUID.isEmpty,
+ let matchedUID = matched.coreAudioUID, !matchedUID.isEmpty
+ {
+ return originalUID == matchedUID
+ }
+ return matched.captureDeviceID == original.captureDeviceID
+ }()
+ if isSamePhysicalDevice {
+ var merged = matched
+ merged.role = original.role
+ merged.avCaptureDeviceID = original.avCaptureDeviceID
+ ?? (original.identitySchemaVersion == nil ? original.captureDeviceID : nil)
+ return MeetingCaptureDeviceElection(identity: merged, role: original.role)
}
// A built-in mic is inherently the owner's; .unknown would block VPIO on upgrades.
let role: MeetingMicrophoneRole = matched.coreAudioUID == "BuiltInMicrophoneDevice" ? .personal : .unknown
@@ -1740,33 +2379,27 @@ nonisolated struct MeetingCaptureDeviceElection: Sendable, Equatable {
}
}
-/// Pure discard rule extracted from the upgrade's ring-flush for hardware-free testing.
-nonisolated enum MeetingUpgradeRingDiscard {
- static func discardingSamples(atOrBehind boundary: CMTime?, from samples: [CMSampleBuffer]) -> [CMSampleBuffer] {
- guard let boundary else { return samples }
- return samples.filter { CMSampleBufferGetPresentationTimeStamp($0) > boundary }
- }
-}
-
/// Owns a swappable mic component (VPIO or AVCaptureSession) plus an app-audio-only SCStream that never stops across a swap.
private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRuntime, @unchecked Sendable {
private static let totalStartDeadlineSeconds: Double = 10
private static let firstBufferDeadlineSeconds: Double = 2
private static let downgradeSettleSeconds: Double = 0.75
- private static let upgradeSettleSeconds: Double = 0.75
- private static let upgradeDwellSeconds: Double = 10
- private static let maxUpgradeAttemptsPerSession = 3
- private static let upgradeBackoffBaseSeconds: Double = 5
- private static let oldProducerStopSupervisionSeconds: Double = 0.3
private enum Phase: Equatable {
case starting
case vpio
case transitioning
+ case committing
case avCapture
case failed
}
+ private enum DowngradeCommitDecision {
+ case committed
+ case stale
+ case failed(String)
+ }
+
private let application: MeetingApplicationIdentity
private let originalMicrophone: MeetingMicrophoneIdentity
private let applicationWriter: MeetingAudioChunkWriter
@@ -1787,12 +2420,17 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
private var watchdogTask: Task<Void, Never>?
private var outputRouteListener: MeetingOutputRouteListener?
private var transitionTask: Task<Void, Never>?
- private var dwellTask: Task<Void, Never>?
- private var dwellArming = false
+ private var transitionGeneration: UInt64 = 0
+ private var pendingAVCaptureFailure: (generation: UInt64, detail: String)?
+ private var avCaptureFailureToken: MeetingAVCaptureAttemptToken?
+ private var routeChangeDirty = false
private var stopRequested = false
+ private var routeDrivenFallback = false
+ private var voiceProcessingRecoveryAttempts = 0
+ private var recoveryDwellTask: Task<Void, Never>?
+ /// Parked before `start()` so stop/supersede can never lose ownership of it.
+ private var pendingVoiceProcessingCapture: MeetingMicrophoneCapture?
private var quarantined = false
- private var upgradeAttempts = 0
- private let abortedCaptureDrain = DispatchGroup()
var captureScope: MeetingCaptureScope? {
if let fallback = self.stateLock.withLock({ self.fallbackRuntime }) { return fallback.captureScope }
@@ -1815,6 +2453,12 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
self.liveAudioHandler = liveAudioHandler
}
+ #if DEBUG
+ deinit {
+ AudioTopologyDiagnostics.record(.ownerWillDeinit, owner: .meetingCaptureEngine, queueRole: .callbackCurrent, phase: .runtimeStop)
+ }
+ #endif
+
static func make(
application: MeetingApplicationIdentity,
microphone: MeetingMicrophoneIdentity,
@@ -1856,7 +2500,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
defer { self.outputRouteListener = nil }
return (self.gate, self.micCapture, self.outputRouteListener)
}
- _ = staleListener // released here, outside the lock
+ await staleListener?.stop()
gateToAbort.abort()
await captureToStop.stop()
await self.stopAppOnlyIfStarted()
@@ -1942,7 +2586,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
guard let next = self.gate.continueCommitFlush() else { break }
batch = next
}
- self.startOutputRouteWatch()
+ await self.startOutputRouteWatch()
self.startWatchdog()
}
@@ -1983,6 +2627,19 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
return false
}
+ private func waitForBufferProgress(
+ timeoutSeconds: Double,
+ using component: MeetingAVCaptureMicrophoneComponent
+ ) async -> Bool {
+ let baseline = component.sampleCount()
+ let deadline = Date().addingTimeInterval(timeoutSeconds)
+ while Date() < deadline {
+ if component.sampleCount() > baseline { return true }
+ try? await Task.sleep(nanoseconds: 50_000_000)
+ }
+ return false
+ }
+
private func startWatchdog() {
let task = Task { [weak self] in
guard let self else { return }
@@ -2008,25 +2665,94 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
self.stateLock.withLock { self.watchdogTask = task }
}
+ private func startAVCaptureWatchdog(
+ generation: UInt64,
+ component: MeetingAVCaptureMicrophoneComponent
+ ) {
+ let task = Task { [weak self] in
+ guard let self else { return }
+ var lastCount = component.sampleCount()
+ var lastProgress = Date()
+ while !Task.isCancelled {
+ try? await Task.sleep(nanoseconds: 1_000_000_000)
+ if Task.isCancelled { return }
+ let stillOwnsComponent = self.stateLock.withLock {
+ self.phase == .avCapture
+ && self.transitionGeneration == generation
+ && self.avCaptureComponent === component
+ }
+ guard stillOwnsComponent else { return }
+ let current = component.sampleCount()
+ if current != lastCount {
+ lastCount = current
+ lastProgress = Date()
+ continue
+ }
+ if MeetingVoiceProcessingWatchdog.isStalled(
+ secondsSinceLastEmittedBuffer: Date().timeIntervalSince(lastProgress)
+ ) {
+ self.beginAVCaptureRebind(
+ detail: "The active microphone stopped delivering audio buffers.",
+ forced: true
+ )
+ return
+ }
+ }
+ }
+ self.stateLock.withLock { self.watchdogTask = task }
+ }
+
/// Registered once: `handleRouteChange()` re-reads `phase` on every fire, so it never needs re-arming.
- private func startOutputRouteWatch() {
- let listener = MeetingOutputRouteListener { [weak self] in
+ private func startOutputRouteWatch() async {
+ let listener = await MeetingOutputRouteListener.make { [weak self] in
self?.handleRouteChange()
}
- self.stateLock.withLock { self.outputRouteListener = listener }
+ let retained = self.stateLock.withLock { () -> Bool in
+ guard !self.stopRequested else { return false }
+ self.outputRouteListener = listener
+ return true
+ }
+ if !retained {
+ await listener?.stop()
+ }
}
private func handleRouteChange() {
- let currentPhase = self.stateLock.withLock { self.phase }
- switch currentPhase {
- case .vpio:
+ enum Action { case evaluateVPIO, evaluateAVCapture, none }
+ let action = self.stateLock.withLock { () -> Action in
+ switch self.phase {
+ case .vpio:
+ return .evaluateVPIO
+ case .avCapture:
+ // Claim the observation before releasing the lock. If recovery wins
+ // the next lock acquisition, finishTransition() will replay it
+ // against whichever backend commits.
+ self.routeChangeDirty = true
+ return .evaluateAVCapture
+ case .transitioning, .committing:
+ self.routeChangeDirty = true
+ return .none
+ case .starting, .failed:
+ return .none
+ }
+ }
+ switch action {
+ case .evaluateVPIO:
let reason = MeetingCapturePathDecider.outputRouteDeclineReason(MeetingCaptureEngine.currentOutputRouteSnapshot())
if let reason {
- self.beginDowngradeTransition(reason: "Output route changed: \(reason)")
+ self.beginDowngradeTransition(
+ reason: "Output route changed: \(reason)",
+ routeDriven: true
+ )
}
- case .avCapture:
- self.armUpgradeDwellIfViable()
- default:
+ case .evaluateAVCapture:
+ // The backend stays sticky, but the physical input must follow the live
+ // system default when a headset is attached or removed.
+ self.beginAVCaptureRebind(
+ detail: "The default microphone route changed.",
+ forced: false
+ )
+ case .none:
break
}
}
@@ -2065,7 +2791,10 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
case .defaultInputChanged:
guard let requestedUID = self.originalMicrophone.coreAudioUID else { return }
if AudioDevice.getDefaultInputDevice()?.uid != requestedUID {
- self.beginDowngradeTransition(reason: "The default input device changed away from the requested microphone.")
+ self.beginDowngradeTransition(
+ reason: "The default input device changed away from the requested microphone.",
+ routeDriven: true
+ )
}
case .overload:
self.eventHandler(.interrupted(kind: .voiceProcessingOverload, trackID: nil, detail: "Voice-processing microphone reported an overload."))
@@ -2074,73 +2803,129 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
}
- private func handleAVCaptureFailureSignal(detail: String) {
- guard self.stateLock.withLock({ self.phase == .avCapture }) else { return }
+ private func handleAVCaptureFailureSignal(
+ detail: String,
+ generation: UInt64,
+ token: MeetingAVCaptureAttemptToken
+ ) {
let shouldEmit = self.stateLock.withLock { () -> Bool in
- guard !self.interruptedThisEra else { return false }
- self.interruptedThisEra = true
- self.phase = .failed
- return true
+ guard MeetingAVCaptureFailureAdmission.accepts(
+ currentGeneration: self.transitionGeneration,
+ eventGeneration: generation,
+ activeToken: self.avCaptureFailureToken,
+ eventToken: token,
+ stopRequested: self.stopRequested
+ ) else { return false }
+ switch self.phase {
+ case .transitioning, .committing:
+ if self.pendingAVCaptureFailure == nil {
+ self.pendingAVCaptureFailure = (generation, detail)
+ }
+ return false
+ case .avCapture:
+ guard !self.interruptedThisEra else { return false }
+ self.interruptedThisEra = true
+ self.phase = .failed
+ return true
+ case .starting, .vpio, .failed:
+ return false
+ }
}
guard shouldEmit else { return }
self.eventHandler(.interrupted(kind: .captureStoppedUnexpectedly, trackID: nil, detail: detail))
}
- private func beginDowngradeTransition(reason: String) {
- self.beginDowngradeTransition(detail: reason)
+ private func invalidateAVCaptureFailureToken(
+ _ token: MeetingAVCaptureAttemptToken,
+ generation: UInt64
+ ) {
+ self.stateLock.withLock {
+ guard self.transitionGeneration == generation,
+ self.avCaptureFailureToken === token
+ else { return }
+ self.avCaptureFailureToken = nil
+ if self.pendingAVCaptureFailure?.generation == generation {
+ self.pendingAVCaptureFailure = nil
+ }
+ }
+ }
+
+ private func beginDowngradeTransition(reason: String, routeDriven: Bool = false) {
+ self.beginDowngradeTransition(detail: reason, routeDriven: routeDriven)
}
- private func beginDowngradeTransition(detail: String) {
- let shouldStart = self.stateLock.withLock { () -> Bool in
- guard self.phase == .vpio, !self.stopRequested else { return false }
+ private func beginDowngradeTransition(detail: String, routeDriven: Bool = false) {
+ let watchdog = self.stateLock.withLock { () -> Task<Void, Never>? in
+ guard self.phase == .vpio, !self.stopRequested else { return nil }
self.phase = .transitioning
- return true
- }
- guard shouldStart else { return }
- let cancellables = self.stateLock.withLock { () -> [Task<Void, Never>?] in
- defer { self.dwellTask = nil; self.watchdogTask = nil }
- return [self.dwellTask, self.watchdogTask]
- }
- cancellables.forEach { $0?.cancel() }
- let task = Task { [weak self] in
- guard let self else { return }
- await self.runDowngrade(detail: detail)
+ self.transitionGeneration &+= 1
+ let generation = self.transitionGeneration
+ self.pendingAVCaptureFailure = nil
+ self.routeChangeDirty = false
+ self.routeDrivenFallback = routeDriven
+ let task = Task { [weak self] in
+ guard let self else { return }
+ await self.runDowngrade(detail: detail, generation: generation)
+ }
+ self.transitionTask = task
+ defer { self.watchdogTask = nil }
+ return self.watchdogTask
}
- self.stateLock.withLock { self.transitionTask = task }
+ watchdog?.cancel()
}
- private func runDowngrade(detail: String) async {
+ private func runDowngrade(detail: String, generation: UInt64) async {
try? await Task.sleep(nanoseconds: UInt64(Self.downgradeSettleSeconds * 1_000_000_000))
- guard !(self.stateLock.withLock({ self.stopRequested })) else {
- self.finishTransition()
+ guard self.ownsDowngradeTransition(generation) else {
+ self.finishTransition(generation)
return
}
let oldFence = self.stateLock.withLock { self.currentFence }
- let oldMicCapture = self.stateLock.withLock { self.micCapture }
+ var oldMicCapture: MeetingMicrophoneCapture? = self.stateLock.withLock { self.micCapture }
await oldFence.quiesce()
- let statsSnapshot = await oldMicCapture.statistics()
+ let statsSnapshot = await oldMicCapture!.statistics()
let drift = MeetingClockDriftRecord(
cumulativeAbsorbedSeconds: statsSnapshot.cumulativeAbsorbedCorrectionSeconds,
elapsedValidHostSeconds: MeetingVoiceProcessingDriftSnapshot.elapsedValidHostSeconds(statsSnapshot),
eligible: MeetingVoiceProcessingDriftSnapshot.isEligible(statsSnapshot)
)
- // Detached, supervised: releases the HAL device before the new client opens it, without blocking the swap.
- oldMicCapture.stopDetachedSupervised(seconds: Self.oldProducerStopSupervisionSeconds) { [weak self] completedInTime in
- guard let self, !completedInTime else { return }
- self.stateLock.withLock { self.quarantined = true }
+ // Hard lifetime barrier: VPIO, its tap, and its Core Audio listeners must be
+ // completely gone before any AVFoundation discovery/session construction.
+ await oldMicCapture!.stop()
+ self.stateLock.withLock {
+ if self.micCapture === oldMicCapture {
+ self.micCapture = MeetingMicrophoneCapture()
+ }
}
+ oldMicCapture = nil
- guard !(self.stateLock.withLock({ self.stopRequested })) else {
- self.finishTransition()
+ guard self.ownsDowngradeTransition(generation) else {
+ self.finishTransition(generation)
return
}
- let elected = MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ let elected = await MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ guard self.ownsDowngradeTransition(generation) else {
+ self.finishTransition(generation)
+ return
+ }
let newFence = MeetingEraAcceptanceFence()
let newGate = MeetingVoiceProcessingCommitGate()
let microphoneWriter = self.microphoneWriter
let liveAudioHandler = self.liveAudioHandler
+ let failureToken = MeetingAVCaptureAttemptToken()
+ guard self.stateLock.withLock({ () -> Bool in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested
+ else { return false }
+ self.avCaptureFailureToken = failureToken
+ return true
+ }) else {
+ self.finishTransition(generation)
+ return
+ }
let component: MeetingAVCaptureMicrophoneComponent
do {
component = try MeetingAVCaptureMicrophoneComponent(
@@ -2157,26 +2942,105 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
},
onTerminalStop: { [weak self] detail in
- self?.handleAVCaptureFailureSignal(detail: detail)
+ self?.handleAVCaptureFailureSignal(
+ detail: detail,
+ generation: generation,
+ token: failureToken
+ )
}
)
+ } catch {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ self.failPendingDowngrade(
+ generation: generation,
+ detail: "Microphone unavailable after a route change: \(error.localizedDescription)"
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ do {
try await component.start()
} catch {
- self.stateLock.withLock { self.phase = .failed }
- self.eventHandler(.interrupted(
- kind: .captureStoppedUnexpectedly, trackID: nil,
+ // `stop()` is a true control-queue barrier: if startRunning was still
+ // executing, retirement cannot finish (or release the meeting lease)
+ // until it returns and stopRunning has run behind it.
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await component.stop()
+ self.failPendingDowngrade(
+ generation: generation,
detail: "Microphone unavailable after a route change: \(error.localizedDescription)"
- ))
- self.finishTransition()
+ )
+ self.finishTransition(generation)
return
}
- if self.stateLock.withLock({ self.stopRequested }) {
+ guard await self.waitForBufferProgress(
+ timeoutSeconds: Self.firstBufferDeadlineSeconds,
+ using: component
+ ) else {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
try? await component.stop()
- self.finishTransition()
+ self.failPendingDowngrade(
+ generation: generation,
+ detail: "The replacement microphone started but delivered no audio buffers."
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ // Promote the bounded gate before the runtime's point of no return. If its
+ // cap was exhausted while AVCapture started, fail with no era/event side
+ // effects. Once promoted, arrivals are retained uncapped and in order
+ // while metadata and the coordinator acknowledgement complete.
+ guard var pendingBatch = newGate.beginCommitFlush() else {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await component.stop()
+ self.failPendingDowngrade(
+ generation: generation,
+ detail: "Microphone buffering overflowed before the route-change handoff could commit."
+ )
+ self.finishTransition(generation)
return
}
+ // Point of no return. Before this lock succeeds, stop may cancel the
+ // transition with no published method/era. After it succeeds, stop waits
+ // for the complete metadata -> acknowledgement -> gate-flush transaction.
+ let commitDecision = self.stateLock.withLock { () -> DowngradeCommitDecision in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested
+ else { return .stale }
+ if let failure = self.pendingAVCaptureFailure, failure.generation == generation {
+ self.pendingAVCaptureFailure = nil
+ self.phase = .failed
+ return .failed(failure.detail)
+ }
+ self.currentFence = newFence
+ self.avCaptureComponent = component
+ self.phase = .committing
+ self.interruptedThisEra = false
+ return .committed
+ }
+ switch commitDecision {
+ case .stale:
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await component.stop()
+ self.finishTransition(generation)
+ return
+ case let .failed(failureDetail):
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await component.stop()
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly, trackID: nil, detail: failureDetail
+ ))
+ self.finishTransition(generation)
+ return
+ case .committed:
+ break
+ }
+
let boundary = await self.microphoneWriter.beginSplice()
// A nil or non-monotonic boundary would misattribute audio; skip the era record instead.
@@ -2210,90 +3074,464 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
)
}
- self.stateLock.withLock {
- self.currentFence = newFence
- self.avCaptureComponent = component
- self.phase = .avCapture
- self.interruptedThisEra = false
- }
-
// Ack before flushing: readers must re-arm before any post-boundary sample reaches them.
let ack = MeetingCaptureMethodChangeAck()
self.eventHandler(.captureMethodChanged(trackID: self.microphoneWriter.trackID, method: .avCaptureSession, ack: ack))
await ack.wait()
- if var batch = newGate.beginCommitFlush() {
- while true {
- await self.flushBatch(batch)
- guard let next = newGate.continueCommitFlush() else { break }
- batch = next
+ while true {
+ await self.flushBatch(pendingBatch)
+ guard let next = newGate.continueCommitFlush() else { break }
+ pendingBatch = next
+ }
+
+ let terminalFailure = self.stateLock.withLock { () -> String? in
+ guard self.phase == .committing, self.transitionGeneration == generation else { return nil }
+ if let failure = self.pendingAVCaptureFailure, failure.generation == generation {
+ self.pendingAVCaptureFailure = nil
+ self.interruptedThisEra = true
+ self.phase = .failed
+ return failure.detail
}
+ self.phase = .avCapture
+ return nil
+ }
+ if let terminalFailure {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly, trackID: nil, detail: terminalFailure
+ ))
+ } else {
+ self.startAVCaptureWatchdog(generation: generation, component: component)
}
- self.finishTransition()
- self.armUpgradeDwellIfViable()
+ self.finishTransition(generation)
+ if terminalFailure == nil { self.armSafeVoiceProcessingRecoveryIfViable() }
}
- private func legacyEra(from track: MeetingAudioTrack) -> MeetingCaptureEra {
- MeetingCaptureEra(
- method: track.captureMethod ?? .voiceProcessing,
- deviceUID: self.originalMicrophone.coreAudioUID,
- deviceName: self.originalMicrophone.displayName,
- roleAtElection: self.originalMicrophone.role,
- startSeconds: 0,
- settledConfig: track.voiceProcessingConfig,
- clockDrift: track.clockDrift
- )
+ private func beginAVCaptureRebind(detail: String, forced: Bool) {
+ let watchdog = self.stateLock.withLock { () -> Task<Void, Never>? in
+ guard self.phase == .avCapture,
+ self.avCaptureComponent != nil,
+ !self.stopRequested
+ else { return nil }
+ let previousGeneration = self.transitionGeneration
+ self.phase = .transitioning
+ self.transitionGeneration &+= 1
+ let generation = self.transitionGeneration
+ self.pendingAVCaptureFailure = nil
+ self.routeChangeDirty = false
+ if forced { self.routeDrivenFallback = false }
+ let task = Task { [weak self] in
+ guard let self else { return }
+ await self.runAVCaptureRebind(
+ detail: detail,
+ forced: forced,
+ generation: generation,
+ previousGeneration: previousGeneration
+ )
+ }
+ self.transitionTask = task
+ defer { self.watchdogTask = nil }
+ return self.watchdogTask
+ }
+ watchdog?.cancel()
}
- /// `dwellArming` reserves the slot in the SAME critical section as the `dwellTask == nil` check.
- private func armUpgradeDwellIfViable() {
- guard self.stateLock.withLock({
- guard self.phase == .avCapture, self.dwellTask == nil, !self.dwellArming, !self.quarantined,
- !self.stopRequested, self.upgradeAttempts < Self.maxUpgradeAttemptsPerSession
+ private func runAVCaptureRebind(
+ detail: String,
+ forced: Bool,
+ generation: UInt64,
+ previousGeneration: UInt64
+ ) async {
+ try? await Task.sleep(nanoseconds: UInt64(Self.downgradeSettleSeconds * 1_000_000_000))
+ guard self.ownsDowngradeTransition(generation),
+ let oldComponent = self.stateLock.withLock({ self.avCaptureComponent })
+ else {
+ self.finishTransition(generation)
+ return
+ }
+
+ // The settle window coalesces everything observed so far. Clear the dirty
+ // bit immediately before the snapshot so only changes racing the snapshot
+ // or the subsequent handoff need a replay.
+ let readyToSnapshot = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested
else { return false }
- self.dwellArming = true
+ self.routeChangeDirty = false
return true
- }) else { return }
- let decision = MeetingCapturePathDecider.decide(
- mode: .onlineCall,
- microphone: self.originalMicrophone,
- outputRoute: MeetingCaptureEngine.currentOutputRouteSnapshot()
- )
- guard case .voiceProcessing = decision else {
- self.stateLock.withLock { self.dwellArming = false }
+ }
+ guard readyToSnapshot else {
+ self.finishTransition(generation)
return
}
-
- let attemptIndex = self.stateLock.withLock { self.upgradeAttempts }
- let backoffSeconds = attemptIndex == 0 ? 0 : Self.upgradeBackoffBaseSeconds * pow(2, Double(attemptIndex - 1))
- let task = Task { [weak self] in
- try? await Task.sleep(nanoseconds: UInt64(backoffSeconds * 1_000_000_000))
- try? await Task.sleep(nanoseconds: UInt64(Self.upgradeDwellSeconds * 1_000_000_000))
- guard let self, !Task.isCancelled else { return }
- self.stateLock.withLock { self.dwellTask = nil }
- guard !(self.stateLock.withLock({ self.stopRequested })), self.stateLock.withLock({ self.phase == .avCapture }) else { return }
- let stillViable = MeetingCapturePathDecider.decide(
- mode: .onlineCall,
- microphone: self.originalMicrophone,
- outputRoute: MeetingCaptureEngine.currentOutputRouteSnapshot()
- )
- guard case .voiceProcessing = stillViable else { return }
- await self.attemptUpgrade()
+ let elected = await MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ guard self.ownsDowngradeTransition(generation) else {
+ self.finishTransition(generation)
+ return
}
- self.stateLock.withLock { self.dwellTask = task; self.dwellArming = false }
- }
- private func attemptUpgrade() async {
- let shouldStart = self.stateLock.withLock { () -> Bool in
- guard self.phase == .avCapture, !self.stopRequested else { return false }
- self.phase = .transitioning
- self.upgradeAttempts += 1
- return true
+ guard MeetingAVCaptureRebindDecision.shouldRebind(
+ activeCoreAudioUID: oldComponent.deviceUID,
+ electedCoreAudioUID: elected.identity.coreAudioUID,
+ forced: forced
+ ) else {
+ let restored = self.stateLock.withLock { () -> (restored: Bool, replay: Bool) in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.avCaptureComponent === oldComponent
+ else { return (false, false) }
+ let replay = self.routeChangeDirty
+ self.phase = .avCapture
+ self.transitionGeneration = previousGeneration
+ self.routeChangeDirty = false
+ self.transitionTask = nil
+ return (true, replay)
+ }
+ if restored.restored {
+ self.startAVCaptureWatchdog(generation: previousGeneration, component: oldComponent)
+ if restored.replay { self.handleRouteChange() }
+ self.armSafeVoiceProcessingRecoveryIfViable()
+ }
+ return
}
- guard shouldStart else { return }
- let task = Task { [weak self] in
- guard let self else { return }
+
+ DebugLogger.shared.log(
+ "Meeting AVCapture input rebind: \(oldComponent.deviceName) -> \(elected.identity.displayName) (\(detail))",
+ source: "MeetingCaptureEngine"
+ )
+
+ // This is the same ownership invariant as the VPIO downgrade: retire the
+ // old producer completely before AVFoundation discovers or starts another.
+ let oldFence = self.stateLock.withLock { self.currentFence }
+ await oldFence.quiesce()
+ try? await oldComponent.stop()
+
+ guard self.ownsDowngradeTransition(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+
+ let microphoneWriter = self.microphoneWriter
+ let liveAudioHandler = self.liveAudioHandler
+ var selectedElection: MeetingCaptureDeviceElection?
+ var selectedFence: MeetingEraAcceptanceFence?
+ var selectedGate: MeetingVoiceProcessingCommitGate?
+ var selectedComponent: MeetingAVCaptureMicrophoneComponent?
+ var selectedFailureToken: MeetingAVCaptureAttemptToken?
+ var lastCandidateFailure = "The replacement microphone was unavailable."
+
+ // CoreAudio and AVFoundation do not settle atomically during Bluetooth
+ // removal. Re-elect once after a bounded delay instead of permanently
+ // failing the meeting on a transient catalog mismatch.
+ for (attempt, retryDelay) in [0.0, 0.75].enumerated() {
+ if retryDelay > 0 {
+ try? await Task.sleep(nanoseconds: UInt64(retryDelay * 1_000_000_000))
+ }
+ guard self.ownsDowngradeTransition(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+ let candidateElection = attempt == 0
+ ? elected
+ : await MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ guard self.ownsDowngradeTransition(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+ self.stateLock.withLock {
+ if self.transitionGeneration == generation { self.pendingAVCaptureFailure = nil }
+ }
+
+ let candidateFence = MeetingEraAcceptanceFence()
+ let candidateGate = MeetingVoiceProcessingCommitGate()
+ let failureToken = MeetingAVCaptureAttemptToken()
+ let tokenInstalled = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested
+ else { return false }
+ self.avCaptureFailureToken = failureToken
+ return true
+ }
+ guard tokenInstalled else {
+ self.finishTransition(generation)
+ return
+ }
+ let candidate: MeetingAVCaptureMicrophoneComponent
+ do {
+ candidate = try MeetingAVCaptureMicrophoneComponent(
+ microphone: candidateElection.identity,
+ onSample: { sample in
+ guard candidateFence.beginSample() else { return }
+ defer { candidateFence.endSample() }
+ switch candidateGate.offer(sample) {
+ case .passthrough:
+ microphoneWriter.enqueue(sample)
+ liveAudioHandler?(.microphone, sample)
+ case .buffered, .aborted:
+ break
+ }
+ },
+ onTerminalStop: { [weak self] failure in
+ self?.handleAVCaptureFailureSignal(
+ detail: failure,
+ generation: generation,
+ token: failureToken
+ )
+ }
+ )
+ } catch {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ lastCandidateFailure = error.localizedDescription
+ continue
+ }
+
+ do {
+ try await candidate.start()
+ } catch {
+ // Even a failed start is retired through the same serialized
+ // barrier before another candidate is constructed.
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await candidate.stop()
+ lastCandidateFailure = error.localizedDescription
+ continue
+ }
+
+ guard await self.waitForBufferProgress(
+ timeoutSeconds: Self.firstBufferDeadlineSeconds,
+ using: candidate
+ ) else {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await candidate.stop()
+ lastCandidateFailure = "The microphone started but delivered no audio buffers."
+ continue
+ }
+
+ let candidateFailed = self.stateLock.withLock {
+ self.pendingAVCaptureFailure?.generation == generation
+ }
+ if candidateFailed {
+ lastCandidateFailure = self.stateLock.withLock {
+ self.pendingAVCaptureFailure?.detail ?? "The microphone stopped before commit."
+ }
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await candidate.stop()
+ continue
+ }
+
+ selectedElection = candidateElection
+ selectedFence = candidateFence
+ selectedGate = candidateGate
+ selectedComponent = candidate
+ selectedFailureToken = failureToken
+ break
+ }
+
+ guard let elected = selectedElection,
+ let newFence = selectedFence,
+ let newGate = selectedGate,
+ let component = selectedComponent,
+ let failureToken = selectedFailureToken
+ else {
+ self.failPendingDowngrade(
+ generation: generation,
+ detail: "Microphone rebind failed after retry: \(lastCandidateFailure)"
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ guard var pendingBatch = newGate.beginCommitFlush() else {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await component.stop()
+ self.failPendingDowngrade(
+ generation: generation,
+ detail: "Microphone buffering overflowed before the input rebind could commit."
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ let commitDecision = self.stateLock.withLock { () -> DowngradeCommitDecision in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.avCaptureComponent === oldComponent,
+ self.avCaptureFailureToken === failureToken
+ else { return .stale }
+ if let failure = self.pendingAVCaptureFailure, failure.generation == generation {
+ self.pendingAVCaptureFailure = nil
+ self.phase = .failed
+ return .failed(failure.detail)
+ }
+ self.currentFence = newFence
+ self.avCaptureComponent = component
+ self.phase = .committing
+ self.interruptedThisEra = false
+ return .committed
+ }
+ switch commitDecision {
+ case .stale:
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await component.stop()
+ self.finishTransition(generation)
+ return
+ case let .failed(failureDetail):
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ try? await component.stop()
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly, trackID: nil, detail: failureDetail
+ ))
+ self.finishTransition(generation)
+ return
+ case .committed:
+ break
+ }
+
+ let boundary = await self.microphoneWriter.beginSplice()
+ do {
+ try await self.microphoneWriter.updateTrackMetadata { track in
+ var eras = track.captureEras ?? [self.legacyEra(from: track)]
+ if let boundary, boundary.seconds > (eras.last?.startSeconds ?? -.infinity) {
+ eras.append(MeetingCaptureEra(
+ method: .avCaptureSession,
+ deviceUID: elected.identity.coreAudioUID,
+ deviceName: elected.identity.displayName,
+ roleAtElection: elected.role,
+ startSeconds: boundary.seconds
+ ))
+ }
+ track.captureEras = eras
+ track.captureMethod = .avCaptureSession
+ }
+ } catch {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ await newFence.quiesce()
+ try? await component.stop()
+ let shouldEmit = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .committing, self.transitionGeneration == generation else { return false }
+ self.phase = .failed
+ self.interruptedThisEra = true
+ return true
+ }
+ if shouldEmit {
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly,
+ trackID: nil,
+ detail: "Could not persist the microphone rebind: \(error.localizedDescription)"
+ ))
+ }
+ self.finishTransition(generation)
+ return
+ }
+
+ // Re-arm live microphone segmentation before any sample from the new
+ // physical source is released, even though the backend method is unchanged.
+ let ack = MeetingCaptureMethodChangeAck()
+ self.eventHandler(.captureMethodChanged(
+ trackID: self.microphoneWriter.trackID,
+ method: .avCaptureSession,
+ ack: ack
+ ))
+ await ack.wait()
+
+ while true {
+ await self.flushBatch(pendingBatch)
+ guard let next = newGate.continueCommitFlush() else { break }
+ pendingBatch = next
+ }
+
+ let terminalFailure = self.stateLock.withLock { () -> String? in
+ guard self.phase == .committing, self.transitionGeneration == generation else { return nil }
+ if let failure = self.pendingAVCaptureFailure, failure.generation == generation {
+ self.pendingAVCaptureFailure = nil
+ self.interruptedThisEra = true
+ self.phase = .failed
+ return failure.detail
+ }
+ self.phase = .avCapture
+ return nil
+ }
+ if let terminalFailure {
+ self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly, trackID: nil, detail: terminalFailure
+ ))
+ } else {
+ self.startAVCaptureWatchdog(generation: generation, component: component)
+ }
+ self.finishTransition(generation)
+ if terminalFailure == nil { self.armSafeVoiceProcessingRecoveryIfViable() }
+ }
+
+ private func legacyEra(from track: MeetingAudioTrack) -> MeetingCaptureEra {
+ MeetingCaptureEra(
+ method: track.captureMethod ?? .voiceProcessing,
+ deviceUID: self.originalMicrophone.coreAudioUID,
+ deviceName: self.originalMicrophone.displayName,
+ roleAtElection: self.originalMicrophone.role,
+ startSeconds: 0,
+ settledConfig: track.voiceProcessingConfig,
+ clockDrift: track.clockDrift
+ )
+ }
+
+ #if false
+ // Retained only as source history during the transition rollout. Compiled out so
+ // an AVCapture fallback cannot create a second VPIO owner in this meeting.
+ /// `dwellArming` reserves the slot in the SAME critical section as the `dwellTask == nil` check.
+ private func armUpgradeDwellIfViable() {
+ guard MeetingCaptureTransitionPolicy.allowsWithinSessionVoiceProcessingUpgrade else { return }
+ guard self.stateLock.withLock({
+ guard self.phase == .avCapture, self.dwellTask == nil, !self.dwellArming, !self.quarantined,
+ !self.stopRequested, self.upgradeAttempts < Self.maxUpgradeAttemptsPerSession
+ else { return false }
+ self.dwellArming = true
+ return true
+ }) else { return }
+ let decision = MeetingCapturePathDecider.decide(
+ mode: .onlineCall,
+ microphone: self.originalMicrophone,
+ outputRoute: MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
+ guard case .voiceProcessing = decision else {
+ self.stateLock.withLock { self.dwellArming = false }
+ return
+ }
+
+ let attemptIndex = self.stateLock.withLock { self.upgradeAttempts }
+ let backoffSeconds = attemptIndex == 0 ? 0 : Self.upgradeBackoffBaseSeconds * pow(2, Double(attemptIndex - 1))
+ let task = Task { [weak self] in
+ try? await Task.sleep(nanoseconds: UInt64(backoffSeconds * 1_000_000_000))
+ try? await Task.sleep(nanoseconds: UInt64(Self.upgradeDwellSeconds * 1_000_000_000))
+ guard let self, !Task.isCancelled else { return }
+ self.stateLock.withLock { self.dwellTask = nil }
+ guard !(self.stateLock.withLock({ self.stopRequested })), self.stateLock.withLock({ self.phase == .avCapture }) else { return }
+ let stillViable = MeetingCapturePathDecider.decide(
+ mode: .onlineCall,
+ microphone: self.originalMicrophone,
+ outputRoute: MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
+ guard case .voiceProcessing = stillViable else { return }
+ await self.attemptUpgrade()
+ }
+ self.stateLock.withLock { self.dwellTask = task; self.dwellArming = false }
+ }
+
+ private func attemptUpgrade() async {
+ let shouldStart = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .avCapture, !self.stopRequested else { return false }
+ self.phase = .transitioning
+ self.upgradeAttempts += 1
+ return true
+ }
+ guard shouldStart else { return }
+ let task = Task { [weak self] in
+ guard let self else { return }
await self.runUpgrade()
}
self.stateLock.withLock { self.transitionTask = task }
@@ -2315,7 +3553,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
return
}
- let elected = MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ let elected = await MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
let candidateMicCapture = MeetingMicrophoneCapture()
let candidateGate = MeetingVoiceProcessingCommitGate()
let candidateFence = MeetingEraAcceptanceFence()
@@ -2443,7 +3681,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
self.stateLock.withLock { self.phase = .avCapture }
self.finishTransition()
- self.armUpgradeDwellIfViable()
+ // Sticky fallback intentionally does not schedule another VPIO owner.
}
private func drainAbortedCaptures() async {
@@ -2453,29 +3691,667 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
}
}
+ #endif
- private func finishTransition() {
- self.stateLock.withLock { self.transitionTask = nil }
+ // MARK: - Stop-first AVCapture -> VPIO recovery
+
+ private func armSafeVoiceProcessingRecoveryIfViable() {
+ let outputRoute = MeetingCaptureEngine.currentOutputRouteSnapshot()
+ let task = self.stateLock.withLock { () -> Task<Void, Never>? in
+ guard self.phase == .avCapture,
+ self.avCaptureComponent != nil,
+ self.recoveryDwellTask == nil,
+ !self.stopRequested,
+ !self.quarantined,
+ MeetingCaptureTransitionPolicy.shouldAttemptVoiceProcessingRecovery(
+ routeDrivenFallback: self.routeDrivenFallback,
+ attempts: self.voiceProcessingRecoveryAttempts,
+ microphone: self.originalMicrophone,
+ outputRoute: outputRoute
+ )
+ else { return nil }
+ let task = Task { [weak self] in
+ try? await Task.sleep(nanoseconds: UInt64(
+ MeetingVoiceProcessingUpgradeDwell.seconds * 1_000_000_000
+ ))
+ guard let self, !Task.isCancelled else { return }
+ await self.beginSafeVoiceProcessingRecovery()
+ }
+ self.recoveryDwellTask = task
+ return task
+ }
+ _ = task // retained by runtime; silence the intentional local discard
+ }
+
+ private func beginSafeVoiceProcessingRecovery() async {
+ let elected = await MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ let outputRoute = MeetingCaptureEngine.currentOutputRouteSnapshot()
+ let started = self.stateLock.withLock { () -> Bool in
+ self.recoveryDwellTask = nil
+ guard self.phase == .avCapture,
+ self.avCaptureComponent != nil,
+ !self.stopRequested,
+ !self.quarantined,
+ MeetingCaptureTransitionPolicy.shouldAttemptVoiceProcessingRecovery(
+ routeDrivenFallback: self.routeDrivenFallback,
+ attempts: self.voiceProcessingRecoveryAttempts,
+ microphone: elected.identity,
+ outputRoute: outputRoute
+ )
+ else { return false }
+ self.phase = .transitioning
+ self.transitionGeneration &+= 1
+ let generation = self.transitionGeneration
+ self.voiceProcessingRecoveryAttempts += 1
+ self.routeChangeDirty = false
+ self.pendingAVCaptureFailure = nil
+ self.avCaptureFailureToken = nil
+ let task = Task { [weak self] in
+ guard let self else { return }
+ await self.runSafeVoiceProcessingRecovery(generation: generation)
+ }
+ self.transitionTask = task
+ return true
+ }
+ if started {
+ let watchdog = self.stateLock.withLock { () -> Task<Void, Never>? in
+ defer { self.watchdogTask = nil }
+ return self.watchdogTask
+ }
+ watchdog?.cancel()
+ }
+ }
+
+ private func ownsSafeRecovery(_ generation: UInt64) -> Bool {
+ self.stateLock.withLock {
+ (self.phase == .transitioning || self.phase == .committing)
+ && self.transitionGeneration == generation
+ && !self.stopRequested
+ }
+ }
+
+ private func runSafeVoiceProcessingRecovery(generation: UInt64) async {
+ guard self.ownsSafeRecovery(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+
+ let oldFence = self.stateLock.withLock { self.currentFence }
+ var retiringComponent = self.stateLock.withLock { self.avCaptureComponent }
+ guard retiringComponent != nil else {
+ self.failPendingDowngrade(
+ generation: generation,
+ detail: "Voice-processing recovery had no active fallback microphone to retire."
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ await oldFence.quiesce()
+ do {
+ try await retiringComponent?.stop()
+ } catch {
+ self.failSafeRecovery(
+ generation: generation,
+ detail: "Could not retire fallback microphone capture: \(error.localizedDescription)"
+ )
+ self.finishTransition(generation)
+ return
+ }
+ guard retiringComponent?.isFullyRetired() == true else {
+ self.failSafeRecovery(
+ generation: generation,
+ detail: "Fallback microphone teardown could not be proven; recovery was quarantined."
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ // Remove every runtime strong reference, then drop the final local reference.
+ // `MeetingAVCaptureMicrophoneComponent.stop()` has already dismantled the
+ // session graph and removed observers on its serialized control queue.
+ let cleared = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ self.avCaptureComponent === retiringComponent
+ else { return false }
+ self.avCaptureComponent = nil
+ self.avCaptureFailureToken = nil
+ self.pendingAVCaptureFailure = nil
+ return true
+ }
+ retiringComponent = nil
+ guard cleared, self.ownsSafeRecovery(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+
+ let boundary = await self.microphoneWriter.beginSplice()
+ guard self.ownsSafeRecovery(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+
+ // Re-elect only after AVFoundation has released the old physical device.
+ let elected = await MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ guard self.ownsSafeRecovery(generation),
+ MeetingCapturePathDecider.decide(
+ mode: .onlineCall,
+ microphone: elected.identity,
+ outputRoute: MeetingCaptureEngine.currentOutputRouteSnapshot()
+ ) == .voiceProcessing
+ else {
+ await self.restoreAVCaptureAfterRecoveryFailure(
+ generation: generation,
+ boundary: boundary,
+ detail: "The audio route changed before voice processing could restart."
+ )
+ return
+ }
+
+ // Optional by design: every rollback path explicitly drops the final local
+ // reference after stop() and before constructing a fresh AVCaptureSession.
+ var candidate: MeetingMicrophoneCapture? = MeetingMicrophoneCapture()
+ let candidateGate = MeetingVoiceProcessingCommitGate()
+ let candidateFence = MeetingEraAcceptanceFence()
+ let microphoneWriter = self.microphoneWriter
+ let liveAudioHandler = self.liveAudioHandler
+ let parked = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.avCaptureComponent == nil
+ else { return false }
+ guard let candidate else { return false }
+ self.pendingVoiceProcessingCapture = candidate
+ self.micCapture = candidate
+ return true
+ }
+ guard parked else {
+ await candidate?.stop()
+ candidate = nil
+ self.finishTransition(generation)
+ return
+ }
+
+ let outcome: MeetingMicrophoneBindingOutcome
+ do {
+ outcome = try await candidate!.start(
+ microphone: elected.identity,
+ authorizationPreflighted: true,
+ onSample: { sample in
+ guard candidateFence.beginSample() else { return }
+ defer { candidateFence.endSample() }
+ switch candidateGate.offer(sample) {
+ case .passthrough:
+ microphoneWriter.enqueue(sample)
+ liveAudioHandler?(.microphone, sample)
+ case .buffered, .aborted:
+ break
+ }
+ },
+ onEvent: { [weak self, weak candidate] event in
+ guard let candidate else { return }
+ self?.handleMicEvent(event, owner: candidate)
+ }
+ )
+ } catch {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ await self.restoreAVCaptureAfterRecoveryFailure(
+ generation: generation,
+ boundary: boundary,
+ detail: "Voice processing could not restart: \(error.localizedDescription)"
+ )
+ return
+ }
+ if case .unavailable = outcome {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ await self.restoreAVCaptureAfterRecoveryFailure(
+ generation: generation,
+ boundary: boundary,
+ detail: "Voice-processing microphone capture was unavailable."
+ )
+ return
+ }
+
+ guard self.ownsSafeRecovery(generation),
+ await self.waitForBufferProgress(
+ timeoutSeconds: Self.firstBufferDeadlineSeconds,
+ using: candidate!
+ ),
+ self.ownsSafeRecovery(generation),
+ MeetingCapturePathDecider.decide(
+ mode: .onlineCall,
+ microphone: elected.identity,
+ outputRoute: MeetingCaptureEngine.currentOutputRouteSnapshot()
+ ) == .voiceProcessing
+ else {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ await self.restoreAVCaptureAfterRecoveryFailure(
+ generation: generation,
+ boundary: boundary,
+ detail: "Voice processing did not become ready on the settled audio route."
+ )
+ return
+ }
+
+ guard var pendingBatch = candidateGate.beginCommitFlush() else {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ await self.restoreAVCaptureAfterRecoveryFailure(
+ generation: generation,
+ boundary: boundary,
+ detail: "Voice-processing buffering overflowed before recovery could commit."
+ )
+ return
+ }
+ let mayCommit = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.pendingVoiceProcessingCapture === candidate,
+ self.avCaptureComponent == nil,
+ self.pendingAVCaptureFailure == nil
+ else { return false }
+ self.phase = .committing
+ return true
+ }
+ guard mayCommit else {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ self.finishTransition(generation)
+ return
+ }
+
+ let settled = await candidate!.settledConfiguration()
+ do {
+ try await self.microphoneWriter.updateTrackMetadata { track in
+ var eras = track.captureEras ?? [self.legacyEra(from: track)]
+ if let boundary, boundary.seconds > (eras.last?.startSeconds ?? -.infinity) {
+ eras.append(MeetingCaptureEra(
+ method: .voiceProcessing,
+ deviceUID: elected.identity.coreAudioUID,
+ deviceName: elected.identity.displayName,
+ roleAtElection: elected.role,
+ startSeconds: boundary.seconds,
+ settledConfig: settled
+ ))
+ }
+ track.captureEras = eras
+ track.captureMethod = .voiceProcessing
+ track.voiceProcessingConfig = settled
+ }
+ } catch {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ await self.restoreAVCaptureAfterRecoveryFailure(
+ generation: generation,
+ boundary: boundary,
+ detail: "Could not persist voice-processing recovery: \(error.localizedDescription)"
+ )
+ return
+ }
+
+ let ack = MeetingCaptureMethodChangeAck()
+ self.eventHandler(.captureMethodChanged(
+ trackID: self.microphoneWriter.trackID,
+ method: .voiceProcessing,
+ ack: ack
+ ))
+ await ack.wait()
+ guard self.ownsSafeRecovery(generation) else {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ self.finishTransition(generation)
+ return
+ }
+
+ while true {
+ await self.flushBatch(pendingBatch)
+ guard let next = candidateGate.continueCommitFlush() else { break }
+ pendingBatch = next
+ }
+
+ let committed = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .committing,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.pendingVoiceProcessingCapture === candidate,
+ self.pendingAVCaptureFailure == nil
+ else { return false }
+ self.currentFence = candidateFence
+ self.gate = candidateGate
+ self.pendingVoiceProcessingCapture = nil
+ self.phase = .vpio
+ self.routeDrivenFallback = false
+ self.interruptedThisEra = false
+ return true
+ }
+ guard committed else {
+ await self.retirePendingVoiceProcessingCapture(generation: generation)
+ candidate = nil
+ self.finishTransition(generation)
+ return
+ }
+ DebugLogger.shared.log(
+ "Meeting microphone recovered voice processing on \(elected.identity.displayName).",
+ source: "MeetingCaptureEngine"
+ )
+ self.startWatchdog()
+ self.finishTransition(generation)
+ }
+
+ private func retirePendingVoiceProcessingCapture(generation: UInt64) async {
+ let capture = self.stateLock.withLock { self.pendingVoiceProcessingCapture }
+ await capture?.stop()
+ self.stateLock.withLock {
+ guard self.transitionGeneration == generation,
+ self.pendingVoiceProcessingCapture === capture
+ else { return }
+ self.pendingVoiceProcessingCapture = nil
+ self.micCapture = MeetingMicrophoneCapture()
+ }
+ }
+
+ private func restoreAVCaptureAfterRecoveryFailure(
+ generation: UInt64,
+ boundary: MeetingMediaTime?,
+ detail: String
+ ) async {
+ guard self.ownsSafeRecovery(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+ let elected = await MeetingCaptureDeviceElection.elect(original: self.originalMicrophone)
+ guard self.ownsSafeRecovery(generation) else {
+ self.finishTransition(generation)
+ return
+ }
+
+ let fence = MeetingEraAcceptanceFence()
+ let gate = MeetingVoiceProcessingCommitGate()
+ let token = MeetingAVCaptureAttemptToken()
+ let microphoneWriter = self.microphoneWriter
+ let liveAudioHandler = self.liveAudioHandler
+ let component: MeetingAVCaptureMicrophoneComponent
+ do {
+ component = try MeetingAVCaptureMicrophoneComponent(
+ microphone: elected.identity,
+ onSample: { sample in
+ guard fence.beginSample() else { return }
+ defer { fence.endSample() }
+ switch gate.offer(sample) {
+ case .passthrough:
+ microphoneWriter.enqueue(sample)
+ liveAudioHandler?(.microphone, sample)
+ case .buffered, .aborted:
+ break
+ }
+ },
+ onTerminalStop: { [weak self] failure in
+ self?.handleAVCaptureFailureSignal(
+ detail: failure,
+ generation: generation,
+ token: token
+ )
+ }
+ )
+ } catch {
+ self.failSafeRecovery(
+ generation: generation,
+ detail: "\(detail) Fallback microphone recreation failed: \(error.localizedDescription)"
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ let parked = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .transitioning || self.phase == .committing,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.pendingVoiceProcessingCapture == nil,
+ self.avCaptureComponent == nil
+ else { return false }
+ self.phase = .transitioning
+ self.avCaptureComponent = component
+ self.avCaptureFailureToken = token
+ self.pendingAVCaptureFailure = nil
+ return true
+ }
+ guard parked else {
+ try? await component.stop()
+ self.finishTransition(generation)
+ return
+ }
+
+ do {
+ try await component.start()
+ } catch {
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.failSafeRecovery(
+ generation: generation,
+ detail: "\(detail) Fallback microphone could not start: \(error.localizedDescription)"
+ )
+ self.finishTransition(generation)
+ return
+ }
+ guard self.ownsSafeRecovery(generation),
+ await self.waitForBufferProgress(
+ timeoutSeconds: Self.firstBufferDeadlineSeconds,
+ using: component
+ ),
+ self.ownsSafeRecovery(generation),
+ var pendingBatch = gate.beginCommitFlush()
+ else {
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.failSafeRecovery(
+ generation: generation,
+ detail: "\(detail) Fallback microphone delivered no audio."
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ let promotion = self.stateLock.withLock { () -> DowngradeCommitDecision in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.avCaptureComponent === component,
+ self.avCaptureFailureToken === token
+ else { return .stale }
+ if let failure = self.pendingAVCaptureFailure, failure.generation == generation {
+ return .failed(failure.detail)
+ }
+ self.phase = .committing
+ return .committed
+ }
+ switch promotion {
+ case .committed:
+ break
+ case .stale:
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.finishTransition(generation)
+ return
+ case let .failed(failureDetail):
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.failSafeRecovery(generation: generation, detail: failureDetail)
+ self.finishTransition(generation)
+ return
+ }
+ do {
+ try await self.microphoneWriter.updateTrackMetadata { track in
+ var eras = track.captureEras ?? [self.legacyEra(from: track)]
+ if let boundary, boundary.seconds > (eras.last?.startSeconds ?? -.infinity) {
+ eras.append(MeetingCaptureEra(
+ method: .avCaptureSession,
+ deviceUID: elected.identity.coreAudioUID,
+ deviceName: elected.identity.displayName,
+ roleAtElection: elected.role,
+ startSeconds: boundary.seconds
+ ))
+ }
+ track.captureEras = eras
+ track.captureMethod = .avCaptureSession
+ }
+ } catch {
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.failSafeRecovery(
+ generation: generation,
+ detail: "\(detail) Fallback microphone metadata could not be saved."
+ )
+ self.finishTransition(generation)
+ return
+ }
+
+ let ack = MeetingCaptureMethodChangeAck()
+ self.eventHandler(.captureMethodChanged(
+ trackID: self.microphoneWriter.trackID,
+ method: .avCaptureSession,
+ ack: ack
+ ))
+ await ack.wait()
+ guard self.ownsSafeRecovery(generation) else {
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.finishTransition(generation)
+ return
+ }
+ while true {
+ await self.flushBatch(pendingBatch)
+ guard let next = gate.continueCommitFlush() else { break }
+ pendingBatch = next
+ }
+
+ let commitDecision = self.stateLock.withLock { () -> DowngradeCommitDecision in
+ guard self.phase == .committing,
+ self.transitionGeneration == generation,
+ !self.stopRequested,
+ self.avCaptureComponent === component,
+ self.avCaptureFailureToken === token
+ else { return .stale }
+ if let failure = self.pendingAVCaptureFailure, failure.generation == generation {
+ return .failed(failure.detail)
+ }
+ self.currentFence = fence
+ self.phase = .avCapture
+ // Route-driven recovery remains eligible for a bounded retry. The
+ // policy's attempt cap prevents churn while allowing transient HAL
+ // settling failures after Bluetooth removal to recover echo control.
+ self.interruptedThisEra = false
+ return .committed
+ }
+ switch commitDecision {
+ case .committed:
+ break
+ case .stale:
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.finishTransition(generation)
+ return
+ case let .failed(failureDetail):
+ self.invalidateAVCaptureFailureToken(token, generation: generation)
+ try? await component.stop()
+ self.failSafeRecovery(generation: generation, detail: failureDetail)
+ self.finishTransition(generation)
+ return
+ }
+ DebugLogger.shared.warning(
+ "\(detail) Continued with fallback microphone capture.",
+ source: "MeetingCaptureEngine"
+ )
+ self.startAVCaptureWatchdog(generation: generation, component: component)
+ self.finishTransition(generation)
+ self.armSafeVoiceProcessingRecoveryIfViable()
+ }
+
+ private func failSafeRecovery(generation: UInt64, detail: String) {
+ let shouldEmit = self.stateLock.withLock { () -> Bool in
+ guard self.transitionGeneration == generation, !self.stopRequested else { return false }
+ self.pendingVoiceProcessingCapture = nil
+ self.avCaptureComponent = nil
+ self.avCaptureFailureToken = nil
+ self.pendingAVCaptureFailure = nil
+ self.quarantined = true
+ self.phase = .failed
+ self.interruptedThisEra = true
+ return true
+ }
+ if shouldEmit {
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly,
+ trackID: nil,
+ detail: detail
+ ))
+ }
+ }
+
+ private func ownsDowngradeTransition(_ generation: UInt64) -> Bool {
+ self.stateLock.withLock {
+ self.phase == .transitioning
+ && self.transitionGeneration == generation
+ && !self.stopRequested
+ }
+ }
+
+ private func failPendingDowngrade(generation: UInt64, detail: String) {
+ let shouldEmit = self.stateLock.withLock { () -> Bool in
+ guard self.phase == .transitioning,
+ self.transitionGeneration == generation,
+ !self.stopRequested
+ else { return false }
+ self.pendingAVCaptureFailure = nil
+ self.phase = .failed
+ self.interruptedThisEra = true
+ return true
+ }
+ if shouldEmit {
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly, trackID: nil, detail: detail
+ ))
+ }
+ }
+
+ private func finishTransition(_ generation: UInt64) {
+ let shouldReplayRouteChange = self.stateLock.withLock { () -> Bool in
+ guard self.transitionGeneration == generation else { return false }
+ self.transitionTask = nil
+ guard (self.phase == .avCapture || self.phase == .vpio),
+ self.routeChangeDirty,
+ !self.stopRequested
+ else { return false }
+ self.routeChangeDirty = false
+ return true
+ }
+ if shouldReplayRouteChange { self.handleRouteChange() }
}
func stop() async throws {
- let dwell = self.stateLock.withLock { () -> Task<Void, Never>? in
+ let recoveryDwell = self.stateLock.withLock { () -> Task<Void, Never>? in
self.stopRequested = true
- defer { self.dwellTask = nil }
- return self.dwellTask
+ self.avCaptureFailureToken = nil
+ self.pendingAVCaptureFailure = nil
+ defer { self.recoveryDwellTask = nil }
+ return self.recoveryDwellTask
}
- dwell?.cancel()
+ recoveryDwell?.cancel()
if let transitionTask = self.stateLock.withLock({ self.transitionTask }) {
await transitionTask.value
}
- await self.drainAbortedCaptures()
let (watchdog, staleListener) = self.stateLock.withLock { () -> (Task<Void, Never>?, MeetingOutputRouteListener?) in
defer { self.watchdogTask = nil; self.outputRouteListener = nil }
return (self.watchdogTask, self.outputRouteListener)
}
watchdog?.cancel()
- _ = staleListener // released here, outside the lock
+ await staleListener?.stop()
if let fallback = self.stateLock.withLock({ self.fallbackRuntime }) {
try await fallback.stop()
@@ -2486,7 +4362,13 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
guard phase == .vpio || phase == .avCapture else {
let (gateToAbort, captureToStop) = self.stateLock.withLock { (self.gate, self.micCapture) }
gateToAbort.abort()
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingMicrophone, queueRole: .actorControl, phase: .microphoneStop)
+ #endif
await captureToStop.stop()
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingMicrophone, queueRole: .actorControl, phase: .microphoneStop)
+ #endif
if let component = self.stateLock.withLock({ self.avCaptureComponent }) {
try? await component.stop()
}
@@ -2497,7 +4379,13 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
if phase == .vpio {
// Quiesce first — stats must not be snapshotted while the tap can still emit.
let micCapture = self.stateLock.withLock { self.micCapture }
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingMicrophone, queueRole: .actorControl, phase: .microphoneStop)
+ #endif
await micCapture.stop()
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingMicrophone, queueRole: .actorControl, phase: .microphoneStop)
+ #endif
let statsSnapshot = await micCapture.statistics()
let drift = MeetingClockDriftRecord(
cumulativeAbsorbedSeconds: statsSnapshot.cumulativeAbsorbedCorrectionSeconds,
@@ -2513,10 +4401,20 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
}
} else if let component = self.stateLock.withLock({ self.avCaptureComponent }) {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .avCapture, queueRole: .actorControl, phase: .avCaptureStop)
+ #endif
try? await component.stop()
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .avCapture, queueRole: .actorControl, phase: .avCaptureStop)
+ #endif
}
if let appOnly = self.stateLock.withLock({ self.appOnlyRuntime }) {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .screenCapture, queueRole: .actorControl, phase: .screenCaptureStop)
+ defer { AudioTopologyDiagnostics.record(.phaseEnd, owner: .screenCapture, queueRole: .actorControl, phase: .screenCaptureStop) }
+ #endif
try await appOnly.stop()
}
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingMicrophoneCapture.swift b/Sources/Fluid/Services/Meeting/MeetingMicrophoneCapture.swift
index bdcd97bf..8be19fcb 100644
--- a/Sources/Fluid/Services/Meeting/MeetingMicrophoneCapture.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingMicrophoneCapture.swift
@@ -327,6 +327,22 @@ nonisolated enum MeetingMicrophoneCaptureEvent: Sendable {
nonisolated enum MeetingMicrophoneCaptureError: Error, Sendable {
case alreadyStarted
+ case lifecycleInterrupted
+}
+
+/// Core Audio invokes property listeners on a shared serial delivery queue. Listener callbacks
+/// must return before meeting policy performs any HAL reads or capture transitions.
+nonisolated enum MeetingMicrophoneEventExecution {
+ private static let queue = DispatchQueue(
+ label: "com.fluidvoice.meeting-microphone-events",
+ qos: .userInitiated
+ )
+
+ static func afterHALCallback(_ work: @escaping @Sendable () -> Void) {
+ AudioTopologyListenerExecution.deliveryQueue.async {
+ Self.queue.async(execute: work)
+ }
+ }
}
// MARK: - Stats
@@ -623,6 +639,9 @@ actor MeetingMicrophoneCapture {
private var engine: AVAudioEngine?
private var tapState: MeetingMicrophoneTapState?
private var isRunning = false
+ /// Non-nil from the instant a start reserves the actor until that generation is stopped or fails.
+ /// This closes actor-reentrancy holes while `start()` awaits format settling/listener installation.
+ private var activeGeneration: UInt64?
private var settled: MeetingMicrophoneSettledConfig?
private var configObserver: NSObjectProtocol?
private var defaultInputListenerToken: AudioObjectPropertyListenerBlock?
@@ -631,11 +650,13 @@ actor MeetingMicrophoneCapture {
func start(
microphone: MeetingMicrophoneIdentity,
+ authorizationPreflighted: Bool = false,
onSample: @escaping @Sendable (CMSampleBuffer) -> Void,
onEvent: @escaping @Sendable (MeetingMicrophoneCaptureEvent) -> Void = { _ in }
) async throws -> MeetingMicrophoneBindingOutcome {
- guard !self.isRunning else { throw MeetingMicrophoneCaptureError.alreadyStarted }
+ guard self.activeGeneration == nil else { throw MeetingMicrophoneCaptureError.alreadyStarted }
let currentGeneration = self.generationBox.advance()
+ self.activeGeneration = currentGeneration
#if DEBUG
var diagnosticsPhaseEnded = false
@@ -647,19 +668,21 @@ actor MeetingMicrophoneCapture {
}
}
#endif
- let authorizationStatus = AVCaptureDevice.authorizationStatus(for: .audio)
+ let authorizationStatus: AVAuthorizationStatus = authorizationPreflighted
+ ? .authorized
+ : AVCaptureDevice.authorizationStatus(for: .audio)
#if DEBUG
AudioTopologyDiagnostics.record(.avfAuthorizationEnd, owner: .meetingMicrophone, queueRole: .actorControl, phase: .catalog, status: Int32(authorizationStatus.rawValue), generation: currentGeneration)
#endif
guard authorizationStatus == .authorized else {
- return self.fail(microphone: microphone, reason: "Microphone permission is not granted.")
+ return self.fail(microphone: microphone, reason: "Microphone permission is not granted.", generation: currentGeneration)
}
// Missing UID means the name match already wasn't unique; retrying it is vacuous.
guard let requestedUID = microphone.coreAudioUID, !requestedUID.isEmpty,
let requestedDeviceID = AudioDevice.listInputDevices().first(where: { $0.uid == requestedUID })?.id
else {
- return self.fail(microphone: microphone, reason: "No CoreAudio device UID for '\(microphone.displayName)'.")
+ return self.fail(microphone: microphone, reason: "No CoreAudio device UID for '\(microphone.displayName)'.", generation: currentGeneration)
}
let engine = AVAudioEngine()
@@ -678,7 +701,7 @@ actor MeetingMicrophoneCapture {
#if DEBUG
AudioTopologyDiagnostics.record(.vpioEnableEnd, owner: .meetingMicrophone, objectID: requestedDeviceID, queueRole: .actorControl, phase: .vpio, status: -1, generation: currentGeneration)
#endif
- return self.fail(microphone: microphone, reason: "Voice processing could not be enabled: \(error.localizedDescription)")
+ return self.fail(microphone: microphone, reason: "Voice processing could not be enabled: \(error.localizedDescription)", generation: currentGeneration)
}
var ducking = AVAudioVoiceProcessingOtherAudioDuckingConfiguration()
ducking.enableAdvancedDucking = false
@@ -686,7 +709,7 @@ actor MeetingMicrophoneCapture {
input.voiceProcessingOtherAudioDuckingConfiguration = ducking
guard let audioUnit = input.audioUnit else {
- return self.fail(microphone: microphone, reason: "AVAudioEngine.inputNode has no backing AudioUnit.")
+ return self.fail(microphone: microphone, reason: "AVAudioEngine.inputNode has no backing AudioUnit.", generation: currentGeneration)
}
// TODO(probe): bind order is a Phase-1 measurement; this hard-codes one attempt.
@@ -702,7 +725,7 @@ actor MeetingMicrophoneCapture {
// 10 minutes of zero callbacks with healthy telemetry.
let preStartFormat = input.outputFormat(forBus: 0)
guard preStartFormat.sampleRate > 0, preStartFormat.channelCount > 0 else {
- return self.fail(microphone: microphone, reason: "Input format invalid before start (\(preStartFormat)).")
+ return self.fail(microphone: microphone, reason: "Input format invalid before start (\(preStartFormat)).", generation: currentGeneration)
}
let tapState = MeetingMicrophoneTapState(statsBox: self.statsBox, canonicalFormat: Self.canonicalFormat, onSample: onSample)
self.tapState = tapState
@@ -732,7 +755,7 @@ actor MeetingMicrophoneCapture {
AudioTopologyDiagnostics.record(.engineStartEnd, owner: .meetingMicrophone, objectID: requestedDeviceID, queueRole: .actorControl, phase: .engine, status: -1, generation: currentGeneration)
#endif
input.removeTap(onBus: 0)
- return self.fail(microphone: microphone, reason: "AVAudioEngine failed to start: \(error.localizedDescription)")
+ return self.fail(microphone: microphone, reason: "AVAudioEngine failed to start: \(error.localizedDescription)", generation: currentGeneration)
}
let outcome = MeetingMicrophoneBindDecision.outcome(
@@ -745,13 +768,26 @@ actor MeetingMicrophoneCapture {
if case let .unavailable(reason) = outcome {
input.removeTap(onBus: 0)
engine.stop()
- return self.fail(microphone: microphone, reason: reason)
+ return self.fail(microphone: microphone, reason: reason, generation: currentGeneration)
}
+ // Publish the generation-owned resources before the first suspension so `stop()` can
+ // atomically detach and terminate this exact engine if it interleaves with startup.
+ self.engine = engine
+ self.isRunning = true
guard let sourceFormat = await self.validatedInputFormat(input: input) else {
+ guard self.ownsGeneration(currentGeneration, engine: engine) else {
+ throw MeetingMicrophoneCaptureError.lifecycleInterrupted
+ }
input.removeTap(onBus: 0)
engine.stop()
- return self.fail(microphone: microphone, reason: "Input format never became valid (persistent 0 Hz).")
+ self.engine = nil
+ self.tapState = nil
+ self.isRunning = false
+ return self.fail(microphone: microphone, reason: "Input format never became valid (persistent 0 Hz).", generation: currentGeneration)
+ }
+ guard self.ownsGeneration(currentGeneration, engine: engine) else {
+ throw MeetingMicrophoneCaptureError.lifecycleInterrupted
}
if sourceFormat != preStartFormat {
// Post-start renegotiation (HAL settling): reattach at the settled format.
@@ -762,8 +798,6 @@ actor MeetingMicrophoneCapture {
}
}
- self.engine = engine
- self.isRunning = true
self.settled = MeetingMicrophoneSettledConfig(
requestedMicrophone: microphone,
bindingOutcome: outcome,
@@ -775,7 +809,16 @@ actor MeetingMicrophoneCapture {
voiceProcessingEnabled: true,
outputChannelDecision: "mono (explicit downmix); observed ch=1 on today's SCStream mic, not derived from SCStream's stereo capture config"
)
- self.registerEventObservers(engine: engine, outcome: outcome, requestedDeviceID: requestedDeviceID, onEvent: onEvent)
+ await self.registerEventObservers(
+ engine: engine,
+ outcome: outcome,
+ requestedDeviceID: requestedDeviceID,
+ generation: currentGeneration,
+ onEvent: onEvent
+ )
+ guard self.ownsGeneration(currentGeneration, engine: engine) else {
+ throw MeetingMicrophoneCaptureError.lifecycleInterrupted
+ }
#if DEBUG
AudioTopologyDiagnostics.record(.readiness, owner: .meetingMicrophone, objectID: requestedDeviceID, queueRole: .actorControl, phase: .engine, status: noErr, generation: currentGeneration)
AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingMicrophone, queueRole: .actorControl, phase: .handoff, status: noErr, generation: currentGeneration)
@@ -785,23 +828,48 @@ actor MeetingMicrophoneCapture {
}
func stop() async {
- guard self.isRunning else { return }
+ guard self.activeGeneration != nil else { return }
self.isRunning = false
- self.tearDownEventObservers() // first: stop-time churn must not poison a stats snapshot
let stopGeneration = self.generationBox.advance()
+ self.activeGeneration = nil
+
+ // Snapshot and detach every generation-owned reference before the first await. A new start
+ // may legally enter while listener removal is suspended and must remain completely isolated.
+ let oldEngine = self.engine
+ let oldConfigObserver = self.configObserver
+ let oldDefaultInputListenerToken = self.defaultInputListenerToken
+ let oldOverloadListenerToken = self.overloadListenerToken
+ let oldOverloadListenerDeviceID = self.overloadListenerDeviceID
+ self.engine = nil
+ self.tapState = nil
+ self.configObserver = nil
+ self.defaultInputListenerToken = nil
+ self.overloadListenerToken = nil
+ self.overloadListenerDeviceID = nil
- if let engine = self.engine {
- engine.inputNode.removeTap(onBus: 0)
+ if let oldConfigObserver {
+ NotificationCenter.default.removeObserver(oldConfigObserver)
+ }
+ if let oldEngine {
+ oldEngine.inputNode.removeTap(onBus: 0)
#if DEBUG
AudioTopologyDiagnostics.record(.engineStopBegin, owner: .meetingMicrophone, queueRole: .actorControl, phase: .engine, generation: stopGeneration)
#endif
- engine.stop()
+ oldEngine.stop()
+ // Explicitly tear down the VPIO Audio Unit. Relying on AVAudioEngine
+ // deallocation leaves device ownership to ARC timing and can overlap a
+ // subsequent AVCaptureSession construction during route recovery.
+ try? oldEngine.inputNode.setVoiceProcessingEnabled(false)
#if DEBUG
AudioTopologyDiagnostics.record(.engineStopEnd, owner: .meetingMicrophone, queueRole: .actorControl, phase: .engine, status: noErr, generation: stopGeneration)
#endif
}
- self.engine = nil
- self.tapState = nil
+ await Self.removeEventObservers(
+ defaultInputToken: oldDefaultInputListenerToken,
+ overloadToken: oldOverloadListenerToken,
+ overloadDeviceID: oldOverloadListenerDeviceID,
+ generation: stopGeneration
+ )
}
/// Fire-and-forget, bounded teardown. `onComplete(false)` means the supervision window elapsed first.
@@ -823,7 +891,24 @@ actor MeetingMicrophoneCapture {
self.settled
}
- private func fail(microphone: MeetingMicrophoneIdentity, reason: String) -> MeetingMicrophoneBindingOutcome {
+ private func ownsGeneration(_ generation: UInt64, engine: AVAudioEngine) -> Bool {
+ self.activeGeneration == generation
+ && self.generationBox.isCurrent(generation)
+ && self.engine === engine
+ && self.isRunning
+ }
+
+ private func fail(
+ microphone: MeetingMicrophoneIdentity,
+ reason: String,
+ generation: UInt64
+ ) -> MeetingMicrophoneBindingOutcome {
+ if self.activeGeneration == generation, self.generationBox.isCurrent(generation) {
+ self.activeGeneration = nil
+ self.isRunning = false
+ self.engine = nil
+ self.tapState = nil
+ }
let outcome = MeetingMicrophoneBindingOutcome.unavailable(reason: reason)
self.settled = MeetingMicrophoneSettledConfig(
requestedMicrophone: microphone,
@@ -856,78 +941,110 @@ actor MeetingMicrophoneCapture {
engine: AVAudioEngine,
outcome: MeetingMicrophoneBindingOutcome,
requestedDeviceID: AudioObjectID,
+ generation: UInt64,
onEvent: @escaping @Sendable (MeetingMicrophoneCaptureEvent) -> Void
- ) {
+ ) async {
let statsBox = self.statsBox
+ let generationBox = self.generationBox
// queue: nil — see ASRService.registerEngineConfigurationChangeObserver for why synchronous
// delivery is load-bearing (avoids a deadlock against the engine's own serial queue).
self.configObserver = NotificationCenter.default.addObserver(
forName: .AVAudioEngineConfigurationChange, object: engine, queue: nil
) { [weak tapState = self.tapState] _ in
+ guard generationBox.isCurrent(generation) else { return }
statsBox.mutate { $0.configurationChangeEvents += 1 }
tapState?.requestTimelineReset()
onEvent(.configurationChanged)
}
if case .defaultMatchesRequested = outcome {
- var address = AudioObjectPropertyAddress(
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultInputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
let token: AudioObjectPropertyListenerBlock = { _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .meetingMicrophone, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingMicrophone, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingMicrophone, objectID: AudioObjectID(kAudioObjectSystemObject), selector: kAudioHardwarePropertyDefaultInputDevice, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
- statsBox.mutate { $0.defaultInputChangeEvents += 1 }
- onEvent(.defaultInputChanged)
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ guard generationBox.isCurrent(generation) else { return }
+ statsBox.mutate { $0.defaultInputChangeEvents += 1 }
+ onEvent(.defaultInputChanged)
+ }
}
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingMicrophone, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener)
#endif
- let status = AudioObjectAddPropertyListenerBlock(AudioObjectID(kAudioObjectSystemObject), &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.add(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: address,
+ token: token
+ )
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingMicrophone, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener, status: status)
#endif
- if status == noErr {
+ if status == noErr, self.isRunning, self.generationBox.isCurrent(generation) {
self.defaultInputListenerToken = token
+ } else if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: address,
+ token: token
+ )
}
}
- if case .boundVerified = outcome {
- var address = AudioObjectPropertyAddress(
+ if case .boundVerified = outcome, self.isRunning, self.generationBox.isCurrent(generation) {
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioDeviceProcessorOverload,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
let token: AudioObjectPropertyListenerBlock = { _, _ in
#if DEBUG
- AudioTopologyDiagnostics.record(.callback, owner: .meetingMicrophone, objectID: requestedDeviceID, selector: kAudioDeviceProcessorOverload, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .mainDelivery)
+ AudioTopologyDiagnostics.record(.callbackBegin, owner: .meetingMicrophone, objectID: requestedDeviceID, selector: kAudioDeviceProcessorOverload, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation)
+ defer { AudioTopologyDiagnostics.record(.callbackEnd, owner: .meetingMicrophone, objectID: requestedDeviceID, selector: kAudioDeviceProcessorOverload, scope: kAudioObjectPropertyScopeGlobal, element: kAudioObjectPropertyElementMain, queueRole: .callbackCurrent, generation: generation) }
#endif
- statsBox.mutate { $0.overloadCount += 1 }
- onEvent(.overload)
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ guard generationBox.isCurrent(generation) else { return }
+ statsBox.mutate { $0.overloadCount += 1 }
+ onEvent(.overload)
+ }
}
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddBegin, owner: .meetingMicrophone, objectID: requestedDeviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener)
#endif
- let status = AudioObjectAddPropertyListenerBlock(requestedDeviceID, &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.add(
+ objectID: requestedDeviceID,
+ address: address,
+ token: token
+ )
#if DEBUG
AudioTopologyDiagnostics.record(.listenerAddEnd, owner: .meetingMicrophone, objectID: requestedDeviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener, status: status)
#endif
- if status == noErr {
+ if status == noErr, self.isRunning, self.generationBox.isCurrent(generation) {
self.overloadListenerToken = token
self.overloadListenerDeviceID = requestedDeviceID
+ } else if status == noErr {
+ _ = await AudioTopologyListenerExecution.remove(
+ objectID: requestedDeviceID,
+ address: address,
+ token: token
+ )
}
}
}
- private func tearDownEventObservers() {
- if let configObserver = self.configObserver {
- NotificationCenter.default.removeObserver(configObserver)
- self.configObserver = nil
- }
- if let token = self.defaultInputListenerToken {
- var address = AudioObjectPropertyAddress(
+ private static func removeEventObservers(
+ defaultInputToken: AudioObjectPropertyListenerBlock?,
+ overloadToken: AudioObjectPropertyListenerBlock?,
+ overloadDeviceID: AudioObjectID?,
+ generation: UInt64
+ ) async {
+ if let token = defaultInputToken {
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDefaultInputDevice,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
@@ -935,14 +1052,17 @@ actor MeetingMicrophoneCapture {
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingMicrophone, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener)
#endif
- let status = AudioObjectRemovePropertyListenerBlock(AudioObjectID(kAudioObjectSystemObject), &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.remove(
+ objectID: AudioObjectID(kAudioObjectSystemObject),
+ address: address,
+ token: token
+ )
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingMicrophone, objectID: AudioObjectID(kAudioObjectSystemObject), selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener, status: status)
#endif
- self.defaultInputListenerToken = nil
}
- if let token = self.overloadListenerToken, let deviceID = self.overloadListenerDeviceID {
- var address = AudioObjectPropertyAddress(
+ if let token = overloadToken, let deviceID = overloadDeviceID {
+ let address = AudioObjectPropertyAddress(
mSelector: kAudioDeviceProcessorOverload,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
@@ -950,12 +1070,14 @@ actor MeetingMicrophoneCapture {
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveBegin, owner: .meetingMicrophone, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener)
#endif
- let status = AudioObjectRemovePropertyListenerBlock(deviceID, &address, DispatchQueue.main, token)
+ let status = await AudioTopologyListenerExecution.remove(
+ objectID: deviceID,
+ address: address,
+ token: token
+ )
#if DEBUG
AudioTopologyDiagnostics.record(.listenerRemoveEnd, owner: .meetingMicrophone, objectID: deviceID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .actorControl, phase: .listener, status: status)
#endif
- self.overloadListenerToken = nil
- self.overloadListenerDeviceID = nil
}
}
@@ -1020,9 +1142,9 @@ enum MeetingMicrophonePhase1Probe {
let microphone: MeetingMicrophoneIdentity
if let overrideUID = ProcessInfo.processInfo.environment["FLUIDVOICE_MIC_PHASE1_DEVICE"] {
- microphone = try MeetingCaptureSourceCatalog.defaultMicrophone(preferredCoreAudioUID: overrideUID)
+ microphone = try await MeetingCaptureSourceCatalog.defaultMicrophone(preferredCoreAudioUID: overrideUID)
} else {
- microphone = try MeetingCaptureSourceCatalog.defaultMicrophone()
+ microphone = try await MeetingCaptureSourceCatalog.defaultMicrophone()
}
var writerBackpressureEvents = 0
diff --git a/Sources/Fluid/Services/Meeting/MeetingModels.swift b/Sources/Fluid/Services/Meeting/MeetingModels.swift
index cc8b1fa8..5af4b26b 100644
--- a/Sources/Fluid/Services/Meeting/MeetingModels.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingModels.swift
@@ -120,10 +120,10 @@ nonisolated struct MeetingRecordingDefaults: Codable, Equatable, Sendable {
func savedMicrophone(in identities: [MeetingMicrophoneIdentity]) -> MeetingMicrophoneIdentity? {
guard self.isConfigured else { return nil }
- return self.microphoneCaptureDeviceID.flatMap { captureDeviceID in
- identities.first(where: { $0.captureDeviceID == captureDeviceID })
- } ?? self.microphoneCoreAudioUID.flatMap { coreAudioUID in
+ return self.microphoneCoreAudioUID.flatMap { coreAudioUID in
identities.first(where: { $0.coreAudioUID == coreAudioUID })
+ } ?? self.microphoneCaptureDeviceID.flatMap { captureDeviceID in
+ identities.first(where: { $0.captureDeviceID == captureDeviceID })
}
}
}
@@ -155,21 +155,31 @@ nonisolated struct MeetingApplicationIdentity: Codable, Equatable, Sendable {
}
nonisolated struct MeetingMicrophoneIdentity: Codable, Equatable, Sendable {
- /// `AVCaptureDevice.uniqueID`, which is the identifier ScreenCaptureKit accepts.
+ /// Stable product-selection identifier. New identities use the Core Audio UID;
+ /// manifests written before identity schema 2 may contain an AVCaptureDevice uniqueID.
var captureDeviceID: String
- /// Kept only for reconciling the existing FluidVoice Core Audio preference.
+ /// Stable Core Audio UID used by VPIO and FluidVoice microphone preferences.
var coreAudioUID: String?
+ /// Resolved only inside an AVFoundation/ScreenCaptureKit capture adapter, after
+ /// the dictation-to-meeting audio handoff has completed.
+ var avCaptureDeviceID: String?
+ /// Nil identifies a legacy manifest whose `captureDeviceID` may be an AV uniqueID.
+ var identitySchemaVersion: Int?
var displayName: String
var role: MeetingMicrophoneRole
init(
captureDeviceID: String,
coreAudioUID: String? = nil,
+ avCaptureDeviceID: String? = nil,
+ identitySchemaVersion: Int? = 2,
displayName: String,
role: MeetingMicrophoneRole = .unknown
) {
self.captureDeviceID = captureDeviceID
self.coreAudioUID = coreAudioUID
+ self.avCaptureDeviceID = avCaptureDeviceID
+ self.identitySchemaVersion = identitySchemaVersion
self.displayName = displayName
self.role = role
}
@@ -1016,7 +1026,13 @@ nonisolated struct MeetingProcessingResult: Sendable {
/// No pipelineVersion bump for batch de-drift: this checkpoint snapshots only the application-audio
/// pass; the mic pass re-derives its correction deterministically from `clockDrift` every run.
nonisolated struct MeetingProcessingCheckpoint: Codable, Equatable, Sendable {
- static let currentVersion = 1
+ static let currentVersion = 2
+
+ /// Immutable identity of the diarization implementation used to produce persisted speaker
+ /// state. A dependency or configuration change must update this value so stale checkpoints
+ /// cannot be resumed with different clustering behavior.
+ static let currentDiarizationFingerprint =
+ "FluidAudio-offline-v1@3fd63887eef1dc25edea8263ce4b44aa854d898b;config=community-default-v1"
nonisolated struct ChunkFingerprint: Codable, Equatable, Sendable {
var id: MeetingAudioChunkID
@@ -1035,6 +1051,7 @@ nonisolated struct MeetingProcessingCheckpoint: Codable, Equatable, Sendable {
var asrProvider: String
var asrModel: String
var languageCode: String
+ var diarizationFingerprint: String
var completedTrackID: MeetingAudioTrackID
var trackFingerprints: [MeetingAudioTrackID: [ChunkFingerprint]]
var speakers: [MeetingSessionSpeaker]
@@ -1058,6 +1075,7 @@ nonisolated struct MeetingProcessingCheckpoint: Codable, Equatable, Sendable {
provider: String,
model: String,
language: String,
+ diarizationFingerprint: String = Self.currentDiarizationFingerprint,
completedTrackID: MeetingAudioTrackID,
expectedFingerprints: [MeetingAudioTrackID: [ChunkFingerprint]]
) -> Bool {
@@ -1067,6 +1085,7 @@ nonisolated struct MeetingProcessingCheckpoint: Codable, Equatable, Sendable {
&& self.asrProvider == provider
&& self.asrModel == model
&& self.languageCode == language
+ && self.diarizationFingerprint == diarizationFingerprint
&& self.completedTrackID == completedTrackID
&& self.trackFingerprints == expectedFingerprints
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
index e7f1522a..321796e6 100644
--- a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
@@ -145,6 +145,10 @@ nonisolated struct MeetingSpeakerEmbeddingIndex: Sendable {
)
}
+ mutating func replace(prototype: Prototype) {
+ self.prototypes[prototype.speakerID] = prototype
+ }
+
func prototype(for speakerID: SessionSpeakerID) -> Prototype? {
self.prototypes[speakerID]
}
@@ -177,6 +181,338 @@ nonisolated struct MeetingSpeakerEmbeddingIndex: Sendable {
}
}
+/// Deterministically joins per-chunk diarizer labels into session identities.
+///
+/// This is deliberately a pure value type. It has no knowledge of transcription or capture
+/// state, and it never uses observation order as an identity signal. Quality is a diarizer
+/// quality metric, not a probability; it is used with speech duration only for the explicit
+/// eligibility gates below.
+nonisolated struct MeetingGlobalSpeakerStitcher: Sendable {
+ nonisolated struct Observation: Sendable, Equatable {
+ let key: String
+ let chunkKey: String
+ let trackKind: MeetingAudioTrackKind
+ let localLabel: String
+ let embedding: [Float]
+ let quality: Float
+ let effectiveSpeechDuration: TimeInterval
+
+ init(
+ key: String,
+ chunkKey: String,
+ trackKind: MeetingAudioTrackKind,
+ localLabel: String,
+ embedding: [Float],
+ quality: Float,
+ effectiveSpeechDuration: TimeInterval
+ ) {
+ self.key = key
+ self.chunkKey = chunkKey
+ self.trackKind = trackKind
+ self.localLabel = localLabel
+ self.embedding = embedding
+ self.quality = quality
+ self.effectiveSpeechDuration = effectiveSpeechDuration
+ }
+ }
+
+ nonisolated struct Prototype: Sendable, Equatable {
+ let speakerID: SessionSpeakerID
+ let embedding: [Float]
+ let observationCount: Int
+ let averageQuality: Float
+ let effectiveSpeechDuration: TimeInterval
+ let containsReliableCore: Bool
+ }
+
+ nonisolated struct Result: Sendable, Equatable {
+ let speakerIDByObservationKey: [String: SessionSpeakerID]
+ let prototypes: [SessionSpeakerID: Prototype]
+ }
+
+ /// These thresholds are intentionally named and injectable so behavior can be pinned by tests.
+ struct Configuration: Sendable, Equatable {
+ var coreMaximumDistance: Float = 0.35
+ var deferredMaximumDistance: Float = 0.25
+ var deferredAmbiguityMargin: Float = 0.10
+ var coreMinimumQuality: Float = 0.65
+ var coreMinimumDuration: TimeInterval = 1.0
+ var coreMaximumPairwiseDistance: Float = 0.35
+ var bootstrapMinimumQuality: Float = 0.45
+ var bootstrapMinimumDuration: TimeInterval = 2.0
+ var prototypeUpdateMaximumDistance: Float = 0.12
+ // These can be lower than the core gates because the much tighter distance gate above
+ // supplies the additional evidence required to update an established prototype.
+ var prototypeUpdateMinimumQuality: Float = 0.55
+ var prototypeUpdateMinimumDuration: TimeInterval = 0.5
+ }
+
+ let configuration: Configuration
+
+ init(configuration: Configuration = Configuration()) {
+ self.configuration = configuration
+ }
+
+ func stitch(_ observations: [Observation]) -> Result {
+ var assignments: [String: SessionSpeakerID] = [:]
+ var prototypes: [SessionSpeakerID: Prototype] = [:]
+
+ for trackKind in MeetingAudioTrackKind.allCases {
+ let trackObservations = observations
+ .filter { $0.trackKind == trackKind }
+ .sorted(by: Self.observationPrecedes)
+ guard !trackObservations.isEmpty else { continue }
+
+ var clusters: [Cluster] = []
+ for observation in trackObservations where self.isReliableCore(observation) {
+ guard let normalized = Self.normalized(observation.embedding) else { continue }
+ clusters.append(Cluster(observation: observation, embedding: normalized))
+ }
+
+ // Avoid a hard regression to "Unknown" when a quiet track has no strict core at all.
+ // Bootstrap only from longer, valid observations; once any strict core exists this
+ // relaxed path is disabled for the entire track.
+ if clusters.isEmpty {
+ for observation in trackObservations where self.isBootstrapCore(observation) {
+ guard let normalized = Self.normalized(observation.embedding) else { continue }
+ clusters.append(Cluster(observation: observation, embedding: normalized))
+ }
+ }
+
+ // Merge reliable cores only. The closest eligible pair is selected with stable
+ // tie-breaks, and a cluster merge is rejected if it would violate cannot-link.
+ while let pair = self.closestMergeablePair(in: clusters) {
+ let right = clusters.remove(at: pair.right)
+ let left = clusters.remove(at: pair.left)
+ clusters.append(Self.merged(left, right))
+ clusters.sort { $0.identityKey < $1.identityKey }
+ }
+
+ for index in clusters.indices {
+ for observation in clusters[index].observations {
+ assignments[observation.key] = clusters[index].speakerID
+ }
+ }
+
+ // Deferred observations can attach to cores, never to another deferred observation.
+ // A failed attachment gets its own deterministic identity and is not added as a core.
+ let coreObservationKeys = Set(clusters.flatMap { $0.observations.map(\.key) })
+ for observation in trackObservations where !coreObservationKeys.contains(observation.key) {
+ guard let normalized = Self.normalized(observation.embedding) else {
+ let singleton = Self.singletonCluster(for: observation)
+ assignments[observation.key] = singleton.speakerID
+ prototypes[singleton.speakerID] = singleton.prototype()
+ continue
+ }
+
+ let candidates = clusters.indices.compactMap { index -> (index: Int, distance: Float)? in
+ let cluster = clusters[index]
+ guard self.canLink(observation, to: cluster),
+ let distance = MeetingSpeakerEmbeddingIndex.cosineDistance(normalized, cluster.embedding)
+ else { return nil }
+ return (index, distance)
+ }.sorted {
+ if $0.distance == $1.distance {
+ return clusters[$0.index].identityKey < clusters[$1.index].identityKey
+ }
+ return $0.distance < $1.distance
+ }
+
+ guard let closest = candidates.first,
+ closest.distance <= self.configuration.deferredMaximumDistance,
+ candidates.dropFirst().first.map({ $0.distance - closest.distance >= self.configuration.deferredAmbiguityMargin }) ?? true
+ else {
+ let singleton = Self.singletonCluster(for: observation)
+ assignments[observation.key] = singleton.speakerID
+ prototypes[singleton.speakerID] = singleton.prototype()
+ continue
+ }
+
+ let clusterIndex = closest.index
+ let clusterID = clusters[clusterIndex].speakerID
+ assignments[observation.key] = clusterID
+ // Cannot-link membership is independent from whether this observation is trusted
+ // enough to move the speaker prototype.
+ clusters[clusterIndex].recordConstraint(observation)
+ if closest.distance <= self.configuration.prototypeUpdateMaximumDistance,
+ self.isEligibleForPrototypeUpdate(observation)
+ {
+ clusters[clusterIndex].add(observation: observation, embedding: normalized)
+ }
+ }
+
+ for cluster in clusters {
+ prototypes[cluster.speakerID] = cluster.prototype()
+ }
+ }
+
+ return Result(
+ speakerIDByObservationKey: assignments,
+ prototypes: prototypes
+ )
+ }
+
+ private struct Cluster {
+ var speakerID: SessionSpeakerID
+ var identityKey: String
+ var observations: [Observation]
+ var constrainedLabelsByChunk: [String: Set<String>]
+ var embedding: [Float]
+ var containsReliableCore: Bool
+
+ init(observation: Observation, embedding: [Float]) {
+ self.speakerID = MeetingGlobalSpeakerStitcher.stableUUID(trackKind: observation.trackKind, key: observation.key)
+ self.identityKey = observation.key
+ self.observations = [observation]
+ self.constrainedLabelsByChunk = [observation.chunkKey: [observation.localLabel]]
+ self.embedding = embedding
+ self.containsReliableCore = true
+ }
+
+ mutating func add(observation: Observation, embedding: [Float]) {
+ let count = Float(self.observations.count)
+ let combined = zip(self.embedding, embedding).map { ($0 * count + $1) / (count + 1) }
+ self.embedding = MeetingGlobalSpeakerStitcher.normalized(combined) ?? self.embedding
+ self.observations.append(observation)
+ }
+
+ mutating func recordConstraint(_ observation: Observation) {
+ self.constrainedLabelsByChunk[observation.chunkKey, default: []].insert(observation.localLabel)
+ }
+
+ func prototype() -> Prototype {
+ Prototype(
+ speakerID: self.speakerID,
+ embedding: self.embedding,
+ observationCount: self.observations.count,
+ averageQuality: self.observations.reduce(Float.zero) { $0 + $1.quality } / Float(self.observations.count),
+ effectiveSpeechDuration: self.observations.reduce(0) { $0 + max(0, $1.effectiveSpeechDuration) },
+ containsReliableCore: self.containsReliableCore
+ )
+ }
+ }
+
+ private func isReliableCore(_ observation: Observation) -> Bool {
+ guard Self.normalized(observation.embedding) != nil else { return false }
+ return observation.quality >= self.configuration.coreMinimumQuality
+ && observation.effectiveSpeechDuration >= self.configuration.coreMinimumDuration
+ }
+
+ private func isBootstrapCore(_ observation: Observation) -> Bool {
+ guard Self.normalized(observation.embedding) != nil else { return false }
+ return observation.quality >= self.configuration.bootstrapMinimumQuality
+ && observation.effectiveSpeechDuration >= self.configuration.bootstrapMinimumDuration
+ }
+
+ private func isEligibleForPrototypeUpdate(_ observation: Observation) -> Bool {
+ observation.quality >= self.configuration.prototypeUpdateMinimumQuality
+ && observation.effectiveSpeechDuration >= self.configuration.prototypeUpdateMinimumDuration
+ }
+
+ private func canLink(_ observation: Observation, to cluster: Cluster) -> Bool {
+ guard let labels = cluster.constrainedLabelsByChunk[observation.chunkKey] else { return true }
+ return labels.allSatisfy { $0 == observation.localLabel }
+ }
+
+ private func closestMergeablePair(in clusters: [Cluster]) -> (left: Int, right: Int)? {
+ var best: (left: Int, right: Int, distance: Float)?
+ for left in clusters.indices {
+ for right in clusters.indices where right > left {
+ guard !clusters[left].observations.contains(where: { first in
+ clusters[right].observations.contains { second in
+ first.chunkKey == second.chunkKey && first.localLabel != second.localLabel
+ }
+ }),
+ self.coreClustersRespectDiameter(clusters[left], clusters[right]),
+ let distance = MeetingSpeakerEmbeddingIndex.cosineDistance(
+ clusters[left].embedding, clusters[right].embedding
+ ),
+ distance <= self.configuration.coreMaximumDistance
+ else { continue }
+ let candidate = (left, right, distance)
+ if best == nil || distance < best!.distance
+ || (distance == best!.distance && Self.pairPrecedes(candidate, best!, clusters: clusters))
+ {
+ best = candidate
+ }
+ }
+ }
+ return best.map { ($0.left, $0.right) }
+ }
+
+ private func coreClustersRespectDiameter(_ left: Cluster, _ right: Cluster) -> Bool {
+ left.observations.allSatisfy { first in
+ right.observations.allSatisfy { second in
+ guard let distance = MeetingSpeakerEmbeddingIndex.cosineDistance(first.embedding, second.embedding) else {
+ return false
+ }
+ return distance <= self.configuration.coreMaximumPairwiseDistance
+ }
+ }
+ }
+
+ private static func pairPrecedes(
+ _ lhs: (Int, Int, Float),
+ _ rhs: (left: Int, right: Int, distance: Float),
+ clusters: [Cluster]
+ ) -> Bool {
+ let leftKey = clusters[lhs.0].identityKey + "\u{0}" + clusters[lhs.1].identityKey
+ let rightKey = clusters[rhs.left].identityKey + "\u{0}" + clusters[rhs.right].identityKey
+ return leftKey < rightKey
+ }
+
+ private static func merged(_ left: Cluster, _ right: Cluster) -> Cluster {
+ let ordered = (left.observations + right.observations).sorted(by: Self.observationPrecedes)
+ var merged = Cluster(observation: ordered[0], embedding: left.identityKey <= right.identityKey ? left.embedding : right.embedding)
+ merged.speakerID = MeetingGlobalSpeakerStitcher.stableUUID(trackKind: ordered[0].trackKind, key: ordered[0].key)
+ merged.identityKey = ordered[0].key
+ merged.observations = []
+ merged.constrainedLabelsByChunk = left.constrainedLabelsByChunk
+ for (chunkKey, labels) in right.constrainedLabelsByChunk {
+ merged.constrainedLabelsByChunk[chunkKey, default: []].formUnion(labels)
+ }
+ merged.containsReliableCore = left.containsReliableCore || right.containsReliableCore
+ for observation in ordered {
+ guard let normalized = normalized(observation.embedding) else { continue }
+ if merged.observations.isEmpty {
+ merged.embedding = normalized
+ }
+ merged.add(observation: observation, embedding: normalized)
+ }
+ return merged
+ }
+
+ private static func singletonCluster(for observation: Observation) -> Cluster {
+ let embedding = Self.normalized(observation.embedding) ?? []
+ var cluster = Cluster(observation: observation, embedding: embedding)
+ cluster.containsReliableCore = false
+ return cluster
+ }
+
+ private static func observationPrecedes(_ lhs: Observation, _ rhs: Observation) -> Bool {
+ let left = [lhs.key, lhs.chunkKey, lhs.localLabel, lhs.trackKind.rawValue].joined(separator: "\u{0}")
+ let right = [rhs.key, rhs.chunkKey, rhs.localLabel, rhs.trackKind.rawValue].joined(separator: "\u{0}")
+ return left < right
+ }
+
+ private static func normalized(_ embedding: [Float]) -> [Float]? {
+ guard let distance = MeetingSpeakerEmbeddingIndex.cosineDistance(embedding, embedding), distance.isFinite else { return nil }
+ let magnitude = sqrt(embedding.reduce(Float.zero) { $0 + $1 * $1 })
+ guard magnitude > 0, magnitude.isFinite else { return nil }
+ return embedding.map { $0 / magnitude }
+ }
+
+ private static func stableUUID(trackKind: MeetingAudioTrackKind, key: String) -> UUID {
+ var bytes = Array(SHA256.hash(data: Data("global-speaker:\(trackKind.rawValue):\(key)".utf8)).prefix(16))
+ bytes[6] = (bytes[6] & 0x0f) | 0x50
+ bytes[8] = (bytes[8] & 0x3f) | 0x80
+ return UUID(uuid: (
+ bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
+ bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]
+ ))
+ }
+}
+
nonisolated enum MeetingTurnSelection {
/// Drops empty-text turns; nil when all were empty (caller falls back to whole-chunk).
static func keepingNonEmpty<T>(
@@ -320,7 +656,7 @@ private final class MeetingProviderLanguagePin {
final class MeetingProcessingPipeline: MeetingProcessingControlling {
/// Bump whenever classification rules change so a resumed run can't mix rules mid-session.
/// Era-awareness did NOT bump this: the mic pass re-runs every time, and multi-era tracks cannot predate this build.
- static let pipelineVersion = 7
+ static let pipelineVersion = 8
/// Per-turn engines starve on short turns, and an all-empty chunk collapses to unlabeled.
static let perTurnTurnMergeGapSeconds = 5.0
@@ -381,6 +717,14 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
)
}
+ nonisolated static func trustedMicrophoneObservationKeys(
+ from turns: [StagedMicrophoneTurn]
+ ) -> Set<String> {
+ Set(turns.compactMap { turn in
+ turn.isLikelyEcho ? nil : turn.diarizationObservationKey
+ })
+ }
+
/// Why "You" was or was not elected. Without this the only visible symptom is an entire
/// microphone track collapsing to "Microphone / Unknown", which has several distinct causes.
nonisolated static func logLocalSpeakerElection(
@@ -548,6 +892,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
var index: Int
var clusterID: SessionSpeakerID
var clusterLabel: String
+ var diarizationObservationKey: String? = nil
var start: TimeInterval
var end: TimeInterval
var text: String
@@ -574,6 +919,16 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
}
}
+ private struct PendingDiarizedTurn {
+ let observation: MeetingGlobalSpeakerStitcher.Observation
+ let trackID: MeetingAudioTrackID
+ let chunkID: MeetingAudioChunkID
+ let index: Int
+ let start: TimeInterval
+ let end: TimeInterval
+ let text: String
+ }
+
// MARK: - Echo signal veto
nonisolated struct ReferenceSpan: Equatable {
@@ -895,26 +1250,21 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
return id
}
- mutating func resolveCluster(
- trackKind: MeetingAudioTrackKind,
- newClusterKey: String,
- profile: SpeakerDiarizationService.SpeakerProfile?,
- excluding: Set<SessionSpeakerID>
- ) -> (speakerID: SessionSpeakerID, matchedExisting: Bool) {
- var index = self.embeddingIndexByTrack[trackKind] ?? MeetingSpeakerEmbeddingIndex()
- let existing = profile.flatMap {
- index.bestMatch(for: $0.embedding, excluding: excluding)
- }
- let speakerID = existing ?? MeetingProcessingPipeline.stableUUID("speaker:\(newClusterKey)")
- if let profile {
- index.observe(
- speakerID: speakerID,
- embedding: profile.embedding,
- quality: profile.averageQuality
- )
+ mutating func applyGlobalStitch(
+ _ result: MeetingGlobalSpeakerStitcher.Result,
+ trackKind: MeetingAudioTrackKind
+ ) {
+ var index = MeetingSpeakerEmbeddingIndex()
+ for prototype in result.prototypes.values.sorted(by: { $0.speakerID.uuidString < $1.speakerID.uuidString }) {
+ guard prototype.containsReliableCore, !prototype.embedding.isEmpty else { continue }
+ index.replace(prototype: MeetingSpeakerEmbeddingIndex.Prototype(
+ speakerID: prototype.speakerID,
+ embedding: prototype.embedding,
+ observationCount: prototype.observationCount,
+ averageQuality: prototype.averageQuality
+ ))
}
self.embeddingIndexByTrack[trackKind] = index
- return (speakerID, existing != nil)
}
mutating func addResolvedSpeaker(
@@ -1026,7 +1376,9 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
asrProvider: provider.name,
asrModel: selectedModel.rawValue,
languageCode: languagePin.languageCode,
- diarizationModel: SpeakerDiarizationService.isSupported ? "FluidAudio-offline-v1" : nil,
+ diarizationModel: SpeakerDiarizationService.isSupported
+ ? MeetingProcessingCheckpoint.currentDiarizationFingerprint
+ : nil,
lastCompletedTrackID: nil,
errorCode: nil
)
@@ -1097,6 +1449,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
asrProvider: provider.name,
asrModel: selectedModel.rawValue,
languageCode: languagePin.languageCode,
+ diarizationFingerprint: MeetingProcessingCheckpoint.currentDiarizationFingerprint,
completedTrackID: applicationTrack.id,
trackFingerprints: expectedFingerprints,
speakers: accumulator.speakers,
@@ -1198,11 +1551,13 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
skippedChunkIDs: inout [MeetingAudioChunkID]
) async throws {
let diarizer = SpeakerDiarizationService()
+ var pendingTurns: [PendingDiarizedTurn] = []
for chunk in track.chunks where chunk.finalizationState == .finalized {
// Snapshot so a mid-chunk partial mutation (e.g. some turns transcribed before an
// unreadable read) never leaks into the accumulated result for a skipped chunk.
let accumulatorSnapshot = accumulator
let remoteSpeechSnapshot = remoteSpeech
+ let pendingTurnsSnapshot = pendingTurns
do {
try await self.processApplicationChunk(
chunk,
@@ -1210,16 +1565,85 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
diarizer: diarizer,
context: context,
accumulator: &accumulator,
- remoteSpeech: &remoteSpeech
+ remoteSpeech: &remoteSpeech,
+ pendingTurns: &pendingTurns
)
} catch {
if error is CancellationError { throw error }
guard case MeetingProcessingError.audioUnreadable = error else { throw error }
accumulator = accumulatorSnapshot
remoteSpeech = remoteSpeechSnapshot
+ pendingTurns = pendingTurnsSnapshot
skippedChunkIDs.append(chunk.id)
}
}
+ self.finishApplicationTrack(track, pendingTurns: pendingTurns, accumulator: &accumulator)
+ }
+
+ private func finishApplicationTrack(
+ _ track: MeetingAudioTrack,
+ pendingTurns: [PendingDiarizedTurn],
+ accumulator: inout Accumulator
+ ) {
+ guard !pendingTurns.isEmpty else { return }
+ var observationsByKey: [String: MeetingGlobalSpeakerStitcher.Observation] = [:]
+ for pending in pendingTurns { observationsByKey[pending.observation.key] = pending.observation }
+ let result = MeetingGlobalSpeakerStitcher().stitch(observationsByKey.values.sorted { $0.key < $1.key })
+ accumulator.applyGlobalStitch(result, trackKind: .applicationAudio)
+
+ let orderedTurns = pendingTurns.sorted {
+ if $0.start == $1.start { return $0.observation.key < $1.observation.key }
+ return $0.start < $1.start
+ }
+ var emittedSpeakerIDs = Set<SessionSpeakerID>()
+ for pending in orderedTurns {
+ guard let speakerID = result.speakerIDByObservationKey[pending.observation.key] else {
+ DebugLogger.shared.warning(
+ "Meeting speaker stitch omitted application observation \(pending.observation.key); preserving text as unattributed",
+ source: "MeetingProcessingPipeline"
+ )
+ let fallbackSpeakerID = accumulator.speaker(
+ key: "meeting-audio",
+ displayName: "Meeting Audio",
+ trackKind: .applicationAudio,
+ isLocalUser: false,
+ clusterID: nil
+ )
+ accumulator.segments.append(Self.segment(
+ key: "remote:\(pending.chunkID.uuidString):\(pending.index)",
+ trackID: track.id,
+ speakerID: fallbackSpeakerID,
+ start: pending.start,
+ end: pending.end,
+ text: pending.text,
+ overlap: .none
+ ))
+ continue
+ }
+ if emittedSpeakerIDs.insert(speakerID).inserted {
+ let firstLabel = orderedTurns.first {
+ result.speakerIDByObservationKey[$0.observation.key] == speakerID
+ }?.observation.localLabel
+ accumulator.addResolvedSpeaker(
+ id: speakerID,
+ key: "remote-cluster:\(speakerID.uuidString)",
+ displayName: "Speaker \(accumulator.nextRemoteSpeaker)",
+ clusterID: firstLabel,
+ trackKind: .applicationAudio,
+ isLocalUser: false
+ )
+ accumulator.nextRemoteSpeaker += 1
+ }
+ accumulator.segments.append(Self.segment(
+ key: "remote:\(pending.chunkID.uuidString):\(pending.index)",
+ trackID: track.id,
+ speakerID: speakerID,
+ start: pending.start,
+ end: pending.end,
+ text: pending.text,
+ overlap: .none
+ ))
+ }
}
private func processApplicationChunk(
@@ -1228,7 +1652,8 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
diarizer: SpeakerDiarizationService,
context: ProcessingContext,
accumulator: inout Accumulator,
- remoteSpeech: inout [TimelineInterval]
+ remoteSpeech: inout [TimelineInterval],
+ pendingTurns: inout [PendingDiarizedTurn]
) async throws {
let url = chunk.fileURL(relativeTo: context.sessionDirectory)
let chunkOffset = chunk.presentationStart.seconds - context.origin
@@ -1265,45 +1690,34 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
end: chunkOffset + turn.endSeconds
)
})
- var speakerIDByLabel: [String: SessionSpeakerID] = [:]
- var usedSpeakerIDs = Set<SessionSpeakerID>()
- for (_, turn, _) in transcribedTurns where speakerIDByLabel[turn.speakerLabel] == nil {
- let resolved = accumulator.resolveCluster(
- trackKind: .applicationAudio,
- newClusterKey: "remote:\(chunk.id.uuidString):\(turn.speakerLabel)",
- profile: diarization.profilesByLabel[turn.speakerLabel],
- excluding: usedSpeakerIDs
- )
- let displayName: String
- if resolved.matchedExisting {
- displayName = "Speaker"
- } else {
- displayName = "Speaker \(accumulator.nextRemoteSpeaker)"
- accumulator.nextRemoteSpeaker += 1
+ let observationsByLabel: [String: MeetingGlobalSpeakerStitcher.Observation] = Dictionary(
+ uniqueKeysWithValues: Set(transcribedTurns.map { $0.turn.speakerLabel }).sorted().map { label in
+ let profile = diarization.profilesByLabel[label]
+ return (label, MeetingGlobalSpeakerStitcher.Observation(
+ key: "application:\(chunk.id.uuidString):\(label)",
+ chunkKey: chunk.id.uuidString,
+ trackKind: .applicationAudio,
+ localLabel: label,
+ embedding: profile?.embedding ?? [],
+ quality: profile?.averageQuality ?? 0,
+ effectiveSpeechDuration: transcribedTurns
+ .filter { $0.turn.speakerLabel == label }
+ .reduce(0) { $0 + max(0, $1.turn.durationSeconds) }
+ ))
}
- accumulator.addResolvedSpeaker(
- id: resolved.speakerID,
- key: "remote-cluster:\(resolved.speakerID.uuidString)",
- displayName: displayName,
- clusterID: turn.speakerLabel,
- trackKind: .applicationAudio,
- isLocalUser: false
- )
- speakerIDByLabel[turn.speakerLabel] = resolved.speakerID
- usedSpeakerIDs.insert(resolved.speakerID)
- }
+ )
for (index, turn, text) in transcribedTurns {
let absoluteStart = chunkOffset + turn.startSeconds
let absoluteEnd = chunkOffset + turn.endSeconds
- guard let speakerID = speakerIDByLabel[turn.speakerLabel] else { continue }
- accumulator.segments.append(Self.segment(
- key: "remote:\(chunk.id.uuidString):\(index)",
+ guard let observation = observationsByLabel[turn.speakerLabel] else { continue }
+ pendingTurns.append(PendingDiarizedTurn(
+ observation: observation,
trackID: track.id,
- speakerID: speakerID,
+ chunkID: chunk.id,
+ index: index,
start: absoluteStart,
end: absoluteEnd,
- text: text,
- overlap: .none
+ text: text
))
}
return
@@ -1368,6 +1782,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
)
)
var stagedTurns: [StagedMicrophoneTurn] = []
+ var pendingTurns: [PendingDiarizedTurn] = []
var echoMonitor = EchoMonitorAccumulator()
let eras = Self.microphoneEras(for: track, origin: context.origin)
// Era 0 has no writer-boundary anchor, so de-drift anchors it at the track's own first PTS.
@@ -1375,6 +1790,9 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
for chunk in track.chunks where chunk.finalizationState == .finalized {
let accumulatorSnapshot = accumulator
let stagedTurnsSnapshot = stagedTurns
+ let pendingTurnsSnapshot = pendingTurns
+ let epochStateSnapshot = epochState
+ let echoMonitorSnapshot = echoMonitor
do {
try await self.processMicrophoneChunk(
chunk,
@@ -1388,17 +1806,55 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
epochState: &epochState,
echoMonitor: &echoMonitor,
accumulator: &accumulator,
- stagedTurns: &stagedTurns
+ stagedTurns: &stagedTurns,
+ pendingTurns: &pendingTurns
)
} catch {
if error is CancellationError { throw error }
guard case MeetingProcessingError.audioUnreadable = error else { throw error }
accumulator = accumulatorSnapshot
stagedTurns = stagedTurnsSnapshot
+ pendingTurns = pendingTurnsSnapshot
+ epochState = epochStateSnapshot
+ echoMonitor = echoMonitorSnapshot
skippedChunkIDs.append(chunk.id)
}
}
+ // Apply the existing text-only echo evidence before building microphone prototypes.
+ // A chunk-local label represented only by confirmed far-end echo must not become a
+ // first-class voice profile or compete in the later "You" election.
+ let profileEligibleTurns = session.selectedMicrophone.role == .personal
+ ? Self.rejectingFarField(stagedTurns, eras: eras)
+ : stagedTurns
+ let preclassified = Self.classifyMicrophoneTurns(
+ profileEligibleTurns,
+ eras: eras,
+ applicationTrackID: applicationTrack?.id,
+ segments: accumulator.segments,
+ fallbackSpeakerID: accumulator.speakerIDByKey["meeting-audio"]
+ )
+ stagedTurns = preclassified.turns
+ let trustedObservationKeys = Self.trustedMicrophoneObservationKeys(from: stagedTurns)
+ var observationsByKey: [String: MeetingGlobalSpeakerStitcher.Observation] = [:]
+ for pending in pendingTurns where trustedObservationKeys.contains(pending.observation.key) {
+ observationsByKey[pending.observation.key] = pending.observation
+ }
+ let stitchResult = MeetingGlobalSpeakerStitcher().stitch(observationsByKey.values.sorted { $0.key < $1.key })
+ accumulator.applyGlobalStitch(stitchResult, trackKind: .microphone)
+ for index in stagedTurns.indices {
+ if let observationKey = stagedTurns[index].diarizationObservationKey,
+ let speakerID = stitchResult.speakerIDByObservationKey[observationKey]
+ {
+ stagedTurns[index].clusterID = speakerID
+ } else if let observationKey = stagedTurns[index].diarizationObservationKey {
+ DebugLogger.shared.warning(
+ "Meeting speaker stitch omitted microphone observation \(observationKey); keeping it outside trusted prototypes",
+ source: "MeetingProcessingPipeline"
+ )
+ }
+ }
+
self.finishMicrophoneTrack(
track,
session: session,
@@ -1422,7 +1878,8 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
epochState: inout EchoDelayEpochState,
echoMonitor: inout EchoMonitorAccumulator,
accumulator: inout Accumulator,
- stagedTurns: inout [StagedMicrophoneTurn]
+ stagedTurns: inout [StagedMicrophoneTurn],
+ pendingTurns: inout [PendingDiarizedTurn]
) async throws {
let url = chunk.fileURL(relativeTo: context.sessionDirectory)
let chunkOffset = chunk.presentationStart.seconds - context.origin
@@ -1449,17 +1906,33 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
source: "MeetingProcessingPipeline"
)
}
- var clusterIDByLabel: [String: SessionSpeakerID] = [:]
- var usedClusterIDs = Set<SessionSpeakerID>()
- for (_, turn, _) in transcribedTurns where clusterIDByLabel[turn.speakerLabel] == nil {
- let resolved = accumulator.resolveCluster(
- trackKind: .microphone,
- newClusterKey: "microphone:\(chunk.id.uuidString):\(turn.speakerLabel)",
- profile: diarization.profilesByLabel[turn.speakerLabel],
- excluding: usedClusterIDs
- )
- clusterIDByLabel[turn.speakerLabel] = resolved.speakerID
- usedClusterIDs.insert(resolved.speakerID)
+ let observationsByLabel: [String: MeetingGlobalSpeakerStitcher.Observation] = Dictionary(
+ uniqueKeysWithValues: Set(transcribedTurns.map { $0.turn.speakerLabel }).sorted().map { label in
+ let profile = diarization.profilesByLabel[label]
+ return (label, MeetingGlobalSpeakerStitcher.Observation(
+ key: "microphone:\(chunk.id.uuidString):\(label)",
+ chunkKey: chunk.id.uuidString,
+ trackKind: .microphone,
+ localLabel: label,
+ embedding: profile?.embedding ?? [],
+ quality: profile?.averageQuality ?? 0,
+ effectiveSpeechDuration: transcribedTurns
+ .filter { $0.turn.speakerLabel == label }
+ .reduce(0) { $0 + max(0, $1.turn.durationSeconds) }
+ ))
+ }
+ )
+ for (index, turn, text) in transcribedTurns {
+ guard let observation = observationsByLabel[turn.speakerLabel] else { continue }
+ pendingTurns.append(PendingDiarizedTurn(
+ observation: observation,
+ trackID: track.id,
+ chunkID: chunk.id,
+ index: index,
+ start: 0,
+ end: 0,
+ text: text
+ ))
}
var signalVerdictByIndex: [Int: TurnEchoVerdict] = [:]
@@ -1523,12 +1996,13 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
let overlapsRemote = remoteSpeech.contains {
min(end, $0.end) - max(start, $0.start) > 0.15
}
- guard let clusterID = clusterIDByLabel[turn.speakerLabel] else { continue }
+ let observationKey = "microphone:\(chunk.id.uuidString):\(turn.speakerLabel)"
stagedTurns.append(StagedMicrophoneTurn(
chunkID: chunk.id,
index: index,
- clusterID: clusterID,
+ clusterID: Self.stableUUID("pending-microphone:\(observationKey)"),
clusterLabel: turn.speakerLabel,
+ diarizationObservationKey: observationKey,
start: start,
end: end,
text: text,
@@ -1707,9 +2181,10 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
accumulator: inout Accumulator
) {
let eras = Self.microphoneEras(for: track, origin: origin)
- let stagedTurns = session.selectedMicrophone.role == .personal
- ? Self.rejectingFarField(allStagedTurns, eras: eras)
- : allStagedTurns
+ // Personal-mic turns were already near-field filtered before profile stitching. Applying
+ // the relative loudness gate again would recompute its reference from a narrower set and
+ // could reject a turn that passed the original, correctly calibrated gate.
+ let stagedTurns = allStagedTurns
if session.mode == .inRoom {
for turn in stagedTurns {
diff --git a/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
index f24a28f2..6ed962b6 100644
--- a/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
@@ -20,8 +20,8 @@ struct MeetingAudioActivityLease: Equatable, Sendable {
@MainActor
protocol MeetingAudioActivityArbitrating: AnyObject {
- func acquireMeetingCapture() throws -> MeetingAudioActivityLease
- func release(_ lease: MeetingAudioActivityLease)
+ func acquireMeetingCapture() async throws -> MeetingAudioActivityLease
+ func release(_ lease: MeetingAudioActivityLease) async
}
/// Matches ASRService's exclusive-activity lease API so the arbiter can depend on a protocol
@@ -30,6 +30,8 @@ protocol MeetingAudioActivityArbitrating: AnyObject {
protocol ASRActivityLeasing: AnyObject {
func acquireExclusiveActivity(_ activity: ASRExclusiveActivity) throws -> ASRActivityLease
func releaseExclusiveActivity(_ lease: ASRActivityLease)
+ func prepareMeetingAudioHandoff(_ lease: ASRActivityLease) async throws
+ func completeMeetingAudioHandback(_ lease: ASRActivityLease) async
}
extension ASRService: ASRActivityLeasing {}
@@ -40,12 +42,14 @@ final class AudioActivityArbiter: MeetingAudioActivityArbitrating {
private var meetingLease: MeetingAudioActivityLease?
private var sharedActivityLease: ASRActivityLease?
private weak var activeASRService: (any ASRActivityLeasing)?
+ private var releaseTask: Task<Void, Never>?
+ private var releaseTaskLease: MeetingAudioActivityLease?
init(asrServiceProvider: @escaping @MainActor () throws -> any ASRActivityLeasing) {
self.asrServiceProvider = asrServiceProvider
}
- func acquireMeetingCapture() throws -> MeetingAudioActivityLease {
+ func acquireMeetingCapture() async throws -> MeetingAudioActivityLease {
guard self.meetingLease == nil else { throw MeetingCoordinatorError.recordingAlreadyActive }
let asrService = try self.asrServiceProvider()
let sharedLease = try asrService.acquireExclusiveActivity(.meeting)
@@ -53,17 +57,50 @@ final class AudioActivityArbiter: MeetingAudioActivityArbitrating {
self.activeASRService = asrService
self.sharedActivityLease = sharedLease
self.meetingLease = lease
- return lease
+ do {
+ try await asrService.prepareMeetingAudioHandoff(sharedLease)
+ try Task.checkCancellation()
+ guard self.meetingLease == lease,
+ self.sharedActivityLease == sharedLease
+ else { throw CancellationError() }
+ return lease
+ } catch {
+ if self.meetingLease == lease, self.sharedActivityLease == sharedLease {
+ // Rollback retains the ASR lease until every partially-started handoff
+ // operation is drained; only then can dictation or another meeting enter.
+ await asrService.completeMeetingAudioHandback(sharedLease)
+ self.activeASRService = nil
+ self.sharedActivityLease = nil
+ self.meetingLease = nil
+ }
+ throw error
+ }
}
- func release(_ lease: MeetingAudioActivityLease) {
+ func release(_ lease: MeetingAudioActivityLease) async {
+ if self.releaseTaskLease == lease, let releaseTask = self.releaseTask {
+ await releaseTask.value
+ return
+ }
guard self.meetingLease == lease else { return }
- if let sharedActivityLease = self.sharedActivityLease {
- self.activeASRService?.releaseExclusiveActivity(sharedActivityLease)
+ let sharedActivityLease = self.sharedActivityLease
+ let asrService = self.activeASRService
+ let releaseTask = Task { @MainActor in
+ if let sharedActivityLease {
+ await asrService?.completeMeetingAudioHandback(sharedActivityLease)
+ }
+ }
+ self.releaseTaskLease = lease
+ self.releaseTask = releaseTask
+ await releaseTask.value
+ guard self.releaseTaskLease == lease else { return }
+ if self.meetingLease == lease, self.sharedActivityLease == sharedActivityLease {
+ self.activeASRService = nil
+ self.sharedActivityLease = nil
+ self.meetingLease = nil
}
- self.activeASRService = nil
- self.sharedActivityLease = nil
- self.meetingLease = nil
+ self.releaseTask = nil
+ self.releaseTaskLease = nil
}
}
@@ -209,7 +246,7 @@ final class MeetingSessionCoordinator: ObservableObject {
preferredWindowID: UInt32?,
requirePreferredApplication: Bool = false
) async throws -> MeetingCaptureConfiguration {
- let microphone = try MeetingCaptureSourceCatalog.defaultMicrophone(
+ let microphone = try await MeetingCaptureSourceCatalog.defaultMicrophone(
preferredCoreAudioUID: self.preferredMicrophoneUID()
)
var application: MeetingApplicationIdentity?
@@ -246,7 +283,24 @@ final class MeetingSessionCoordinator: ObservableObject {
throw MeetingCoordinatorError.maintenanceInProgress
}
- let lease = try self.audioArbiter.acquireMeetingCapture()
+ // Permission prompts are outside the exclusive audio lease. A prompt can remain
+ // open indefinitely; reserving meeting audio first would unnecessarily block dictation.
+ try await self.capture.preflightPermissions()
+ try Task.checkCancellation()
+ guard self.activeSession == nil, !self.hasBlockingActivityTask else {
+ throw MeetingCoordinatorError.recordingAlreadyActive
+ }
+ guard !self.isMutatingSession else {
+ throw MeetingCoordinatorError.maintenanceInProgress
+ }
+
+ let lease = try await self.audioArbiter.acquireMeetingCapture()
+ do {
+ try Task.checkCancellation()
+ } catch {
+ await self.audioArbiter.release(lease)
+ throw error
+ }
self.activityLease = lease
let generation = UUID()
self.captureGeneration = generation
@@ -353,7 +407,7 @@ final class MeetingSessionCoordinator: ObservableObject {
self.degradeReason = nil
self.operationGeneration = nil
self.state = .failed(session.id, failure)
- self.releaseActivityLease()
+ await self.releaseActivityLease()
throw error
}
}
@@ -448,7 +502,7 @@ final class MeetingSessionCoordinator: ObservableObject {
// Acquire before displacing/adopting so a throw here never strands activeSession mid-transition.
if self.activityLease == nil {
- self.activityLease = try self.audioArbiter.acquireMeetingCapture()
+ self.activityLease = try await self.audioArbiter.acquireMeetingCapture()
}
// Target is valid — only now displace the passive recovery offer it replaces.
@@ -474,7 +528,7 @@ final class MeetingSessionCoordinator: ObservableObject {
private func beginProcessingRetry(_ inputSession: MeetingSession) async throws -> MeetingSession {
var session = inputSession
if self.activityLease == nil {
- self.activityLease = try self.audioArbiter.acquireMeetingCapture()
+ self.activityLease = try await self.audioArbiter.acquireMeetingCapture()
}
let generation = UUID()
self.operationGeneration = generation
@@ -1113,7 +1167,7 @@ final class MeetingSessionCoordinator: ObservableObject {
await self.flushQueuedPersistence()
await self.capture.shutdownForTermination()
await self.tearDownLiveTranscription()
- self.releaseActivityLease()
+ await self.releaseActivityLease()
return
}
@@ -1161,17 +1215,30 @@ final class MeetingSessionCoordinator: ObservableObject {
self.activeSession = nil
self.trackHealth = [:]
self.state = .idle
- self.releaseActivityLease()
+ await self.releaseActivityLease()
return
} catch {
// Preserve today's behavior: reinstall as interrupted despite the save failure.
}
self.activeSession = session
self.state = .interrupted(session.id)
- self.releaseActivityLease()
+ await self.releaseActivityLease()
}
private func performStopAndTranscribe(generation: UUID) async throws -> MeetingSession {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCoordinator, queueRole: .mainControl, phase: .coordinatorStop)
+ var diagnosticsStopSucceeded = false
+ defer {
+ AudioTopologyDiagnostics.record(
+ .phaseEnd,
+ owner: .meetingCoordinator,
+ queueRole: .mainControl,
+ phase: .coordinatorStop,
+ status: diagnosticsStopSucceeded ? 0 : -1
+ )
+ }
+ #endif
guard var session = self.activeSession else { throw MeetingCoordinatorError.noActiveMeeting }
self.state = .stopping(session.id)
session.state = .stopping
@@ -1183,8 +1250,17 @@ final class MeetingSessionCoordinator: ObservableObject {
let stopResult: MeetingCaptureStopResult
do {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCoordinator, queueRole: .mainControl, phase: .captureStop)
+ #endif
stopResult = try await self.capture.stop(sessionID: session.id)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCoordinator, queueRole: .mainControl, phase: .captureStop)
+ #endif
} catch {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCoordinator, queueRole: .mainControl, phase: .captureStop, status: -1)
+ #endif
await self.handleStopFailure(error, session: session, generation: generation)
throw error
}
@@ -1201,10 +1277,27 @@ final class MeetingSessionCoordinator: ObservableObject {
self.activeSession = session
self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) })
try? await self.store.save(session)
- return try await self.process(session, generation: generation)
+ let processed = try await self.process(session, generation: generation)
+ #if DEBUG
+ diagnosticsStopSucceeded = true
+ #endif
+ return processed
}
private func process(_ inputSession: MeetingSession, generation: UUID) async throws -> MeetingSession {
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCoordinator, queueRole: .mainControl, phase: .processing)
+ var diagnosticsProcessingSucceeded = false
+ defer {
+ AudioTopologyDiagnostics.record(
+ .phaseEnd,
+ owner: .meetingCoordinator,
+ queueRole: .mainControl,
+ phase: .processing,
+ status: diagnosticsProcessingSucceeded ? 0 : -1
+ )
+ }
+ #endif
var session = inputSession
do {
let sessionDirectory = try await self.store.sessionDirectory(for: session.id)
@@ -1253,9 +1346,12 @@ final class MeetingSessionCoordinator: ObservableObject {
self.latestCompletedSession = session
self.state = .completed(session.id)
self.operationGeneration = nil
- self.releaseActivityLease()
+ await self.releaseActivityLease()
// afterTranscription policy deletes here; sweep failures are logged, never affect this result.
Task { @MainActor [weak self] in await self?.sweepExpiredAudio() }
+ #if DEBUG
+ diagnosticsProcessingSucceeded = true
+ #endif
return session
} catch {
guard self.operationGeneration == generation else {
@@ -1269,7 +1365,7 @@ final class MeetingSessionCoordinator: ObservableObject {
self.activeSession = session
self.state = .interrupted(session.id)
self.operationGeneration = nil
- self.releaseActivityLease()
+ await self.releaseActivityLease()
throw error
}
let failure = Self.failure(from: error, domain: .processing, recoverable: true)
@@ -1285,7 +1381,7 @@ final class MeetingSessionCoordinator: ObservableObject {
self.activeSession = session
self.state = .failed(session.id, failure)
self.operationGeneration = nil
- self.releaseActivityLease()
+ await self.releaseActivityLease()
throw error
}
}
@@ -1444,7 +1540,7 @@ final class MeetingSessionCoordinator: ObservableObject {
self.activeSession = session
self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) })
self.state = .interrupted(session.id)
- self.releaseActivityLease()
+ await self.releaseActivityLease()
self.interruptionTask = nil
}
@@ -1484,7 +1580,7 @@ final class MeetingSessionCoordinator: ObservableObject {
self.activeSession = session
self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) })
self.state = .interrupted(session.id)
- self.releaseActivityLease()
+ await self.releaseActivityLease()
}
private func enqueuePersistence(_ session: MeetingSession) {
@@ -1517,9 +1613,13 @@ final class MeetingSessionCoordinator: ObservableObject {
self.liveTranscript = .empty
}
- private func releaseActivityLease() {
+ private func releaseActivityLease() async {
guard let lease = self.activityLease else { return }
- self.audioArbiter.release(lease)
+ #if DEBUG
+ AudioTopologyDiagnostics.record(.phaseBegin, owner: .meetingCoordinator, queueRole: .mainControl, phase: .activityLease)
+ defer { AudioTopologyDiagnostics.record(.phaseEnd, owner: .meetingCoordinator, queueRole: .mainControl, phase: .activityLease) }
+ #endif
+ await self.audioArbiter.release(lease)
self.activityLease = nil
}
diff --git a/Sources/Fluid/UI/AppNavigationState.swift b/Sources/Fluid/UI/AppNavigationState.swift
index 68f2acd7..9d8a84b6 100644
--- a/Sources/Fluid/UI/AppNavigationState.swift
+++ b/Sources/Fluid/UI/AppNavigationState.swift
@@ -12,7 +12,8 @@ enum SidebarItem: Hashable {
case voiceEngine
case aiEnhancements
case cleanupStyles
- case meetingTools
+ case fileTranscription
+ case meetingTranscription
case customDictionary
case stats
case history
diff --git a/Sources/Fluid/UI/MeetingTranscriptionView.swift b/Sources/Fluid/UI/MeetingTranscriptionView.swift
index d26f661b..b29674e4 100644
--- a/Sources/Fluid/UI/MeetingTranscriptionView.swift
+++ b/Sources/Fluid/UI/MeetingTranscriptionView.swift
@@ -451,9 +451,10 @@ struct MeetingTranscriptionView: View {
}
if AVCaptureDevice.authorizationStatus(for: .audio) == .authorized {
- let identities = MeetingCaptureSourceCatalog.availableMicrophones()
+ let snapshot = await MeetingCaptureSourceCatalog.microphoneSnapshot()
+ let identities = snapshot.identities
self.microphones = identities.map(MeetingMicrophoneOption.init)
- self.selectPreferredMicrophone(from: identities)
+ self.selectPreferredMicrophone(from: identities, systemDefaultUID: snapshot.defaultCoreAudioUID)
} else {
self.microphones = []
self.setupDraft.selectedMicrophoneID = nil
@@ -497,7 +498,10 @@ struct MeetingTranscriptionView: View {
self.regenerateDefaultTitleIfNeeded()
}
- private func selectPreferredMicrophone(from identities: [MeetingMicrophoneIdentity]) {
+ private func selectPreferredMicrophone(
+ from identities: [MeetingMicrophoneIdentity],
+ systemDefaultUID: String?
+ ) {
if let selectedID = setupDraft.selectedMicrophoneID,
identities.contains(where: { $0.captureDeviceID == selectedID })
{
@@ -506,14 +510,15 @@ struct MeetingTranscriptionView: View {
let defaults = SettingsStore.shared.meetingRecordingDefaults
let savedMicrophone = defaults.savedMicrophone(in: identities)
+ let preferredInputUID = SettingsStore.shared.preferredInputDeviceUID
// Default preselection follows the system's current input, not the remembered device.
let selection = MeetingMicrophonePreselection.select(
identities: identities,
savedDeviceID: savedMicrophone?.captureDeviceID,
savedRole: defaults.microphoneRole,
- systemDefaultUID: AudioDevice.getDefaultInputDevice()?.uid,
- preferredInputUID: SettingsStore.shared.preferredInputDeviceUID,
- systemDefaultCaptureID: AVCaptureDevice.default(for: .audio)?.uniqueID
+ systemDefaultUID: systemDefaultUID,
+ preferredInputUID: preferredInputUID,
+ systemDefaultCaptureID: nil
)
self.setupDraft.selectedMicrophoneID = selection.deviceID
self.setupDraft.microphoneRole = selection.role
diff --git a/Tests/FluidDictationIntegrationTests/AudioTopologyDiagnosticsTests.swift b/Tests/FluidDictationIntegrationTests/AudioTopologyDiagnosticsTests.swift
index d3b6bad3..f00e5e17 100644
--- a/Tests/FluidDictationIntegrationTests/AudioTopologyDiagnosticsTests.swift
+++ b/Tests/FluidDictationIntegrationTests/AudioTopologyDiagnosticsTests.swift
@@ -3,6 +3,12 @@ import CoreAudio
import XCTest
final class AudioTopologyDiagnosticsTests: XCTestCase {
+ func testInputAvailabilityRequiresAtLeastOneLiveEnumeratedDevice() {
+ XCTAssertFalse(AudioInputAvailabilityPolicy.hasAvailableInput(liveness: []))
+ XCTAssertFalse(AudioInputAvailabilityPolicy.hasAvailableInput(liveness: [false, false]))
+ XCTAssertTrue(AudioInputAvailabilityPolicy.hasAvailableInput(liveness: [false, true]))
+ }
+
override func setUp() {
super.setUp()
fv_audio_topology_trace_reset()
@@ -81,6 +87,8 @@ final class AudioTopologyDiagnosticsTests: XCTestCase {
(.recoveryBegin, .recoveryEnd),
(.phaseBegin, .phaseEnd),
(.probeCycleBegin, .probeCycleEnd),
+ (.callbackBegin, .callbackEnd),
+ (.mainHopBegin, .mainHopEnd),
]
for (begin, end) in pairs {
@@ -99,6 +107,38 @@ final class AudioTopologyDiagnosticsTests: XCTestCase {
XCTAssertTrue(AudioTopologyDiagnostics.hasOpenTopologyPhase(Self.snapshot().events))
}
+ func testDifferentPropertyAddressCannotCloseOpenBoundary() {
+ _ = Self.record(.listenerRemoveBegin, objectID: 17, selector: 100)
+ _ = Self.record(.listenerRemoveEnd, objectID: 17, selector: 200)
+ XCTAssertTrue(AudioTopologyDiagnostics.hasOpenTopologyPhase(Self.snapshot().events))
+
+ _ = Self.record(.listenerRemoveEnd, objectID: 17, selector: 100)
+ XCTAssertFalse(AudioTopologyDiagnostics.hasOpenTopologyPhase(Self.snapshot().events))
+ }
+
+ func testInterleavedListenerBoundariesRemainOpenUntilBothEnd() {
+ _ = Self.record(.listenerRemoveBegin, objectID: 17, selector: 100)
+ _ = Self.record(.listenerRemoveBegin, objectID: 17, selector: 200)
+ _ = Self.record(.listenerRemoveEnd, objectID: 17, selector: 100)
+ XCTAssertTrue(AudioTopologyDiagnostics.hasOpenTopologyPhase(Self.snapshot().events))
+
+ _ = Self.record(.listenerRemoveEnd, objectID: 17, selector: 200)
+ XCTAssertFalse(AudioTopologyDiagnostics.hasOpenTopologyPhase(Self.snapshot().events))
+ }
+
+ func testMainThreadFlagIsCapturedPerEvent() async throws {
+ await MainActor.run {
+ _ = Self.record(.callbackBegin, objectID: 101)
+ }
+ await Task.detached {
+ _ = Self.record(.callbackBegin, objectID: 202)
+ }.value
+
+ let events = Self.snapshot().events
+ XCTAssertEqual(events.first(where: { $0.objectID == 101 })?.isMainThread, 1)
+ XCTAssertEqual(events.first(where: { $0.objectID == 202 })?.isMainThread, 0)
+ }
+
func testClockAnchorContainsBothTimeDomains() throws {
let line = AudioTopologyDiagnostics.clockAnchorLine(
continuousTime: 123,
@@ -258,17 +298,283 @@ final class AudioTopologyDiagnosticsTests: XCTestCase {
}
}
+ func testRetainedAvailabilityRegistrationSurvivesUnchangedTopologyRefresh() {
+ let registration = AudioInputAvailabilityListenerIdentity(
+ deviceID: 41,
+ uid: "input-a",
+ epoch: 1,
+ lifecycleGeneration: 7
+ )
+
+ // Reconciliation generations are deliberately absent from callback validity. An
+ // unchanged registration remains current after any number of topology snapshots.
+ XCTAssertTrue(
+ AudioInputAvailabilityListenerPolicy.callbackIsCurrent(
+ captured: registration,
+ registered: registration,
+ installed: true
+ )
+ )
+ }
+
+ func testFailedOvertakenAvailabilityAddRequestsFreshReconciliation() {
+ XCTAssertTrue(
+ AudioInputAvailabilityListenerPolicy.shouldRetryStaleCompletion(
+ ownsPendingMarker: true,
+ installed: true,
+ capturedLifecycleGeneration: 3,
+ currentLifecycleGeneration: 3,
+ capturedUID: "input-a",
+ desiredUID: "input-a",
+ hasRegistration: false,
+ reconciliationIsStale: true
+ )
+ )
+ }
+
+ func testSuccessfulOvertakenAddAfterSameUIDReconnectIsRetriedNotAccepted() {
+ // R1 starts an add for N/A. R2 observes absence. R3 observes the same N/A again.
+ // The R1 completion is stale even though AudioObjectID and UID match, so production
+ // routes it through cleanup and uses this policy to request a new reconciliation.
+ XCTAssertFalse(
+ AudioInputAvailabilityListenerPolicy.completionCanInstall(
+ statusSucceeded: true,
+ ownsPendingMarker: true,
+ installed: true,
+ capturedLifecycleGeneration: 4,
+ currentLifecycleGeneration: 4,
+ capturedUID: "input-a",
+ desiredUID: "input-a",
+ reconciliationIsCurrent: false
+ )
+ )
+ XCTAssertTrue(
+ AudioInputAvailabilityListenerPolicy.shouldRetryStaleCompletion(
+ ownsPendingMarker: true,
+ installed: true,
+ capturedLifecycleGeneration: 4,
+ currentLifecycleGeneration: 4,
+ capturedUID: "input-a",
+ desiredUID: "input-a",
+ hasRegistration: false,
+ reconciliationIsStale: true
+ )
+ )
+ }
+
+ func testServiceRestartRejectsOldAvailabilityWorkAndAcceptsNewLifecycle() {
+ let old = AudioInputAvailabilityListenerIdentity(
+ deviceID: 41,
+ uid: "input-a",
+ epoch: 4,
+ lifecycleGeneration: 8
+ )
+ let replacement = AudioInputAvailabilityListenerIdentity(
+ deviceID: 41,
+ uid: "input-a",
+ epoch: 5,
+ lifecycleGeneration: 9
+ )
+
+ XCTAssertFalse(
+ AudioInputAvailabilityListenerPolicy.callbackIsCurrent(
+ captured: old,
+ registered: replacement,
+ installed: true
+ )
+ )
+ XCTAssertFalse(
+ AudioInputAvailabilityListenerPolicy.shouldRetryStaleCompletion(
+ ownsPendingMarker: true,
+ installed: true,
+ capturedLifecycleGeneration: 8,
+ currentLifecycleGeneration: 9,
+ capturedUID: "input-a",
+ desiredUID: "input-a",
+ hasRegistration: false,
+ reconciliationIsStale: true
+ )
+ )
+ XCTAssertTrue(
+ AudioInputAvailabilityListenerPolicy.callbackIsCurrent(
+ captured: replacement,
+ registered: replacement,
+ installed: true
+ )
+ )
+ }
+
+ func testReusedAudioObjectIDReplacesUIDAndRejectsOldCallback() {
+ let old = AudioInputAvailabilityListenerIdentity(
+ deviceID: 77,
+ uid: "old-uid",
+ epoch: 10,
+ lifecycleGeneration: 2
+ )
+ let replacement = AudioInputAvailabilityListenerIdentity(
+ deviceID: 77,
+ uid: "new-uid",
+ epoch: 11,
+ lifecycleGeneration: 2
+ )
+
+ XCTAssertTrue(
+ AudioInputAvailabilityListenerPolicy.registrationNeedsReplacement(
+ registeredUID: old.uid,
+ desiredUID: replacement.uid
+ )
+ )
+ XCTAssertFalse(
+ AudioInputAvailabilityListenerPolicy.callbackIsCurrent(
+ captured: old,
+ registered: replacement,
+ installed: true
+ )
+ )
+ XCTAssertTrue(
+ AudioInputAvailabilityListenerPolicy.callbackIsCurrent(
+ captured: replacement,
+ registered: replacement,
+ installed: true
+ )
+ )
+ }
+
+ func testASRHardwareCallbackDuringRegistrationIsLatchedThenHandled() {
+ XCTAssertEqual(
+ ASRHardwareListenerPolicy.disposition(
+ isTerminating: false,
+ lifecycleMatches: true,
+ isInstalled: false,
+ pendingIdentityMatches: true
+ ),
+ .latchUntilInstalled
+ )
+ XCTAssertEqual(
+ ASRHardwareListenerPolicy.disposition(
+ isTerminating: false,
+ lifecycleMatches: true,
+ isInstalled: true,
+ pendingIdentityMatches: false
+ ),
+ .handle
+ )
+ }
+
+ func testASRStaleOrFailedRegistrationCallbackIsIgnored() {
+ XCTAssertEqual(
+ ASRHardwareListenerPolicy.disposition(
+ isTerminating: false,
+ lifecycleMatches: false,
+ isInstalled: false,
+ pendingIdentityMatches: true
+ ),
+ .ignore
+ )
+ XCTAssertEqual(
+ ASRHardwareListenerPolicy.disposition(
+ isTerminating: false,
+ lifecycleMatches: true,
+ isInstalled: false,
+ pendingIdentityMatches: false
+ ),
+ .ignore
+ )
+ }
+
+ func testASRLivenessResultCannotAffectReplacementDeviceOrLifecycle() {
+ XCTAssertFalse(
+ ASRHardwareListenerPolicy.selectedDeviceQueryCanApply(
+ isTerminating: false,
+ capturedLifecycle: 3,
+ currentLifecycle: 3,
+ capturedEpoch: 8,
+ currentEpoch: 10,
+ capturedDeviceID: 41,
+ currentDeviceID: 52,
+ listenerIsInstalled: true
+ )
+ )
+ XCTAssertFalse(
+ ASRHardwareListenerPolicy.selectedDeviceQueryCanApply(
+ isTerminating: false,
+ capturedLifecycle: 3,
+ currentLifecycle: 4,
+ capturedEpoch: 8,
+ currentEpoch: 8,
+ capturedDeviceID: 41,
+ currentDeviceID: 41,
+ listenerIsInstalled: true
+ )
+ )
+ XCTAssertTrue(
+ ASRHardwareListenerPolicy.selectedDeviceQueryCanApply(
+ isTerminating: false,
+ capturedLifecycle: 4,
+ currentLifecycle: 4,
+ capturedEpoch: 10,
+ currentEpoch: 10,
+ capturedDeviceID: 52,
+ currentDeviceID: 52,
+ listenerIsInstalled: true
+ )
+ )
+ }
+
+ func testASRDeviceListQueryUsesLastRequestWins() {
+ XCTAssertFalse(
+ ASRHardwareListenerPolicy.deviceListQueryCanApply(
+ isTerminating: false,
+ capturedLifecycle: 4,
+ currentLifecycle: 4,
+ capturedQueryGeneration: 10,
+ currentQueryGeneration: 11
+ )
+ )
+ XCTAssertTrue(
+ ASRHardwareListenerPolicy.deviceListQueryCanApply(
+ isTerminating: false,
+ capturedLifecycle: 4,
+ currentLifecycle: 4,
+ capturedQueryGeneration: 11,
+ currentQueryGeneration: 11
+ )
+ )
+ }
+
+ func testASRDeviceListQueryCannotApplyAfterResetOrTermination() {
+ XCTAssertFalse(
+ ASRHardwareListenerPolicy.deviceListQueryCanApply(
+ isTerminating: false,
+ capturedLifecycle: 4,
+ currentLifecycle: 5,
+ capturedQueryGeneration: 11,
+ currentQueryGeneration: 11
+ )
+ )
+ XCTAssertFalse(
+ ASRHardwareListenerPolicy.deviceListQueryCanApply(
+ isTerminating: true,
+ capturedLifecycle: 5,
+ currentLifecycle: 5,
+ capturedQueryGeneration: 12,
+ currentQueryGeneration: 12
+ )
+ )
+ }
+
@discardableResult
private static func record(
_ event: AudioTopologyTraceEvent,
objectID: AudioObjectID,
+ selector: AudioObjectPropertySelector = kAudioDevicePropertyDeviceIsAlive,
status: OSStatus = AudioTopologyDiagnostics.noStatus
) -> UInt64 {
fv_audio_topology_trace_record(
event.rawValue,
AudioTopologyTraceOwner.audioHardwareObserver.rawValue,
objectID,
- kAudioDevicePropertyDeviceIsAlive,
+ selector,
kAudioObjectPropertyScopeGlobal,
kAudioObjectPropertyElementMain,
AudioTopologyTraceQueueRole.mainControl.rawValue,
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift b/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
index 2653dd7d..66464f12 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
@@ -29,7 +29,7 @@ private final class FakeMicActivity: MicActivitySignalProviding {
private final class FakeAudioProcessActivity: AudioProcessActivityProviding {
var activeBundleIdentifiers: Set<String> = []
- func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) -> Bool {
+ func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) async -> Bool {
!self.activeBundleIdentifiers.isDisjoint(with: bundleIdentifiers)
}
}
@@ -411,7 +411,7 @@ final class MeetingAutoDetectorTests: XCTestCase {
XCTAssertEqual(h.prompts.count, 1)
}
- func testProcessAudioConfirmsMutedZoomJoinWithoutDeviceMicEdge() {
+ func testProcessAudioConfirmsMutedZoomJoinWithoutDeviceMicEdge() async {
let h = DetectorHarness()
h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "us.zoom.xos", processID: 5), at: h.clock.now())
h.detector.handleWindowSnapshot(
@@ -419,7 +419,7 @@ final class MeetingAutoDetectorTests: XCTestCase {
at: h.clock.now()
)
h.audioProcessActivity.activeBundleIdentifiers = ["us.zoom.caphost"]
- h.detector.pollAudioProcessActivity(at: h.clock.now())
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
XCTAssertEqual(h.prompts.count, 1)
}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingEchoVetoTests.swift b/Tests/FluidDictationIntegrationTests/MeetingEchoVetoTests.swift
index 84a20d98..72440ae1 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingEchoVetoTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingEchoVetoTests.swift
@@ -228,6 +228,23 @@ final class MeetingEchoVetoTests: XCTestCase {
XCTAssertFalse(turn(isLikelyEcho: false, signalVerdict: .unknown).effectiveEcho)
}
+ func testTextEchoObservationIsExcludedBeforeMicrophoneStitching() {
+ func turn(key: String, isLikelyEcho: Bool) -> MeetingProcessingPipeline.StagedMicrophoneTurn {
+ MeetingProcessingPipeline.StagedMicrophoneTurn(
+ chunkID: UUID(), index: 0, clusterID: UUID(), clusterLabel: "0",
+ diarizationObservationKey: key,
+ start: 0, end: 1, text: "hi", overlapsRemote: true,
+ isLikelyEcho: isLikelyEcho, echoScored: true
+ )
+ }
+ let keys = MeetingProcessingPipeline.trustedMicrophoneObservationKeys(from: [
+ turn(key: "echo", isLikelyEcho: true),
+ turn(key: "local", isLikelyEcho: false),
+ ])
+
+ XCTAssertEqual(keys, ["local"])
+ }
+
// MARK: - 6. Local-speaker election is unchanged by signal verdicts
func testLocalSpeakerElectionIsUnchangedBySignalVerdict() {
@@ -324,6 +341,7 @@ final class MeetingEchoVetoTests: XCTestCase {
asrProvider: "apple",
asrModel: "base",
languageCode: "en",
+ diarizationFingerprint: MeetingProcessingCheckpoint.currentDiarizationFingerprint,
completedTrackID: trackID,
trackFingerprints: fingerprints,
speakers: [],
diff --git a/Tests/FluidDictationIntegrationTests/MeetingExistingAudioReplayTests.swift b/Tests/FluidDictationIntegrationTests/MeetingExistingAudioReplayTests.swift
new file mode 100644
index 00000000..91e84f7c
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/MeetingExistingAudioReplayTests.swift
@@ -0,0 +1,211 @@
+@testable import FluidVoice_Debug
+import Darwin.Mach
+import Foundation
+import XCTest
+
+/// Opt-in replay of retained meeting recordings through the current batch pipeline.
+///
+/// `FLUIDVOICE_REPLAY_MEETING_DIRS` is a colon-separated list of session directories. Every
+/// source is copied before processing so this test can never rewrite a user's stored meeting.
+@MainActor
+final class MeetingExistingAudioReplayTests: XCTestCase {
+ func testRetainedMeetingsReplayThroughCurrentPipeline() async throws {
+ let environmentDirectories = ProcessInfo.processInfo.environment["FLUIDVOICE_REPLAY_MEETING_DIRS"]
+ // XCTest's working directory is not guaranteed to be the repository root. Resolve the
+ // opt-in file from this source file so a local one-fixture run cannot silently fall back
+ // to the legacy machine-wide manifest.
+ let repositoryRoot = URL(fileURLWithPath: #filePath)
+ .deletingLastPathComponent() // FluidDictationIntegrationTests
+ .deletingLastPathComponent() // Tests
+ .deletingLastPathComponent() // repository root
+ let localManifestPath = repositoryRoot
+ .appendingPathComponent("benchmark_reports/meeting-diarization/replay-directories.txt")
+ .path
+ let defaultManifestPath = FileManager.default.fileExists(atPath: localManifestPath)
+ ? localManifestPath
+ : "/private/tmp/FluidVoiceMeetingReplayDirectories.txt"
+ let manifestPath = ProcessInfo.processInfo.environment["FLUIDVOICE_REPLAY_MEETING_MANIFEST"]
+ ?? defaultManifestPath
+ let manifestDirectories = try? String(contentsOfFile: manifestPath, encoding: .utf8)
+ guard let rawDirectories = environmentDirectories ?? manifestDirectories else {
+ throw XCTSkip("No retained-meeting replay input was provided")
+ }
+ let sourceDirectories = rawDirectories
+ .split(whereSeparator: { $0 == ":" || $0.isNewline })
+ .map { URL(fileURLWithPath: String($0), isDirectory: true) }
+ XCTAssertFalse(sourceDirectories.isEmpty)
+
+ let decoder = JSONDecoder()
+ decoder.dateDecodingStrategy = .iso8601
+ let asrService = ASRService()
+ let meetingLease = try asrService.acquireExclusiveActivity(.meeting)
+ defer { asrService.releaseExclusiveActivity(meetingLease) }
+ let pipeline = MeetingProcessingPipeline(asrServiceProvider: { asrService })
+ var reportLines: [String] = []
+ let reportRoot = URL(
+ fileURLWithPath: ProcessInfo.processInfo.environment["FLUIDVOICE_REPLAY_REPORT_DIR"]
+ ?? "/private/tmp/FluidVoiceMeetingReplayReport",
+ isDirectory: true
+ )
+ try FileManager.default.createDirectory(
+ at: reportRoot,
+ withIntermediateDirectories: true,
+ attributes: [.posixPermissions: 0o700]
+ )
+
+ for sourceDirectory in sourceDirectories {
+ let sourceManifest = sourceDirectory.appendingPathComponent("session.json", isDirectory: false)
+ let sourceSession = try decoder.decode(
+ MeetingSession.self,
+ from: Data(contentsOf: sourceManifest)
+ )
+ let replayDirectory = FileManager.default.temporaryDirectory
+ .appendingPathComponent("FluidVoiceMeetingReplay-\(sourceSession.id.uuidString)-\(UUID().uuidString)", isDirectory: true)
+ try FileManager.default.copyItem(at: sourceDirectory, to: replayDirectory)
+ defer { try? FileManager.default.removeItem(at: replayDirectory) }
+
+ // Force a full pass even when the retained session contains an older checkpoint.
+ try? FileManager.default.removeItem(
+ at: replayDirectory.appendingPathComponent("checkpoint.json", isDirectory: false)
+ )
+ let result: MeetingProcessingResult
+ let startedAt = Date()
+ let residentBytesBefore = Self.currentResidentBytes()
+ do {
+ result = try await pipeline.process(
+ session: sourceSession,
+ sessionDirectory: replayDirectory,
+ progress: { _ in }
+ )
+ } catch {
+ reportLines.append("id=\(sourceSession.id) FAILED error=\(error)")
+ XCTFail("\(sourceSession.id) failed replay: \(error)")
+ continue
+ }
+ let wallTimeSeconds = Date().timeIntervalSince(startedAt)
+ let residentBytesAfter = Self.currentResidentBytes()
+
+ XCTAssertEqual(result.attempt.pipelineVersion, MeetingProcessingPipeline.pipelineVersion)
+ XCTAssertFalse(result.segments.isEmpty, "\(sourceSession.id) produced no transcript segments")
+ XCTAssertFalse(result.speakers.isEmpty, "\(sourceSession.id) produced no speakers")
+ XCTAssertEqual(Set(result.speakers.map(\.id)).count, result.speakers.count)
+
+ let speakerByID = Dictionary(uniqueKeysWithValues: result.speakers.map { ($0.id, $0) })
+ let trackKindByID = Dictionary(uniqueKeysWithValues: sourceSession.audioTracks.map { ($0.id, $0.kind) })
+ for segment in result.segments {
+ XCTAssertFalse(segment.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
+ guard let speakerID = segment.speakerID,
+ let speaker = speakerByID[speakerID] else {
+ XCTFail("\(sourceSession.id) emitted a segment with an unknown speaker")
+ continue
+ }
+ XCTAssertEqual(
+ speaker.trackKind,
+ trackKindByID[segment.sourceTrackID],
+ "\(sourceSession.id) crossed application and microphone speaker identities"
+ )
+ }
+
+ let speakerSummary = result.speakers
+ .sorted { $0.displayName < $1.displayName }
+ .map { "\($0.displayName)[\($0.trackKind.rawValue)]" }
+ .joined(separator: ", ")
+ let summary = "id=\(sourceSession.id) chunks=\(sourceSession.audioTracks.flatMap(\.chunks).count) "
+ + "segments=\(result.segments.count) speakers=\(result.speakers.count) "
+ + "skipped=\(result.skippedChunkIDs.count) wall=\(String(format: "%.3f", wallTimeSeconds))s "
+ + "{\(speakerSummary)}"
+ reportLines.append(summary)
+ print("[meeting-replay] \(summary)")
+
+ let fixtureDirectory = reportRoot.appendingPathComponent(sourceSession.id.uuidString, isDirectory: true)
+ try FileManager.default.createDirectory(
+ at: fixtureDirectory,
+ withIntermediateDirectories: true,
+ attributes: [.posixPermissions: 0o700]
+ )
+ let hypothesisRTTM = result.segments.compactMap { segment -> String? in
+ guard let speakerID = segment.speakerID,
+ let speaker = speakerByID[speakerID],
+ segment.end.seconds > segment.start.seconds else { return nil }
+ return String(
+ format: "SPEAKER %@ 1 %.6f %.6f <NA> <NA> %@ <NA> <NA>",
+ sourceSession.id.uuidString,
+ segment.start.seconds,
+ segment.end.seconds - segment.start.seconds,
+ speaker.displayName.replacingOccurrences(of: " ", with: "-")
+ )
+ }.joined(separator: "\n") + "\n"
+ try Self.writeProtected(
+ Data(hypothesisRTTM.utf8),
+ to: fixtureDirectory.appendingPathComponent("hypothesis.rttm")
+ )
+
+ let speakerText = Dictionary(grouping: result.segments.compactMap { segment -> (String, String)? in
+ guard let speakerID = segment.speakerID, let speaker = speakerByID[speakerID] else { return nil }
+ return (speaker.displayName, segment.text)
+ }, by: { $0.0 }).mapValues { $0.map(\.1).joined(separator: " ") }
+ let speakerTextData = try JSONSerialization.data(
+ withJSONObject: speakerText,
+ options: [.prettyPrinted, .sortedKeys]
+ )
+ try Self.writeProtected(
+ speakerTextData,
+ to: fixtureDirectory.appendingPathComponent("hypothesis-speaker-text.json")
+ )
+
+ let fixtureReport: [String: Any] = [
+ "schemaVersion": 1,
+ "sessionID": sourceSession.id.uuidString,
+ "pipelineVersion": result.attempt.pipelineVersion,
+ "asrProvider": result.attempt.asrProvider ?? "unknown",
+ "asrModel": result.attempt.asrModel ?? "unknown",
+ "languageCode": result.attempt.languageCode ?? "unknown",
+ "diarizationFingerprint": result.attempt.diarizationModel ?? "unsupported",
+ "sourceChunkSHA256": sourceSession.audioTracks.flatMap(\.chunks).map(\.sha256).sorted(),
+ "sourceChunkCount": sourceSession.audioTracks.flatMap(\.chunks).count,
+ "speakerCount": result.speakers.count,
+ "speakerCountByTrack": Dictionary(
+ grouping: result.speakers,
+ by: { $0.trackKind.rawValue }
+ ).mapValues(\.count),
+ "segmentCount": result.segments.count,
+ "segmentCountByTrack": Dictionary(
+ grouping: result.segments,
+ by: { segment in
+ trackKindByID[segment.sourceTrackID]?.rawValue ?? "unknown"
+ }
+ ).mapValues(\.count),
+ "skippedChunkCount": result.skippedChunkIDs.count,
+ "wallTimeSeconds": wallTimeSeconds,
+ "residentBytesBefore": residentBytesBefore,
+ "residentBytesAfter": residentBytesAfter,
+ ]
+ try Self.writeProtected(
+ try JSONSerialization.data(withJSONObject: fixtureReport, options: [.prettyPrinted, .sortedKeys]),
+ to: fixtureDirectory.appendingPathComponent("run.json")
+ )
+ }
+
+ try (reportLines.joined(separator: "\n") + "\n").write(
+ toFile: "/private/tmp/FluidVoiceMeetingReplayReport.txt",
+ atomically: true,
+ encoding: .utf8
+ )
+ }
+
+ private static func currentResidentBytes() -> UInt64 {
+ var information = mach_task_basic_info()
+ var count = mach_msg_type_number_t(MemoryLayout<mach_task_basic_info>.size / MemoryLayout<natural_t>.size)
+ let result = withUnsafeMutablePointer(to: &information) { pointer in
+ pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { rebound in
+ task_info(mach_task_self_, task_flavor_t(MACH_TASK_BASIC_INFO), rebound, &count)
+ }
+ }
+ return result == KERN_SUCCESS ? information.resident_size : 0
+ }
+
+ private static func writeProtected(_ data: Data, to url: URL) throws {
+ try data.write(to: url, options: .atomic)
+ try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path)
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift b/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift
index 83367048..fbfe7f3f 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift
@@ -902,26 +902,24 @@ extension MeetingMicrophoneCaptureTests {
XCTAssertLessThan(Date().timeIntervalSince(start), 0.5)
}
- // MARK: - Upgrade ring-discard boundary
-
- func testRingDiscardDropsSamplesAtOrBehindBoundaryAndKeepsSamplesAfterIt() throws {
- let buffer = self.makeMonoBuffer(frameCount: 480) { _ in 0.1 }
- let before = try XCTUnwrap(meetingMicrophoneSynthesizeSampleBuffer(
- from: buffer, presentationTime: CMTime(value: 0, timescale: 48_000)
- ))
- let atBoundary = try XCTUnwrap(meetingMicrophoneSynthesizeSampleBuffer(
- from: buffer, presentationTime: CMTime(value: 48_000, timescale: 48_000)
- ))
- let after = try XCTUnwrap(meetingMicrophoneSynthesizeSampleBuffer(
- from: buffer, presentationTime: CMTime(value: 48_001, timescale: 48_000)
- ))
- let boundary = CMTime(value: 48_000, timescale: 48_000)
-
- let kept = MeetingUpgradeRingDiscard.discardingSamples(atOrBehind: boundary, from: [before, atBoundary, after])
- XCTAssertEqual(kept.count, 1, "only the strictly-after sample survives")
+ // MARK: - HAL callback isolation
+
+ func testMeetingEventWorkStartsOnlyAfterHALDeliveryCallbackReturns() {
+ let callbackReturned = expectation(description: "HAL callback returned")
+ let eventHandled = expectation(description: "meeting event handled")
+ let state = MeetingMicrophoneCallbackOrderingState()
+
+ // Model the body of a property-listener callback on the exact serial queue passed to HAL.
+ AudioTopologyListenerExecution.deliveryQueue.async {
+ MeetingMicrophoneEventExecution.afterHALCallback {
+ XCTAssertTrue(state.callbackHasReturned)
+ eventHandled.fulfill()
+ }
+ state.markCallbackReturned()
+ callbackReturned.fulfill()
+ }
- let unfiltered = MeetingUpgradeRingDiscard.discardingSamples(atOrBehind: nil, from: [before, atBoundary, after])
- XCTAssertEqual(unfiltered.count, 3, "nil boundary means the writer produced no splice — keep everything")
+ wait(for: [callbackReturned, eventHandled], timeout: 1, enforceOrder: true)
}
// MARK: - Not covered here: phase-machine transitions (file-private, hardware-driven —
@@ -944,3 +942,20 @@ extension MeetingMicrophoneCaptureTests {
}
}
+
+private final class MeetingMicrophoneCallbackOrderingState: @unchecked Sendable {
+ private let lock = NSLock()
+ private var returned = false
+
+ var callbackHasReturned: Bool {
+ self.lock.lock()
+ defer { self.lock.unlock() }
+ return self.returned
+ }
+
+ func markCallbackReturned() {
+ self.lock.lock()
+ self.returned = true
+ self.lock.unlock()
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift b/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
index 7f73f468..61c7dfa9 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
@@ -873,6 +873,7 @@ final class MeetingRecoveryTests: XCTestCase {
asrProvider: "apple",
asrModel: asrModel,
languageCode: "en",
+ diarizationFingerprint: MeetingProcessingCheckpoint.currentDiarizationFingerprint,
completedTrackID: completedTrackID,
trackFingerprints: trackFingerprints,
speakers: [],
@@ -913,6 +914,7 @@ final class MeetingRecoveryTests: XCTestCase {
provider: String = "apple",
model: String = "base",
language: String = "en",
+ diarizationFingerprint: String = MeetingProcessingCheckpoint.currentDiarizationFingerprint,
completedTrackID: MeetingAudioTrackID = trackID,
expectedFingerprints: [MeetingAudioTrackID: [MeetingProcessingCheckpoint.ChunkFingerprint]] = fingerprints,
session: MeetingSession = session
@@ -923,6 +925,7 @@ final class MeetingRecoveryTests: XCTestCase {
provider: provider,
model: model,
language: language,
+ diarizationFingerprint: diarizationFingerprint,
completedTrackID: completedTrackID,
expectedFingerprints: expectedFingerprints
)
@@ -933,6 +936,7 @@ final class MeetingRecoveryTests: XCTestCase {
XCTAssertFalse(isValid(provider: "other"))
XCTAssertFalse(isValid(model: "other"))
XCTAssertFalse(isValid(language: "fr"))
+ XCTAssertFalse(isValid(diarizationFingerprint: "different-diarizer"))
XCTAssertFalse(isValid(completedTrackID: UUID()))
var otherSession = session
otherSession.id = UUID()
@@ -2301,41 +2305,166 @@ final class MeetingRecoveryTests: XCTestCase {
// MARK: - ASR activity arbiter
- func testArbiterDoubleAcquireThrowsRecordingAlreadyActive() throws {
+ func testPermissionPreflightCompletesBeforeMeetingLeaseAcquisition() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let capture = StubCaptureController()
+ let arbiter = StubArbiter()
+ var continuation: CheckedContinuation<Void, Never>?
+ capture.onPreflight = {
+ await withCheckedContinuation { continuation = $0 }
+ }
+ let coordinator = MeetingSessionCoordinator(
+ store: MeetingSessionStore(rootDirectory: dir),
+ capture: capture,
+ processing: StubProcessingController(),
+ audioArbiter: arbiter
+ )
+
+ let start = Task { try await coordinator.startRecording(configuration: self.makeConfiguration()) }
+ while capture.preflightCount == 0 { await Task.yield() }
+ XCTAssertEqual(arbiter.acquireCount, 0, "permission prompt must not hold the meeting audio lease")
+
+ continuation?.resume()
+ _ = try await start.value
+ XCTAssertEqual(arbiter.acquireCount, 1)
+ }
+
+ func testDeniedPermissionNeverAcquiresMeetingLease() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let capture = StubCaptureController()
+ capture.preflightError = MeetingCaptureError.microphonePermissionDenied
+ let arbiter = StubArbiter()
+ let coordinator = MeetingSessionCoordinator(
+ store: MeetingSessionStore(rootDirectory: dir),
+ capture: capture,
+ processing: StubProcessingController(),
+ audioArbiter: arbiter
+ )
+
+ do {
+ _ = try await coordinator.startRecording(configuration: self.makeConfiguration())
+ XCTFail("denied microphone permission must fail before audio ownership")
+ } catch {
+ guard case MeetingCaptureError.microphonePermissionDenied = error else {
+ return XCTFail("Expected microphonePermissionDenied, got \(error)")
+ }
+ }
+ XCTAssertEqual(arbiter.acquireCount, 0)
+ XCTAssertNil(coordinator.activeSession)
+ XCTAssertEqual(coordinator.state, .idle)
+ }
+
+ func testArbiterDoubleAcquireThrowsRecordingAlreadyActive() async throws {
let leasing = FakeASRActivityLeasing()
let arbiter = AudioActivityArbiter { leasing }
- _ = try arbiter.acquireMeetingCapture()
- XCTAssertThrowsError(try arbiter.acquireMeetingCapture()) { error in
+ _ = try await arbiter.acquireMeetingCapture()
+ do {
+ _ = try await arbiter.acquireMeetingCapture()
+ XCTFail("Expected .recordingAlreadyActive")
+ } catch {
guard case MeetingCoordinatorError.recordingAlreadyActive = error else {
return XCTFail("Expected .recordingAlreadyActive, got \(error)")
}
}
XCTAssertEqual(leasing.acquireCount, 1)
+ XCTAssertEqual(leasing.prepareCount, 1)
}
- func testArbiterReleaseThenReacquireSucceeds() throws {
+ func testArbiterRejectsSecondAcquireWhileFirstHandoffIsSuspended() async throws {
+ let leasing = SuspendedHandoffASRActivityLeasing()
+ let arbiter = AudioActivityArbiter { leasing }
+ let firstAcquire = Task { try await arbiter.acquireMeetingCapture() }
+
+ while leasing.prepareStarted == false { await Task.yield() }
+ do {
+ _ = try await arbiter.acquireMeetingCapture()
+ XCTFail("Expected the published preparing lease to reject re-entrant acquisition")
+ } catch {
+ guard case MeetingCoordinatorError.recordingAlreadyActive = error else {
+ return XCTFail("Expected .recordingAlreadyActive, got \(error)")
+ }
+ }
+
+ leasing.resumePrepare()
+ let lease = try await firstAcquire.value
+ await arbiter.release(lease)
+ XCTAssertEqual(leasing.handbackCount, 1)
+ }
+
+ func testCancelledSuspendedHandoffRollsBackBeforeReturningLease() async throws {
+ let leasing = SuspendedHandoffASRActivityLeasing()
+ let arbiter = AudioActivityArbiter { leasing }
+ let acquire = Task { try await arbiter.acquireMeetingCapture() }
+
+ while leasing.prepareStarted == false { await Task.yield() }
+ acquire.cancel()
+ leasing.resumePrepare()
+
+ do {
+ _ = try await acquire.value
+ XCTFail("Canceled handoff must not return a usable meeting lease")
+ } catch is CancellationError {}
+ XCTAssertEqual(leasing.handbackCount, 1)
+
+ let replacement = try await arbiter.acquireMeetingCapture()
+ await arbiter.release(replacement)
+ }
+
+ func testArbiterReleaseThenReacquireSucceeds() async throws {
let leasing = FakeASRActivityLeasing()
let arbiter = AudioActivityArbiter { leasing }
- let lease = try arbiter.acquireMeetingCapture()
- arbiter.release(lease)
+ let lease = try await arbiter.acquireMeetingCapture()
+ await arbiter.release(lease)
XCTAssertEqual(leasing.releaseCount, 1)
+ XCTAssertEqual(leasing.handbackCount, 1)
- let reacquired = try arbiter.acquireMeetingCapture()
+ let reacquired = try await arbiter.acquireMeetingCapture()
XCTAssertNotEqual(lease, reacquired)
XCTAssertEqual(leasing.acquireCount, 2)
}
- func testArbiterReleaseWithStaleLeaseIsNoOp() throws {
+ func testConcurrentDuplicateReleaseCoalescesUntilHandbackCompletes() async throws {
+ let leasing = SuspendedHandbackASRActivityLeasing()
+ let arbiter = AudioActivityArbiter { leasing }
+ let lease = try await arbiter.acquireMeetingCapture()
+
+ let firstRelease = Task { await arbiter.release(lease) }
+ while leasing.handbackStarted == false { await Task.yield() }
+ let duplicateRelease = Task { await arbiter.release(lease) }
+ await Task.yield()
+
+ XCTAssertEqual(leasing.handbackCount, 1, "duplicate release must await the retained handback")
+ do {
+ _ = try await arbiter.acquireMeetingCapture()
+ XCTFail("meeting ownership must remain published until handback finishes")
+ } catch {
+ guard case MeetingCoordinatorError.recordingAlreadyActive = error else {
+ return XCTFail("Expected .recordingAlreadyActive, got \(error)")
+ }
+ }
+
+ leasing.resumeHandback()
+ await firstRelease.value
+ await duplicateRelease.value
+ XCTAssertEqual(leasing.handbackCount, 1)
+
+ let replacement = try await arbiter.acquireMeetingCapture()
+ await arbiter.release(replacement)
+ }
+
+ func testArbiterReleaseWithStaleLeaseIsNoOp() async throws {
let leasing = FakeASRActivityLeasing()
let arbiter = AudioActivityArbiter { leasing }
- let lease = try arbiter.acquireMeetingCapture()
- arbiter.release(MeetingAudioActivityLease(id: UUID())) // stale, not the held lease
+ let lease = try await arbiter.acquireMeetingCapture()
+ await arbiter.release(MeetingAudioActivityLease(id: UUID())) // stale, not the held lease
XCTAssertEqual(leasing.releaseCount, 0, "a stale lease must not release the real ASR activity")
- arbiter.release(lease)
+ await arbiter.release(lease)
XCTAssertEqual(leasing.releaseCount, 1)
}
@@ -2737,6 +2866,15 @@ private final class StubCaptureController: MeetingCaptureControlling, @unchecked
var startResult = MeetingCaptureStartResult(tracks: [], firstPresentationTime: nil)
var startError: Error?
var onStart: (@Sendable () async -> Void)?
+ var preflightError: Error?
+ var onPreflight: (@Sendable () async -> Void)?
+ private(set) var preflightCount = 0
+
+ func preflightPermissions() async throws {
+ self.preflightCount += 1
+ await self.onPreflight?()
+ if let preflightError { throw preflightError }
+ }
func start(
session: MeetingSession,
@@ -2872,8 +3010,12 @@ private final class GatedProcessingController: MeetingProcessingControlling {
@MainActor
private final class StubArbiter: MeetingAudioActivityArbitrating {
- func acquireMeetingCapture() throws -> MeetingAudioActivityLease { MeetingAudioActivityLease(id: UUID()) }
- func release(_ lease: MeetingAudioActivityLease) {}
+ private(set) var acquireCount = 0
+ func acquireMeetingCapture() async throws -> MeetingAudioActivityLease {
+ self.acquireCount += 1
+ return MeetingAudioActivityLease(id: UUID())
+ }
+ func release(_ lease: MeetingAudioActivityLease) async {}
}
/// Records acquire/release calls without touching a real ASRService.
@@ -2881,6 +3023,8 @@ private final class StubArbiter: MeetingAudioActivityArbitrating {
private final class FakeASRActivityLeasing: ASRActivityLeasing {
private(set) var acquireCount = 0
private(set) var releaseCount = 0
+ private(set) var prepareCount = 0
+ private(set) var handbackCount = 0
private var activeLease: ASRActivityLease?
func acquireExclusiveActivity(_ activity: ASRExclusiveActivity) throws -> ASRActivityLease {
@@ -2898,20 +3042,105 @@ private final class FakeASRActivityLeasing: ASRActivityLeasing {
self.releaseCount += 1
self.activeLease = nil
}
+
+ func prepareMeetingAudioHandoff(_ lease: ASRActivityLease) async throws {
+ self.prepareCount += 1
+ }
+ func completeMeetingAudioHandback(_ lease: ASRActivityLease) async {
+ self.handbackCount += 1
+ self.releaseExclusiveActivity(lease)
+ }
+}
+
+@MainActor
+private final class SuspendedHandoffASRActivityLeasing: ASRActivityLeasing {
+ private(set) var prepareStarted = false
+ private(set) var handbackCount = 0
+ private var activeLease: ASRActivityLease?
+ private var prepareContinuation: CheckedContinuation<Void, Never>?
+ private var prepareInvocationCount = 0
+
+ func acquireExclusiveActivity(_ activity: ASRExclusiveActivity) throws -> ASRActivityLease {
+ guard self.activeLease == nil else { throw ASRActivityError.activityInProgress(activity) }
+ let lease = ASRActivityLease(id: UUID(), activity: activity)
+ self.activeLease = lease
+ return lease
+ }
+
+ func releaseExclusiveActivity(_ lease: ASRActivityLease) {
+ guard self.activeLease == lease else { return }
+ self.activeLease = nil
+ }
+
+ func prepareMeetingAudioHandoff(_ lease: ASRActivityLease) async throws {
+ self.prepareInvocationCount += 1
+ self.prepareStarted = true
+ if self.prepareInvocationCount == 1 {
+ await withCheckedContinuation { self.prepareContinuation = $0 }
+ }
+ }
+
+ func resumePrepare() {
+ let continuation = self.prepareContinuation
+ self.prepareContinuation = nil
+ continuation?.resume()
+ }
+
+ func completeMeetingAudioHandback(_ lease: ASRActivityLease) async {
+ self.handbackCount += 1
+ self.releaseExclusiveActivity(lease)
+ }
+}
+
+@MainActor
+private final class SuspendedHandbackASRActivityLeasing: ASRActivityLeasing {
+ private(set) var handbackStarted = false
+ private(set) var handbackCount = 0
+ private var activeLease: ASRActivityLease?
+ private var handbackContinuation: CheckedContinuation<Void, Never>?
+
+ func acquireExclusiveActivity(_ activity: ASRExclusiveActivity) throws -> ASRActivityLease {
+ guard self.activeLease == nil else { throw ASRActivityError.activityInProgress(activity) }
+ let lease = ASRActivityLease(id: UUID(), activity: activity)
+ self.activeLease = lease
+ return lease
+ }
+
+ func releaseExclusiveActivity(_ lease: ASRActivityLease) {
+ guard self.activeLease == lease else { return }
+ self.activeLease = nil
+ }
+
+ func prepareMeetingAudioHandoff(_ lease: ASRActivityLease) async throws {}
+
+ func completeMeetingAudioHandback(_ lease: ASRActivityLease) async {
+ self.handbackCount += 1
+ self.handbackStarted = true
+ if self.handbackCount == 1 {
+ await withCheckedContinuation { self.handbackContinuation = $0 }
+ }
+ self.releaseExclusiveActivity(lease)
+ }
+
+ func resumeHandback() {
+ let continuation = self.handbackContinuation
+ self.handbackContinuation = nil
+ continuation?.resume()
+ }
}
/// Throws on the first `acquireMeetingCapture()` call, then succeeds on every subsequent call.
@MainActor
private final class ThrowOnceArbiter: MeetingAudioActivityArbitrating {
private var hasThrown = false
- func acquireMeetingCapture() throws -> MeetingAudioActivityLease {
+ func acquireMeetingCapture() async throws -> MeetingAudioActivityLease {
guard self.hasThrown else {
self.hasThrown = true
throw MeetingCoordinatorError.dictationActive
}
return MeetingAudioActivityLease(id: UUID())
}
- func release(_ lease: MeetingAudioActivityLease) {}
+ func release(_ lease: MeetingAudioActivityLease) async {}
}
/// Wraps a real `MeetingSessionStore`, suspending `delete(id:)` until `openGate()` is called —
diff --git a/Tests/FluidDictationIntegrationTests/MeetingTimelineDriftProbeTests.swift b/Tests/FluidDictationIntegrationTests/MeetingTimelineDriftProbeTests.swift
index 911526b8..ec0bb8d2 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingTimelineDriftProbeTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingTimelineDriftProbeTests.swift
@@ -52,7 +52,7 @@ final class MeetingTimelineDriftProbeTests: XCTestCase {
try FileManager.default.createDirectory(at: temp, withIntermediateDirectories: true)
defer { try? FileManager.default.removeItem(at: temp) }
- let microphone = try MeetingCaptureSourceCatalog.defaultMicrophone(preferredCoreAudioUID: "BuiltInMicrophoneDevice")
+ let microphone = try await MeetingCaptureSourceCatalog.defaultMicrophone(preferredCoreAudioUID: "BuiltInMicrophoneDevice")
var writerDiscontinuities = 0
let track = MeetingAudioTrack(
id: UUID(), kind: .microphone,
diff --git a/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift b/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
index 3851b5d5..402f312e 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
@@ -13,7 +13,13 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
// MARK: - Decision table (incl. role)
private func mic(uid: String? = "mic-uid", role: MeetingMicrophoneRole = .personal) -> MeetingMicrophoneIdentity {
- MeetingMicrophoneIdentity(captureDeviceID: "dev-1", coreAudioUID: uid, displayName: "Mic", role: role)
+ MeetingMicrophoneIdentity(
+ captureDeviceID: uid ?? "selection-1",
+ coreAudioUID: uid,
+ avCaptureDeviceID: "dev-1",
+ displayName: "Mic",
+ role: role
+ )
}
private func viableRoute() -> MeetingOutputRouteSnapshot {
@@ -176,6 +182,220 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
XCTAssertTrue(MeetingVoiceProcessingWatchdog.isStalled(secondsSinceLastEmittedBuffer: 30))
}
+ func testRouteDrivenFallbackRecoversOnlyOnViableBuiltInRoute() {
+ XCTAssertTrue(MeetingCaptureTransitionPolicy.shouldAttemptVoiceProcessingRecovery(
+ routeDrivenFallback: true,
+ attempts: 0,
+ microphone: self.mic(),
+ outputRoute: self.viableRoute()
+ ))
+ XCTAssertFalse(MeetingCaptureTransitionPolicy.shouldAttemptVoiceProcessingRecovery(
+ routeDrivenFallback: true,
+ attempts: 0,
+ microphone: self.mic(),
+ outputRoute: MeetingOutputRouteSnapshot(
+ deviceExists: true,
+ isBluetooth: true,
+ isBuiltIn: false,
+ isHeadphonesDataSource: false
+ )
+ ))
+ }
+
+ func testWatchdogOrEngineFailureFallbackStaysSticky() {
+ XCTAssertFalse(MeetingCaptureTransitionPolicy.shouldAttemptVoiceProcessingRecovery(
+ routeDrivenFallback: false,
+ attempts: 0,
+ microphone: self.mic(),
+ outputRoute: self.viableRoute()
+ ))
+ }
+
+ func testVoiceProcessingRecoveryAttemptsAreBounded() {
+ XCTAssertFalse(MeetingCaptureTransitionPolicy.shouldAttemptVoiceProcessingRecovery(
+ routeDrivenFallback: true,
+ attempts: MeetingCaptureTransitionPolicy.maximumRouteRecoveryAttempts,
+ microphone: self.mic(),
+ outputRoute: self.viableRoute()
+ ))
+ }
+
+ func testRecoveryDwellCannotArmDuringTransitionStopOrQuarantine() {
+ for blocked in [
+ (false, false, false, false),
+ (true, true, false, false),
+ (true, false, true, false),
+ (true, false, false, true),
+ ] {
+ XCTAssertFalse(MeetingVoiceProcessingUpgradeDwell.shouldArm(
+ phaseIsAVCapture: blocked.0,
+ dwellAlreadyArmed: blocked.1,
+ quarantined: blocked.2,
+ stopRequested: blocked.3,
+ routeDrivenFallback: true,
+ attempts: 0,
+ microphone: self.mic(),
+ outputRoute: self.viableRoute()
+ ))
+ }
+ }
+
+ func testStopOldFirstOwnershipSequenceReleasesAVCaptureBeforeVPIOConstruction() throws {
+ let steps = MeetingVoiceProcessingUpgradeOwnership.stopOldFirstSequence
+ let clearRuntime = try XCTUnwrap(steps.firstIndex(of: .clearRuntimeRef))
+ let clearLocal = try XCTUnwrap(steps.firstIndex(of: .clearLocalRef))
+ let constructVPIO = try XCTUnwrap(steps.firstIndex(of: .constructVoiceProcessing))
+ XCTAssertLessThan(clearRuntime, constructVPIO)
+ XCTAssertLessThan(clearLocal, constructVPIO)
+ XCTAssertFalse(MeetingVoiceProcessingUpgradeOwnership.mayConstructVoiceProcessing(
+ avCaptureRuntimeRef: false,
+ avCaptureLocalRef: true,
+ avCaptureRunning: false,
+ teardownProven: true
+ ))
+ XCTAssertTrue(MeetingVoiceProcessingUpgradeOwnership.mayConstructVoiceProcessing(
+ avCaptureRuntimeRef: false,
+ avCaptureLocalRef: false,
+ avCaptureRunning: false,
+ teardownProven: true
+ ))
+ XCTAssertFalse(MeetingVoiceProcessingUpgradeOwnership.watchdogRunsDuringIntentionalGap)
+ }
+
+ func testRollbackMayPublishAVCaptureOnlyWithALiveComponent() {
+ XCTAssertFalse(MeetingVoiceProcessingUpgradeOwnership.mayPhaseAVCapture(
+ hasComponent: false,
+ componentStopped: false
+ ))
+ XCTAssertFalse(MeetingVoiceProcessingUpgradeOwnership.mayPhaseAVCapture(
+ hasComponent: true,
+ componentStopped: true
+ ))
+ XCTAssertTrue(MeetingVoiceProcessingUpgradeOwnership.mayPhaseAVCapture(
+ hasComponent: true,
+ componentStopped: false
+ ))
+ }
+
+ func testStickyFallbackRebindsWhenPhysicalInputChanges() {
+ XCTAssertTrue(MeetingAVCaptureRebindDecision.shouldRebind(
+ activeCoreAudioUID: "airpods",
+ electedCoreAudioUID: "built-in",
+ forced: false
+ ))
+ }
+
+ func testOutputOnlyRouteChangeDoesNotRebindTheSamePhysicalInput() {
+ XCTAssertFalse(MeetingAVCaptureRebindDecision.shouldRebind(
+ activeCoreAudioUID: "built-in",
+ electedCoreAudioUID: "built-in",
+ forced: false
+ ))
+ }
+
+ func testMissingSettledDefaultDoesNotAbandonTheActiveInput() {
+ XCTAssertFalse(MeetingAVCaptureRebindDecision.shouldRebind(
+ activeCoreAudioUID: "airpods",
+ electedCoreAudioUID: nil,
+ forced: false
+ ))
+ }
+
+ func testNoBufferWatchdogCanRebuildTheSamePhysicalInput() {
+ XCTAssertTrue(MeetingAVCaptureRebindDecision.shouldRebind(
+ activeCoreAudioUID: "built-in",
+ electedCoreAudioUID: "built-in",
+ forced: true
+ ))
+ }
+
+ func testRetiredCandidateTerminalCallbackCannotFailItsSuccessor() {
+ let retired = MeetingAVCaptureAttemptToken()
+ let active = MeetingAVCaptureAttemptToken()
+ XCTAssertFalse(MeetingAVCaptureFailureAdmission.accepts(
+ currentGeneration: 7,
+ eventGeneration: 7,
+ activeToken: active,
+ eventToken: retired,
+ stopRequested: false
+ ))
+ XCTAssertTrue(MeetingAVCaptureFailureAdmission.accepts(
+ currentGeneration: 7,
+ eventGeneration: 7,
+ activeToken: active,
+ eventToken: active,
+ stopRequested: false
+ ))
+ }
+
+ func testStopRejectsEvenTheCurrentCandidateTerminalCallback() {
+ let active = MeetingAVCaptureAttemptToken()
+ XCTAssertFalse(MeetingAVCaptureFailureAdmission.accepts(
+ currentGeneration: 7,
+ eventGeneration: 7,
+ activeToken: active,
+ eventToken: active,
+ stopRequested: true
+ ))
+ }
+
+ func testCaptureRetirementWaitsBehindAnInFlightFrameworkStart() async {
+ let startEntered = DispatchSemaphore(value: 0)
+ let allowStartToReturn = DispatchSemaphore(value: 0)
+ let stopRan = DispatchSemaphore(value: 0)
+ let lifecycle = MeetingSerializedCaptureLifecycle(
+ label: "com.fluidvoice.tests.avcapture-lifecycle",
+ start: {
+ startEntered.signal()
+ allowStartToReturn.wait()
+ return false
+ },
+ stop: { stopRan.signal() }
+ )
+
+ let startTask = Task { await lifecycle.start() }
+ XCTAssertEqual(startEntered.wait(timeout: .now() + 1), .success)
+ let stopTask = Task { await lifecycle.stop() }
+ XCTAssertEqual(
+ stopRan.wait(timeout: .now() + 0.1), .timedOut,
+ "retirement must not bypass an uninterruptible startRunning call"
+ )
+
+ allowStartToReturn.signal()
+ _ = await startTask.value
+ await stopTask.value
+ XCTAssertEqual(stopRan.wait(timeout: .now() + 1), .success)
+ }
+
+ func testCaptureRetirementRejectsAStartSubmittedAfterStopAdmission() async {
+ let startRan = DispatchSemaphore(value: 0)
+ let stopEntered = DispatchSemaphore(value: 0)
+ let allowStopToReturn = DispatchSemaphore(value: 0)
+ let lifecycle = MeetingSerializedCaptureLifecycle(
+ label: "com.fluidvoice.tests.avcapture-retirement-seal",
+ start: {
+ startRan.signal()
+ return true
+ },
+ stop: {
+ stopEntered.signal()
+ allowStopToReturn.wait()
+ }
+ )
+
+ let stopTask = Task { await lifecycle.stop() }
+ XCTAssertEqual(stopEntered.wait(timeout: .now() + 1), .success)
+ let lateStartSucceeded = await lifecycle.start()
+ XCTAssertFalse(lateStartSucceeded)
+ XCTAssertEqual(
+ startRan.wait(timeout: .now() + 0.1), .timedOut,
+ "retirement must permanently seal the lifecycle against late restarts"
+ )
+
+ allowStopToReturn.signal()
+ await stopTask.value
+ }
+
// MARK: - ScreenCaptureKit audio-settings plumbing (rebuild regression)
func testIncludeMicrophoneFalseAttachesNoMicrophoneDevice() {
@@ -192,6 +412,123 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
XCTAssertEqual(configuration.microphoneCaptureDeviceID, "dev-1")
}
+ func testAVCaptureSelectionPrefersExplicitResolvedID() {
+ let candidates = [
+ MeetingAVCaptureDeviceDescriptor(uniqueID: "explicit", displayName: "Mic"),
+ MeetingAVCaptureDeviceDescriptor(uniqueID: "mic-uid", displayName: "Mic"),
+ ]
+ var microphone = self.mic()
+ microphone.avCaptureDeviceID = "explicit"
+ XCTAssertEqual(
+ MeetingAVCaptureDeviceSelection.select(microphone: microphone, candidates: candidates),
+ "explicit"
+ )
+ }
+
+ func testAVCaptureSelectionAcceptsLegacyCaptureIDOnlyForLegacyManifest() {
+ let candidates = [MeetingAVCaptureDeviceDescriptor(uniqueID: "legacy-av", displayName: "Different")]
+ let legacy = MeetingMicrophoneIdentity(
+ captureDeviceID: "legacy-av",
+ coreAudioUID: "core-id",
+ identitySchemaVersion: nil,
+ displayName: "Saved mic"
+ )
+ XCTAssertEqual(
+ MeetingAVCaptureDeviceSelection.select(microphone: legacy, candidates: candidates),
+ "legacy-av"
+ )
+
+ var current = legacy
+ current.identitySchemaVersion = 2
+ XCTAssertNil(MeetingAVCaptureDeviceSelection.select(microphone: current, candidates: candidates))
+ }
+
+ func testAVCaptureSelectionUsesCoreAudioUIDWhenFrameworkIDsMatch() {
+ let microphone = MeetingMicrophoneIdentity(
+ captureDeviceID: "core-id", coreAudioUID: "core-id", displayName: "Mic"
+ )
+ let candidates = [MeetingAVCaptureDeviceDescriptor(uniqueID: "core-id", displayName: "Other")]
+ XCTAssertEqual(
+ MeetingAVCaptureDeviceSelection.select(microphone: microphone, candidates: candidates),
+ "core-id"
+ )
+ }
+
+ func testAVCaptureSelectionUsesOnlyUniqueNormalizedNameFallback() {
+ let microphone = MeetingMicrophoneIdentity(
+ captureDeviceID: "core-id", coreAudioUID: "core-id", displayName: " Studio MÍC "
+ )
+ let unique = [MeetingAVCaptureDeviceDescriptor(uniqueID: "av-id", displayName: "studio mic")]
+ XCTAssertEqual(
+ MeetingAVCaptureDeviceSelection.select(microphone: microphone, candidates: unique),
+ "av-id"
+ )
+
+ let ambiguous = unique + [MeetingAVCaptureDeviceDescriptor(uniqueID: "av-id-2", displayName: "Studio Mic")]
+ XCTAssertNil(MeetingAVCaptureDeviceSelection.select(microphone: microphone, candidates: ambiguous))
+ }
+
+ func testMicrophoneIdentityDecodesLegacyManifestAndRoundTripsCurrentIdentity() throws {
+ let legacyJSON = Data(#"{"captureDeviceID":"legacy-av","coreAudioUID":"core-id","displayName":"Mic","role":"personal"}"#.utf8)
+ let legacy = try JSONDecoder().decode(MeetingMicrophoneIdentity.self, from: legacyJSON)
+ XCTAssertNil(legacy.identitySchemaVersion)
+ XCTAssertNil(legacy.avCaptureDeviceID)
+ XCTAssertEqual(legacy.captureDeviceID, "legacy-av")
+
+ let current = self.mic()
+ let roundTrip = try JSONDecoder().decode(
+ MeetingMicrophoneIdentity.self,
+ from: JSONEncoder().encode(current)
+ )
+ XCTAssertEqual(roundTrip, current)
+ XCTAssertEqual(roundTrip.identitySchemaVersion, 2)
+ }
+
+ func testSameDeviceElectionPreservesLegacyAVMappingAcrossCoreAudioRefresh() {
+ let original = MeetingMicrophoneIdentity(
+ captureDeviceID: "legacy-av",
+ coreAudioUID: "core-id",
+ identitySchemaVersion: nil,
+ displayName: "Shared Name",
+ role: .personal
+ )
+ let refreshed = MeetingMicrophoneIdentity(
+ captureDeviceID: "core-id", coreAudioUID: "core-id", displayName: "Shared Name"
+ )
+ let elected = MeetingCaptureDeviceElection.decide(
+ original: original, catalog: [refreshed], defaultInputUID: "core-id"
+ )
+ XCTAssertEqual(elected.identity.captureDeviceID, "core-id")
+ XCTAssertEqual(elected.identity.avCaptureDeviceID, "legacy-av")
+ XCTAssertEqual(elected.role, .personal)
+
+ let ambiguous = [
+ MeetingAVCaptureDeviceDescriptor(uniqueID: "legacy-av", displayName: "Shared Name"),
+ MeetingAVCaptureDeviceDescriptor(uniqueID: "other-av", displayName: "Shared Name"),
+ ]
+ XCTAssertEqual(
+ MeetingAVCaptureDeviceSelection.select(microphone: elected.identity, candidates: ambiguous),
+ "legacy-av"
+ )
+ }
+
+ func testSameDeviceElectionPreservesCurrentExplicitAVMapping() {
+ let original = MeetingMicrophoneIdentity(
+ captureDeviceID: "core-id",
+ coreAudioUID: "core-id",
+ avCaptureDeviceID: "explicit-av",
+ displayName: "Shared Name",
+ role: .personal
+ )
+ let refreshed = MeetingMicrophoneIdentity(
+ captureDeviceID: "core-id", coreAudioUID: "core-id", displayName: "Shared Name"
+ )
+ let elected = MeetingCaptureDeviceElection.decide(
+ original: original, catalog: [refreshed], defaultInputUID: "core-id"
+ )
+ XCTAssertEqual(elected.identity.avCaptureDeviceID, "explicit-av")
+ }
+
// MARK: - Eligibility
private func makeStats(
diff --git a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift
index 2e266892..c6451c81 100644
--- a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift
+++ b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift
@@ -961,6 +961,179 @@ final class MeetingSpeakerEmbeddingIndexTests: XCTestCase {
}
}
+final class MeetingGlobalSpeakerStitcherTests: XCTestCase {
+ private func observation(
+ _ key: String,
+ chunk: String,
+ track: MeetingAudioTrackKind = .applicationAudio,
+ label: String,
+ embedding: [Float],
+ quality: Float = 0.9,
+ duration: TimeInterval = 3
+ ) -> MeetingGlobalSpeakerStitcher.Observation {
+ MeetingGlobalSpeakerStitcher.Observation(
+ key: key,
+ chunkKey: chunk,
+ trackKind: track,
+ localLabel: label,
+ embedding: embedding,
+ quality: quality,
+ effectiveSpeechDuration: duration
+ )
+ }
+
+ func testInputOrderDoesNotChangeAssignmentsOrPrototypes() {
+ let observations = [
+ self.observation("c", chunk: "chunk-c", label: "Speaker 1", embedding: [0.99, 0.1]),
+ self.observation("a", chunk: "chunk-a", label: "Speaker 0", embedding: [1, 0]),
+ self.observation("b", chunk: "chunk-b", label: "Speaker 4", embedding: [0, 1]),
+ ]
+ let stitcher = MeetingGlobalSpeakerStitcher()
+ let forward = stitcher.stitch(observations)
+ let reverse = stitcher.stitch(Array(observations.reversed()))
+
+ XCTAssertEqual(forward, reverse)
+ XCTAssertEqual(forward.speakerIDByObservationKey["a"], forward.speakerIDByObservationKey["c"])
+ XCTAssertNotEqual(forward.speakerIDByObservationKey["a"], forward.speakerIDByObservationKey["b"])
+ }
+
+ func testSameVoiceAcrossNonadjacentChunksStitches() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("first", chunk: "chunk-1", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("other", chunk: "chunk-2", label: "Speaker 1", embedding: [0, 1]),
+ self.observation("later", chunk: "chunk-3", label: "Speaker 7", embedding: [0.99, 0.12]),
+ ])
+
+ XCTAssertEqual(result.speakerIDByObservationKey["first"], result.speakerIDByObservationKey["later"])
+ XCTAssertNotEqual(result.speakerIDByObservationKey["first"], result.speakerIDByObservationKey["other"])
+ }
+
+ func testNoisyBoundaryObservationAttachesAfterReliableCores() {
+ let boundary = self.observation(
+ "boundary", chunk: "chunk-boundary", label: "Speaker 9", embedding: [0.98, 0.2], quality: 0.2, duration: 0.3
+ )
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("core-a", chunk: "chunk-a", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("core-b", chunk: "chunk-b", label: "Speaker 2", embedding: [0, 1]),
+ boundary,
+ ])
+
+ XCTAssertEqual(result.speakerIDByObservationKey["boundary"], result.speakerIDByObservationKey["core-a"])
+ }
+
+ func testAmbiguousDeferredObservationDoesNotMergeCores() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("core-a", chunk: "chunk-a", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("core-b", chunk: "chunk-b", label: "Speaker 2", embedding: [0.5, 0.866]),
+ self.observation("ambiguous", chunk: "chunk-c", label: "Speaker 8", embedding: [0.866, 0.5], quality: 0.2, duration: 0.2),
+ ])
+
+ XCTAssertNotEqual(result.speakerIDByObservationKey["core-a"], result.speakerIDByObservationKey["core-b"])
+ XCTAssertNotEqual(result.speakerIDByObservationKey["ambiguous"], result.speakerIDByObservationKey["core-a"])
+ XCTAssertNotEqual(result.speakerIDByObservationKey["ambiguous"], result.speakerIDByObservationKey["core-b"])
+ }
+
+ func testSameChunkCannotLinkKeepsDistinctLocalLabelsApart() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("same-a", chunk: "same", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("same-b", chunk: "same", label: "Speaker 2", embedding: [0.99, 0.1]),
+ ])
+
+ XCTAssertNotEqual(result.speakerIDByObservationKey["same-a"], result.speakerIDByObservationKey["same-b"])
+ }
+
+ func testDeferredSameChunkLabelsCannotAttachToTheSameCore() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("core", chunk: "core-chunk", label: "Speaker 0", embedding: [1, 0]),
+ self.observation("weak-a", chunk: "boundary", label: "Speaker 1", embedding: [0.99, 0.1], quality: 0.2, duration: 0.2),
+ self.observation("weak-b", chunk: "boundary", label: "Speaker 2", embedding: [0.98, 0.2], quality: 0.2, duration: 0.2),
+ ])
+
+ let coreID = result.speakerIDByObservationKey["core"]
+ let attachedCount = ["weak-a", "weak-b"].filter {
+ result.speakerIDByObservationKey[$0] == coreID
+ }.count
+ XCTAssertEqual(attachedCount, 1)
+ XCTAssertNotEqual(result.speakerIDByObservationKey["weak-a"], result.speakerIDByObservationKey["weak-b"])
+ }
+
+ func testEligibleDeferredObservationUpdatesPrototypeOnlyThroughTightDistanceGate() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("core", chunk: "chunk-core", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("short", chunk: "chunk-short", label: "Speaker 4", embedding: [0.995, 0.1], quality: 0.8, duration: 0.75),
+ ])
+ let speakerID = result.speakerIDByObservationKey["core"]
+ let prototype = speakerID.flatMap { result.prototypes[$0] }
+
+ XCTAssertEqual(speakerID, result.speakerIDByObservationKey["short"])
+ XCTAssertEqual(prototype?.observationCount, 2)
+ XCTAssertNotEqual(prototype?.embedding, [1, 0])
+ }
+
+ func testCoreMergeRejectsCentroidChainBeyondPairwiseDiameter() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("a", chunk: "chunk-a", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("b", chunk: "chunk-b", label: "Speaker 1", embedding: [0.866, 0.5]),
+ self.observation("c", chunk: "chunk-c", label: "Speaker 1", embedding: [0.5, 0.866]),
+ ])
+
+ let identities = Set(["a", "b", "c"].compactMap { result.speakerIDByObservationKey[$0] })
+ XCTAssertEqual(identities.count, 2)
+ }
+
+ func testInvalidEmbeddingRemainsUntrustedSingleton() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("core", chunk: "chunk-core", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("invalid", chunk: "chunk-invalid", label: "Speaker 2", embedding: []),
+ ])
+ let invalidID = result.speakerIDByObservationKey["invalid"]
+
+ XCTAssertNotEqual(result.speakerIDByObservationKey["core"], invalidID)
+ XCTAssertEqual(invalidID.flatMap { result.prototypes[$0]?.containsReliableCore }, false)
+ XCTAssertEqual(invalidID.flatMap { result.prototypes[$0]?.embedding }, [])
+ }
+
+ func testLongModerateQualityObservationBootstrapsWhenNoStrictCoreExists() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("quiet", chunk: "chunk-quiet", track: .microphone, label: "Speaker 1", embedding: [1, 0], quality: 0.5, duration: 4),
+ ])
+ let speakerID = result.speakerIDByObservationKey["quiet"]
+
+ XCTAssertEqual(speakerID.flatMap { result.prototypes[$0]?.containsReliableCore }, true)
+ }
+
+ func testShortLowQualityObservationDoesNotBootstrap() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("noise", chunk: "chunk-noise", track: .microphone, label: "Speaker 1", embedding: [1, 0], quality: 0.3, duration: 0.5),
+ ])
+ let speakerID = result.speakerIDByObservationKey["noise"]
+
+ XCTAssertEqual(speakerID.flatMap { result.prototypes[$0]?.containsReliableCore }, false)
+ }
+
+ func testWeakObservationDoesNotDistortReliablePrototype() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("core", chunk: "chunk-core", label: "Speaker 1", embedding: [1, 0]),
+ self.observation("weak", chunk: "chunk-weak", label: "Speaker 4", embedding: [0.99, 0.1], quality: 0.2, duration: 0.2),
+ ])
+ let speakerID = result.speakerIDByObservationKey["core"]
+ let prototype = speakerID.flatMap { result.prototypes[$0] }
+
+ XCTAssertEqual(speakerID, result.speakerIDByObservationKey["weak"])
+ XCTAssertEqual(prototype?.observationCount, 1)
+ XCTAssertEqual(prototype?.embedding, [1, 0])
+ }
+
+ func testApplicationAndMicrophoneTracksNeverCrossMatch() {
+ let result = MeetingGlobalSpeakerStitcher().stitch([
+ self.observation("application", chunk: "chunk-a", track: .applicationAudio, label: "Speaker 1", embedding: [1, 0]),
+ self.observation("microphone", chunk: "chunk-m", track: .microphone, label: "Speaker 1", embedding: [1, 0]),
+ ])
+
+ XCTAssertNotEqual(result.speakerIDByObservationKey["application"], result.speakerIDByObservationKey["microphone"])
+ }
+}
+
final class MeetingSessionModelTests: XCTestCase {
func testMeetingRecordingDefaultsRoundTripStableSourceIdentifiers() throws {
let defaults = MeetingRecordingDefaults(
@@ -2282,6 +2455,8 @@ private actor FakeMeetingSessionStore: MeetingSessionStoring {
}
private actor FakeMeetingCaptureController: MeetingCaptureControlling {
+ func preflightPermissions() async throws {}
+
struct Snapshot: Sendable {
var startCallCount: Int
var stopCallCount: Int
@@ -2423,14 +2598,14 @@ private final class FakeMeetingAudioActivityArbiter: MeetingAudioActivityArbitra
private(set) var releaseCallCount = 0
private var activeLease: MeetingAudioActivityLease?
- func acquireMeetingCapture() throws -> MeetingAudioActivityLease {
+ func acquireMeetingCapture() async throws -> MeetingAudioActivityLease {
self.acquireCallCount += 1
let lease = MeetingAudioActivityLease(id: UUID())
self.activeLease = lease
return lease
}
- func release(_ lease: MeetingAudioActivityLease) {
+ func release(_ lease: MeetingAudioActivityLease) async {
guard self.activeLease == lease else { return }
self.activeLease = nil
self.releaseCallCount += 1
diff --git a/Tests/FluidDictationIntegrationTests/SupportedFileExtensionsTests.swift b/Tests/FluidDictationIntegrationTests/SupportedFileExtensionsTests.swift
index beefd67f..2f177c84 100644
--- a/Tests/FluidDictationIntegrationTests/SupportedFileExtensionsTests.swift
+++ b/Tests/FluidDictationIntegrationTests/SupportedFileExtensionsTests.swift
@@ -11,7 +11,7 @@ import XCTest
@MainActor
final class SupportedFileExtensionsTests: XCTestCase {
func testWhatsAppOpusExtensionsAreAccepted() {
- let supported = MeetingTranscriptionService.supportedFileExtensions
+ let supported = FileTranscriptionService.supportedFileExtensions
XCTAssertTrue(supported.contains("opus"), "WhatsApp voice notes use .opus — macOS decodes them as Ogg Opus")
XCTAssertTrue(supported.contains("oga"), ".oga is an alternate Ogg audio extension macOS decodes natively")
@@ -19,7 +19,7 @@ final class SupportedFileExtensionsTests: XCTestCase {
}
func testCommonFormatsRemainAccepted() {
- let supported = MeetingTranscriptionService.supportedFileExtensions
+ let supported = FileTranscriptionService.supportedFileExtensions
for ext in ["wav", "mp3", "m4a", "mp4", "mov", "flac", "aac"] {
XCTAssertTrue(supported.contains(ext), ".\(ext) must remain accepted")
@@ -27,7 +27,7 @@ final class SupportedFileExtensionsTests: XCTestCase {
}
func testOnlyDecodableExtensionsAreAccepted() {
- let supported = MeetingTranscriptionService.supportedFileExtensions
+ let supported = FileTranscriptionService.supportedFileExtensions
// Every accepted extension must map to a UTType AVFoundation reports as decodable
// audio/video — the set must not drift into subtitles, playlists, or arbitrary types.
diff --git a/tools/debug_distribution/Collect FluidVoice Logs.command b/tools/debug_distribution/Collect FluidVoice Logs.command
new file mode 100755
index 00000000..11a212a3
--- /dev/null
+++ b/tools/debug_distribution/Collect FluidVoice Logs.command
@@ -0,0 +1,43 @@
+#!/bin/zsh
+
+set -u
+
+fv_log_dir="${HOME}/Desktop/FluidVoice Debug Logs"
+fv_timestamp="$(date '+%Y%m%d-%H%M%S')"
+fv_collection="${fv_log_dir}/collection-${fv_timestamp}"
+fv_zip="${fv_log_dir}/FluidVoice-Diagnostics-${fv_timestamp}.zip"
+
+mkdir -p "${fv_collection}"
+
+find "${fv_log_dir}" -maxdepth 1 -type f \
+ \( -name 'fluidvoice-audio-topology-*.jsonl' -o \
+ -name 'fluidvoice-audio-topology-*.stall' -o \
+ -name 'fluidvoice-audio-topology-stall-*.jsonl' \) \
+ -exec cp -p {} "${fv_collection}/" \;
+
+fv_pid="$(pgrep -f '/Applications/FluidVoice Debug.app/Contents/MacOS/FluidVoice Debug' | head -1)"
+if [[ -n "${fv_pid}" ]]; then
+ sample "${fv_pid}" 8 -file "${fv_collection}/FluidVoice-process-sample.txt" >/dev/null 2>&1 || true
+ ps -p "${fv_pid}" -o pid,ppid,%cpu,%mem,state,lstart,command > "${fv_collection}/FluidVoice-process-state.txt" 2>&1 || true
+else
+ echo "FluidVoice Debug was not running when logs were collected." > "${fv_collection}/FluidVoice-process-state.txt"
+fi
+
+log show --last 20m --style compact \
+ --predicate 'process == "FluidVoice Debug"' \
+ > "${fv_collection}/FluidVoice-system-log.txt" 2>&1 || true
+
+system_profiler SPAudioDataType \
+ > "${fv_collection}/macOS-audio-devices.txt" 2>&1 || true
+
+sw_vers > "${fv_collection}/macOS-version.txt" 2>&1 || true
+
+ditto -c -k --sequesterRsrc --keepParent "${fv_collection}" "${fv_zip}"
+
+echo "Diagnostic bundle created:"
+echo "${fv_zip}"
+echo
+echo "No meeting audio or transcript files are intentionally included."
+echo "Review the ZIP before sharing because application and system logs can still contain private metadata."
+open -R "${fv_zip}"
+read -r "?Press Return to close."
diff --git a/tools/debug_distribution/README - FluidVoice Diagnostics.txt b/tools/debug_distribution/README - FluidVoice Diagnostics.txt
new file mode 100644
index 00000000..a0d57a35
--- /dev/null
+++ b/tools/debug_distribution/README - FluidVoice Diagnostics.txt
@@ -0,0 +1,20 @@
+FluidVoice Debug diagnostics build
+==================================
+
+1. Drag FluidVoice Debug.app to the Applications shortcut in this DMG.
+2. Grant macOS permissions if prompted. Permissions are expected once on a new Mac.
+3. Quit FluidVoice Debug if it is already running.
+4. Control-click "Start FluidVoice Diagnostics.command" and choose Open.
+5. Start a meeting transcription, let it run for at least 30-60 seconds, then end it.
+6. If FluidVoice freezes, leave it running.
+7. Control-click "Collect FluidVoice Logs.command" and choose Open.
+8. Send back the ZIP created inside "FluidVoice Debug Logs" on the Desktop.
+
+The collector intentionally does not copy meeting audio or transcript files. It collects
+the diagnostic trace, a process sample, recent FluidVoice system logs, macOS version,
+and the visible audio-device topology. Review the ZIP before sharing because logs may
+still contain private metadata such as device names and file paths.
+
+This is an internal Apple Development-signed build and is not notarized for public
+distribution. macOS may require Control-click > Open or confirmation in Privacy &
+Security the first time it is opened.
diff --git a/tools/debug_distribution/Start FluidVoice Diagnostics.command b/tools/debug_distribution/Start FluidVoice Diagnostics.command
new file mode 100755
index 00000000..2e78edd6
--- /dev/null
+++ b/tools/debug_distribution/Start FluidVoice Diagnostics.command
@@ -0,0 +1,34 @@
+#!/bin/zsh
+
+set -euo pipefail
+
+fv_app="/Applications/FluidVoice Debug.app"
+fv_log_dir="${HOME}/Desktop/FluidVoice Debug Logs"
+
+if [[ ! -d "${fv_app}" ]]; then
+ echo "FluidVoice Debug is not installed."
+ echo "Drag FluidVoice Debug.app from the DMG into Applications, then run this helper again."
+ read -r "?Press Return to close."
+ exit 1
+fi
+
+if pgrep -f '/Applications/FluidVoice Debug.app/Contents/MacOS/FluidVoice Debug' >/dev/null 2>&1; then
+ echo "FluidVoice Debug is already running."
+ echo "Quit it normally, then run this helper again so diagnostics are enabled at launch."
+ read -r "?Press Return to close."
+ exit 1
+fi
+
+mkdir -p "${fv_log_dir}"
+
+open \
+ --env FLUIDVOICE_AUDIO_TOPOLOGY_DIAGNOSTICS=1 \
+ --env "FLUIDVOICE_AUDIO_TOPOLOGY_TRACE_DIRECTORY=${fv_log_dir}" \
+ --env FLUIDVOICE_AUDIO_TOPOLOGY_STALL_SECONDS=2 \
+ -a "${fv_app}"
+
+echo "FluidVoice Debug started with diagnostics enabled."
+echo "Logs will be written to: ${fv_log_dir}"
+echo
+echo "Reproduce the issue. If the app freezes, leave it running and use Collect FluidVoice Logs.command."
+read -r "?Press Return to close."
diff --git a/tools/meeting_diarization_eval.py b/tools/meeting_diarization_eval.py
new file mode 100644
index 00000000..b0e320ae
--- /dev/null
+++ b/tools/meeting_diarization_eval.py
@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+"""Local-only, deterministic meeting diarization evaluator.
+
+This tool reads RTTM timelines and optional speaker-attributed transcript JSON. It never uploads,
+rewrites, or copies source recordings. Reports contain aggregate metrics and file fingerprints only.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import sys
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+SCHEMA_VERSION = 1
+DEPENDENCY_REVISION = "3fd63887eef1dc25edea8263ce4b44aa854d898b"
+DIARIZATION_FINGERPRINT = (
+ "FluidAudio-offline-v1@3fd63887eef1dc25edea8263ce4b44aa854d898b;"
+ "config=community-default-v1"
+)
+
+
+@dataclass(frozen=True)
+class Segment:
+ speaker: str
+ start: float
+ end: float
+
+
+def resolve(base: Path, value: str) -> Path:
+ path = Path(os.path.expanduser(value))
+ return path if path.is_absolute() else (base / path).resolve()
+
+
+def sha256(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(block)
+ return digest.hexdigest()
+
+
+def read_rttm(path: Path) -> list[Segment]:
+ segments: list[Segment] = []
+ for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ line = raw.strip()
+ if not line or line.startswith("#"):
+ continue
+ fields = line.split()
+ if len(fields) < 8 or fields[0] != "SPEAKER":
+ raise ValueError(f"{path}:{line_number}: expected an RTTM SPEAKER row")
+ start, duration = float(fields[3]), float(fields[4])
+ if start < 0 or duration <= 0 or not math.isfinite(start + duration):
+ raise ValueError(f"{path}:{line_number}: invalid start/duration")
+ segments.append(Segment(fields[7], start, start + duration))
+ if not segments:
+ raise ValueError(f"{path}: contains no speaker segments")
+ return sorted(segments, key=lambda item: (item.start, item.end, item.speaker))
+
+
+def active(segments: list[Segment], midpoint: float) -> set[str]:
+ return {segment.speaker for segment in segments if segment.start <= midpoint < segment.end}
+
+
+def scored_intervals(
+ reference: list[Segment], hypothesis: list[Segment], collar: float, score_overlap: bool
+) -> list[tuple[float, set[str], set[str]]]:
+ boundaries = sorted({point for segment in reference + hypothesis for point in (segment.start, segment.end)})
+ reference_boundaries = [point for segment in reference for point in (segment.start, segment.end)]
+ intervals: list[tuple[float, set[str], set[str]]] = []
+ for left, right in zip(boundaries, boundaries[1:]):
+ if right <= left:
+ continue
+ midpoint = (left + right) / 2
+ if collar and any(abs(midpoint - point) < collar for point in reference_boundaries):
+ continue
+ ref_active, hyp_active = active(reference, midpoint), active(hypothesis, midpoint)
+ if not score_overlap and len(ref_active) > 1:
+ continue
+ intervals.append((right - left, ref_active, hyp_active))
+ return intervals
+
+
+def best_assignment(weights: list[list[float]]) -> list[tuple[int, int]]:
+ """Maximum-weight one-to-one assignment; corpus speaker counts are intentionally small."""
+ rows = len(weights)
+ columns = max((len(row) for row in weights), default=0)
+ size = max(rows, columns)
+ if size > 12:
+ raise ValueError("more than 12 speakers is unsupported by the deterministic evaluator")
+ padded = [row + [0.0] * (size - len(row)) for row in weights]
+ padded += [[0.0] * size for _ in range(size - rows)]
+ @lru_cache(maxsize=None)
+ def solve(row: int, used_columns: int) -> tuple[float, tuple[int, ...]]:
+ if row == size:
+ return 0.0, ()
+ best: tuple[float, tuple[int, ...]] | None = None
+ for column in range(size):
+ if used_columns & (1 << column):
+ continue
+ remaining_score, remaining_columns = solve(row + 1, used_columns | (1 << column))
+ candidate = (padded[row][column] + remaining_score, (column,) + remaining_columns)
+ if best is None or candidate[0] > best[0]:
+ best = candidate
+ assert best is not None
+ return best
+
+ _, columns_by_row = solve(0, 0)
+ return [(row, column) for row, column in enumerate(columns_by_row) if row < rows and column < columns]
+
+
+def overlap_matrix(
+ intervals: list[tuple[float, set[str], set[str]]], references: list[str], hypotheses: list[str]
+) -> list[list[float]]:
+ matrix = [[0.0 for _ in hypotheses] for _ in references]
+ ref_index, hyp_index = {v: i for i, v in enumerate(references)}, {v: i for i, v in enumerate(hypotheses)}
+ for duration, ref_active, hyp_active in intervals:
+ for ref in ref_active:
+ for hyp in hyp_active:
+ matrix[ref_index[ref]][hyp_index[hyp]] += duration
+ return matrix
+
+
+def duration_by_speaker(segments: list[Segment]) -> dict[str, float]:
+ return {
+ speaker: sum(segment.end - segment.start for segment in segments if segment.speaker == speaker)
+ for speaker in sorted({segment.speaker for segment in segments})
+ }
+
+
+def score_timeline(reference: list[Segment], hypothesis: list[Segment], collar: float, score_overlap: bool) -> dict[str, Any]:
+ intervals = scored_intervals(reference, hypothesis, collar, score_overlap)
+ references = sorted({segment.speaker for segment in reference})
+ hypotheses = sorted({segment.speaker for segment in hypothesis})
+ matrix = overlap_matrix(intervals, references, hypotheses)
+ assignment = best_assignment(matrix)
+ hyp_to_ref = {hypotheses[column]: references[row] for row, column in assignment}
+
+ reference_time = miss = false_alarm = confusion = 0.0
+ for duration, ref_active, hyp_active in intervals:
+ reference_time += duration * len(ref_active)
+ miss += duration * max(0, len(ref_active) - len(hyp_active))
+ false_alarm += duration * max(0, len(hyp_active) - len(ref_active))
+ correct = sum(1 for hyp in hyp_active if hyp_to_ref.get(hyp) in ref_active)
+ confusion += duration * max(0, min(len(ref_active), len(hyp_active)) - correct)
+
+ ref_durations = duration_by_speaker(reference)
+ hyp_durations = duration_by_speaker(hypothesis)
+ mapped_column = {row: column for row, column in assignment}
+ jaccard_errors, recovered, coverage_and_purity = [], {}, {}
+ for row, speaker in enumerate(references):
+ column = mapped_column.get(row)
+ intersection = matrix[row][column] if column is not None else 0.0
+ hyp_duration = hyp_durations.get(hypotheses[column], 0.0) if column is not None else 0.0
+ union = ref_durations[speaker] + hyp_duration - intersection
+ jaccard_errors.append(1.0 - intersection / union if union else 0.0)
+ coverage = intersection / ref_durations[speaker] if ref_durations[speaker] else 0.0
+ purity = intersection / hyp_duration if hyp_duration else 0.0
+ coverage_and_purity[speaker] = {"coverage": coverage, "purity": purity}
+ recovered[speaker] = coverage >= 0.5 and purity >= 0.5
+
+ buckets = {"<1s": [0, 0], "1-3s": [0, 0], "3-10s": [0, 0], ">10s": [0, 0]}
+ for speaker, duration in ref_durations.items():
+ bucket = "<1s" if duration < 1 else "1-3s" if duration < 3 else "3-10s" if duration < 10 else ">10s"
+ buckets[bucket][1] += 1
+ buckets[bucket][0] += int(recovered[speaker])
+
+ # A meaningful overlap link is at least 250 ms and at least 5% of the shorter speaker.
+ links: list[tuple[int, int]] = []
+ for row, ref in enumerate(references):
+ for column, hyp in enumerate(hypotheses):
+ threshold = max(0.25, 0.05 * min(ref_durations[ref], hyp_durations[hyp]))
+ if matrix[row][column] >= threshold:
+ links.append((row, column))
+ merge_count = sum(1 for column in range(len(hypotheses)) if len({r for r, c in links if c == column}) > 1)
+ split_count = sum(1 for row in range(len(references)) if len({c for r, c in links if r == row}) > 1)
+
+ denominator = reference_time or 1.0
+ return {
+ "collarSeconds": collar,
+ "overlapScored": score_overlap,
+ "referenceSpeakers": len(references),
+ "hypothesisSpeakers": len(hypotheses),
+ "speakerCountError": len(hypotheses) - len(references),
+ "mergeCount": merge_count,
+ "splitCount": split_count,
+ "recoveryByDuration": {
+ name: {"recovered": values[0], "total": values[1], "recall": values[0] / values[1] if values[1] else None}
+ for name, values in buckets.items()
+ },
+ "der": (miss + false_alarm + confusion) / denominator,
+ "derComponents": {
+ "miss": miss / denominator,
+ "falseAlarm": false_alarm / denominator,
+ "confusion": confusion / denominator,
+ },
+ "jer": sum(jaccard_errors) / len(jaccard_errors) if jaccard_errors else 0.0,
+ "speakerCoverageAndPurity": coverage_and_purity,
+ }
+
+
+def words(text: str) -> list[str]:
+ return ["".join(character.lower() for character in token if character.isalnum() or character == "'")
+ for token in text.split() if any(character.isalnum() for character in token)]
+
+
+def edit_distance(left: list[str], right: list[str]) -> int:
+ row = list(range(len(right) + 1))
+ for left_index, left_word in enumerate(left, 1):
+ next_row = [left_index]
+ for right_index, right_word in enumerate(right, 1):
+ next_row.append(min(next_row[-1] + 1, row[right_index] + 1, row[right_index - 1] + (left_word != right_word)))
+ row = next_row
+ return row[-1]
+
+
+def cpwer(reference_path: Path, hypothesis_path: Path) -> dict[str, Any]:
+ reference = json.loads(reference_path.read_text(encoding="utf-8"))
+ hypothesis = json.loads(hypothesis_path.read_text(encoding="utf-8"))
+ ref_texts = [words(value) for _, value in sorted(reference.items())]
+ hyp_texts = [words(value) for _, value in sorted(hypothesis.items())]
+ size = max(len(ref_texts), len(hyp_texts))
+ costs = []
+ for ref_index in range(size):
+ ref_words = ref_texts[ref_index] if ref_index < len(ref_texts) else []
+ costs.append([
+ edit_distance(ref_words, hyp_texts[hyp_index] if hyp_index < len(hyp_texts) else [])
+ for hyp_index in range(size)
+ ])
+ # Convert minimum-cost assignment into maximum-weight assignment.
+ maximum = max((value for row in costs for value in row), default=0)
+ assignment = best_assignment([[maximum - value for value in row] for row in costs])
+ errors = sum(costs[row][column] for row, column in assignment)
+ count = sum(len(value) for value in ref_texts)
+ return {"errors": errors, "referenceWords": count, "cpWER": errors / count if count else 0.0}
+
+
+def validate_manifest(manifest_path: Path, require_files: bool = True) -> dict[str, Any]:
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ if manifest.get("schemaVersion") != SCHEMA_VERSION:
+ raise ValueError(f"schemaVersion must be {SCHEMA_VERSION}")
+ dependency = manifest.get("dependency", {})
+ if dependency.get("revision") != DEPENDENCY_REVISION:
+ raise ValueError("manifest dependency revision does not match the pinned build")
+ if dependency.get("diarizationFingerprint") != DIARIZATION_FINGERPRINT:
+ raise ValueError("manifest diarization fingerprint does not match the pinned build")
+ fixtures = manifest.get("fixtures", [])
+ if not fixtures:
+ raise ValueError("manifest must contain at least one fixture")
+ ids = [fixture.get("id") for fixture in fixtures]
+ if any(not value for value in ids) or len(ids) != len(set(ids)):
+ raise ValueError("fixture ids must be present and unique")
+ if not any(fixture.get("split") == "heldOut" for fixture in fixtures):
+ raise ValueError("manifest must define a heldOut fixture before tuning")
+ base = manifest_path.parent
+ for fixture in fixtures:
+ if fixture.get("split") not in ("development", "heldOut"):
+ raise ValueError(f"{fixture['id']}: split must be development or heldOut")
+ if fixture.get("status", "annotated") == "annotated":
+ for key in ("referenceRTTM", "hypothesisRTTM"):
+ if key not in fixture:
+ raise ValueError(f"{fixture['id']}: annotated fixture is missing {key}")
+ if require_files and not resolve(base, fixture[key]).is_file():
+ raise ValueError(f"{fixture['id']}: missing {key} file")
+ return manifest
+
+
+def score_manifest(manifest_path: Path) -> dict[str, Any]:
+ manifest = validate_manifest(manifest_path)
+ base, reports = manifest_path.parent, []
+ for fixture in manifest["fixtures"]:
+ if fixture.get("status", "annotated") != "annotated":
+ inventory_report: dict[str, Any] = {
+ "id": fixture["id"],
+ "split": fixture["split"],
+ "status": "inventory",
+ "tags": fixture.get("tags", []),
+ }
+ expected = fixture.get("expectedSpeakerCount")
+ observed = fixture.get("observedRemoteSpeakerCount")
+ if isinstance(expected, int):
+ inventory_report["expectedSpeakerCount"] = expected
+ if isinstance(observed, int):
+ inventory_report["observedRemoteSpeakerCount"] = observed
+ if isinstance(expected, int) and isinstance(observed, int):
+ inventory_report["speakerCountError"] = observed - expected
+ for manifest_key, report_key in (
+ ("hypothesisRTTM", "hypothesisSHA256"),
+ ("runReport", "runReportSHA256"),
+ ):
+ if fixture.get(manifest_key):
+ path = resolve(base, fixture[manifest_key])
+ if path.is_file():
+ inventory_report[report_key] = sha256(path)
+ reports.append(inventory_report)
+ continue
+ reference_path = resolve(base, fixture["referenceRTTM"])
+ hypothesis_path = resolve(base, fixture["hypothesisRTTM"])
+ reference, hypothesis = read_rttm(reference_path), read_rttm(hypothesis_path)
+ report: dict[str, Any] = {
+ "id": fixture["id"], "split": fixture["split"], "tags": fixture.get("tags", []),
+ "status": "scored", "referenceSHA256": sha256(reference_path),
+ "hypothesisSHA256": sha256(hypothesis_path),
+ "comparable": score_timeline(reference, hypothesis, 0.25, False),
+ "strict": score_timeline(reference, hypothesis, 0.0, True),
+ }
+ if fixture.get("referenceSpeakerText") and fixture.get("hypothesisSpeakerText"):
+ report["cpWER"] = cpwer(resolve(base, fixture["referenceSpeakerText"]), resolve(base, fixture["hypothesisSpeakerText"]))
+ reports.append(report)
+ return {
+ "schemaVersion": SCHEMA_VERSION,
+ "dependency": manifest["dependency"],
+ "normalization": manifest.get("normalization", {}),
+ "fixtures": reports,
+ }
+
+
+def self_test() -> None:
+ reference = [Segment("A", 0, 2), Segment("B", 2, 4)]
+ perfect = score_timeline(reference, reference, 0, True)
+ assert perfect["der"] == 0 and perfect["mergeCount"] == 0 and perfect["splitCount"] == 0
+ merged = score_timeline(reference, [Segment("X", 0, 4)], 0, True)
+ assert merged["mergeCount"] == 1 and merged["speakerCountError"] == -1 and merged["der"] == 0.5
+ split = score_timeline([Segment("A", 0, 4)], [Segment("X", 0, 2), Segment("Y", 2, 4)], 0, True)
+ assert split["splitCount"] == 1 and split["speakerCountError"] == 1 and split["der"] == 0.5
+ print("meeting_diarization_eval: self-test passed")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ subparsers.add_parser("self-test")
+ validate = subparsers.add_parser("validate")
+ validate.add_argument("manifest", type=Path)
+ score = subparsers.add_parser("score")
+ score.add_argument("manifest", type=Path)
+ score.add_argument("--output", type=Path)
+ arguments = parser.parse_args()
+ if arguments.command == "self-test":
+ self_test()
+ return 0
+ manifest_path = arguments.manifest.resolve()
+ if arguments.command == "validate":
+ manifest = validate_manifest(manifest_path)
+ print(f"valid manifest: {len(manifest['fixtures'])} fixtures")
+ return 0
+ report = score_manifest(manifest_path)
+ encoded = json.dumps(report, indent=2, sort_keys=True) + "\n"
+ if arguments.output:
+ arguments.output.parent.mkdir(parents=True, exist_ok=True)
+ arguments.output.write_text(encoded, encoding="utf-8")
+ print(arguments.output)
+ else:
+ print(encoded, end="")
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except (OSError, ValueError, json.JSONDecodeError) as error:
+ print(f"error: {error}", file=sys.stderr)
+ raise SystemExit(2)
diff --git a/tools/meeting_diarization_eval_manifest.example.json b/tools/meeting_diarization_eval_manifest.example.json
new file mode 100644
index 00000000..7b0c9bc3
--- /dev/null
+++ b/tools/meeting_diarization_eval_manifest.example.json
@@ -0,0 +1,35 @@
+{
+ "schemaVersion": 1,
+ "dependency": {
+ "repository": "https://github.com/altic-dev/FluidAudio.git",
+ "revision": "3fd63887eef1dc25edea8263ce4b44aa854d898b",
+ "diarizationFingerprint": "FluidAudio-offline-v1@3fd63887eef1dc25edea8263ce4b44aa854d898b;config=community-default-v1"
+ },
+ "normalization": {
+ "asrProvider": "record-per-run",
+ "asrModel": "record-per-run",
+ "pipelineVersion": "record-per-run",
+ "text": "lowercase-alphanumeric-apostrophe-v1"
+ },
+ "fixtures": [
+ {
+ "id": "five-speaker-brief-return",
+ "split": "development",
+ "status": "inventory",
+ "sessionDirectory": "/local/private/path/to/copied-session",
+ "expectedSpeakerCount": 5,
+ "recurringSpeakers": ["brief-speaker"],
+ "overlapRegions": [],
+ "captureDiscontinuities": [],
+ "knownMissingIntervals": [],
+ "tags": ["short-speaker", "recurring-speaker", "five-speakers"]
+ },
+ {
+ "id": "held-out-created-before-tuning",
+ "split": "heldOut",
+ "status": "inventory",
+ "sessionDirectory": "/local/private/path/to/held-out-session",
+ "tags": ["held-out"]
+ }
+ ]
+}