Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

fix(meeting): harden capture lifecycle and fixed caption overlays

argus-vault1809cd3ddd91
diff --git a/Sources/Fluid/Services/AppServices.swift b/Sources/Fluid/Services/AppServices.swift
index 13013e26..2a17f5f4 100644
--- a/Sources/Fluid/Services/AppServices.swift
+++ b/Sources/Fluid/Services/AppServices.swift
@@ -231,6 +231,7 @@ final class AppServices: ObservableObject {
/// Read-only meeting check that does not initialize the meeting stack.
var hasActiveMeetingSessionActivity: Bool {
guard let coordinator = self._meetingSessionCoordinator else { return false }
+ if coordinator.hasPendingStart { return true }
switch coordinator.state {
case .preparing, .recording, .recordingDegraded, .stopping, .processing:
return true
diff --git a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveModels.swift b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveModels.swift
index 23a79e60..25e7b3f8 100644
--- a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveModels.swift
+++ b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveModels.swift
@@ -2,8 +2,9 @@ import AVFoundation
import CoreMedia
import Foundation
-/// "You" is the microphone track, "Them" is the application-audio track. No diarization live —
-/// the offline pipeline resolves real speaker identity and replaces this transcript on completion.
+/// Internal live-only source tags: `.you` is the microphone track and `.them` is application audio.
+/// The UI deliberately presents neutral source labels because live capture has no speaker identity;
+/// the offline pipeline resolves speakers and replaces this transcript on completion.
nonisolated enum MeetingLiveSpeaker: Sendable, Equatable {
case you
case them
@@ -133,7 +134,7 @@ nonisolated enum MeetingLiveTimeConversion {
}
}
-/// Thread-safe, set-once-per-session capture origin shared by both tracks so "You" and "Them"
+/// Thread-safe, set-once-per-session capture origin shared by both source tracks so their
/// timestamps land on the same clock.
final nonisolated class MeetingLiveOriginBox: @unchecked Sendable {
private let lock = NSLock()
@@ -160,7 +161,7 @@ final nonisolated class MeetingLiveOriginBox: @unchecked Sendable {
/// Echo suppression: on speakers, the far end bleeds into the microphone track. This is text-only
/// (the offline signal scorer needs data live capture can't produce), comparing a mic utterance
-/// against recently finalized "Them" text.
+/// against recently finalized application-audio text.
nonisolated enum MeetingLiveEchoFilter {
static func recentThemText(
from themUtterances: [MeetingLiveUtterance],
diff --git a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift
index fea11d56..f49f74c5 100644
--- a/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift
+++ b/Sources/Fluid/Services/Meeting/LiveTranscription/MeetingLiveTranscriptionCoordinator.swift
@@ -36,7 +36,7 @@ final nonisolated class MeetingLiveTranscriptionCoordinator: @unchecked Sendable
self.onUpdate = onUpdate
}
- /// Clears any in-progress "You" partial at a capture-side mic splice, so it can't straddle eras.
+ /// Clears any in-progress microphone partial at a capture-side splice, so it can't straddle eras.
func resetMicrophoneUtterance() {
self.publish { $0.settingPartial(nil, for: .you) }
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingAppRegistry.swift b/Sources/Fluid/Services/Meeting/MeetingAppRegistry.swift
index 34dc858f..64cc1549 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAppRegistry.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAppRegistry.swift
@@ -60,18 +60,6 @@ nonisolated enum MeetingAppRegistry {
self.tier(forBundleIdentifier: bundleIdentifier) == .browserTier2
}
- static func audioProcessBundleIdentifiers(for bundleIdentifier: String) -> Set<String> {
- self.audioProcessBundleIdentifiersByPrimaryBundleIdentifier[bundleIdentifier] ?? [bundleIdentifier]
- }
-
- private static let audioProcessBundleIdentifiersByPrimaryBundleIdentifier: [String: Set<String>] = [
- "us.zoom.xos": ["us.zoom.xos", "us.zoom.caphost"],
- "com.microsoft.teams2": ["com.microsoft.teams2"],
- "com.microsoft.teams": ["com.microsoft.teams"],
- "com.cisco.webexmeetingsapp": ["com.cisco.webexmeetingsapp"],
- "Cisco-Systems.Spark": ["Cisco-Systems.Spark"],
- ]
-
private static let zoomMeetingTitles: Set<String> = [
// en
"Zoom", "Zoom Meeting", "Zoom Webinar",
diff --git a/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift b/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift
index 39a71fb7..882f2741 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift
@@ -181,6 +181,26 @@ final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable {
}
}
+ /// Safety metadata must fail closed even when its manifest write fails. Publishing the
+ /// conservative candidate in memory ensures the track returned by `stop()` cannot resurrect
+ /// the prior optimistic era; the caller must still terminate capture after the thrown error.
+ func updateTrackMetadataFailClosed(_ mutate: @escaping (inout MeetingAudioTrack) -> Void) async throws {
+ try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<Void, Error>) in
+ self.queue.async { [self] in
+ var candidate = self.track
+ mutate(&candidate)
+ do {
+ try self.persistTrackManifest(candidate)
+ self.track = candidate
+ continuation.resume()
+ } catch {
+ self.track = candidate
+ continuation.resume(throwing: error)
+ }
+ }
+ }
+ }
+
/// Splice point for a capture-side swap: finalizes the active chunk, returns its `presentationEnd`. No-op unless `.accepting`.
func beginSplice() async -> MeetingMediaTime? {
await withCheckedContinuation { (continuation: CheckedContinuation<MeetingMediaTime?, Never>) in
diff --git a/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift b/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
index 77d4607d..547c8cc9 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAutoDetectSignals.swift
@@ -2,6 +2,7 @@ import AppKit
import ApplicationServices
import AVFoundation
import CoreAudio
+import Darwin
import Foundation
// MARK: - Protocols (unit-testable via fakes; no CoreAudio/AX/NSWorkspace in tests)
@@ -51,7 +52,62 @@ protocol MicActivitySignalProviding: AnyObject {
@MainActor
protocol AudioProcessActivityProviding: AnyObject {
- func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) async -> Bool
+ func snapshot() async -> AudioProcessActivitySnapshot
+}
+
+nonisolated struct AudioProcessDescriptor: Sendable, Equatable {
+ var processID: Int32
+ var bundleIdentifier: String?
+ var executablePath: String?
+ var isInputRunning: Bool
+ var isOutputRunning: Bool
+}
+
+nonisolated struct MeetingProcessOwner: Sendable, Equatable {
+ var processID: Int32
+ var bundleIdentifier: String
+ var bundlePath: String
+}
+
+nonisolated struct AudioProcessActivitySnapshot: Sendable, Equatable {
+ enum QueryState: Sendable, Equatable { case valid, unknown }
+ var processes: [AudioProcessDescriptor]
+ var owners: [MeetingProcessOwner]
+ var queryState: QueryState
+}
+
+/// Maps HAL process objects to the registered native application that owns them. Bundle IDs from
+/// HAL are advisory: helpers frequently report their own (or stale) IDs. Ownership is accepted
+/// only when the live executable is contained in the owning app bundle after symlink resolution.
+/// Ambiguous, missing, or malformed data is deliberately rejected.
+nonisolated enum MeetingAudioProcessResolver {
+ static func activeOwnerInputByPID(snapshot: AudioProcessActivitySnapshot) -> [Int32: Bool]? {
+ guard snapshot.queryState == .valid else { return nil }
+ var matches: [Int32: Bool] = [:]
+ for process in snapshot.processes where process.isInputRunning || process.isOutputRunning {
+ guard let path = canonicalPath(process.executablePath), process.processID > 0 else { return nil }
+ let candidates = snapshot.owners.filter { owner in
+ guard MeetingAppRegistry.isNativeMeetingApp(bundleIdentifier: owner.bundleIdentifier),
+ let ownerPath = canonicalPath(owner.bundlePath), ownerPath.hasSuffix(".app")
+ else { return false }
+ guard path.hasPrefix(ownerPath + "/Contents/") else { return false }
+ // The owner list is built from live NSRunningApplication instances; requiring a
+ // positive PID prevents stale/placeholder identities from matching.
+ return owner.processID > 0
+ }
+ guard candidates.count <= 1 else { return nil }
+ guard let owner = candidates.first else { continue }
+ matches[owner.processID] = (matches[owner.processID] ?? false) || process.isInputRunning
+ }
+ return matches
+ }
+
+ private static func canonicalPath(_ path: String?) -> String? {
+ guard let path, !path.isEmpty else { return nil }
+ guard path.hasPrefix("/") else { return nil }
+ let url = URL(fileURLWithPath: path).resolvingSymlinksInPath().standardizedFileURL
+ return url.path.hasPrefix("/") ? url.path : nil
+ }
}
nonisolated struct WindowSnapshot: Sendable, Equatable {
@@ -487,18 +543,62 @@ final class CoreAudioMicActivitySignal: MicActivitySignalProviding {
@MainActor
final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
- func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) async -> Bool {
- await AudioTopologyListenerExecution.perform {
- guard #available(macOS 14.4, *) else { return false }
- return Self.processObjectIDs().contains { processID in
- guard let bundleIdentifier = Self.bundleIdentifier(for: processID), bundleIdentifiers.contains(bundleIdentifier) else { return false }
- return Self.isRunning(processID, selector: kAudioProcessPropertyIsRunningInput)
- || Self.isRunning(processID, selector: kAudioProcessPropertyIsRunningOutput)
+ func snapshot() async -> AudioProcessActivitySnapshot {
+ let owners = NSWorkspace.shared.runningApplications.compactMap { app -> MeetingProcessOwner? in
+ guard let bundle = app.bundleIdentifier,
+ MeetingAppRegistry.isNativeMeetingApp(bundleIdentifier: bundle),
+ !app.isTerminated, app.processIdentifier > 0,
+ let path = app.bundleURL?.path else { return nil }
+ return MeetingProcessOwner(processID: app.processIdentifier, bundleIdentifier: bundle, bundlePath: path)
+ }
+ return await AudioTopologyListenerExecution.perform {
+ guard #available(macOS 14.4, *) else {
+ return AudioProcessActivitySnapshot(processes: [], owners: [], queryState: .unknown)
+ }
+ guard let processIDs = Self.processObjectIDs() else {
+ return AudioProcessActivitySnapshot(processes: [], owners: owners, queryState: .unknown)
+ }
+ var queryFailed = false
+ let processes = processIDs.compactMap { processObjectID -> AudioProcessDescriptor? in
+ guard let input = Self.isRunning(processObjectID, selector: kAudioProcessPropertyIsRunningInput),
+ let output = Self.isRunning(processObjectID, selector: kAudioProcessPropertyIsRunningOutput)
+ else { queryFailed = true; return nil }
+ guard input || output else { return nil }
+ guard let pid = Self.processID(for: processObjectID), pid > 0,
+ let path = Self.executablePath(for: pid),
+ Self.processID(for: processObjectID) == pid
+ else { queryFailed = true; return nil }
+ return AudioProcessDescriptor(
+ processID: pid,
+ bundleIdentifier: Self.bundleIdentifier(for: processObjectID),
+ executablePath: path,
+ isInputRunning: input,
+ isOutputRunning: output
+ )
}
+ // Missing executable identity is not equivalent to silence. Keep the whole sample
+ // unknown so the detector preserves continuity without inventing an inactive edge.
+ let state: AudioProcessActivitySnapshot.QueryState = queryFailed ? .unknown : .valid
+ return AudioProcessActivitySnapshot(processes: processes, owners: owners, queryState: state)
}
}
- nonisolated private static func processObjectIDs() -> [AudioObjectID] {
+ nonisolated private static func processID(for objectID: AudioObjectID) -> Int32? {
+ var address = AudioObjectPropertyAddress(mSelector: kAudioProcessPropertyPID, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMain)
+ var pid: pid_t = 0
+ var size = UInt32(MemoryLayout<pid_t>.size)
+ guard AudioObjectGetPropertyData(objectID, &address, 0, nil, &size, &pid) == noErr else { return nil }
+ return Int32(pid)
+ }
+
+ nonisolated private static func executablePath(for pid: Int32) -> String? {
+ var buffer = [CChar](repeating: 0, count: Int(PATH_MAX))
+ let length = proc_pidpath(pid, &buffer, UInt32(buffer.count))
+ guard length > 0 else { return nil }
+ return String(cString: buffer)
+ }
+
+ nonisolated private static func processObjectIDs() -> [AudioObjectID]? {
var address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyProcessObjectList,
mScope: kAudioObjectPropertyScopeGlobal,
@@ -513,7 +613,8 @@ final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
#if DEBUG
AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: sizeStatus)
#endif
- guard sizeStatus == noErr else { return [] }
+ guard sizeStatus == noErr, size % UInt32(MemoryLayout<AudioObjectID>.size) == 0 else { return nil }
+ guard size > 0 else { return [] }
var processIDs = Array(repeating: AudioObjectID(), count: Int(size) / MemoryLayout<AudioObjectID>.size)
#if DEBUG
AudioTopologyDiagnostics.record(.halQueryBegin, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener)
@@ -522,8 +623,8 @@ final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
#if DEBUG
AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: systemObject, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: dataStatus)
#endif
- guard dataStatus == noErr else { return [] }
- return processIDs
+ guard dataStatus == noErr, size % UInt32(MemoryLayout<AudioObjectID>.size) == 0 else { return nil }
+ return Array(processIDs.prefix(Int(size) / MemoryLayout<AudioObjectID>.size))
}
nonisolated private static func bundleIdentifier(for processID: AudioObjectID) -> String? {
@@ -545,7 +646,7 @@ final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
return value as String
}
- nonisolated private static func isRunning(_ processID: AudioObjectID, selector: AudioObjectPropertySelector) -> Bool {
+ nonisolated private static func isRunning(_ processID: AudioObjectID, selector: AudioObjectPropertySelector) -> Bool? {
var address = AudioObjectPropertyAddress(
mSelector: selector,
mScope: kAudioObjectPropertyScopeGlobal,
@@ -560,7 +661,8 @@ final class CoreAudioProcessActivityProvider: AudioProcessActivityProviding {
#if DEBUG
AudioTopologyDiagnostics.record(.halQueryEnd, owner: .meetingDetector, objectID: processID, selector: address.mSelector, scope: address.mScope, element: address.mElement, queueRole: .dedicatedControl, phase: .listener, status: status)
#endif
- return status == noErr && value != 0
+ guard status == noErr, size == MemoryLayout<UInt32>.size else { return nil }
+ return value != 0
}
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift b/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
index dc886327..c1856858 100644
--- a/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingAutoDetector.swift
@@ -56,6 +56,7 @@ final class MeetingAutoDetector {
var audioEvidenceAt: Date?
var audioEvidenceSource: AudioEvidenceSource?
var processAudioWasActive = false
+ var processAudioObserved = false
var processAudioInactiveAt: Date?
var windowEvidenceAt: Date?
var windowEvidenceKey: String?
@@ -105,6 +106,7 @@ final class MeetingAutoDetector {
private var titleEnrichmentGeneration: [Int32: UInt64] = [:]
private var titleEnrichmentInFlight: Set<Int32> = []
private var lastHealth: Health?
+ private var audioQueryUnknown = false
init(
workspaceEvents: any WorkspaceEventsProviding,
@@ -284,19 +286,68 @@ final class MeetingAutoDetector {
}
private func pollAudioProcessActivity(at now: Date, expectedRunGeneration: UInt64?) async {
- for (pid, record) in self.records where record.tier == .nativeTier1 {
- let isActive = await self.audioProcessActivity.isAudioActive(forBundleIdentifiers: MeetingAppRegistry.audioProcessBundleIdentifiers(for: record.bundleIdentifier))
+ guard !Task.isCancelled,
+ expectedRunGeneration.map({ self.runGeneration == $0 && self.pollTask != nil && self.isNativeDetectionEnabled() }) ?? true
+ else { return }
+ let candidates = self.records.filter { $0.value.tier == .nativeTier1 }
+ let snapshot = await self.audioProcessActivity.snapshot()
+ guard !Task.isCancelled,
+ expectedRunGeneration.map({ self.runGeneration == $0 && self.pollTask != nil && self.isNativeDetectionEnabled() }) ?? true
+ else { return }
+ guard let activeInputByPID = MeetingAudioProcessResolver.activeOwnerInputByPID(snapshot: snapshot) else {
+ if !self.audioQueryUnknown {
+ DebugLogger.shared.warning("audio-query-unavailable native confirmation paused", source: "MeetingAutoDetector")
+ }
+ self.audioQueryUnknown = true
+ for (pid, record) in candidates where self.records[pid]?.incarnation == record.incarnation {
+ self.records[pid]?.processAudioInactiveAt = nil
+ }
+ return
+ }
+ if self.audioQueryUnknown {
+ DebugLogger.shared.info("audio-query-recovered native activity reconciliation resumed", source: "MeetingAutoDetector")
+ }
+ self.audioQueryUnknown = false
+ for (pid, record) in candidates {
guard !Task.isCancelled,
expectedRunGeneration.map({ self.runGeneration == $0 && self.pollTask != nil && self.isNativeDetectionEnabled() }) ?? true,
self.records[pid]?.incarnation == record.incarnation
else { continue }
+ let isActive = activeInputByPID[pid] != nil
+ let hasInput = activeInputByPID[pid] == true
+ guard !isActive || snapshot.owners.contains(where: { $0.processID == pid && $0.bundleIdentifier == record.bundleIdentifier }) else { continue }
+ // Generic helpers often expose output-only media while the app is idle. Require
+ // input for a new generic episode; an already-established episode is enough
+ // to make output-only activity meaningful. Zoom's muted output-only path remains
+ // supported because Zoom also requires an explicit meeting-window title.
+ let hasEstablishedEpisode = self.episodesByKey.values.contains { $0.pid == pid }
+ if isActive && !hasInput && record.bundleIdentifier != "us.zoom.xos" && !hasEstablishedEpisode {
+ self.handleAudioProcessActivity(false, pid: pid, at: now)
+ continue
+ }
self.handleAudioProcessActivity(isActive, pid: pid, at: now)
}
}
func handleAudioProcessActivity(_ isActive: Bool, pid: Int32, at now: Date) {
guard var record = self.records[pid], record.tier == .nativeTier1 else { return }
- guard isActive != record.processAudioWasActive else { return }
+ record.processAudioObserved = true
+ if isActive && record.processAudioWasActive {
+ // A continuous stream is still fresh evidence. Refreshing here allows a late window
+ // snapshot/activation to confirm without synthesizing duplicate episodes.
+ record.audioEvidenceAt = now
+ record.audioEvidenceSource = .process
+ record.processAudioInactiveAt = nil
+ self.records[pid] = record
+ self.attemptConfirm(pid: pid, at: now)
+ return
+ }
+ if !isActive && !record.processAudioWasActive {
+ // Inactivity must be continuously observed; an unknown snapshot resets this timer.
+ if record.processAudioInactiveAt == nil { record.processAudioInactiveAt = now }
+ self.records[pid] = record
+ return
+ }
record.processAudioWasActive = isActive
if isActive {
record.processAudioInactiveAt = nil
@@ -437,6 +488,9 @@ final class MeetingAutoDetector {
let windowEvidenceAt = record.windowEvidenceAt,
let evidenceKey = record.windowEvidenceKey
else { return }
+ if record.tier == .nativeTier1, self.audioQueryUnknown {
+ return // a transient HAL failure is not an inactive edge, but cannot confirm anew
+ }
guard now.timeIntervalSince(audioEvidenceAt) <= Self.confirmDeadlineSeconds else {
self.logConfirmRejected("deadline", bundleIdentifier: record.bundleIdentifier)
return
@@ -600,12 +654,16 @@ final class MeetingAutoDetector {
// the meeting's own evidence persists, never merely because our recording stopped.
// Mic release ≥60s ends episodes too — Teams/Webex main windows persist after a call, so
// window loss alone would keep their episodes alive (and back-to-back meetings silent) forever.
- let micReleasedLongEnough = self.lastMicReleaseAt.map { now.timeIntervalSince($0) >= Self.micReleaseEpisodeEndSeconds } == true
for (key, episode) in self.episodesByKey {
let ended: Bool
if let record = self.records[episode.pid] {
let windowGone = record.windowLostAt.map { now.timeIntervalSince($0) >= Self.micReleaseEpisodeEndSeconds } == true
- let processAudioInactiveLongEnough = record.processAudioInactiveAt.map { now.timeIntervalSince($0) >= Self.micReleaseEpisodeEndSeconds } == true
+ let processAudioInactiveLongEnough = !self.audioQueryUnknown &&
+ (record.processAudioInactiveAt.map { now.timeIntervalSince($0) >= Self.micReleaseEpisodeEndSeconds } == true)
+ // A global mic release can belong to another app/device transition. Process
+ // audio is app-specific and therefore wins over that global signal.
+ let micReleasedLongEnough = (record.processAudioObserved || (record.tier == .nativeTier1 && self.audioQueryUnknown)) ? false :
+ (self.lastMicReleaseAt.map { now.timeIntervalSince($0) >= Self.micReleaseEpisodeEndSeconds } == true)
ended = windowGone || micReleasedLongEnough || processAudioInactiveLongEnough
} else {
ended = true // the app quit
@@ -634,6 +692,7 @@ final class MeetingAutoDetector {
// MARK: Disarm on lock / sleep / fast-user-switch — mic bits are unreliable across these.
func disarmAndClearTransientState() {
+ self.audioQueryUnknown = false
let unconsumedIDs = self.episodesByKey.values.compactMap { episode in
episode.consumed ? nil : episode.id
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
index 82f1b207..4e6aa789 100644
--- a/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift
@@ -75,22 +75,35 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
let microphoneWriter = writersByKind[.microphone]
else { throw MeetingCaptureError.applicationNotSelected }
+ let outputRoute = Self.currentOutputRouteSnapshot()
let decision = MeetingCapturePathDecider.decide(
mode: configuration.mode,
microphone: configuration.microphone,
- outputRoute: Self.currentOutputRouteSnapshot()
+ outputRoute: outputRoute
)
switch decision {
case let .screenCaptureKit(reason):
+ let protection = MeetingRawMicrophoneProtection.classify(outputRoute)
DebugLogger.shared.log(
"Meeting voice-processing capture declined: \(reason)",
source: "MeetingCaptureEngine"
)
eventHandler(.interrupted(kind: .voiceProcessingDeclined, trackID: nil, detail: reason))
+ try await microphoneWriter.updateTrackMetadata { track in
+ track.captureEras = [MeetingCaptureEra(
+ method: .screenCaptureKit,
+ deviceUID: configuration.microphone.coreAudioUID,
+ deviceName: configuration.microphone.displayName,
+ roleAtElection: configuration.microphone.role,
+ echoProtection: protection,
+ startSeconds: 0
+ )]
+ }
runtime = try await ScreenCaptureMeetingRuntime.make(
application: application,
microphone: configuration.microphone,
includeMicrophone: true,
+ microphoneEchoProtection: protection,
applicationWriter: applicationWriter,
microphoneWriter: microphoneWriter,
eventHandler: eventHandler,
@@ -326,8 +339,42 @@ actor MeetingCaptureEngine: MeetingCaptureControlling {
deviceExists: true,
isBluetooth: device.isBluetooth,
isBuiltIn: device.isBuiltIn,
- isHeadphonesDataSource: AudioDevice.outputDataSourceIsHeadphones(device.id)
+ isHeadphonesDataSource: AudioDevice.outputDataSourceIsHeadphones(device.id),
+ terminalTypes: Self.outputTerminalTypes(deviceID: device.id)
+ )
+ }
+
+ private static func outputTerminalTypes(deviceID: AudioObjectID) -> Set<UInt32> {
+ var streamsAddress = AudioObjectPropertyAddress(
+ mSelector: kAudioDevicePropertyStreams,
+ mScope: kAudioObjectPropertyScopeOutput,
+ mElement: kAudioObjectPropertyElementMain
+ )
+ var byteCount: UInt32 = 0
+ guard AudioObjectGetPropertyDataSize(deviceID, &streamsAddress, 0, nil, &byteCount) == noErr,
+ byteCount >= UInt32(MemoryLayout<AudioStreamID>.size)
+ else { return [] }
+ var streams = [AudioStreamID](
+ repeating: 0,
+ count: Int(byteCount) / MemoryLayout<AudioStreamID>.size
)
+ let streamStatus = streams.withUnsafeMutableBytes { bytes in
+ AudioObjectGetPropertyData(deviceID, &streamsAddress, 0, nil, &byteCount, bytes.baseAddress!)
+ }
+ guard streamStatus == noErr else { return [] }
+ return Set(streams.compactMap { streamID in
+ var terminalAddress = AudioObjectPropertyAddress(
+ mSelector: kAudioStreamPropertyTerminalType,
+ mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain
+ )
+ var terminalType: UInt32 = kAudioStreamTerminalTypeUnknown
+ var terminalSize = UInt32(MemoryLayout<UInt32>.size)
+ guard AudioObjectGetPropertyData(
+ streamID, &terminalAddress, 0, nil, &terminalSize, &terminalType
+ ) == noErr else { return nil }
+ return terminalType
+ })
}
private nonisolated static func preflightStorage(at directory: URL) throws {
@@ -390,6 +437,10 @@ nonisolated enum MeetingWindowSelector {
private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCaptureRuntime, SCStreamOutput, SCStreamDelegate,
@unchecked Sendable
{
+ /// HAL notifications can arrive while the default input/output pair is still converging.
+ /// Admission stays closed until the latest route revision has settled.
+ private static let rawRouteSettleSeconds: Double = 0.75
+
private struct BuiltStream {
let stream: SCStream
let delegateProxy: ScreenCaptureRuntimeDelegateProxy
@@ -403,6 +454,7 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
private let microphone: MeetingMicrophoneIdentity
/// Immutable for the runtime's lifetime: a rebuild never drops/adds the mic.
private let includeMicrophone: Bool
+ private let microphoneTranscriptGate: MeetingMicrophoneTranscriptGate
private let applicationWriter: MeetingAudioChunkWriter
private let microphoneWriter: MeetingAudioChunkWriter
private let eventHandler: @Sendable (MeetingCaptureEvent) -> Void
@@ -415,6 +467,10 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
private var rebuilding = false
private var pendingStream: SCStream?
private var pendingStreamFailed = false
+ private var outputRouteListener: MeetingOutputRouteListener?
+ private var outputRouteRevision: UInt64 = 0
+ private var routeSafetyPersistenceFailed = false
+ private let routeProtectionTasks = MeetingAsyncTaskChain()
var captureScope: MeetingCaptureScope? {
self.stateLock.withLock { self.scope }
@@ -426,6 +482,7 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
application: MeetingApplicationIdentity,
microphone: MeetingMicrophoneIdentity,
includeMicrophone: Bool,
+ microphoneEchoProtection: MeetingMicrophoneEchoProtection,
applicationWriter: MeetingAudioChunkWriter,
microphoneWriter: MeetingAudioChunkWriter,
eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void,
@@ -436,6 +493,9 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
self.application = application
self.microphone = microphone
self.includeMicrophone = includeMicrophone
+ self.microphoneTranscriptGate = MeetingMicrophoneTranscriptGate(
+ includeMicrophone ? .unprotected : microphoneEchoProtection
+ )
self.applicationWriter = applicationWriter
self.microphoneWriter = microphoneWriter
self.eventHandler = eventHandler
@@ -447,6 +507,7 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
application: MeetingApplicationIdentity,
microphone: MeetingMicrophoneIdentity,
includeMicrophone: Bool = true,
+ microphoneEchoProtection: MeetingMicrophoneEchoProtection = .unprotected,
applicationWriter: MeetingAudioChunkWriter,
microphoneWriter: MeetingAudioChunkWriter,
eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void,
@@ -466,6 +527,7 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
application: application,
microphone: resolvedMicrophone,
includeMicrophone: includeMicrophone,
+ microphoneEchoProtection: microphoneEchoProtection,
applicationWriter: applicationWriter,
microphoneWriter: microphoneWriter,
eventHandler: eventHandler,
@@ -572,10 +634,54 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
}
func start() async throws {
+ var installedListener: MeetingOutputRouteListener?
+ if self.includeMicrophone {
+ // Arm observation before promoting raw microphone admission. This closes the snapshot →
+ // stream-start hole: a route notification racing startup increments the revision and
+ // prevents the stale protection snapshot from opening the gate.
+ installedListener = await MeetingOutputRouteListener.make { [weak self] in
+ self?.invalidateRawMicrophoneAdmissionAfterRouteChange()
+ }
+ let baselineRevision = self.stateLock.withLock { () -> UInt64 in
+ self.outputRouteListener = installedListener
+ return self.outputRouteRevision
+ }
+ // Listener failure degrades transcript admission, never the recording itself.
+ let protection: MeetingMicrophoneEchoProtection = installedListener == nil
+ ? .unprotected
+ : MeetingRawMicrophoneProtection.classify(MeetingCaptureEngine.currentOutputRouteSnapshot())
+ do {
+ try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
+ guard var eras = track.captureEras, !eras.isEmpty else { return }
+ eras[eras.index(before: eras.endIndex)].echoProtection = protection
+ track.captureEras = eras
+ }
+ } catch {
+ await installedListener?.stop()
+ self.stateLock.withLock { self.outputRouteListener = nil }
+ throw error
+ }
+ await self.routeProtectionTasks.drain()
+ let routeStayedStable = self.stateLock.withLock {
+ self.outputRouteRevision == baselineRevision && !self.stopping
+ }
+ if !routeStayedStable {
+ // A callback may have raced and completed before the startup metadata write. Make
+ // the final persisted state fail-closed as well as the already-closed live gate.
+ try await self.microphoneWriter.updateTrackMetadata { track in
+ guard var eras = track.captureEras, !eras.isEmpty else { return }
+ eras[eras.index(before: eras.endIndex)].echoProtection = .unprotected
+ track.captureEras = eras
+ }
+ }
+ self.microphoneTranscriptGate.update(routeStayedStable ? protection : .unprotected)
+ }
let currentStream = self.stateLock.withLock { self.stream }
do {
try await currentStream.startCapture()
} catch {
+ await installedListener?.stop()
+ self.stateLock.withLock { self.outputRouteListener = nil }
throw MeetingCaptureError.captureStartFailed(error.localizedDescription)
}
}
@@ -586,6 +692,12 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
defer { AudioTopologyDiagnostics.record(.phaseEnd, owner: .screenCapture, queueRole: .actorControl, phase: .screenCaptureStop) }
#endif
self.markStopping()
+ let routeListener = self.stateLock.withLock { () -> MeetingOutputRouteListener? in
+ defer { self.outputRouteListener = nil }
+ return self.outputRouteListener
+ }
+ await routeListener?.stop()
+ await self.routeProtectionTasks.drain()
let (currentStream, streamAlreadyDead) = self.stateLock.withLock { (self.stream, self.rebuilding) }
// Mid-rebuild the current stream already stopped on its own; stopCapture would only time out.
guard !streamAlreadyDead else { return }
@@ -624,7 +736,9 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
self.liveAudioHandler?(.applicationAudio, sampleBuffer)
case .microphone:
self.microphoneWriter.enqueue(sampleBuffer)
- self.liveAudioHandler?(.microphone, sampleBuffer)
+ if self.microphoneTranscriptGate.observeAndAdmit(sampleBuffer) {
+ self.liveAudioHandler?(.microphone, sampleBuffer)
+ }
case .screen:
break
@unknown default:
@@ -632,6 +746,127 @@ private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCa
}
}
+ /// A raw ScreenCaptureKit microphone has no producer handoff. Close live admission at the
+ /// observation boundary and persist a new unprotected era from that point forward. Historical
+ /// audio remains classified by the route on which it was actually captured.
+ private func invalidateRawMicrophoneAdmissionAfterRouteChange() {
+ let revision = self.stateLock.withLock { () -> UInt64? in
+ guard !self.stopping else { return nil }
+ self.outputRouteRevision &+= 1
+ return self.outputRouteRevision
+ }
+ guard let revision else { return }
+ let boundary = self.microphoneTranscriptGate.invalidate()
+ self.routeProtectionTasks.enqueue { [weak self] in
+ guard let self else { return }
+ let invalidationPersisted: Bool
+ do {
+ try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
+ guard var eras = track.captureEras, !eras.isEmpty else { return }
+ if let boundary {
+ if boundary.unsafeStartSeconds > (eras.last?.startSeconds ?? -.infinity) {
+ var unsafeEra = eras[eras.index(before: eras.endIndex)]
+ unsafeEra.echoProtection = .unprotected
+ unsafeEra.startSeconds = boundary.unsafeStartSeconds
+ unsafeEra.settledConfig = nil
+ unsafeEra.clockDrift = nil
+ eras.append(unsafeEra)
+ } else {
+ eras[eras.index(before: eras.endIndex)].echoProtection = .unprotected
+ }
+ } else {
+ // No sample boundary exists yet, but admission remains fail-closed while
+ // the input/output route pair settles.
+ eras[eras.index(before: eras.endIndex)].echoProtection = .unprotected
+ }
+ track.captureEras = eras
+ }
+ invalidationPersisted = true
+ } catch {
+ invalidationPersisted = false
+ DebugLogger.shared.warning(
+ "Raw microphone route safety could not be persisted; transcript admission remains closed: \(error.localizedDescription)",
+ source: "MeetingCaptureEngine"
+ )
+ let shouldEmit = self.stateLock.withLock { () -> Bool in
+ guard !self.routeSafetyPersistenceFailed else { return false }
+ self.routeSafetyPersistenceFailed = true
+ self.stopping = true
+ return true
+ }
+ if shouldEmit {
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly,
+ trackID: self.microphoneWriter.trackID,
+ detail: "Microphone route safety metadata could not be saved; recording was stopped fail-closed."
+ ))
+ }
+ }
+
+ guard invalidationPersisted else { return }
+ try? await Task.sleep(nanoseconds: UInt64(Self.rawRouteSettleSeconds * 1_000_000_000))
+ let isLatestBeforeClassification = self.stateLock.withLock {
+ self.outputRouteRevision == revision && !self.stopping
+ }
+ guard isLatestBeforeClassification else { return }
+
+ let settledProtection = MeetingRawMicrophoneProtection.classify(
+ MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
+ guard settledProtection.admitsTranscript else {
+ self.eventHandler(.interrupted(
+ kind: .microphoneChanged,
+ trackID: self.microphoneWriter.trackID,
+ detail: "Microphone transcript admission remains disabled because the output route is not echo-protected."
+ ))
+ return
+ }
+ // A nil boundary means the route changed before the first microphone buffer. In that
+ // case no historical samples can be blessed accidentally, so era zero may be updated
+ // directly after the settled route has been positively classified.
+ let resumeAt = self.microphoneTranscriptGate.invalidate()?.observedSeconds
+
+ let resumePersisted: Bool
+ var wroteResumeEra = false
+ do {
+ try await self.microphoneWriter.updateTrackMetadata { track in
+ guard var eras = track.captureEras, !eras.isEmpty else { return }
+ if let resumeAt {
+ guard resumeAt > (eras.last?.startSeconds ?? -.infinity) else { return }
+ var resumedEra = eras[eras.index(before: eras.endIndex)]
+ resumedEra.echoProtection = settledProtection
+ resumedEra.startSeconds = resumeAt
+ eras.append(resumedEra)
+ } else {
+ eras[eras.index(before: eras.endIndex)].echoProtection = settledProtection
+ }
+ track.captureEras = eras
+ wroteResumeEra = true
+ }
+ resumePersisted = wroteResumeEra
+ } catch {
+ resumePersisted = false
+ DebugLogger.shared.warning(
+ "Settled raw microphone protection could not be persisted; transcript admission remains closed: \(error.localizedDescription)",
+ source: "MeetingCaptureEngine"
+ )
+ }
+ let isLatestAfterPersistence = self.stateLock.withLock {
+ self.outputRouteRevision == revision && !self.stopping
+ }
+ if resumePersisted, isLatestAfterPersistence {
+ self.microphoneTranscriptGate.update(settledProtection)
+ }
+ self.eventHandler(.interrupted(
+ kind: .microphoneChanged,
+ trackID: self.microphoneWriter.trackID,
+ detail: resumePersisted && isLatestAfterPersistence
+ ? "Microphone transcript admission resumed after the output route settled."
+ : "Microphone transcript admission remains disabled after the output route changed."
+ ))
+ }
+ }
+
func handleStreamStop(_ stoppedStream: SCStream, error: Error) {
let shouldRebuild = self.stateLock.withLock { () -> Bool in
if stoppedStream === self.pendingStream {
@@ -1759,10 +1994,113 @@ nonisolated struct MeetingOutputRouteSnapshot: Sendable, Equatable {
var isBluetooth: Bool
var isBuiltIn: Bool
var isHeadphonesDataSource: Bool
+ var terminalTypes: Set<UInt32> = []
+}
+
+nonisolated enum MeetingRawMicrophoneProtection {
+ /// Closed must be positively identified. Unknown, aggregate, HDMI, virtual, and speaker
+ /// routes remain unprotected even when they are not the built-in speakers.
+ static func classify(_ route: MeetingOutputRouteSnapshot) -> MeetingMicrophoneEchoProtection {
+ guard route.deviceExists else { return .unprotected }
+ if (route.isBuiltIn && route.isHeadphonesDataSource)
+ || route.terminalTypes.contains(kAudioStreamTerminalTypeHeadphones)
+ {
+ return .acousticallyClosed
+ }
+ return .unprotected
+ }
}
-/// Pure decision table. Requires `role == .personal` — the election does too; role-blind VPIO
-/// would capture cleanly but never attribute.
+final nonisolated class MeetingMicrophoneTranscriptGate: @unchecked Sendable {
+ /// Core Audio notifications are asynchronous relative to the physical route switch. Rewind the
+ /// persisted cut slightly so callback latency cannot bless the first speaker-leak buffers.
+ static let routeNotificationSafetySeconds: Double = 0.5
+ private let lock = NSLock()
+ private var protection: MeetingMicrophoneEchoProtection
+ private var latestPresentationSeconds: Double?
+
+ init(_ protection: MeetingMicrophoneEchoProtection = .unprotected) {
+ self.protection = protection
+ }
+
+ func update(_ protection: MeetingMicrophoneEchoProtection) {
+ self.lock.withLock { self.protection = protection }
+ }
+
+ func admitsTranscript() -> Bool {
+ self.lock.withLock { self.protection.admitsTranscript }
+ }
+
+ /// Atomically observes the sample boundary and decides live admission. A route callback taking
+ /// the same lock therefore has an exact conservative cut: samples ordered after invalidation
+ /// cannot leak into live transcription under the prior route's protection.
+ func observeAndAdmit(_ sampleBuffer: CMSampleBuffer) -> Bool {
+ let time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
+ return self.lock.withLock {
+ if time.isValid, time.isNumeric {
+ let seconds = CMTimeGetSeconds(time)
+ if seconds.isFinite {
+ self.latestPresentationSeconds = max(self.latestPresentationSeconds ?? seconds, seconds)
+ }
+ }
+ return self.protection.admitsTranscript
+ }
+ }
+
+ /// Closes admission first and returns the last observed sample PTS. Using that PTS as the next
+ /// era boundary deliberately excludes the final pre-notification buffer as a safety margin.
+ struct InvalidationBoundary: Equatable, Sendable {
+ let unsafeStartSeconds: Double
+ let observedSeconds: Double
+ }
+
+ func invalidate() -> InvalidationBoundary? {
+ self.lock.withLock {
+ self.protection = .unprotected
+ return self.latestPresentationSeconds.map { observed in
+ InvalidationBoundary(
+ unsafeStartSeconds: max(0, observed - Self.routeNotificationSafetySeconds),
+ observedSeconds: observed
+ )
+ }
+ }
+ }
+
+ func latestObservedSeconds() -> Double? {
+ self.lock.withLock { self.latestPresentationSeconds }
+ }
+}
+
+/// Serializes route-protection manifest updates without ever blocking an audio callback.
+final nonisolated class MeetingAsyncTaskChain: @unchecked Sendable {
+ private let lock = NSLock()
+ private var tail: Task<Void, Never>?
+ private var generation: UInt64 = 0
+
+ @discardableResult
+ func enqueue(_ operation: @escaping @Sendable () async -> Void) -> Task<Void, Never> {
+ self.lock.withLock {
+ self.generation &+= 1
+ let predecessor = self.tail
+ let task = Task {
+ await predecessor?.value
+ await operation()
+ }
+ self.tail = task
+ return task
+ }
+ }
+
+ func drain() async {
+ while true {
+ let snapshot = self.lock.withLock { (self.generation, self.tail) }
+ await snapshot.1?.value
+ if self.lock.withLock({ self.generation == snapshot.0 }) { return }
+ }
+ }
+}
+
+/// Pure decision table. Capture safety is independent of semantic speaker ownership.
nonisolated enum MeetingCapturePathDecider {
static func decide(
mode: MeetingCaptureMode,
@@ -1772,9 +2110,6 @@ nonisolated enum MeetingCapturePathDecider {
guard mode == .onlineCall else {
return .screenCaptureKit(reason: "Voice-processing capture only applies to online-call recordings.")
}
- guard microphone.role == .personal else {
- return .screenCaptureKit(reason: "Microphone role is not set to personal.")
- }
guard let coreAudioUID = microphone.coreAudioUID, !coreAudioUID.isEmpty else {
return .screenCaptureKit(reason: "The selected microphone has no CoreAudio device UID.")
}
@@ -2324,7 +2659,8 @@ nonisolated enum MeetingVoiceProcessingDriftSnapshot {
}
-/// Device election for a capture-side swap: prefers the system default input; role transfers only when it equals the saved/original device.
+/// Device election for a capture-side swap. Role is retained in the schema for old manifests but
+/// never inferred or used by new capture decisions.
nonisolated struct MeetingCaptureDeviceElection: Sendable, Equatable {
var identity: MeetingMicrophoneIdentity
var role: MeetingMicrophoneRole
@@ -2345,10 +2681,14 @@ nonisolated struct MeetingCaptureDeviceElection: Sendable, Equatable {
defaultInputUID: String?
) -> MeetingCaptureDeviceElection {
guard let defaultInputUID else {
- return MeetingCaptureDeviceElection(identity: original, role: original.role)
+ var neutral = original
+ neutral.role = .unknown
+ return MeetingCaptureDeviceElection(identity: neutral, role: .unknown)
}
guard let matched = catalog.first(where: { $0.coreAudioUID == defaultInputUID }) else {
- return MeetingCaptureDeviceElection(identity: original, role: original.role)
+ var neutral = original
+ neutral.role = .unknown
+ return MeetingCaptureDeviceElection(identity: neutral, role: .unknown)
}
let isSamePhysicalDevice = {
if let originalUID = original.coreAudioUID, !originalUID.isEmpty,
@@ -2360,21 +2700,19 @@ nonisolated struct MeetingCaptureDeviceElection: Sendable, Equatable {
}()
if isSamePhysicalDevice {
var merged = matched
- merged.role = original.role
+ merged.role = .unknown
merged.avCaptureDeviceID = original.avCaptureDeviceID
?? (original.identitySchemaVersion == nil ? original.captureDeviceID : nil)
- return MeetingCaptureDeviceElection(identity: merged, role: original.role)
+ return MeetingCaptureDeviceElection(identity: merged, role: .unknown)
}
- // A built-in mic is inherently the owner's; .unknown would block VPIO on upgrades.
- let role: MeetingMicrophoneRole = matched.coreAudioUID == "BuiltInMicrophoneDevice" ? .personal : .unknown
return MeetingCaptureDeviceElection(
identity: MeetingMicrophoneIdentity(
captureDeviceID: matched.captureDeviceID,
coreAudioUID: matched.coreAudioUID,
displayName: matched.displayName,
- role: role
+ role: .unknown
),
- role: role
+ role: .unknown
)
}
}
@@ -2424,8 +2762,13 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
private var pendingAVCaptureFailure: (generation: UInt64, detail: String)?
private var avCaptureFailureToken: MeetingAVCaptureAttemptToken?
private var routeChangeDirty = false
+ private var routeSafetyPersistenceFailed = false
private var stopRequested = false
private var routeDrivenFallback = false
+ private let routeProtectionTasks = MeetingAsyncTaskChain()
+ /// Pessimistically closed as soon as a route change is observed. A producer may reopen live
+ /// admission only as part of the same commit that persists its new era.
+ private let transcriptGate = MeetingMicrophoneTranscriptGate()
private var voiceProcessingRecoveryAttempts = 0
private var recoveryDwellTask: Task<Void, Never>?
/// Parked before `start()` so stop/supersede can never lose ownership of it.
@@ -2504,10 +2847,34 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
gateToAbort.abort()
await captureToStop.stop()
await self.stopAppOnlyIfStarted()
+ let fallbackProtection = MeetingRawMicrophoneProtection.classify(
+ MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
+ let preFallbackTrack = await self.microphoneWriter.snapshot()
+ guard preFallbackTrack.chunks.isEmpty,
+ preFallbackTrack.captureEras?.isEmpty != false
+ else {
+ throw MeetingCaptureError.captureStartFailed(
+ "Voice-processing fallback refused because microphone audio provenance was already committed."
+ )
+ }
+ try await self.microphoneWriter.updateTrackMetadata { track in
+ track.captureMethod = .screenCaptureKit
+ track.voiceProcessingConfig = nil
+ track.captureEras = [MeetingCaptureEra(
+ method: .screenCaptureKit,
+ deviceUID: self.originalMicrophone.coreAudioUID,
+ deviceName: self.originalMicrophone.displayName,
+ roleAtElection: self.originalMicrophone.role,
+ echoProtection: fallbackProtection,
+ startSeconds: 0
+ )]
+ }
let fallback = try await ScreenCaptureMeetingRuntime.make(
application: self.application,
microphone: self.originalMicrophone,
includeMicrophone: true,
+ microphoneEchoProtection: fallbackProtection,
applicationWriter: self.applicationWriter,
microphoneWriter: self.microphoneWriter,
eventHandler: self.eventHandler,
@@ -2522,6 +2889,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
private func attemptVoiceProcessingStart() async throws {
let microphoneWriter = self.microphoneWriter
let liveAudioHandler = self.liveAudioHandler
+ let transcriptGate = self.transcriptGate
let gate = self.gate
let fence = self.currentFence
let capture = self.stateLock.withLock { self.micCapture }
@@ -2533,7 +2901,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
switch gate.offer(sample) {
case .passthrough:
microphoneWriter.enqueue(sample)
- liveAudioHandler?(.microphone, sample)
+ if transcriptGate.observeAndAdmit(sample) { liveAudioHandler?(.microphone, sample) }
case .buffered, .aborted:
break
}
@@ -2577,10 +2945,12 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
deviceUID: self.originalMicrophone.coreAudioUID,
deviceName: self.originalMicrophone.displayName,
roleAtElection: self.originalMicrophone.role,
+ echoProtection: .voiceProcessed,
startSeconds: 0,
settledConfig: settled
)]
}
+ self.transcriptGate.update(.voiceProcessed)
while true {
await self.flushBatch(batch)
guard let next = self.gate.continueCommitFlush() else { break }
@@ -2609,7 +2979,9 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
private func flushBatch(_ samples: [CMSampleBuffer]) async {
for (index, sample) in samples.enumerated() {
self.microphoneWriter.enqueue(sample)
- self.liveAudioHandler?(.microphone, sample)
+ if self.transcriptGate.observeAndAdmit(sample) {
+ self.liveAudioHandler?(.microphone, sample)
+ }
if index % 4 == 3 {
try? await Task.sleep(nanoseconds: 5_000_000)
}
@@ -2719,17 +3091,26 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
private func handleRouteChange() {
enum Action { case evaluateVPIO, evaluateAVCapture, none }
+ let outputRoute = MeetingCaptureEngine.currentOutputRouteSnapshot()
+ let vpioDeclineReason = MeetingCapturePathDecider.outputRouteDeclineReason(outputRoute)
let action = self.stateLock.withLock { () -> Action in
+ guard !self.stopRequested else { return .none }
switch self.phase {
case .vpio:
+ // VPIO remains echo-safe across notifications that do not actually make its route
+ // unsupported. This filters unrelated/default-device property churn.
+ guard vpioDeclineReason != nil else { return .none }
+ self.persistRouteInvalidation(self.transcriptGate.invalidate())
return .evaluateVPIO
case .avCapture:
+ self.persistRouteInvalidation(self.transcriptGate.invalidate())
// Claim the observation before releasing the lock. If recovery wins
// the next lock acquisition, finishTransition() will replay it
// against whichever backend commits.
self.routeChangeDirty = true
return .evaluateAVCapture
case .transitioning, .committing:
+ self.persistRouteInvalidation(self.transcriptGate.invalidate())
self.routeChangeDirty = true
return .none
case .starting, .failed:
@@ -2738,8 +3119,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
switch action {
case .evaluateVPIO:
- let reason = MeetingCapturePathDecider.outputRouteDeclineReason(MeetingCaptureEngine.currentOutputRouteSnapshot())
- if let reason {
+ if let reason = vpioDeclineReason {
self.beginDowngradeTransition(
reason: "Output route changed: \(reason)",
routeDriven: true
@@ -2757,6 +3137,78 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
}
+ /// Persists the same fail-closed boundary used by live admission. The task chain keeps rapid
+ /// attach/detach notifications ordered without blocking the Core Audio notification callback.
+ private func persistRouteInvalidation(_ boundary: MeetingMicrophoneTranscriptGate.InvalidationBoundary?) {
+ self.routeProtectionTasks.enqueue { [weak self] in
+ guard let self else { return }
+ do {
+ try await self.microphoneWriter.updateTrackMetadataFailClosed { track in
+ guard var eras = track.captureEras, !eras.isEmpty else { return }
+ if let boundary,
+ boundary.unsafeStartSeconds > (eras.last?.startSeconds ?? -.infinity)
+ {
+ var unprotected = eras[eras.index(before: eras.endIndex)]
+ unprotected.echoProtection = .unprotected
+ unprotected.startSeconds = boundary.unsafeStartSeconds
+ unprotected.settledConfig = nil
+ unprotected.clockDrift = nil
+ eras.append(unprotected)
+ } else {
+ eras[eras.index(before: eras.endIndex)].echoProtection = .unprotected
+ }
+ track.captureEras = eras
+ }
+ } catch {
+ let shouldEmit = self.stateLock.withLock { () -> Bool in
+ guard !self.routeSafetyPersistenceFailed else { return false }
+ self.routeSafetyPersistenceFailed = true
+ self.phase = .failed
+ self.interruptedThisEra = true
+ return true
+ }
+ if shouldEmit {
+ self.eventHandler(.interrupted(
+ kind: .captureStoppedUnexpectedly,
+ trackID: self.microphoneWriter.trackID,
+ detail: "Microphone route safety metadata could not be saved; recording was stopped fail-closed."
+ ))
+ }
+ }
+ }
+ }
+
+ /// Reopens a raw era only after a matching protected boundary is durable. The gap between the
+ /// rewound invalidation cut and this observation absorbs route-notification latency.
+ private func persistRawRouteResume(_ protection: MeetingMicrophoneEchoProtection) async -> Bool {
+ guard protection.admitsTranscript,
+ let boundarySeconds = self.transcriptGate.latestObservedSeconds()
+ else { return false }
+ await self.routeProtectionTasks.drain()
+ var recorded = false
+ try? await self.microphoneWriter.updateTrackMetadata { track in
+ guard var eras = track.captureEras, !eras.isEmpty,
+ boundarySeconds > (eras.last?.startSeconds ?? -.infinity)
+ else { return }
+ var resumed = eras[eras.index(before: eras.endIndex)]
+ resumed.echoProtection = protection
+ resumed.startSeconds = boundarySeconds
+ resumed.settledConfig = nil
+ resumed.clockDrift = nil
+ eras.append(resumed)
+ track.captureEras = eras
+ recorded = true
+ }
+ return recorded
+ }
+
+ /// A route event observed during a commit keeps the gate closed; `finishTransition` replays it
+ /// against the newly committed backend instead of briefly admitting audio under stale facts.
+ private func publishTranscriptProtection(_ protection: MeetingMicrophoneEchoProtection) {
+ let mayOpen = self.stateLock.withLock { !self.routeChangeDirty && !self.stopRequested }
+ self.transcriptGate.update(mayOpen ? protection : .unprotected)
+ }
+
private func handleMicEvent(_ event: MeetingMicrophoneCaptureEvent, owner: MeetingMicrophoneCapture) {
let (phase, isOwner) = self.stateLock.withLock { (self.phase, self.micCapture === owner) }
guard isOwner else { return }
@@ -2857,6 +3309,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
private func beginDowngradeTransition(detail: String, routeDriven: Bool = false) {
let watchdog = self.stateLock.withLock { () -> Task<Void, Never>? in
guard self.phase == .vpio, !self.stopRequested else { return nil }
+ self.transcriptGate.update(.unprotected)
self.phase = .transitioning
self.transitionGeneration &+= 1
let generation = self.transitionGeneration
@@ -2914,6 +3367,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
let newGate = MeetingVoiceProcessingCommitGate()
let microphoneWriter = self.microphoneWriter
let liveAudioHandler = self.liveAudioHandler
+ let transcriptGate = self.transcriptGate
let failureToken = MeetingAVCaptureAttemptToken()
guard self.stateLock.withLock({ () -> Bool in
guard self.phase == .transitioning,
@@ -2936,7 +3390,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
switch newGate.offer(sample) {
case .passthrough:
microphoneWriter.enqueue(sample)
- liveAudioHandler?(.microphone, sample)
+ if transcriptGate.observeAndAdmit(sample) { liveAudioHandler?(.microphone, sample) }
case .buffered, .aborted:
break
}
@@ -3041,7 +3495,11 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
break
}
+ await self.routeProtectionTasks.drain()
let boundary = await self.microphoneWriter.beginSplice()
+ let rawProtection = MeetingRawMicrophoneProtection.classify(
+ MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
// A nil or non-monotonic boundary would misattribute audio; skip the era record instead.
var recordedNewEra = false
@@ -3058,6 +3516,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
deviceUID: elected.identity.coreAudioUID,
deviceName: elected.identity.displayName,
roleAtElection: elected.role,
+ echoProtection: rawProtection,
startSeconds: boundary.seconds
))
recordedNewEra = true
@@ -3072,6 +3531,8 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
"Meeting downgrade: no valid splice boundary (nil or non-monotonic) — the AVCapture era was not recorded; turns after this point stay attributed to the prior era.",
source: "MeetingCaptureEngine"
)
+ } else {
+ self.publishTranscriptProtection(rawProtection)
}
// Ack before flushing: readers must re-arm before any post-boundary sample reaches them.
@@ -3115,6 +3576,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
self.avCaptureComponent != nil,
!self.stopRequested
else { return nil }
+ self.transcriptGate.update(.unprotected)
let previousGeneration = self.transitionGeneration
self.phase = .transitioning
self.transitionGeneration &+= 1
@@ -3192,6 +3654,12 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
return (true, replay)
}
if restored.restored {
+ let protection = MeetingRawMicrophoneProtection.classify(
+ MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
+ if await self.persistRawRouteResume(protection) {
+ self.publishTranscriptProtection(protection)
+ }
self.startAVCaptureWatchdog(generation: previousGeneration, component: oldComponent)
if restored.replay { self.handleRouteChange() }
self.armSafeVoiceProcessingRecoveryIfViable()
@@ -3217,6 +3685,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
let microphoneWriter = self.microphoneWriter
let liveAudioHandler = self.liveAudioHandler
+ let transcriptGate = self.transcriptGate
var selectedElection: MeetingCaptureDeviceElection?
var selectedFence: MeetingEraAcceptanceFence?
var selectedGate: MeetingVoiceProcessingCommitGate?
@@ -3271,7 +3740,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
switch candidateGate.offer(sample) {
case .passthrough:
microphoneWriter.enqueue(sample)
- liveAudioHandler?(.microphone, sample)
+ if transcriptGate.observeAndAdmit(sample) { liveAudioHandler?(.microphone, sample) }
case .buffered, .aborted:
break
}
@@ -3392,7 +3861,12 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
break
}
+ await self.routeProtectionTasks.drain()
let boundary = await self.microphoneWriter.beginSplice()
+ let rawProtection = MeetingRawMicrophoneProtection.classify(
+ MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
+ var recordedNewEra = false
do {
try await self.microphoneWriter.updateTrackMetadata { track in
var eras = track.captureEras ?? [self.legacyEra(from: track)]
@@ -3402,12 +3876,15 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
deviceUID: elected.identity.coreAudioUID,
deviceName: elected.identity.displayName,
roleAtElection: elected.role,
+ echoProtection: rawProtection,
startSeconds: boundary.seconds
))
+ recordedNewEra = true
}
track.captureEras = eras
track.captureMethod = .avCaptureSession
}
+ if recordedNewEra { self.publishTranscriptProtection(rawProtection) }
} catch {
self.invalidateAVCaptureFailureToken(failureToken, generation: generation)
await newFence.quiesce()
@@ -3469,11 +3946,13 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
private func legacyEra(from track: MeetingAudioTrack) -> MeetingCaptureEra {
- MeetingCaptureEra(
- method: track.captureMethod ?? .voiceProcessing,
+ let method = track.captureMethod ?? .voiceProcessing
+ return MeetingCaptureEra(
+ method: method,
deviceUID: self.originalMicrophone.coreAudioUID,
deviceName: self.originalMicrophone.displayName,
roleAtElection: self.originalMicrophone.role,
+ echoProtection: method == .voiceProcessing ? .voiceProcessed : .legacyUnclassified,
startSeconds: 0,
settledConfig: track.voiceProcessingConfig,
clockDrift: track.clockDrift
@@ -3600,6 +4079,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
let oldComponent = self.stateLock.withLock { self.avCaptureComponent }
await oldFence.quiesce()
+ await self.routeProtectionTasks.drain()
let boundary = await self.microphoneWriter.beginSplice()
let boundaryTime = boundary.map { CMTime(value: $0.value, timescale: $0.timescale) }
@@ -3621,6 +4101,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
deviceUID: elected.identity.coreAudioUID,
deviceName: elected.identity.displayName,
roleAtElection: elected.role,
+ echoProtection: .voiceProcessed,
startSeconds: boundary.seconds,
settledConfig: settled
))
@@ -3635,6 +4116,8 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
"Meeting upgrade: no valid splice boundary (nil or non-monotonic) — the voice-processing era was not recorded; turns after this point stay attributed to the prior era.",
source: "MeetingCaptureEngine"
)
+ } else {
+ self.publishTranscriptProtection(.voiceProcessed)
}
// Awaited: the MainActor hop (filter re-arm, history entry) must land before the gate opens.
@@ -3739,6 +4222,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
outputRoute: outputRoute
)
else { return false }
+ self.transcriptGate.update(.unprotected)
self.phase = .transitioning
self.transitionGeneration &+= 1
let generation = self.transitionGeneration
@@ -3826,6 +4310,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
return
}
+ await self.routeProtectionTasks.drain()
let boundary = await self.microphoneWriter.beginSplice()
guard self.ownsSafeRecovery(generation) else {
self.finishTransition(generation)
@@ -3856,6 +4341,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
let candidateFence = MeetingEraAcceptanceFence()
let microphoneWriter = self.microphoneWriter
let liveAudioHandler = self.liveAudioHandler
+ let transcriptGate = self.transcriptGate
let parked = self.stateLock.withLock { () -> Bool in
guard self.phase == .transitioning,
self.transitionGeneration == generation,
@@ -3885,7 +4371,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
switch candidateGate.offer(sample) {
case .passthrough:
microphoneWriter.enqueue(sample)
- liveAudioHandler?(.microphone, sample)
+ if transcriptGate.observeAndAdmit(sample) { liveAudioHandler?(.microphone, sample) }
case .buffered, .aborted:
break
}
@@ -3967,6 +4453,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
let settled = await candidate!.settledConfiguration()
+ var recordedNewEra = false
do {
try await self.microphoneWriter.updateTrackMetadata { track in
var eras = track.captureEras ?? [self.legacyEra(from: track)]
@@ -3976,14 +4463,17 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
deviceUID: elected.identity.coreAudioUID,
deviceName: elected.identity.displayName,
roleAtElection: elected.role,
+ echoProtection: .voiceProcessed,
startSeconds: boundary.seconds,
settledConfig: settled
))
+ recordedNewEra = true
}
track.captureEras = eras
track.captureMethod = .voiceProcessing
track.voiceProcessingConfig = settled
}
+ if recordedNewEra { self.publishTranscriptProtection(.voiceProcessed) }
} catch {
await self.retirePendingVoiceProcessingCapture(generation: generation)
candidate = nil
@@ -4076,7 +4566,11 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
let token = MeetingAVCaptureAttemptToken()
let microphoneWriter = self.microphoneWriter
let liveAudioHandler = self.liveAudioHandler
+ let transcriptGate = self.transcriptGate
let component: MeetingAVCaptureMicrophoneComponent
+ let rawProtection = MeetingRawMicrophoneProtection.classify(
+ MeetingCaptureEngine.currentOutputRouteSnapshot()
+ )
do {
component = try MeetingAVCaptureMicrophoneComponent(
microphone: elected.identity,
@@ -4086,7 +4580,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
switch gate.offer(sample) {
case .passthrough:
microphoneWriter.enqueue(sample)
- liveAudioHandler?(.microphone, sample)
+ if transcriptGate.observeAndAdmit(sample) { liveAudioHandler?(.microphone, sample) }
case .buffered, .aborted:
break
}
@@ -4185,6 +4679,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
self.finishTransition(generation)
return
}
+ var recordedNewEra = false
do {
try await self.microphoneWriter.updateTrackMetadata { track in
var eras = track.captureEras ?? [self.legacyEra(from: track)]
@@ -4194,12 +4689,15 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
deviceUID: elected.identity.coreAudioUID,
deviceName: elected.identity.displayName,
roleAtElection: elected.role,
+ echoProtection: rawProtection,
startSeconds: boundary.seconds
))
+ recordedNewEra = true
}
track.captureEras = eras
track.captureMethod = .avCaptureSession
}
+ if recordedNewEra { self.publishTranscriptProtection(rawProtection) }
} catch {
self.invalidateAVCaptureFailureToken(token, generation: generation)
try? await component.stop()
@@ -4352,6 +4850,7 @@ private final nonisolated class VoiceProcessingMeetingRuntime: MeetingCaptureRun
}
watchdog?.cancel()
await staleListener?.stop()
+ await self.routeProtectionTasks.drain()
if let fallback = self.stateLock.withLock({ self.fallbackRuntime }) {
try await fallback.stop()
diff --git a/Sources/Fluid/Services/Meeting/MeetingModels.swift b/Sources/Fluid/Services/Meeting/MeetingModels.swift
index 5af4b26b..a447a697 100644
--- a/Sources/Fluid/Services/Meeting/MeetingModels.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingModels.swift
@@ -89,6 +89,17 @@ nonisolated enum MeetingTranscriptOverlap: String, Codable, Sendable {
case ambiguous
}
+nonisolated enum MeetingTranscriptCoverageGapReason: String, Codable, Sendable {
+ case unprotectedMicrophone
+}
+
+nonisolated struct MeetingTranscriptCoverageGap: Codable, Equatable, Sendable {
+ var trackID: MeetingAudioTrackID
+ var start: TimeInterval
+ var end: TimeInterval
+ var reason: MeetingTranscriptCoverageGapReason
+}
+
nonisolated enum MeetingMicrophoneRole: String, Codable, CaseIterable, Sendable {
case personal
case shared
@@ -185,7 +196,8 @@ nonisolated struct MeetingMicrophoneIdentity: Codable, Equatable, Sendable {
}
}
-/// Pure decision for the new-meeting sheet's default microphone: the system default input wins; a remembered role only carries over when that default IS the saved device.
+/// Pure decision for the new-meeting sheet's default microphone. Semantic ownership is legacy
+/// metadata only; all new selections are neutral.
nonisolated enum MeetingMicrophonePreselection {
static func select(
identities: [MeetingMicrophoneIdentity],
@@ -204,10 +216,9 @@ nonisolated enum MeetingMicrophonePreselection {
} ?? identities.first
guard let matched else { return (nil, .unknown) }
- if matched.captureDeviceID == savedDeviceID { return (matched.captureDeviceID, savedRole) }
- // A built-in mic is inherently the owner's; .unknown would silently disable voice processing.
- let role: MeetingMicrophoneRole = matched.coreAudioUID == "BuiltInMicrophoneDevice" ? .personal : .unknown
- return (matched.captureDeviceID, role)
+ _ = savedDeviceID
+ _ = savedRole
+ return (matched.captureDeviceID, .unknown)
}
}
@@ -355,6 +366,21 @@ nonisolated enum MeetingAudioTrackCaptureMethod: String, Codable, Sendable {
case voiceProcessing
}
+/// Whether microphone audio from a capture era is safe to use as speech evidence.
+/// This is deliberately independent of speaker identity or microphone ownership.
+nonisolated enum MeetingMicrophoneEchoProtection: String, Codable, Sendable {
+ case voiceProcessed
+ case acousticallyClosed
+ /// Pre-v9 audio that was historically transcribed without route evidence. It remains eligible
+ /// for backward compatibility, but must never be rewritten as positively verified protection.
+ case legacyUnclassified
+ case unprotected
+
+ var admitsTranscript: Bool {
+ self != .unprotected
+ }
+}
+
nonisolated struct MeetingClockDriftRecord: Codable, Equatable, Sendable {
var cumulativeAbsorbedSeconds: Double
var elapsedValidHostSeconds: Double
@@ -367,9 +393,67 @@ nonisolated struct MeetingCaptureEra: Codable, Equatable, Sendable {
var deviceUID: String?
var deviceName: String?
var roleAtElection: MeetingMicrophoneRole
+ var echoProtection: MeetingMicrophoneEchoProtection = .unprotected
var startSeconds: Double
var settledConfig: MeetingMicrophoneSettledConfig? = nil
var clockDrift: MeetingClockDriftRecord? = nil
+
+ private enum CodingKeys: String, CodingKey {
+ case method, deviceUID, deviceName, roleAtElection, echoProtection
+ case startSeconds, settledConfig, clockDrift
+ }
+
+ init(
+ method: MeetingAudioTrackCaptureMethod,
+ deviceUID: String?,
+ deviceName: String?,
+ roleAtElection: MeetingMicrophoneRole,
+ echoProtection: MeetingMicrophoneEchoProtection = .unprotected,
+ startSeconds: Double,
+ settledConfig: MeetingMicrophoneSettledConfig? = nil,
+ clockDrift: MeetingClockDriftRecord? = nil
+ ) {
+ self.method = method
+ self.deviceUID = deviceUID
+ self.deviceName = deviceName
+ self.roleAtElection = roleAtElection
+ self.echoProtection = echoProtection
+ self.startSeconds = startSeconds
+ self.settledConfig = settledConfig
+ self.clockDrift = clockDrift
+ }
+
+ init(from decoder: Decoder) throws {
+ let container = try decoder.container(keyedBy: CodingKeys.self)
+ self.method = try container.decode(MeetingAudioTrackCaptureMethod.self, forKey: .method)
+ self.deviceUID = try container.decodeIfPresent(String.self, forKey: .deviceUID)
+ self.deviceName = try container.decodeIfPresent(String.self, forKey: .deviceName)
+ self.roleAtElection = try container.decodeIfPresent(MeetingMicrophoneRole.self, forKey: .roleAtElection) ?? .unknown
+ let rawProtection = try container.decodeIfPresent(String.self, forKey: .echoProtection)
+ if let rawProtection {
+ self.echoProtection = MeetingMicrophoneEchoProtection(rawValue: rawProtection) ?? .unprotected
+ } else {
+ // Version-8 manifests predate echo-admission metadata and were already eligible for
+ // transcription. Preserve eligibility without laundering unknown provenance into a
+ // positive closed-route assertion when the manifest is written again.
+ self.echoProtection = self.method == .voiceProcessing ? .voiceProcessed : .legacyUnclassified
+ }
+ self.startSeconds = try container.decode(Double.self, forKey: .startSeconds)
+ self.settledConfig = try container.decodeIfPresent(MeetingMicrophoneSettledConfig.self, forKey: .settledConfig)
+ self.clockDrift = try container.decodeIfPresent(MeetingClockDriftRecord.self, forKey: .clockDrift)
+ }
+
+ func encode(to encoder: Encoder) throws {
+ var container = encoder.container(keyedBy: CodingKeys.self)
+ try container.encode(self.method, forKey: .method)
+ try container.encodeIfPresent(self.deviceUID, forKey: .deviceUID)
+ try container.encodeIfPresent(self.deviceName, forKey: .deviceName)
+ try container.encode(self.roleAtElection, forKey: .roleAtElection)
+ try container.encode(self.echoProtection, forKey: .echoProtection)
+ try container.encode(self.startSeconds, forKey: .startSeconds)
+ try container.encodeIfPresent(self.settledConfig, forKey: .settledConfig)
+ try container.encodeIfPresent(self.clockDrift, forKey: .clockDrift)
+ }
}
/// Consumes and discards one malformed `captureEras` array element.
@@ -475,13 +559,27 @@ extension MeetingAudioTrack: Codable {
guard container.contains(key) else { return nil }
var unkeyed = try container.nestedUnkeyedContainer(forKey: key)
var eras: [MeetingCaptureEra] = []
+ var encounteredMalformedEra = false
while !unkeyed.isAtEnd {
if let era = try? unkeyed.decode(MeetingCaptureEra.self) {
eras.append(era)
} else {
+ encounteredMalformedEra = true
_ = try? unkeyed.decode(MeetingCaptureEraSkippedElement.self)
}
}
+ if encounteredMalformedEra {
+ // Skipping an unknown middle era would let a neighbouring protected era extend across
+ // an interval whose safety is unknowable. Fail closed for the complete legacy track.
+ return [MeetingCaptureEra(
+ method: .avCaptureSession,
+ deviceUID: nil,
+ deviceName: nil,
+ roleAtElection: .unknown,
+ echoProtection: .unprotected,
+ startSeconds: 0
+ )]
+ }
return eras
}
@@ -640,6 +738,8 @@ nonisolated struct MeetingSession: Codable, Identifiable, Equatable, Sendable {
var audioTracks: [MeetingAudioTrack]
var speakers: [MeetingSessionSpeaker]
var transcriptSegments: [MeetingTranscriptSegment]
+ /// Audio intervals intentionally excluded from the transcript. Optional for schema-1 sessions.
+ var transcriptCoverageGaps: [MeetingTranscriptCoverageGap]? = nil
var retention: MeetingRetentionState
var processingAttempts: [MeetingProcessingAttempt]
var updatedAt: Date
@@ -669,6 +769,7 @@ nonisolated struct MeetingSession: Codable, Identifiable, Equatable, Sendable {
self.audioTracks = []
self.speakers = []
self.transcriptSegments = []
+ self.transcriptCoverageGaps = nil
// Informational; coordinator stamps it post-init from the current policy.
self.retention = MeetingRetentionState(
audioDeletedAt: nil,
@@ -1016,6 +1117,7 @@ nonisolated struct MeetingProcessingResult: Sendable {
var segments: [MeetingTranscriptSegment]
var attempt: MeetingProcessingAttempt
var skippedChunkIDs: [MeetingAudioChunkID] = []
+ var coverageGaps: [MeetingTranscriptCoverageGap] = []
}
/// Resume point written after the application-audio pass so a retry can skip re-diarizing and
diff --git a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
index 321796e6..7d80406f 100644
--- a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift
@@ -656,7 +656,7 @@ private final class MeetingProviderLanguagePin {
final class MeetingProcessingPipeline: MeetingProcessingControlling {
/// Bump whenever classification rules change so a resumed run can't mix rules mid-session.
/// Era-awareness did NOT bump this: the mic pass re-runs every time, and multi-era tracks cannot predate this build.
- static let pipelineVersion = 8
+ static let pipelineVersion = 10
/// Per-turn engines starve on short turns, and an all-empty chunk collapses to unlabeled.
static let perTurnTurnMergeGapSeconds = 5.0
@@ -895,6 +895,10 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
var diarizationObservationKey: String? = nil
var start: TimeInterval
var end: TimeInterval
+ /// Original capture clock coordinates. Echo-protection era boundaries live in this domain;
+ /// display/merge timestamps above may be de-drifted.
+ var rawStart: TimeInterval? = nil
+ var rawEnd: TimeInterval? = nil
var text: String
var overlapsRemote: Bool
/// Loudness of this turn's microphone audio, for the near-field gate. Zero when unmeasured.
@@ -1467,7 +1471,14 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
progress(.transcribing)
var microphoneSkippedChunkIDs: [MeetingAudioChunkID] = []
+ var coverageGaps: [MeetingTranscriptCoverageGap] = []
if let microphoneTrack = session.audioTracks.first(where: { $0.kind == .microphone }) {
+ if session.mode == .onlineCall {
+ coverageGaps = Self.microphoneCoverageGaps(
+ track: microphoneTrack,
+ origin: context.origin
+ )
+ }
try await self.processMicrophoneTrack(
microphoneTrack,
session: session,
@@ -1500,7 +1511,8 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
speakers: accumulator.speakers,
segments: accumulator.segments,
attempt: attempt,
- skippedChunkIDs: skippedChunkIDs
+ skippedChunkIDs: skippedChunkIDs,
+ coverageGaps: coverageGaps
)
}
@@ -1797,6 +1809,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
try await self.processMicrophoneChunk(
chunk,
track: track,
+ requiresEchoProtection: session.mode == .onlineCall,
eras: eras,
era0AnchorRelative: era0AnchorRelative,
diarizer: diarizer,
@@ -1821,12 +1834,26 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
}
}
- // Apply the existing text-only echo evidence before building microphone prototypes.
- // A chunk-local label represented only by confirmed far-end echo must not become a
- // first-class voice profile or compete in the later "You" election.
- let profileEligibleTurns = session.selectedMicrophone.role == .personal
- ? Self.rejectingFarField(stagedTurns, eras: eras)
+ let admittedTurns = session.mode == .onlineCall
+ ? stagedTurns.filter {
+ Self.microphoneIntervalIsAdmitted(
+ start: $0.rawStart ?? $0.start,
+ end: $0.rawEnd ?? $0.end,
+ eras: eras
+ )
+ }
: stagedTurns
+ if admittedTurns.count != stagedTurns.count {
+ DebugLogger.shared.warning(
+ "Meeting microphone admission excluded \(stagedTurns.count - admittedTurns.count) turn(s) intersecting unprotected capture eras.",
+ source: "MeetingProcessingPipeline"
+ )
+ }
+ stagedTurns = admittedTurns
+
+ // Echo evidence remains the only semantic filter before prototype construction. Relative
+ // loudness is not ownership: a quieter person on the microphone side is still legitimate.
+ let profileEligibleTurns = stagedTurns
let preclassified = Self.classifyMicrophoneTurns(
profileEligibleTurns,
eras: eras,
@@ -1869,6 +1896,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
private func processMicrophoneChunk(
_ chunk: MeetingAudioChunk,
track: MeetingAudioTrack,
+ requiresEchoProtection: Bool,
eras: [MeetingCaptureEra],
era0AnchorRelative: TimeInterval,
diarizer: SpeakerDiarizationService,
@@ -2005,6 +2033,8 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
diarizationObservationKey: observationKey,
start: start,
end: end,
+ rawStart: rawStart,
+ rawEnd: chunkOffset + turn.endSeconds,
text: text,
overlapsRemote: overlapsRemote,
rms: turnLoudness.indices.contains(offset) ? turnLoudness[offset] : 0,
@@ -2041,6 +2071,15 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
languagePin: context.languagePin
) {
let duration = chunkDuration
+ guard !requiresEchoProtection || Self.microphoneIntervalIsAdmitted(
+ start: chunkOffset, end: chunkOffset + duration, eras: eras
+ ) else {
+ DebugLogger.shared.warning(
+ "Meeting microphone fallback transcript excluded because its chunk intersects an unprotected capture era.",
+ source: "MeetingProcessingPipeline"
+ )
+ return
+ }
let speakerID = accumulator.speaker(
key: "microphone-unknown",
displayName: "Microphone / Unknown",
@@ -2064,11 +2103,15 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
/// `captureEras[].startSeconds` is raw PTS, converted here to origin-relative; era 0 is pinned to `-infinity`.
nonisolated static func microphoneEras(for track: MeetingAudioTrack, origin: TimeInterval) -> [MeetingCaptureEra] {
guard let rawEras = track.captureEras, !rawEras.isEmpty else {
+ let legacyMethod = track.captureMethod ?? .avCaptureSession
return [MeetingCaptureEra(
- method: track.captureMethod ?? .avCaptureSession,
+ method: legacyMethod,
deviceUID: nil,
deviceName: nil,
roleAtElection: .unknown,
+ // A missing era array is a pre-v9 recording. Grandfather its previous eligibility
+ // while preserving that its route protection was never positively observed.
+ echoProtection: legacyMethod == .voiceProcessing ? .voiceProcessed : .legacyUnclassified,
startSeconds: -.infinity,
settledConfig: track.voiceProcessingConfig,
clockDrift: track.clockDrift
@@ -2093,6 +2136,48 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
eras.indices.contains(index + 1) ? eras[index + 1].startSeconds : nil
}
+ /// A diarized turn is indivisible at this stage. Any overlap with an unprotected era rejects
+ /// the complete turn; assigning solely from its start would leak audio across a splice.
+ nonisolated static func microphoneIntervalIsAdmitted(
+ start: TimeInterval,
+ end: TimeInterval,
+ eras: [MeetingCaptureEra]
+ ) -> Bool {
+ guard end > start, !eras.isEmpty else { return false }
+ for index in eras.indices {
+ let eraStart = eras[index].startSeconds
+ let eraEnd = Self.eraEndRelative(afterIndex: index, eras: eras) ?? .infinity
+ guard min(end, eraEnd) > max(start, eraStart) else { continue }
+ if !eras[index].echoProtection.admitsTranscript { return false }
+ }
+ return true
+ }
+
+ nonisolated static func microphoneCoverageGaps(
+ track: MeetingAudioTrack,
+ origin: TimeInterval
+ ) -> [MeetingTranscriptCoverageGap] {
+ guard let first = track.chunks.map(\.presentationStart.seconds).min(),
+ let last = track.chunks.map(\.presentationEnd.seconds).max(),
+ last > first
+ else { return [] }
+ let trackStart = first - origin
+ let trackEnd = last - origin
+ let eras = Self.microphoneEras(for: track, origin: origin)
+ return eras.indices.compactMap { index in
+ guard !eras[index].echoProtection.admitsTranscript else { return nil }
+ let start = max(trackStart, eras[index].startSeconds)
+ let end = min(trackEnd, Self.eraEndRelative(afterIndex: index, eras: eras) ?? trackEnd)
+ guard end > start else { return nil }
+ return MeetingTranscriptCoverageGap(
+ trackID: track.id,
+ start: start,
+ end: end,
+ reason: .unprotectedMicrophone
+ )
+ }
+ }
+
nonisolated static func classifyMicrophoneTurns(
_ turns: [StagedMicrophoneTurn],
eras: [MeetingCaptureEra],
@@ -2235,36 +2320,12 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
)
let prototypeSpeakerIDs = accumulator.embeddingIndexByTrack[.microphone]?.speakerIDs ?? []
- let localClusterID = session.selectedMicrophone.role == .personal
- ? Self.selectLocalCluster(from: classifiedTurns, prototypeSpeakerIDs: prototypeSpeakerIDs)
- : nil
- Self.logLocalSpeakerElection(
- turns: classifiedTurns,
- prototypeSpeakerIDs: prototypeSpeakerIDs,
- index: accumulator.embeddingIndexByTrack[.microphone],
- role: session.selectedMicrophone.role,
- elected: localClusterID,
- captureMethod: eras.last?.method,
- widenedIndices: widenedIndices,
- captureEras: track.captureEras
- )
+ _ = widenedIndices // retained as echo-classification diagnostics; no ownership election
Self.logEchoMonitorSummary(
turns: classifiedTurns,
monitor: echoMonitor,
applicationTrackHadSpeech: applicationTrackHadSpeech
)
- if let localClusterID,
- let evidenceTurn = classifiedTurns.first(where: { $0.clusterID == localClusterID })
- {
- accumulator.addResolvedSpeaker(
- id: localClusterID,
- key: "local-user",
- displayName: "You",
- clusterID: evidenceTurn.clusterLabel,
- trackKind: .microphone,
- isLocalUser: true
- )
- }
let unknownSpeakerID = accumulator.speaker(
key: "microphone-unknown",
displayName: "Microphone / Unknown",
@@ -2276,12 +2337,8 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
// echo-demoted turns stay in the anonymous bucket (bleed must not become a "person").
var micSpeakerOrdinalByCluster: [SessionSpeakerID: Int] = [:]
for turn in classifiedTurns {
- let isLocalCluster = turn.clusterID == localClusterID
- let isCleanLocalTurn = isLocalCluster && !turn.effectiveEcho
let speakerID: SessionSpeakerID
- if isCleanLocalTurn {
- speakerID = localClusterID ?? unknownSpeakerID
- } else if !isLocalCluster, !turn.effectiveEcho, prototypeSpeakerIDs.contains(turn.clusterID) {
+ if !turn.effectiveEcho, prototypeSpeakerIDs.contains(turn.clusterID) {
if micSpeakerOrdinalByCluster[turn.clusterID] == nil {
let ordinal = micSpeakerOrdinalByCluster.count + 1
micSpeakerOrdinalByCluster[turn.clusterID] = ordinal
@@ -2306,7 +2363,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {
end: turn.end,
text: turn.text,
overlap: turn.overlapsRemote ? .overlapsOtherTrack : .none,
- isLikelyEcho: isLocalCluster ? false : turn.effectiveEcho
+ isLikelyEcho: turn.effectiveEcho
))
}
}
diff --git a/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
index 6ed962b6..a7c6e891 100644
--- a/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
+++ b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift
@@ -128,6 +128,11 @@ final class MeetingSessionCoordinator: ObservableObject {
private var degradeReason: DegradeReason?
private var activityLease: MeetingAudioActivityLease?
+ /// Reservation held from the public start entry point through every async start step.
+ /// This closes the preflight/lease window where two starts could otherwise pass guards.
+ private var startReservation: UUID? {
+ willSet { if (newValue != nil) != (self.startReservation != nil) { self.objectWillChange.send() } }
+ }
private var liveTranscriptionCoordinator: MeetingLiveTranscriptionCoordinator?
private var captureGeneration: UUID?
private var operationGeneration: UUID?
@@ -188,13 +193,33 @@ final class MeetingSessionCoordinator: ObservableObject {
}
}
+ var hasPendingStart: Bool { self.startReservation != nil }
+
+ /// A recovery offer is not audio ownership. Only matching terminal states may
+ /// be displaced; preparing/stopping and inconsistent states remain blocked.
+ private var canBeginNewRecording: Bool {
+ guard self.activityLease == nil else { return false }
+ switch self.state {
+ case .idle, .completed:
+ return self.activeSession == nil
+ case let .failed(id, _):
+ return self.activeSession == nil ||
+ (self.activeSession?.id == id && self.activeSession?.state == .failed)
+ case let .interrupted(id):
+ return self.activeSession == nil ||
+ (self.activeSession?.id == id && self.activeSession?.state == .interrupted)
+ case .preparing, .recording, .recordingDegraded, .stopping, .processing:
+ return false
+ }
+ }
+
var isProcessing: Bool {
if case .processing = self.state { return true }
return false
}
private var hasBlockingActivityTask: Bool {
- self.stopTask != nil || self.interruptionTask != nil || self.terminationTask != nil || self.retryTask != nil
+ self.hasBlockingActivityTaskExceptStart || self.startReservation != nil
}
/// True when no capture/stop/interruption/termination/retry activity is in flight and the
@@ -274,9 +299,44 @@ final class MeetingSessionCoordinator: ObservableObject {
@discardableResult
func startRecording(configuration: MeetingCaptureConfiguration) async throws -> MeetingSession {
+ return try await self.startRecording(configuration: configuration, passiveOffer: nil, resolvingSourceID: nil)
+ }
+
+ private func startRecording(
+ configuration: MeetingCaptureConfiguration,
+ passiveOffer: MeetingSession?,
+ resolvingSourceID: MeetingSessionID?
+ ) async throws -> MeetingSession {
+ guard self.startReservation == nil else { throw MeetingCoordinatorError.recordingAlreadyActive }
+ guard !self.hasBlockingActivityTask,
+ self.canBeginNewRecording, !self.isDeleting
+ else { throw MeetingCoordinatorError.recordingAlreadyActive }
+ let reservation = UUID()
+ self.startReservation = reservation
+ defer { if self.startReservation == reservation { self.startReservation = nil } }
await self.ensureRestored()
+ let passiveOffer = passiveOffer ?? self.activeSession.flatMap { offer in
+ (offer.state == .interrupted || offer.state == .failed) ? offer : nil
+ }
+ return try await self.performStartRecording(
+ configuration: configuration, passiveOffer: passiveOffer,
+ resolvingSourceID: resolvingSourceID, reservation: reservation
+ )
+ }
+
+ private func performStartRecording(
+ configuration: MeetingCaptureConfiguration,
+ passiveOffer: MeetingSession?,
+ resolvingSourceID: MeetingSessionID?,
+ reservation: UUID
+ ) async throws -> MeetingSession {
+ try Task.checkCancellation()
try configuration.validate()
- guard self.activeSession == nil, !self.hasBlockingActivityTask else {
+ guard self.startReservation == reservation,
+ self.activeSession?.id == passiveOffer?.id,
+ !self.hasBlockingActivityTaskExceptStart,
+ self.canBeginNewRecording, !self.isDeleting
+ else {
throw MeetingCoordinatorError.recordingAlreadyActive
}
guard !self.isMutatingSession else {
@@ -287,7 +347,11 @@ final class MeetingSessionCoordinator: ObservableObject {
// open indefinitely; reserving meeting audio first would unnecessarily block dictation.
try await self.capture.preflightPermissions()
try Task.checkCancellation()
- guard self.activeSession == nil, !self.hasBlockingActivityTask else {
+ guard self.startReservation == reservation,
+ self.activeSession?.id == passiveOffer?.id,
+ !self.hasBlockingActivityTaskExceptStart,
+ self.canBeginNewRecording, !self.isDeleting
+ else {
throw MeetingCoordinatorError.recordingAlreadyActive
}
guard !self.isMutatingSession else {
@@ -297,6 +361,12 @@ final class MeetingSessionCoordinator: ObservableObject {
let lease = try await self.audioArbiter.acquireMeetingCapture()
do {
try Task.checkCancellation()
+ guard self.startReservation == reservation,
+ self.activeSession?.id == passiveOffer?.id,
+ !self.hasBlockingActivityTaskExceptStart,
+ self.canBeginNewRecording, !self.isDeleting, !self.isMutatingSession else {
+ throw CancellationError()
+ }
} catch {
await self.audioArbiter.release(lease)
throw error
@@ -306,6 +376,11 @@ final class MeetingSessionCoordinator: ObservableObject {
self.captureGeneration = generation
self.degradeReason = nil
self.operationGeneration = generation
+ if passiveOffer != nil {
+ self.activeSession = nil
+ self.trackHealth = [:]
+ self.state = .idle
+ }
let timebase = Self.makeTimebase()
var session = MeetingSession(configuration: configuration, timebase: timebase)
session.retention.retainAudioUntil = SettingsStore.shared.meetingAudioRetentionPolicy
@@ -332,6 +407,11 @@ final class MeetingSessionCoordinator: ObservableObject {
do {
try await self.store.create(session)
let sessionDirectory = try await self.store.sessionDirectory(for: session.id)
+ guard self.startReservation == reservation,
+ self.operationGeneration == generation,
+ self.terminationTask == nil,
+ !Task.isCancelled
+ else { throw CancellationError() }
captureStartAttempted = true
let startResult = try await self.capture.start(
session: session,
@@ -363,6 +443,36 @@ final class MeetingSessionCoordinator: ObservableObject {
self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) })
self.state = .recording(session.id)
try await self.store.save(session)
+ guard self.startReservation == reservation,
+ self.operationGeneration == generation, !Task.isCancelled else {
+ throw CancellationError()
+ }
+ if let resolvingSourceID {
+ // Keep the start reservation held while serializing the explicit
+ // Record Again resolution against queued corrections/deletes.
+ await self.flushQueuedPersistence()
+ guard self.startReservation == reservation,
+ self.operationGeneration == generation, !Task.isCancelled else {
+ throw CancellationError()
+ }
+ if var source = try? await self.store.load(id: resolvingSourceID) {
+ guard self.startReservation == reservation,
+ self.operationGeneration == generation, !Task.isCancelled else {
+ throw CancellationError()
+ }
+ source.recoveryResolvedAt = Date()
+ source.updatedAt = Date()
+ do {
+ try await self.store.save(source)
+ } catch {
+ DebugLogger.shared.log("Record Again source resolution failed; previous audio remains recoverable: \(error)")
+ }
+ }
+ }
+ guard self.startReservation == reservation,
+ self.operationGeneration == generation, !Task.isCancelled else {
+ throw CancellationError()
+ }
Task { @MainActor [weak self] in await self?.sweepExpiredAudio() }
return session
} catch {
@@ -408,10 +518,26 @@ final class MeetingSessionCoordinator: ObservableObject {
self.operationGeneration = nil
self.state = .failed(session.id, failure)
await self.releaseActivityLease()
+ if let passiveOffer,
+ self.startReservation == reservation,
+ self.terminationTask == nil,
+ !hasRecoverableAudio,
+ self.activeSession == nil
+ {
+ self.activeSession = passiveOffer
+ self.trackHealth = Dictionary(uniqueKeysWithValues: passiveOffer.audioTracks.map { ($0.kind, $0.health) })
+ self.state = passiveOffer.state == .interrupted
+ ? .interrupted(passiveOffer.id)
+ : .failed(passiveOffer.id, passiveOffer.failures.last ?? failure)
+ }
throw error
}
}
+ private var hasBlockingActivityTaskExceptStart: Bool {
+ self.stopTask != nil || self.interruptionTask != nil || self.terminationTask != nil || self.retryTask != nil
+ }
+
@discardableResult
func stopAndTranscribe() async throws -> MeetingSession {
if let stopTask = self.stopTask { return try await stopTask.value }
@@ -469,7 +595,8 @@ final class MeetingSessionCoordinator: ObservableObject {
self.terminationTask == nil,
!self.isProcessing,
!self.isRecording,
- !self.isDeleting
+ !self.isDeleting,
+ self.startReservation == nil
else { throw MeetingCoordinatorError.activityInProgress }
guard !self.isMutatingSession else { throw MeetingCoordinatorError.maintenanceInProgress }
@@ -495,7 +622,8 @@ final class MeetingSessionCoordinator: ObservableObject {
self.terminationTask == nil,
!self.isProcessing,
!self.isRecording,
- !self.isDeleting
+ !self.isDeleting,
+ self.startReservation == nil
else { throw MeetingCoordinatorError.activityInProgress }
guard !self.isMutatingSession else { throw MeetingCoordinatorError.maintenanceInProgress }
guard session.hasRetryableAudio else { throw MeetingCoordinatorError.noRecoverableAudio }
@@ -599,45 +727,19 @@ final class MeetingSessionCoordinator: ObservableObject {
_ snapshot: MeetingSession,
configuration: MeetingCaptureConfiguration
) async throws -> MeetingSession {
- self.activeSession = nil
- self.trackHealth = [:]
- self.state = .idle
- do {
- let newSession = try await self.startRecording(configuration: configuration)
- var resolved = snapshot
- resolved.recoveryResolvedAt = Date()
- resolved.updatedAt = Date()
- try? await self.store.save(resolved)
- Task { @MainActor [weak self] in await self?.sweepExpiredAudio() }
+ let newSession = try await self.startRecording(
+ configuration: configuration, passiveOffer: snapshot, resolvingSourceID: snapshot.id
+ )
return newSession
- } catch {
- if self.activeSession == nil {
- self.activeSession = snapshot
- self.trackHealth = Dictionary(uniqueKeysWithValues: snapshot.audioTracks.map { ($0.kind, $0.health) })
- self.state = snapshot.state == .interrupted
- ? .interrupted(snapshot.id)
- : .failed(snapshot.id, snapshot.failures.last ?? Self.failure(from: error, domain: .capture, recoverable: true))
- }
- throw error
- }
}
private func recordAgainFromHistory(
sourceID: MeetingSessionID,
configuration: MeetingCaptureConfiguration
) async throws -> MeetingSession {
- let newSession = try await self.startRecording(configuration: configuration)
- if var source = try? await self.store.load(id: sourceID) {
- source.recoveryResolvedAt = Date()
- source.updatedAt = Date()
- do {
- try await self.store.save(source)
- Task { @MainActor [weak self] in await self?.sweepExpiredAudio() }
- } catch {
- DebugLogger.shared.log("recordAgain: could not stamp recoveryResolvedAt for \(sourceID): \(error)")
- }
- }
- return newSession
+ return try await self.startRecording(
+ configuration: configuration, passiveOffer: nil, resolvingSourceID: sourceID
+ )
}
// MARK: - Transcript corrections
@@ -1150,6 +1252,9 @@ final class MeetingSessionCoordinator: ObservableObject {
}
private func performShutdownForTermination() async {
+ // Invalidate a suspended permission/lease start before draining capture. Its
+ // post-await ownership checks will then release any acquired lease and abort.
+ self.startReservation = nil
// Bounded: termination must never hang on a stuck mutation save.
for _ in 0..<100 {
guard self.isMutatingSession else { break }
@@ -1312,6 +1417,7 @@ final class MeetingSessionCoordinator: ObservableObject {
}
session.speakers = result.speakers
session.transcriptSegments = result.segments
+ session.transcriptCoverageGaps = result.coverageGaps.isEmpty ? nil : result.coverageGaps
session.processingAttempts.removeAll {
$0.completedAt == nil && $0.id != result.attempt.id
}
@@ -1337,6 +1443,21 @@ final class MeetingSessionCoordinator: ObservableObject {
}
}
}
+ if !result.coverageGaps.isEmpty {
+ let excludedSeconds = result.coverageGaps.reduce(0.0) { total, gap in
+ total + max(0, gap.end - gap.start)
+ }
+ session.events.append(MeetingSessionEvent(
+ id: UUID(),
+ occurredAt: Date(),
+ kind: .microphoneChanged,
+ trackID: result.coverageGaps.first?.trackID,
+ detail: String(
+ format: "%.1f seconds of unprotected microphone audio were excluded from the transcript.",
+ excludedSeconds
+ )
+ ))
+ }
session.state = .completed
session.updatedAt = Date()
await self.flushQueuedPersistence()
diff --git a/Sources/Fluid/UI/MeetingLiveCaptionBubbles.swift b/Sources/Fluid/UI/MeetingLiveCaptionBubbles.swift
index e2fa0f56..1815059d 100644
--- a/Sources/Fluid/UI/MeetingLiveCaptionBubbles.swift
+++ b/Sources/Fluid/UI/MeetingLiveCaptionBubbles.swift
@@ -122,30 +122,34 @@ struct MeetingLiveBubbleRow: View, Equatable {
lhs.row == rhs.row && lhs.hidesPartialFromAccessibility == rhs.hidesPartialFromAccessibility
}
- private var isYou: Bool { self.row.speaker == .you }
+ private var isMicrophone: Bool { self.row.speaker == .you }
- /// Same washes as the final transcript: accent for You, the first speaker-palette pastel for Them.
+ private var sourceLabel: String {
+ self.isMicrophone ? "Microphone" : "Meeting audio"
+ }
+
+ /// Distinguish capture sources without claiming that either source belongs to a person.
private var fill: Color {
- let base = self.isYou ? self.theme.palette.accent : MeetingSpeakerPalette.tint(forSpeakerIndex: 0)
+ let base = self.isMicrophone ? self.theme.palette.accent : MeetingSpeakerPalette.tint(forSpeakerIndex: 0)
return base.opacity(self.row.isPartial ? 0.05 : 0.10)
}
var body: some View {
- VStack(alignment: self.isYou ? .trailing : .leading, spacing: 4) {
+ VStack(alignment: self.isMicrophone ? .trailing : .leading, spacing: 4) {
if self.row.showsLabel {
- Text(self.isYou ? "You" : "Them")
+ Text(self.sourceLabel)
.font(self.theme.typography.captionStrong)
- .foregroundStyle(self.isYou ? self.theme.palette.accent : self.theme.palette.secondaryText)
+ .foregroundStyle(self.isMicrophone ? self.theme.palette.accent : self.theme.palette.secondaryText)
}
Text(self.row.text)
// The partial→final color swap solidifies in place; it must never animate.
.transaction { $0.animation = nil }
.meetingBubbleStyle(.final(fill: self.fill, foreground: self.row.isPartial ? self.theme.palette.secondaryText : self.theme.palette.primaryText))
}
- .frame(maxWidth: .infinity, alignment: self.isYou ? .trailing : .leading)
- .padding(self.isYou ? .leading : .trailing, 32)
+ .frame(maxWidth: .infinity, alignment: self.isMicrophone ? .trailing : .leading)
+ .padding(self.isMicrophone ? .leading : .trailing, 32)
.accessibilityElement(children: .combine)
- .accessibilityLabel("\(self.isYou ? "You" : "Them"), \(self.row.text)")
+ .accessibilityLabel("\(self.sourceLabel), \(self.row.text)")
.accessibilityHidden(self.hidesPartialFromAccessibility && self.row.isPartial)
}
}
diff --git a/Sources/Fluid/UI/MeetingOverlayModels.swift b/Sources/Fluid/UI/MeetingOverlayModels.swift
index 589771af..4328a627 100644
--- a/Sources/Fluid/UI/MeetingOverlayModels.swift
+++ b/Sources/Fluid/UI/MeetingOverlayModels.swift
@@ -43,6 +43,9 @@ enum MeetingOverlayPresentation: Equatable, Sendable {
/// moves as caption content grows or shrinks.
static let captionsFooterHeight: CGFloat = 35
+ /// Always reserved, even when hover controls are hidden, so text never moves on hover.
+ static let captionsControlsHeight: CGFloat = 38
+
/// Canonical caption text viewport height: the remainder of the captions canvas above the
/// footer. viewport + footer must equal `captions.visibleSize.height`.
static let captionsViewportHeight: CGFloat = MeetingOverlayPresentation.captions.visibleSize.height
@@ -62,9 +65,10 @@ enum MeetingOverlayPresentation: Equatable, Sendable {
- (2 * MeetingOverlayPresentation.captionsHorizontalInset)
/// Caption text content height: the viewport height minus the top and bottom insets. The live
- /// text and empty state are clipped to exactly this rectangle so an overflowing fixedSize Text
- /// can never consume the top or bottom inset.
+ /// renderer selects complete recent lines within this rectangle; the empty state and safety
+ /// clip use the same bounds so neither can consume the top or bottom inset.
static let captionsContentHeight: CGFloat = MeetingOverlayPresentation.captionsViewportHeight
+ - MeetingOverlayPresentation.captionsControlsHeight
- MeetingOverlayPresentation.captionsTopInset
- MeetingOverlayPresentation.captionsBottomInset
}
diff --git a/Sources/Fluid/UI/MeetingRecordingPill.swift b/Sources/Fluid/UI/MeetingRecordingPill.swift
index 7e69e5c0..536ffd3a 100644
--- a/Sources/Fluid/UI/MeetingRecordingPill.swift
+++ b/Sources/Fluid/UI/MeetingRecordingPill.swift
@@ -406,6 +406,9 @@ struct MeetingRecordingPillContent: View {
let rows = MeetingLiveBubbleComposer.rows(for: self.coordinator.liveTranscript)
return VStack(spacing: 0) {
VStack(spacing: 0) {
+ Color.clear
+ .frame(height: MeetingOverlayPresentation.captionsControlsHeight)
+
Color.clear
.frame(height: MeetingOverlayPresentation.captionsTopInset)
@@ -422,24 +425,24 @@ struct MeetingRecordingPillContent: View {
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .center)
.accessibilityHidden(true)
} else {
- Text(self.rollingSubtitle(rows: rows))
- .font(.system(size: 14, design: .monospaced))
- .lineSpacing(4)
- .foregroundStyle(Color.white.opacity(0.94))
- // fixedSize is load-bearing — a height-constrained Text truncates its END,
- // freezing the display on the oldest words instead of clipping the top.
- .fixedSize(horizontal: false, vertical: true)
- .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .bottomLeading)
- .accessibilityHidden(true)
+ MeetingRollingCaptionView(
+ text: self.rollingSubtitle(rows: rows),
+ foregroundColor: NSColor.white.withAlphaComponent(0.94)
+ )
+ .frame(
+ width: MeetingOverlayPresentation.captionsContentWidth,
+ height: MeetingOverlayPresentation.captionsContentHeight,
+ alignment: .bottomLeading
+ )
+ .accessibilityHidden(true)
}
}
.frame(
width: MeetingOverlayPresentation.captionsContentWidth,
height: MeetingOverlayPresentation.captionsContentHeight
)
- // Clip strictly to the content rectangle so an overflowing fixedSize Text can never
- // consume the top or bottom inset — clipping only at the outer viewport let overflow
- // eat into the top inset before reaching that boundary.
+ // Clip strictly to the content rectangle; the TextKit view itself also draws only
+ // the selected suffix inside this frame.
.clipped()
Color.clear
@@ -462,7 +465,7 @@ struct MeetingRecordingPillContent: View {
height: MeetingOverlayPresentation.captions.visibleSize.height
)
.background(self.captionsSurface)
- // Pure text island; controls surface only on hover so it reads as subtitles, not a widget.
+ // Hover controls stay inside their permanently reserved top strip, never over text.
.overlay(alignment: .topTrailing) {
if self.isHoveringStrip {
HStack(spacing: 8) {
@@ -567,9 +570,9 @@ struct MeetingRecordingPillContent: View {
return NSApplication.shared.applicationIconImage
}
- /// One rolling word stream across all turns — no bubbles, no speaker structure, just the last
- /// words spoken. Oldest words fall off the front; "—" marks a speaker change, subtitle-style.
- private func rollingSubtitle(rows: [MeetingLiveBubbleComposer.Row], maxWords: Int = 42) -> String {
+ /// One rolling stream across the most recent rows. TextKit selects complete wrapped lines;
+ /// this input bound prevents an ever-growing live transcript from retaining excess storage.
+ private func rollingSubtitle(rows: [MeetingLiveBubbleComposer.Row]) -> String {
var words: [String] = []
var previousSpeaker: MeetingLiveSpeaker?
for row in rows.suffix(8) {
@@ -577,9 +580,9 @@ struct MeetingRecordingPillContent: View {
words.append("—")
}
previousSpeaker = row.speaker
- words.append(contentsOf: row.text.split(separator: " ").map(String.init))
+ words.append(row.text)
}
- return words.suffix(maxWords).joined(separator: " ")
+ return String(words.joined(separator: " ").suffix(MeetingRollingCaptionLayout.maximumInputCharacters))
}
private func stopRecording() {
diff --git a/Sources/Fluid/UI/MeetingRollingCaptionView.swift b/Sources/Fluid/UI/MeetingRollingCaptionView.swift
new file mode 100644
index 00000000..b1337206
--- /dev/null
+++ b/Sources/Fluid/UI/MeetingRollingCaptionView.swift
@@ -0,0 +1,134 @@
+import AppKit
+import SwiftUI
+
+/// A fixed-size caption viewport. TextKit performs one unbounded layout, then the same layout
+/// manager draws only the maximal suffix of complete wrapped lines whose ink fits the viewport.
+struct MeetingRollingCaptionView: NSViewRepresentable {
+ let text: String
+ var foregroundColor: NSColor = .white
+
+ func makeNSView(context: Context) -> MeetingRollingCaptionNSView {
+ MeetingRollingCaptionNSView()
+ }
+
+ func updateNSView(_ nsView: MeetingRollingCaptionNSView, context: Context) {
+ nsView.foregroundColor = self.foregroundColor
+ nsView.text = self.text
+ }
+
+ func sizeThatFits(_ proposal: ProposedViewSize, nsView: MeetingRollingCaptionNSView, context: Context) -> CGSize {
+ MeetingRollingCaptionLayout.contentSize
+ }
+}
+
+struct MeetingRollingCaptionLayout {
+ static let contentSize = CGSize(width: MeetingOverlayPresentation.captionsContentWidth, height: MeetingOverlayPresentation.captionsContentHeight)
+ static let font = NSFont.monospacedSystemFont(ofSize: 14, weight: .regular)
+ static let lineSpacing: CGFloat = 4
+ static let maximumInputCharacters = 4096
+
+ let attributedText: NSAttributedString
+ let textStorage: NSTextStorage
+ let selectedGlyphRange: NSRange
+ let translation: CGPoint
+ let inkBounds: CGRect
+ let layoutManager: NSLayoutManager
+
+ static func inputSuffix(_ text: String) -> String {
+ String(text.suffix(Self.maximumInputCharacters))
+ }
+
+ static func make(_ text: String, width: CGFloat = contentSize.width, height: CGFloat = contentSize.height, foregroundColor: NSColor = .white) -> MeetingRollingCaptionLayout {
+ let boundedText = Self.inputSuffix(text)
+ let paragraph = NSMutableParagraphStyle()
+ paragraph.lineSpacing = Self.lineSpacing
+ paragraph.lineBreakMode = .byWordWrapping
+ let attributed = NSAttributedString(string: boundedText, attributes: [
+ .font: Self.font,
+ .foregroundColor: foregroundColor,
+ .paragraphStyle: paragraph,
+ ])
+ let storage = NSTextStorage(attributedString: attributed)
+ let manager = NSLayoutManager()
+ let container = NSTextContainer(size: NSSize(width: width, height: 10_000_000))
+ container.lineFragmentPadding = 0
+ storage.addLayoutManager(manager)
+ manager.addTextContainer(container)
+ // Font fallback and RTL glyphs can extend past their line container. Refine the
+ // wrap width, never the viewport or font, until the actual selected ink fits.
+ // Every pass lays out the original bounded string; drawing retains that same manager.
+ for _ in 0..<8 {
+ guard width > 0, height > 0, container.containerSize.width > 0 else { break }
+ manager.ensureLayout(for: container)
+ var lines: [(glyphRange: NSRange, ink: CGRect)] = []
+ manager.enumerateLineFragments(forGlyphRange: NSRange(location: 0, length: manager.numberOfGlyphs)) { _, usedRect, container, glyphRange, _ in
+ guard glyphRange.length > 0 else { return }
+ let ink = usedRect.union(manager.boundingRect(forGlyphRange: glyphRange, in: container))
+ lines.append((glyphRange, ink))
+ }
+ guard !lines.isEmpty else { break }
+ var first = lines.count
+ var selectedInk = CGRect.null
+ for index in stride(from: lines.count - 1, through: 0, by: -1) {
+ let candidate = selectedInk.union(lines[index].ink)
+ if candidate.height > height { break }
+ selectedInk = candidate
+ first = index
+ }
+ guard first < lines.count else { break }
+ if selectedInk.width > width {
+ container.containerSize.width -= selectedInk.width - width + 0.5
+ continue
+ }
+ let selected = NSUnionRange(lines[first].glyphRange, lines[lines.count - 1].glyphRange)
+ let translation = CGPoint(x: -selectedInk.minX, y: height - selectedInk.maxY)
+ return Self(attributedText: attributed, textStorage: storage, selectedGlyphRange: selected, translation: translation, inkBounds: selectedInk.offsetBy(dx: translation.x, dy: translation.y), layoutManager: manager)
+ }
+ // A glyph taller/wider than the entire viewport cannot be shown as a complete line.
+ return Self(attributedText: attributed, textStorage: storage, selectedGlyphRange: NSRange(location: 0, length: 0), translation: .zero, inkBounds: .zero, layoutManager: manager)
+ }
+}
+
+final class MeetingRollingCaptionNSView: NSView {
+ var text = "" { didSet { if oldValue != text { invalidateCaption() } } }
+ var foregroundColor = NSColor.white { didSet { if !oldValue.isEqual(foregroundColor) { invalidateCaption() } } }
+ private var cachedText = ""
+ private var cachedSize: CGSize = .zero
+ private var cachedColor: NSColor = .clear
+ private var cachedLayout: MeetingRollingCaptionLayout?
+
+ override var isFlipped: Bool { true }
+ override var intrinsicContentSize: NSSize { NSSize(width: NSView.noIntrinsicMetric, height: NSView.noIntrinsicMetric) }
+
+ override func draw(_ dirtyRect: NSRect) {
+ let captionLayout = self.captionLayout
+ guard captionLayout.selectedGlyphRange.length > 0 else { return }
+ let context = NSGraphicsContext.current?.cgContext
+ context?.saveGState()
+ context?.clip(to: self.bounds)
+ context?.translateBy(x: captionLayout.translation.x, y: captionLayout.translation.y)
+ captionLayout.layoutManager.drawBackground(forGlyphRange: captionLayout.selectedGlyphRange, at: .zero)
+ captionLayout.layoutManager.drawGlyphs(forGlyphRange: captionLayout.selectedGlyphRange, at: .zero)
+ context?.restoreGState()
+ }
+
+ override func layout() {
+ super.layout()
+ if self.bounds.size != self.cachedSize { self.invalidateCaption() }
+ }
+
+ private var captionLayout: MeetingRollingCaptionLayout {
+ if let cachedLayout, cachedText == text, cachedSize == bounds.size, cachedColor.isEqual(foregroundColor) { return cachedLayout }
+ let value = MeetingRollingCaptionLayout.make(text, width: max(1, bounds.width), height: max(1, bounds.height), foregroundColor: foregroundColor)
+ cachedText = text
+ cachedSize = bounds.size
+ cachedColor = foregroundColor
+ cachedLayout = value
+ return value
+ }
+
+ private func invalidateCaption() {
+ cachedLayout = nil
+ needsDisplay = true
+ }
+}
diff --git a/Sources/Fluid/UI/MeetingTranscriptionView.swift b/Sources/Fluid/UI/MeetingTranscriptionView.swift
index b29674e4..7d50e451 100644
--- a/Sources/Fluid/UI/MeetingTranscriptionView.swift
+++ b/Sources/Fluid/UI/MeetingTranscriptionView.swift
@@ -28,7 +28,6 @@ struct MeetingTranscriptionSetupDraft: Equatable {
var title: String = Self.defaultTitle(mode: .onlineCall, applicationDisplayName: nil)
var selectedApplicationID: String?
var selectedMicrophoneID: String?
- var microphoneRole: MeetingMicrophoneRole = .unknown
/// Once true, the default title stops following the selected application/mode.
var titleWasEdited = false
@@ -39,7 +38,6 @@ struct MeetingTranscriptionSetupDraft: Equatable {
self.selectedApplicationID = nil
// Left unset here: selectPreferredMicrophone decides once identities load.
self.selectedMicrophoneID = nil
- self.microphoneRole = .unknown
}
static func defaultTitle(mode: MeetingCaptureMode, applicationDisplayName: String?) -> String {
@@ -521,7 +519,6 @@ struct MeetingTranscriptionView: View {
systemDefaultCaptureID: nil
)
self.setupDraft.selectedMicrophoneID = selection.deviceID
- self.setupDraft.microphoneRole = selection.role
}
private func selectPreferredApplication(from identities: [MeetingApplicationIdentity]) {
@@ -584,7 +581,7 @@ struct MeetingTranscriptionView: View {
}
var microphone = microphoneOption.identity
- microphone.role = self.setupDraft.mode == .onlineCall ? self.setupDraft.microphoneRole : .unknown
+ microphone.role = .unknown
let applicationOption = self.applications.first(where: { $0.id == self.setupDraft.selectedApplicationID })
if self.setupDraft.mode == .onlineCall, applicationOption == nil { return nil }
@@ -776,14 +773,14 @@ struct MeetingTranscriptionView: View {
let match = self.microphones.first(where: { $0.identity.coreAudioUID == coreAudioUID })
{
var identity = match.identity
- identity.role = session.selectedMicrophone.role
+ identity.role = .unknown
return identity
}
guard let match = self.microphones.first(where: { $0.identity.captureDeviceID == session.selectedMicrophone.captureDeviceID }) else {
return nil
}
var identity = match.identity
- identity.role = session.selectedMicrophone.role
+ identity.role = .unknown
return identity
}
@@ -982,7 +979,7 @@ struct MeetingTranscriptionView: View {
applicationDisplayName: applicationDisplayName,
microphoneCaptureDeviceID: microphone.identity.captureDeviceID,
microphoneCoreAudioUID: microphone.identity.coreAudioUID,
- microphoneRole: self.setupDraft.microphoneRole
+ microphoneRole: .unknown
)
let previousRetentionPolicy = settings.meetingAudioRetentionPolicy
@@ -1864,23 +1861,6 @@ private struct MeetingRecordingSettingsSheet: View {
.accessibilityLabel("Default meeting microphone")
}
- if self.draft.mode == .onlineCall {
- Divider()
- MeetingAdaptiveSetupRow(
- title: "Microphone use",
- detail: "A personal mic can identify clean speech as You."
- ) {
- Picker("Microphone use", selection: self.$draft.microphoneRole) {
- Text("Only me").tag(MeetingMicrophoneRole.personal)
- Text("Shared").tag(MeetingMicrophoneRole.shared)
- Text("Not sure").tag(MeetingMicrophoneRole.unknown)
- }
- .labelsHidden()
- .frame(width: 320, alignment: .trailing)
- .accessibilityLabel("Default microphone use")
- }
- }
-
Divider()
MeetingAdaptiveSetupRow(title: "Language") {
Text("English")
@@ -2090,17 +2070,8 @@ private struct MeetingSetupCanvas: View {
?? "Choose microphone"
}
- private var microphoneRoleName: String? {
- guard self.draft.mode == .onlineCall else { return nil }
- switch self.draft.microphoneRole {
- case .personal: return "Personal mic"
- case .shared: return "Shared mic"
- case .unknown: return "Mic use not set"
- }
- }
-
private var setupSummary: String {
- [self.applicationName, self.microphoneName, self.microphoneRoleName]
+ [self.applicationName, self.microphoneName]
.compactMap { $0 }
.joined(separator: " · ")
}
@@ -2371,8 +2342,8 @@ private struct MeetingRecordingCanvas: View {
}
}
-/// Live captions during recording only — speaker identity is unresolved here (You/Them by track,
-/// not by diarization); the offline pipeline replaces this view's content entirely on completion.
+/// Live captions during recording only — rows identify capture source, not a person; the offline
+/// diarization pipeline replaces this view's content entirely on completion.
private struct MeetingLiveTranscriptCard: View {
let snapshot: MeetingLiveTranscriptSnapshot
@@ -2787,12 +2758,12 @@ private struct MeetingResultCanvas: View {
return "\(speaker.displayName) (You)"
}
- // The whole mic feed reads as "my side" of the chat, not just segments attributed to You.
+ // Only explicitly persisted legacy ownership metadata receives the local-user treatment.
private func isLocalSegment(_ segment: MeetingTranscriptSegment) -> Bool {
guard let speakerID = segment.speakerID,
let speaker = self.session.speakers.first(where: { $0.id == speakerID })
else { return false }
- return speaker.isLocalUser || speaker.trackKind == .microphone
+ return speaker.isLocalUser
}
@ViewBuilder
diff --git a/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift b/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
index 66464f12..0b3f79fd 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingAutoDetectorTests.swift
@@ -28,9 +28,26 @@ private final class FakeMicActivity: MicActivitySignalProviding {
@MainActor
private final class FakeAudioProcessActivity: AudioProcessActivityProviding {
var activeBundleIdentifiers: Set<String> = []
-
- func isAudioActive(forBundleIdentifiers bundleIdentifiers: Set<String>) async -> Bool {
- !self.activeBundleIdentifiers.isDisjoint(with: bundleIdentifiers)
+ var snapshotOverride: AudioProcessActivitySnapshot?
+ var snapshotCount = 0
+ var beforeSnapshot: (() -> Void)?
+
+ func snapshot() async -> AudioProcessActivitySnapshot {
+ self.snapshotCount += 1
+ self.beforeSnapshot?()
+ if let snapshotOverride { return snapshotOverride }
+ let processes = self.activeBundleIdentifiers.map { bundle in
+ let owner = bundle == "us.zoom.caphost" ? "us.zoom.xos" : bundle
+ return AudioProcessDescriptor(processID: 55, bundleIdentifier: bundle,
+ executablePath: "/Applications/\(owner).app/Contents/Helpers/\(bundle)",
+ isInputRunning: bundle != "us.zoom.caphost", isOutputRunning: true)
+ }
+ let owners = self.activeBundleIdentifiers.map { bundle in
+ let owner = bundle == "us.zoom.caphost" ? "us.zoom.xos" : bundle
+ return MeetingProcessOwner(processID: 5, bundleIdentifier: owner,
+ bundlePath: "/Applications/\(owner).app")
+ }
+ return AudioProcessActivitySnapshot(processes: processes, owners: owners, queryState: .valid)
}
}
@@ -120,6 +137,183 @@ private final class DetectorHarness {
@MainActor
final class MeetingAutoDetectorTests: XCTestCase {
+ // MARK: Generic HAL ownership resolver
+
+ func testAudioResolverMapsHelpersInsideEachRegisteredNativeBundle() {
+ let apps = [
+ ("com.microsoft.teams2", "/Applications/Microsoft Teams.app"),
+ ("com.microsoft.teams", "/Applications/Classic Teams.app"),
+ ("us.zoom.xos", "/Applications/zoom.us.app"),
+ ("com.cisco.webexmeetingsapp", "/Applications/Webex.app"),
+ ("Cisco-Systems.Spark", "/Applications/Webex New.app"),
+ ]
+ for (bundle, root) in apps {
+ let result = MeetingAudioProcessResolver.activeOwnerInputByPID(snapshot: .init(
+ processes: [.init(processID: 900, bundleIdentifier: "helper.unknown", executablePath: root + "/Contents/Helpers/Module.app/Contents/MacOS/Module", isInputRunning: true, isOutputRunning: false)],
+ owners: [.init(processID: 42, bundleIdentifier: bundle, bundlePath: root)], queryState: .valid))
+ XCTAssertEqual(result?[42], true, "helper attribution should work for \(bundle)")
+ }
+ }
+
+ func testAudioResolverRejectsPrefixSpoofRelativeMissingAndAmbiguousOwners() {
+ func resolve(_ processPath: String?, owners: [MeetingProcessOwner]) -> [Int32: Bool]? {
+ MeetingAudioProcessResolver.activeOwnerInputByPID(snapshot: .init(
+ processes: [.init(processID: 1, bundleIdentifier: nil, executablePath: processPath, isInputRunning: true, isOutputRunning: false)],
+ owners: owners, queryState: .valid))
+ }
+ let owner = MeetingProcessOwner(processID: 10, bundleIdentifier: "com.microsoft.teams2", bundlePath: "/Applications/Teams.app")
+ XCTAssertEqual(resolve("/Applications/Teams.app.evil/Contents/MacOS/x", owners: [owner]), [:])
+ XCTAssertNil(resolve(nil, owners: [owner]))
+ XCTAssertNil(resolve("Applications/Teams.app/Contents/MacOS/x", owners: [owner]))
+ XCTAssertNil(resolve("/Applications/Teams.app/Contents/MacOS/x", owners: [owner, .init(processID: 11, bundleIdentifier: "com.microsoft.teams2", bundlePath: "/Applications/Teams.app")]))
+ XCTAssertEqual(resolve("/Applications/Teams.app/Contents/MacOS/x", owners: [.init(processID: 10, bundleIdentifier: "com.google.Chrome", bundlePath: "/Applications/Teams.app")]), [:], "browser owners must never become native audio owners")
+ XCTAssertEqual(resolve("/Applications/Teams.app/Contents/MacOS/x", owners: [.init(processID: 10, bundleIdentifier: "com.microsoft.teams2", bundlePath: "relative/Teams.app")]), [:])
+ }
+
+ func testAudioResolverRejectsBrowserOwnerAndUnknownQuery() {
+ let browser = MeetingProcessOwner(processID: 3, bundleIdentifier: "com.google.Chrome", bundlePath: "/Applications/Chrome.app")
+ let process = AudioProcessDescriptor(processID: 4, bundleIdentifier: nil, executablePath: "/Applications/Chrome.app/Contents/MacOS/Chrome", isInputRunning: true, isOutputRunning: true)
+ XCTAssertNil(MeetingAudioProcessResolver.activeOwnerInputByPID(snapshot: .init(processes: [process], owners: [browser], queryState: .unknown)))
+ }
+
+ func testAudioResolverRejectsSymlinkEscapingBundle() throws {
+ let root = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("meeting-resolver-\(UUID().uuidString)")
+ let app = root.appendingPathComponent("Teams.app")
+ let outside = root.appendingPathComponent("outside")
+ try FileManager.default.createDirectory(at: app, withIntermediateDirectories: true)
+ try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true)
+ try Data().write(to: outside.appendingPathComponent("Module"))
+ let link = app.appendingPathComponent("Contents")
+ try FileManager.default.createSymbolicLink(at: link, withDestinationURL: outside)
+ defer { try? FileManager.default.removeItem(at: root) }
+ let result = MeetingAudioProcessResolver.activeOwnerInputByPID(snapshot: .init(
+ processes: [.init(processID: 1, bundleIdentifier: nil, executablePath: link.path + "/Module", isInputRunning: true, isOutputRunning: false)],
+ owners: [.init(processID: 2, bundleIdentifier: "com.microsoft.teams2", bundlePath: app.path)], queryState: .valid))
+ XCTAssertEqual(result, [:])
+ }
+
+ func testGenericOutputOnlyHelperDoesNotCreateNewPrompt() async {
+ let h = DetectorHarness()
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.detector.handleWindowSnapshot([.init(processID: 5, windowID: 9, title: nil, layer: 0)], at: h.clock.now())
+ h.audioProcessActivity.snapshotOverride = .init(
+ processes: [.init(processID: 55, bundleIdentifier: "teams.helper", executablePath: "/Applications/com.microsoft.teams2.app/Contents/Helpers/Module", isInputRunning: false, isOutputRunning: true)],
+ owners: [.init(processID: 5, bundleIdentifier: "com.microsoft.teams2", bundlePath: "/Applications/com.microsoft.teams2.app")], queryState: .valid)
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ XCTAssertTrue(h.prompts.isEmpty)
+ }
+
+ func testNativePollTakesOneSnapshot() async {
+ let h = DetectorHarness()
+ h.detector.handleWorkspaceEvent(.init(kind: .launched, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.detector.handleWorkspaceEvent(.init(kind: .launched, bundleIdentifier: "us.zoom.xos", processID: 6), at: h.clock.now())
+ h.audioProcessActivity.snapshotOverride = .init(processes: [], owners: [], queryState: .valid)
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ XCTAssertEqual(h.audioProcessActivity.snapshotCount, 1)
+ }
+
+ func testSnapshotCannotApplyToReusedPIDAfterTermination() async {
+ let h = DetectorHarness()
+ h.detector.handleWorkspaceEvent(.init(kind: .launched, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.audioProcessActivity.snapshotOverride = .init(
+ processes: [.init(processID: 55, bundleIdentifier: "teams.helper", executablePath: "/Applications/Teams.app/Contents/Helpers/Module", isInputRunning: true, isOutputRunning: true)],
+ owners: [.init(processID: 5, bundleIdentifier: "com.microsoft.teams2", bundlePath: "/Applications/Teams.app")], queryState: .valid)
+ h.audioProcessActivity.beforeSnapshot = {
+ h.detector.handleWorkspaceEvent(.init(kind: .terminated, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.detector.handleWindowSnapshot([.init(processID: 5, windowID: 7, title: nil, layer: 0)], at: h.clock.now())
+ }
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ XCTAssertTrue(h.prompts.isEmpty)
+ }
+
+ func testUnknownAudioQueryCannotCreateNativePrompt() async {
+ let h = DetectorHarness()
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.detector.handleAudioProcessActivity(true, pid: 5, at: h.clock.now())
+ h.audioProcessActivity.snapshotOverride = .init(processes: [], owners: [], queryState: .unknown)
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ h.detector.handleWindowSnapshot([.init(processID: 5, windowID: 1, title: nil, layer: 0)], at: h.clock.now())
+ XCTAssertTrue(h.prompts.isEmpty)
+ }
+
+ func testUnknownQueryResetsContinuousInactivityAndPreservesEpisode() async {
+ let h = DetectorHarness()
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "us.zoom.xos", processID: 5), at: h.clock.now())
+ h.detector.handleWindowSnapshot([.init(processID: 5, windowID: 1, title: "Zoom", layer: 0)], at: h.clock.now())
+ h.detector.handleAudioProcessActivity(true, pid: 5, at: h.clock.now())
+ XCTAssertEqual(h.prompts.count, 1)
+ h.detector.handleAudioProcessActivity(false, pid: 5, at: h.clock.now())
+ h.detector.handleMicEdge(.init(isActive: false), at: h.clock.now())
+ h.advance(30)
+ h.audioProcessActivity.snapshotOverride = .init(processes: [], owners: [], queryState: .unknown)
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ h.advance(70)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertTrue(h.invalidated.isEmpty)
+ h.audioProcessActivity.snapshotOverride = .init(processes: [], owners: [], queryState: .valid)
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ h.detector.tick(at: h.clock.now())
+ XCTAssertTrue(h.invalidated.isEmpty)
+ h.advance(61)
+ h.detector.tick(at: h.clock.now())
+ XCTAssertEqual(h.invalidated, [h.prompts[0].episodeID])
+ }
+
+ func testUnknownNativeSnapshotDoesNotBlockBrowserURLDetection() async {
+ let h = DetectorHarness()
+ h.audioProcessActivity.snapshotOverride = .init(processes: [], owners: [], queryState: .unknown)
+ await h.detector.pollAudioProcessActivity(at: h.clock.now())
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "com.google.Chrome", processID: 8), at: h.clock.now())
+ h.detector.handleMicEdge(.init(isActive: true), at: h.clock.now())
+ h.detector.handleBrowserTabURL(.init(host: "meet.google.com", path: "/abc-defg-hij"), pid: 8, bundleIdentifier: "com.google.Chrome", at: h.clock.now())
+ XCTAssertEqual(h.prompts.count, 1)
+ XCTAssertEqual(h.prompts.first?.tier, .browserTier2)
+ }
+
+ func testLiveAudioProcessOwnershipProbe() async throws {
+ guard ProcessInfo.processInfo.environment["FV_DETECTOR_LIVE_PROBE"] == "1" else {
+ throw XCTSkip("Opt-in read-only live Core Audio ownership probe")
+ }
+ let snapshot = await CoreAudioProcessActivityProvider().snapshot()
+ XCTAssertEqual(snapshot.queryState, .valid)
+ let result = try XCTUnwrap(MeetingAudioProcessResolver.activeOwnerInputByPID(snapshot: snapshot))
+ for owner in snapshot.owners {
+ print("FV_DETECTOR_PROBE owner=\(owner.bundleIdentifier) pid=\(owner.processID) active=\(result[owner.processID] != nil) input=\(result[owner.processID] == true)")
+ }
+ for process in snapshot.processes where process.bundleIdentifier?.contains("teams") == true {
+ print("FV_DETECTOR_PROBE process=\(process.bundleIdentifier ?? "unknown") pid=\(process.processID) input=\(process.isInputRunning) output=\(process.isOutputRunning)")
+ if process.isInputRunning {
+ let owner = try XCTUnwrap(snapshot.owners.first { $0.bundleIdentifier == "com.microsoft.teams2" })
+ XCTAssertEqual(result[owner.processID], true)
+ }
+ }
+ }
+
+ func testActiveBeforeForegroundCanRecoverAfterStaleWindowDeadline() {
+ let h = DetectorHarness()
+ h.detector.handleWorkspaceEvent(.init(kind: .launched, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.detector.handleAudioProcessActivity(true, pid: 5, at: h.clock.now())
+ h.advance(31)
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "com.microsoft.teams2", processID: 5), at: h.clock.now())
+ h.detector.handleWindowSnapshot([.init(processID: 5, windowID: 2, title: nil, layer: 0)], at: h.clock.now())
+ h.detector.handleAudioProcessActivity(true, pid: 5, at: h.clock.now())
+ XCTAssertEqual(h.prompts.count, 1)
+ }
+
+ func testUnrelatedMicReleaseDoesNotEndProcessConfirmedEpisode() {
+ let h = DetectorHarness()
+ h.detector.handleWorkspaceEvent(.init(kind: .activated, bundleIdentifier: "us.zoom.xos", processID: 5), at: h.clock.now())
+ h.detector.handleWindowSnapshot([.init(processID: 5, windowID: 3, title: "Zoom", layer: 0)], at: h.clock.now())
+ h.detector.handleAudioProcessActivity(true, pid: 5, at: h.clock.now())
+ XCTAssertEqual(h.prompts.count, 1)
+ h.detector.handleMicEdge(.init(isActive: false), at: h.clock.now())
+ h.advance(61)
+ h.detector.tick(at: h.clock.now())
+ h.detector.handleAudioProcessActivity(true, pid: 5, at: h.clock.now())
+ XCTAssertEqual(h.prompts.count, 1)
+ }
+
// MARK: Prompt suppression
func testPromptSuppressionAllowsDetectedMeetingAppFullScreen() {
diff --git a/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift b/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift
index fbfe7f3f..b9035aec 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingMicrophoneCaptureTests.swift
@@ -824,8 +824,9 @@ extension MeetingMicrophoneCaptureTests {
XCTAssertNil(decoded.captureMethod)
}
- /// A single malformed era (unknown `method`) is skipped, not fatal to the whole track decode.
- func testUnknownEraMethodRawValueIsSkippedNotFatal() throws {
+ /// An unknown middle era must fail the complete track closed; skipping it would extend a
+ /// neighbouring protected era across audio whose capture safety is unknowable.
+ func testUnknownEraMethodFailsCompleteTrackClosed() throws {
let json = """
{
"id": "\(UUID().uuidString)",
@@ -842,8 +843,9 @@ extension MeetingMicrophoneCaptureTests {
}
"""
let decoded = try JSONDecoder().decode(MeetingAudioTrack.self, from: Data(json.utf8))
- XCTAssertEqual(decoded.captureEras?.count, 1, "the malformed era is skipped, not fatal")
+ XCTAssertEqual(decoded.captureEras?.count, 1)
XCTAssertEqual(decoded.captureEras?.first?.method, .avCaptureSession)
+ XCTAssertEqual(decoded.captureEras?.first?.echoProtection, .unprotected)
}
func testMultiEraTrackCaptureMethodIsAvCaptureSession() {
diff --git a/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift b/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift
index 8fbf2323..59adf457 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingOverlayModelsTests.swift
@@ -1,6 +1,8 @@
@testable import FluidVoice_Debug
+import AppKit
import CoreGraphics
import Foundation
+import SwiftUI
import XCTest
final class MeetingOverlayGeometryTests: XCTestCase {
@@ -28,8 +30,10 @@ final class MeetingOverlayGeometryTests: XCTestCase {
XCTAssertEqual(MeetingOverlayPresentation.captionsHorizontalInset, 16)
XCTAssertEqual(MeetingOverlayPresentation.captionsTopInset, 12)
XCTAssertEqual(MeetingOverlayPresentation.captionsBottomInset, 7)
+ XCTAssertEqual(MeetingOverlayPresentation.captionsControlsHeight, 38)
XCTAssertEqual(
- MeetingOverlayPresentation.captionsTopInset
+ MeetingOverlayPresentation.captionsControlsHeight
+ + MeetingOverlayPresentation.captionsTopInset
+ MeetingOverlayPresentation.captionsContentHeight
+ MeetingOverlayPresentation.captionsBottomInset,
MeetingOverlayPresentation.captionsViewportHeight
@@ -40,7 +44,64 @@ final class MeetingOverlayGeometryTests: XCTestCase {
MeetingOverlayPresentation.captions.visibleSize.width
)
XCTAssertEqual(MeetingOverlayPresentation.captionsContentWidth, 308)
- XCTAssertEqual(MeetingOverlayPresentation.captionsContentHeight, 102)
+ XCTAssertEqual(MeetingOverlayPresentation.captionsContentHeight, 64)
+ }
+
+ func testRollingCaptionLayoutUsesBoundedTextAndKeepsLatestGlyph() {
+ let text = String(repeating: "older words ", count: 500) + "最新の字幕 مرحبا 👩🏽‍💻 é"
+ let layout = MeetingRollingCaptionLayout.make(text)
+ XCTAssertLessThanOrEqual(layout.attributedText.string.count, MeetingRollingCaptionLayout.maximumInputCharacters)
+ XCTAssertNotNil(layout.layoutManager.textStorage)
+ XCTAssertEqual(layout.selectedGlyphRange.upperBound, layout.layoutManager.numberOfGlyphs)
+ XCTAssertEqual(MeetingRollingCaptionLayout.contentSize, CGSize(width: 308, height: 64))
+ XCTAssertTrue(CGRect(origin: .zero, size: MeetingRollingCaptionLayout.contentSize).contains(layout.inkBounds))
+ }
+
+ func testRollingCaptionLayoutHandlesShortUnicodeAndLongUnbrokenTokens() {
+ for text in ["", "短い字幕", "שלום RTL 😀 cafe\u{301}", String(repeating: "x", count: 1200)] {
+ let layout = MeetingRollingCaptionLayout.make(text)
+ XCTAssertNotNil(layout.layoutManager.textStorage)
+ XCTAssertTrue(layout.inkBounds.isEmpty || CGRect(origin: .zero, size: MeetingRollingCaptionLayout.contentSize).contains(layout.inkBounds))
+ }
+ }
+
+ @MainActor
+ func testCaptionHostingFrameNeverGrowsAcrossTextUpdates() {
+ func content(_ text: String) -> some View {
+ VStack(spacing: 0) {
+ Color.clear.frame(height: MeetingOverlayPresentation.captionsControlsHeight)
+ Color.clear.frame(height: MeetingOverlayPresentation.captionsTopInset)
+ MeetingRollingCaptionView(text: text)
+ .frame(width: MeetingOverlayPresentation.captionsContentWidth,
+ height: MeetingOverlayPresentation.captionsContentHeight)
+ Color.clear.frame(height: MeetingOverlayPresentation.captionsBottomInset)
+ Color.gray.frame(height: MeetingOverlayPresentation.captionsFooterHeight)
+ }
+ .frame(width: 340, height: 156)
+ }
+ let host = NSHostingView(rootView: content(""))
+ host.frame = CGRect(x: 0, y: 0, width: 340, height: 156)
+ for text in ["short", String(repeating: "caption growing 👩🏽‍💻 字幕 ", count: 500), "rewritten latest caption"] {
+ host.rootView = content(text)
+ host.layoutSubtreeIfNeeded()
+ XCTAssertEqual(host.frame.size, CGSize(width: 340, height: 156))
+ XCTAssertEqual(host.fittingSize.width, 340, accuracy: 0.01)
+ XCTAssertEqual(host.fittingSize.height, 156, accuracy: 0.01)
+ }
+ }
+
+ func testSelectedRangeStartsAtAWholeLineAndDropsOlderLines() {
+ let layout = MeetingRollingCaptionLayout.make(String(repeating: "Older words wrap into complete lines. ", count: 40) + "Newest words")
+ var lineRanges: [NSRange] = []
+ layout.layoutManager.enumerateLineFragments(forGlyphRange: NSRange(location: 0, length: layout.layoutManager.numberOfGlyphs)) { _, _, _, range, _ in
+ lineRanges.append(range)
+ }
+ XCTAssertGreaterThan(layout.selectedGlyphRange.location, 0)
+ XCTAssertTrue(lineRanges.contains { $0.location == layout.selectedGlyphRange.location })
+ XCTAssertEqual(layout.selectedGlyphRange.upperBound, layout.layoutManager.numberOfGlyphs)
+ XCTAssertLessThanOrEqual(layout.inkBounds.height, MeetingOverlayPresentation.captionsContentHeight)
+ let cannotFit = MeetingRollingCaptionLayout.make("A full line", height: 1)
+ XCTAssertEqual(cannotFit.selectedGlyphRange.length, 0, "Never fall back to a vertically clipped line")
}
func testUnclampedLayoutVisibleSurfaceMatchesCanonicalPresentationSize() {
diff --git a/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift b/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
index 61c7dfa9..cc2d29cd 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingRecoveryTests.swift
@@ -858,6 +858,213 @@ final class MeetingRecoveryTests: XCTestCase {
XCTAssertNil(persistedOld?.recoveryResolvedAt)
}
+ // A recoverable failed capture is passive after its lease is released; a later
+ // auto-detected start must be allowed to replace it without resolving its audio.
+ func testOrdinaryStartReplacesPassiveFailedOfferWithoutResolvingIt() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let old = self.makeSession(
+ state: .failed,
+ endedAt: Date(),
+ audioTracks: [self.makeMicrophoneTrack(chunks: [self.makeFinalizedChunk()])],
+ failures: [MeetingSessionFailure(id: UUID(), occurredAt: Date(), domain: .capture, code: "teams.noSpeech", message: "No speech", recoverable: true)]
+ )
+ try await store.create(old)
+
+ let capture = StubCaptureController()
+ capture.startResult = MeetingCaptureStartResult(tracks: [self.makeMicrophoneTrack(chunks: [])], firstPresentationTime: nil)
+ let arbiter = StubArbiter()
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: capture, processing: StubProcessingController(), audioArbiter: arbiter
+ )
+ await coordinator.ensureRestored()
+ XCTAssertEqual(coordinator.activeSession?.id, old.id)
+
+ let fresh = try await coordinator.startRecording(configuration: self.makeConfiguration(title: "Zoom"))
+ XCTAssertNotEqual(fresh.id, old.id)
+ XCTAssertEqual(coordinator.activeSession?.id, fresh.id)
+ let persistedOld = try await store.load(id: old.id)
+ XCTAssertNil(persistedOld?.recoveryResolvedAt)
+ XCTAssertTrue(persistedOld?.hasFinalizedAudio == true)
+ }
+
+ func testStartFailureRestoresPassiveOfferUnchanged() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let old = self.makeSession(
+ state: .interrupted,
+ endedAt: Date(),
+ audioTracks: [self.makeMicrophoneTrack(chunks: [self.makeFinalizedChunk()])]
+ )
+ try await store.create(old)
+ let capture = StubCaptureController()
+ capture.startError = MeetingCaptureError.applicationNotSelected
+ let arbiter = StubArbiter()
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: capture, processing: StubProcessingController(), audioArbiter: arbiter
+ )
+ await coordinator.ensureRestored()
+ do {
+ _ = try await coordinator.startRecording(configuration: self.makeConfiguration(title: "Retry"))
+ XCTFail("expected capture start to fail")
+ } catch {}
+ XCTAssertEqual(coordinator.activeSession?.id, old.id)
+ XCTAssertEqual(coordinator.activeSession?.audioTracks, old.audioTracks)
+ XCTAssertEqual(coordinator.state, .interrupted(old.id))
+ let persistedOld = try await store.load(id: old.id)
+ XCTAssertNil(persistedOld?.recoveryResolvedAt)
+ }
+
+ func testOrdinaryStartPreservesExpiredInterruptedAudioAndPendingAttempt() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let past = Date(timeIntervalSinceNow: -40 * 24 * 60 * 60)
+ let old = self.makeSession(
+ state: .interrupted, startedAt: past, endedAt: past,
+ audioTracks: [self.makeMicrophoneTrack(chunks: [self.makeFinalizedChunk()])],
+ processingAttempts: [MeetingProcessingAttempt(
+ id: UUID(), startedAt: past, completedAt: nil, stage: .pending,
+ pipelineVersion: MeetingProcessingPipeline.pipelineVersion,
+ asrProvider: nil, asrModel: nil, diarizationModel: nil,
+ lastCompletedTrackID: nil, errorCode: nil
+ )]
+ )
+ try await store.create(old)
+ let sessionDirectory = try await store.sessionDirectory(for: old.id)
+ let audio = sessionDirectory.appendingPathComponent("tracks/microphone/chunk_0.caf")
+ try FileManager.default.createDirectory(at: audio.deletingLastPathComponent(), withIntermediateDirectories: true)
+ try Data(repeating: 1, count: 128).write(to: audio)
+ let capture = StubCaptureController()
+ capture.startResult = MeetingCaptureStartResult(tracks: [self.makeMicrophoneTrack(chunks: [])], firstPresentationTime: nil)
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: capture, processing: StubProcessingController(), audioArbiter: StubArbiter()
+ )
+ await coordinator.ensureRestored()
+ let before = try await store.load(id: old.id)
+ _ = try await coordinator.startRecording(configuration: self.makeConfiguration(title: "Next app"))
+ await coordinator.sweepExpiredAudio()
+ let after = try await store.load(id: old.id)
+ XCTAssertNil(after?.recoveryResolvedAt)
+ XCTAssertNil(after?.retention.audioDeletedAt)
+ XCTAssertEqual(after?.processingAttempts, before?.processingAttempts)
+ XCTAssertEqual(after?.audioTracks, before?.audioTracks)
+ XCTAssertTrue(FileManager.default.fileExists(atPath: audio.path))
+ XCTAssertFalse(coordinator.hasPendingStart)
+ }
+
+ func testDeniedPermissionPreservesPassiveOfferAndReleasesStartReservation() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let old = self.makeSession(state: .interrupted, endedAt: Date(),
+ audioTracks: [self.makeMicrophoneTrack(chunks: [self.makeFinalizedChunk()])])
+ try await store.create(old)
+ let capture = StubCaptureController()
+ capture.preflightError = MeetingCaptureError.microphonePermissionDenied
+ capture.startResult = MeetingCaptureStartResult(tracks: [self.makeMicrophoneTrack(chunks: [])], firstPresentationTime: nil)
+ let arbiter = StubArbiter()
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: capture, processing: StubProcessingController(), audioArbiter: arbiter
+ )
+ await coordinator.ensureRestored()
+ let before = coordinator.activeSession
+ do {
+ _ = try await coordinator.startRecording(configuration: self.makeConfiguration())
+ XCTFail("Permission denial must reject the new recording")
+ } catch {
+ guard case MeetingCaptureError.microphonePermissionDenied = error else {
+ return XCTFail("Unexpected error: \(error)")
+ }
+ }
+ XCTAssertEqual(coordinator.activeSession?.id, before?.id)
+ XCTAssertEqual(coordinator.activeSession?.audioTracks, before?.audioTracks)
+ XCTAssertEqual(arbiter.acquireCount, 0)
+ XCTAssertFalse(coordinator.hasPendingStart)
+ capture.preflightError = nil
+ let fresh = try await coordinator.startRecording(configuration: self.makeConfiguration())
+ XCTAssertNotEqual(fresh.id, old.id)
+ XCTAssertFalse(coordinator.hasPendingStart)
+ }
+
+ func testConcurrentStartAndRecoveryActionsAreBlockedDuringPermissionPreflight() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let store = MeetingSessionStore(rootDirectory: dir)
+ let capture = StubCaptureController()
+ capture.startResult = MeetingCaptureStartResult(tracks: [self.makeMicrophoneTrack(chunks: [])], firstPresentationTime: nil)
+ var continuation: CheckedContinuation<Void, Never>?
+ capture.onPreflight = {
+ await withCheckedContinuation { continuation = $0 }
+ }
+ let arbiter = StubArbiter()
+ let coordinator = MeetingSessionCoordinator(
+ store: store, capture: capture, processing: StubProcessingController(), audioArbiter: arbiter
+ )
+ let start = Task { try await coordinator.startRecording(configuration: self.makeConfiguration()) }
+ while capture.preflightCount == 0 { await Task.yield() }
+
+ do {
+ _ = try await coordinator.retryProcessing(sessionID: UUID())
+ XCTFail("retry must be rejected while start is reserved")
+ } catch let error as MeetingCoordinatorError {
+ guard case .activityInProgress = error else { return XCTFail("expected activityInProgress, got \(error)") }
+ }
+ XCTAssertEqual(arbiter.acquireCount, 0)
+
+ do {
+ _ = try await coordinator.startRecording(configuration: self.makeConfiguration(title: "Second"))
+ XCTFail("duplicate start must be rejected while preflight is suspended")
+ } catch let error as MeetingCoordinatorError {
+ guard case .recordingAlreadyActive = error else {
+ return XCTFail("expected recordingAlreadyActive, got \(error)")
+ }
+ }
+ do {
+ try coordinator.resetForNewMeeting()
+ XCTFail("reset must be rejected while start is reserved")
+ } catch let error as MeetingCoordinatorError {
+ guard case .activityInProgress = error else {
+ return XCTFail("expected activityInProgress, got \(error)")
+ }
+ }
+ do {
+ try await coordinator.deleteSession(id: UUID())
+ XCTFail("delete must be rejected while start is reserved")
+ } catch let error as MeetingCoordinatorError {
+ guard case .activityInProgress = error else {
+ return XCTFail("expected activityInProgress, got \(error)")
+ }
+ }
+ continuation?.resume()
+ _ = try await start.value
+ XCTAssertEqual(capture.preflightCount, 1)
+ }
+
+ func testTerminationInvalidatesSuspendedStartBeforePermissionReturns() async throws {
+ let dir = self.makeTempDirectory()
+ defer { try? FileManager.default.removeItem(at: dir) }
+ let capture = StubCaptureController()
+ var continuation: CheckedContinuation<Void, Never>?
+ capture.onPreflight = { await withCheckedContinuation { continuation = $0 } }
+ let coordinator = MeetingSessionCoordinator(
+ store: MeetingSessionStore(rootDirectory: dir), capture: capture,
+ processing: StubProcessingController(), audioArbiter: StubArbiter()
+ )
+ let start = Task { try await coordinator.startRecording(configuration: self.makeConfiguration()) }
+ while capture.preflightCount == 0 { await Task.yield() }
+ await coordinator.shutdownForTermination()
+ continuation?.resume()
+ do {
+ _ = try await start.value
+ XCTFail("terminated coordinator must not start capture after preflight")
+ } catch {}
+ XCTAssertEqual(capture.preflightCount, 1)
+ XCTAssertEqual(capture.startCount, 0)
+ }
+
// MARK: - Slice 3: checkpoint fixtures
private func makeCheckpoint(
@@ -2309,6 +2516,7 @@ final class MeetingRecoveryTests: XCTestCase {
let dir = self.makeTempDirectory()
defer { try? FileManager.default.removeItem(at: dir) }
let capture = StubCaptureController()
+ capture.startResult = MeetingCaptureStartResult(tracks: [self.makeMicrophoneTrack(chunks: [])], firstPresentationTime: nil)
let arbiter = StubArbiter()
var continuation: CheckedContinuation<Void, Never>?
capture.onPreflight = {
@@ -2869,6 +3077,7 @@ private final class StubCaptureController: MeetingCaptureControlling, @unchecked
var preflightError: Error?
var onPreflight: (@Sendable () async -> Void)?
private(set) var preflightCount = 0
+ private(set) var startCount = 0
func preflightPermissions() async throws {
self.preflightCount += 1
@@ -2883,6 +3092,7 @@ private final class StubCaptureController: MeetingCaptureControlling, @unchecked
eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void,
liveAudioHandler: (@Sendable (MeetingAudioTrackKind, CMSampleBuffer) -> Void)?
) async throws -> MeetingCaptureStartResult {
+ self.startCount += 1
await self.onStart?()
if let startError { throw startError }
return self.startResult
diff --git a/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift b/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
index 402f312e..2df04d13 100644
--- a/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
+++ b/Tests/FluidDictationIntegrationTests/MeetingVoiceProcessingCaptureTests.swift
@@ -1,5 +1,6 @@
@testable import FluidVoice_Debug
import AVFoundation
+import CoreAudio
import CoreMedia
import Foundation
import ScreenCaptureKit
@@ -10,7 +11,7 @@ import XCTest
/// screen-recording/microphone permission and are not exercised here. This covers the pure
/// decision, gate, drift-eligibility, de-drift transform and Codable pieces of Phase 3.
final class MeetingVoiceProcessingCaptureTests: XCTestCase {
- // MARK: - Decision table (incl. role)
+ // MARK: - Decision table
private func mic(uid: String? = "mic-uid", role: MeetingMicrophoneRole = .personal) -> MeetingMicrophoneIdentity {
MeetingMicrophoneIdentity(
@@ -77,20 +78,137 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
XCTAssertEqual(decision, .voiceProcessing)
}
- func testDecisionDeclinesForSharedRole() {
+ func testDecisionIgnoresSharedRoleForCaptureSafety() {
let decision = MeetingCapturePathDecider.decide(
mode: .onlineCall, microphone: self.mic(role: .shared), outputRoute: self.viableRoute()
)
- guard case let .screenCaptureKit(reason) = decision else { return XCTFail("expected decline") }
- XCTAssertTrue(reason.contains("personal"))
+ XCTAssertEqual(decision, .voiceProcessing)
}
- func testDecisionDeclinesForUnknownRole() {
+ func testDecisionIgnoresUnknownRoleForCaptureSafety() {
let decision = MeetingCapturePathDecider.decide(
mode: .onlineCall, microphone: self.mic(role: .unknown), outputRoute: self.viableRoute()
)
- guard case let .screenCaptureKit(reason) = decision else { return XCTFail("expected decline") }
- XCTAssertTrue(reason.contains("personal"))
+ XCTAssertEqual(decision, .voiceProcessing)
+ }
+
+ func testRawRouteRequiresPositiveHeadphoneEvidence() {
+ let unknownBluetooth = MeetingOutputRouteSnapshot(
+ deviceExists: true, isBluetooth: true, isBuiltIn: false,
+ isHeadphonesDataSource: false, terminalTypes: []
+ )
+ XCTAssertEqual(MeetingRawMicrophoneProtection.classify(unknownBluetooth), .unprotected)
+
+ var externalWithoutDataSource = unknownBluetooth
+ externalWithoutDataSource.isHeadphonesDataSource = true
+ XCTAssertEqual(
+ MeetingRawMicrophoneProtection.classify(externalWithoutDataSource),
+ .unprotected,
+ "AudioDevice reports missing data-source properties pessimistically; that alone is not closed-route proof"
+ )
+
+ var headphones = unknownBluetooth
+ headphones.terminalTypes = [kAudioStreamTerminalTypeHeadphones]
+ XCTAssertEqual(MeetingRawMicrophoneProtection.classify(headphones), .acousticallyClosed)
+
+ let builtInHeadphoneJack = MeetingOutputRouteSnapshot(
+ deviceExists: true, isBluetooth: false, isBuiltIn: true,
+ isHeadphonesDataSource: true
+ )
+ XCTAssertEqual(MeetingRawMicrophoneProtection.classify(builtInHeadphoneJack), .acousticallyClosed)
+ }
+
+ func testTranscriptGateFailsClosedAcrossTransition() {
+ let gate = MeetingMicrophoneTranscriptGate(.voiceProcessed)
+ XCTAssertTrue(gate.admitsTranscript())
+ gate.update(.unprotected)
+ XCTAssertFalse(gate.admitsTranscript())
+ gate.update(.acousticallyClosed)
+ XCTAssertTrue(gate.admitsTranscript())
+ }
+
+ func testTurnCrossingUnprotectedEraIsRejectedAsAWhole() {
+ let eras = [
+ MeetingCaptureEra(
+ method: .voiceProcessing, deviceUID: "mic", deviceName: "Mic",
+ roleAtElection: .unknown, echoProtection: .voiceProcessed, startSeconds: -.infinity
+ ),
+ MeetingCaptureEra(
+ method: .avCaptureSession, deviceUID: "mic", deviceName: "Mic",
+ roleAtElection: .unknown, echoProtection: .unprotected, startSeconds: 10
+ ),
+ ]
+ XCTAssertTrue(MeetingProcessingPipeline.microphoneIntervalIsAdmitted(start: 8, end: 9.9, eras: eras))
+ XCTAssertFalse(MeetingProcessingPipeline.microphoneIntervalIsAdmitted(start: 9.9, end: 10.1, eras: eras))
+ XCTAssertFalse(MeetingProcessingPipeline.microphoneIntervalIsAdmitted(start: 10, end: 11, eras: eras))
+ }
+
+ func testLegacyVoiceProcessingEraWithoutProtectionRetainsPositiveVPIOEvidence() throws {
+ let data = Data(#"{"method":"voiceProcessing","roleAtElection":"personal","startSeconds":0}"#.utf8)
+ let era = try JSONDecoder().decode(MeetingCaptureEra.self, from: data)
+ XCTAssertEqual(era.echoProtection, .voiceProcessed)
+ }
+
+ func testLegacyRawEraWithoutProtectionPreservesPreviouslyEligibleRecording() throws {
+ let data = Data(#"{"method":"avCaptureSession","roleAtElection":"personal","startSeconds":0}"#.utf8)
+ let era = try JSONDecoder().decode(MeetingCaptureEra.self, from: data)
+ XCTAssertEqual(era.echoProtection, .legacyUnclassified)
+ let roundTripped = try JSONDecoder().decode(
+ MeetingCaptureEra.self,
+ from: JSONEncoder().encode(era)
+ )
+ XCTAssertEqual(roundTripped.echoProtection, .legacyUnclassified)
+ }
+
+ func testTranscriptGateInvalidationUsesLastObservedSampleBoundary() {
+ let gate = MeetingMicrophoneTranscriptGate(.voiceProcessed)
+ XCTAssertTrue(gate.observeAndAdmit(self.makeSampleBuffer(ptsSeconds: 12.5)))
+ let boundary = gate.invalidate()
+ XCTAssertEqual(
+ boundary?.unsafeStartSeconds ?? -1,
+ 12.5 - MeetingMicrophoneTranscriptGate.routeNotificationSafetySeconds,
+ accuracy: 0.000_001
+ )
+ XCTAssertEqual(boundary?.observedSeconds ?? -1, 12.5, accuracy: 0.000_001)
+ XCTAssertFalse(gate.observeAndAdmit(self.makeSampleBuffer(ptsSeconds: 12.51)))
+ }
+
+ func testCoverageGapPersistsTheUnprotectedTail() {
+ let trackID = UUID()
+ let track = MeetingAudioTrack(
+ id: trackID,
+ kind: .microphone,
+ sourceIdentifier: "mic",
+ sourceDisplayName: "Mic",
+ format: nil,
+ timebase: MeetingTimebaseMetadata(
+ startedHostTime: 0, machTimebaseNumerator: 1,
+ machTimebaseDenominator: 1, firstPresentationTime: nil
+ ),
+ health: .waiting,
+ chunks: [MeetingAudioChunk(
+ id: UUID(), sequence: 0, relativeFilePath: "mic.m4a",
+ presentationStart: MeetingMediaTime(value: 100, timescale: 1),
+ presentationEnd: MeetingMediaTime(value: 120, timescale: 1),
+ discontinuities: [], sha256: "x", byteCount: 1,
+ finalizationState: .finalized
+ )],
+ captureMethod: .avCaptureSession,
+ captureEras: [
+ MeetingCaptureEra(
+ method: .voiceProcessing, deviceUID: "mic", deviceName: "Mic",
+ roleAtElection: .unknown, echoProtection: .voiceProcessed, startSeconds: 0
+ ),
+ MeetingCaptureEra(
+ method: .avCaptureSession, deviceUID: "mic", deviceName: "Mic",
+ roleAtElection: .unknown, echoProtection: .unprotected, startSeconds: 110
+ ),
+ ]
+ )
+ let gaps = MeetingProcessingPipeline.microphoneCoverageGaps(track: track, origin: 100)
+ XCTAssertEqual(gaps, [MeetingTranscriptCoverageGap(
+ trackID: trackID, start: 10, end: 20, reason: .unprotectedMicrophone
+ )])
}
// MARK: - Commit gate
@@ -500,7 +618,8 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
)
XCTAssertEqual(elected.identity.captureDeviceID, "core-id")
XCTAssertEqual(elected.identity.avCaptureDeviceID, "legacy-av")
- XCTAssertEqual(elected.role, .personal)
+ XCTAssertEqual(elected.role, .unknown)
+ XCTAssertEqual(elected.identity.role, .unknown)
let ambiguous = [
MeetingAVCaptureDeviceDescriptor(uniqueID: "legacy-av", displayName: "Shared Name"),
@@ -780,7 +899,7 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
XCTAssertEqual(selection.role, .unknown)
}
- func testPreselectionReusesSavedRoleOnlyWhenDeviceMatchesSaved() {
+ func testPreselectionNeutralizesSavedRoleEvenWhenDeviceMatches() {
let identities = [self.identity("saved", uid: "saved-uid", role: .personal)]
let selection = MeetingMicrophonePreselection.select(
identities: identities,
@@ -791,7 +910,7 @@ final class MeetingVoiceProcessingCaptureTests: XCTestCase {
systemDefaultCaptureID: nil
)
XCTAssertEqual(selection.deviceID, "saved")
- XCTAssertEqual(selection.role, .personal)
+ XCTAssertEqual(selection.role, .unknown)
}
/// System default → preferred input UID → `AVCaptureDevice.default` unique ID → first identity.