Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
fix(meeting): identify the mic owner through speaker bleed
Shreeraman A Kb5133dcaf85b
diff --git a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swiftindex ce516317..ea30f522 100644--- a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift+++ b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift@@ -189,10 +189,10 @@ nonisolated enum MeetingTurnSelection {nonisolated enum MeetingLocalSpeakerEvidenceSelector {static func candidate(- cleanDurationByCluster: [SessionSpeakerID: TimeInterval],+ evidenceDurationByCluster: [SessionSpeakerID: TimeInterval],prototypeSpeakerIDs: Set<SessionSpeakerID>) -> SessionSpeakerID? {- let ranked = cleanDurationByCluster+ let ranked = evidenceDurationByCluster.filter { $0.value >= 1 && prototypeSpeakerIDs.contains($0.key) }.sorted {if $0.value == $1.value { return $0.key.uuidString < $1.key.uuidString }@@ -297,6 +297,28 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {return false}+ /// On speakers nearly everything overlaps, so overlap cannot identify the mic's owner.+ /// What can: speech that is not the far end coming back.+ nonisolated static func localSpeakerEvidence(+ from turns: [(+ clusterID: SessionSpeakerID, start: TimeInterval, end: TimeInterval,+ overlapsRemote: Bool, echoScored: Bool, isEcho: Bool+ )]+ ) -> [SessionSpeakerID: TimeInterval] {+ var supporting: [SessionSpeakerID: TimeInterval] = [:]+ var echoed: [SessionSpeakerID: TimeInterval] = [:]+ for turn in turns {+ let duration = max(0, turn.end - turn.start)+ if turn.isEcho {+ echoed[turn.clusterID, default: 0] += duration+ continue+ }+ guard !turn.overlapsRemote || turn.echoScored else { continue }+ supporting[turn.clusterID, default: 0] += duration+ }+ return supporting.filter { $0.value > (echoed[$0.key] ?? 0) }+ }+nonisolated static func remoteFallbackIntervals(diarizedTurns: [(start: TimeInterval, end: TimeInterval)],chunkOffset: TimeInterval,@@ -330,6 +352,8 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {var text: Stringvar overlapsRemote: Boolvar isLikelyEcho = false+ /// False when nothing scoreable overlapped: absence of a verdict is not evidence.+ var echoScored = false}private struct Accumulator {@@ -826,11 +850,9 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {) async throws {let diarizer = SpeakerDiarizationService()var stagedTurns: [StagedMicrophoneTurn] = []- var cleanDurationByCluster: [SessionSpeakerID: TimeInterval] = [:]for chunk in track.chunks where chunk.finalizationState == .finalized {let accumulatorSnapshot = accumulatorlet stagedTurnsSnapshot = stagedTurns- let cleanDurationByClusterSnapshot = cleanDurationByClusterdo {try await self.processMicrophoneChunk(chunk,@@ -839,15 +861,13 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {context: context,remoteSpeech: remoteSpeech,accumulator: &accumulator,- stagedTurns: &stagedTurns,- cleanDurationByCluster: &cleanDurationByCluster+ stagedTurns: &stagedTurns)} catch {if error is CancellationError { throw error }guard case MeetingProcessingError.audioUnreadable = error else { throw error }accumulator = accumulatorSnapshotstagedTurns = stagedTurnsSnapshot- cleanDurationByCluster = cleanDurationByClusterSnapshotskippedChunkIDs.append(chunk.id)}}@@ -856,7 +876,6 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {track,session: session,stagedTurns: stagedTurns,- cleanDurationByCluster: cleanDurationByCluster,accumulator: &accumulator)}@@ -868,8 +887,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {context: ProcessingContext,remoteSpeech: [TimelineInterval],accumulator: inout Accumulator,- stagedTurns: inout [StagedMicrophoneTurn],- cleanDurationByCluster: inout [SessionSpeakerID: TimeInterval]+ stagedTurns: inout [StagedMicrophoneTurn]) async throws {let url = chunk.fileURL(relativeTo: context.sessionDirectory)let chunkOffset = chunk.presentationStart.seconds - context.origin@@ -924,9 +942,6 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {text: text,overlapsRemote: overlapsRemote))- if !overlapsRemote {- cleanDurationByCluster[clusterID, default: 0] += max(0, end - start)- }}return}@@ -971,7 +986,6 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {_ track: MeetingAudioTrack,session: MeetingSession,stagedTurns: [StagedMicrophoneTurn],- cleanDurationByCluster: [SessionSpeakerID: TimeInterval],accumulator: inout Accumulator) {if session.mode == .inRoom {@@ -1011,14 +1025,44 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {return}+ let applicationTrackID = session.audioTracks.first(where: { $0.kind == .applicationAudio })?.id+ // The 60s remote whole-chunk fallback is too coarse to score against; exclude its speaker.+ let fallbackSpeakerID = accumulator.speakerIDByKey["meeting-audio"]+ var classifiedTurns = stagedTurns+ if let applicationTrackID {+ for index in classifiedTurns.indices where classifiedTurns[index].overlapsRemote {+ let turn = classifiedTurns[index]+ let remoteText = Self.echoCandidateText(+ segments: accumulator.segments,+ speakerIDToExclude: fallbackSpeakerID,+ applicationTrackID: applicationTrackID,+ window: (turn.start, turn.end)+ )+ guard !remoteText.isEmpty else { continue }+ classifiedTurns[index].echoScored = true+ classifiedTurns[index].isLikelyEcho = MeetingEchoDetector.isLikelyEcho(+ micText: turn.text,+ remoteText: remoteText+ )+ }+ }+let localClusterID = session.selectedMicrophone.role == .personal? MeetingLocalSpeakerEvidenceSelector.candidate(- cleanDurationByCluster: cleanDurationByCluster,+ evidenceDurationByCluster: Self.localSpeakerEvidence(+ from: classifiedTurns.map {+ (+ clusterID: $0.clusterID, start: $0.start, end: $0.end,+ overlapsRemote: $0.overlapsRemote, echoScored: $0.echoScored,+ isEcho: $0.isLikelyEcho+ )+ }+ ),prototypeSpeakerIDs: accumulator.embeddingIndexByTrack[.microphone]?.speakerIDs ?? []): nilif let localClusterID,- let evidenceTurn = stagedTurns.first(where: { $0.clusterID == localClusterID })+ let evidenceTurn = classifiedTurns.first(where: { $0.clusterID == localClusterID }){accumulator.addResolvedSpeaker(id: localClusterID,@@ -1036,28 +1080,9 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {isLocalUser: false,clusterID: nil)- let applicationTrackID = session.audioTracks.first(where: { $0.kind == .applicationAudio })?.id- // The 60s remote whole-chunk fallback is too coarse to score against; exclude its speaker.- let fallbackSpeakerID = accumulator.speakerIDByKey["meeting-audio"]- var classifiedTurns = stagedTurns- if let applicationTrackID {- for index in classifiedTurns.indices where classifiedTurns[index].overlapsRemote {- let turn = classifiedTurns[index]- guard turn.clusterID != localClusterID else { continue }- let remoteText = Self.echoCandidateText(- segments: accumulator.segments,- speakerIDToExclude: fallbackSpeakerID,- applicationTrackID: applicationTrackID,- window: (turn.start, turn.end)- )- classifiedTurns[index].isLikelyEcho = MeetingEchoDetector.isLikelyEcho(- micText: turn.text,- remoteText: remoteText- )- }- }for turn in classifiedTurns {- let isCleanLocalTurn = turn.clusterID == localClusterID && !turn.overlapsRemote+ let isLocalCluster = turn.clusterID == localClusterID+ let isCleanLocalTurn = isLocalCluster && !turn.isLikelyEchoaccumulator.segments.append(Self.segment(key: "microphone:\(turn.chunkID.uuidString):\(turn.index)",trackID: track.id,@@ -1066,7 +1091,7 @@ final class MeetingProcessingPipeline: MeetingProcessingControlling {end: turn.end,text: turn.text,overlap: turn.overlapsRemote ? .overlapsOtherTrack : .none,- isLikelyEcho: turn.isLikelyEcho+ isLikelyEcho: isLocalCluster ? false : turn.isLikelyEcho))}}diff --git a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swiftindex a7532b6b..443cc446 100644--- a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift+++ b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift@@ -929,7 +929,7 @@ final class MeetingSpeakerEmbeddingIndexTests: XCTestCase {let localID = UUID()let secondaryID = UUID()let selected = MeetingLocalSpeakerEvidenceSelector.candidate(- cleanDurationByCluster: [localID: 12, secondaryID: 3],+ evidenceDurationByCluster: [localID: 12, secondaryID: 3],prototypeSpeakerIDs: [localID, secondaryID])@@ -940,7 +940,7 @@ final class MeetingSpeakerEmbeddingIndexTests: XCTestCase {let firstID = UUID()let secondID = UUID()let selected = MeetingLocalSpeakerEvidenceSelector.candidate(- cleanDurationByCluster: [firstID: 8, secondID: 7],+ evidenceDurationByCluster: [firstID: 8, secondID: 7],prototypeSpeakerIDs: [firstID, secondID])@@ -950,7 +950,7 @@ final class MeetingSpeakerEmbeddingIndexTests: XCTestCase {func testClusterWithoutPositiveEmbeddingEvidenceDoesNotSelectYou() {let clusterID = UUID()let selected = MeetingLocalSpeakerEvidenceSelector.candidate(- cleanDurationByCluster: [clusterID: 12],+ evidenceDurationByCluster: [clusterID: 12],prototypeSpeakerIDs: [])@@ -1412,6 +1412,61 @@ final class MeetingSessionModelTests: XCTestCase {XCTAssertEqual(result.byTurn[0], "one two three")}+ // MARK: - Identifying "You" when the far end bleeds into the microphone++ private func micTurn(+ _ cluster: UUID, _ start: Double, _ end: Double,+ overlapsRemote: Bool = true, echoScored: Bool = true, isEcho: Bool = false+ ) -> (clusterID: UUID, start: Double, end: Double, overlapsRemote: Bool, echoScored: Bool, isEcho: Bool) {+ (cluster, start, end, overlapsRemote, echoScored, isEcho)+ }++ func testSpeakingOverTheFarEndCountsAsEvidence() {+ // Measured on a real Meet call: 83% of mic time overlapped remote speech.+ let you = UUID(), bleed = UUID()+ let evidence = MeetingProcessingPipeline.localSpeakerEvidence(from: [+ self.micTurn(you, 0, 12),+ self.micTurn(bleed, 12, 40, isEcho: true),+ self.micTurn(you, 40, 46),+ ])+ XCTAssertEqual(evidence[you], 18)+ XCTAssertNil(evidence[bleed], "the far end coming back is never evidence of the mic owner")+ }++ func testClusterThatIsMostlyEchoIsNeverACandidate() {+ // A sole candidate skips the margin rule entirely, so purity must stop it earlier.+ let bleed = UUID()+ let evidence = MeetingProcessingPipeline.localSpeakerEvidence(from: [+ self.micTurn(bleed, 0, 200, isEcho: true),+ self.micTurn(bleed, 200, 240),+ ])+ XCTAssertTrue(evidence.isEmpty)+ XCTAssertNil(+ MeetingLocalSpeakerEvidenceSelector.candidate(+ evidenceDurationByCluster: evidence,+ prototypeSpeakerIDs: [bleed]+ ),+ "a sole candidate skips the margin rule, so purity has to stop it earlier"+ )+ }++ func testUnscoreableOverlapProvesNothingEitherWay() {+ // A remote fallback chunk is excluded from scoring, so its window yields no verdict.+ let bleed = UUID()+ let evidence = MeetingProcessingPipeline.localSpeakerEvidence(from: [+ self.micTurn(bleed, 0, 600, echoScored: false),+ ])+ XCTAssertTrue(evidence.isEmpty, "no remote text to score against is unknown, not clean")+ }++ func testSpeechWithNoRemoteOverlapRemainsEvidenceWithoutScoring() {+ let you = UUID()+ let evidence = MeetingProcessingPipeline.localSpeakerEvidence(from: [+ self.micTurn(you, 0, 9, overlapsRemote: false, echoScored: false),+ ])+ XCTAssertEqual(evidence[you], 9, "the original physical signal still counts")+ }+// MARK: - Per-turn engines: merge gap, and fallbacks that must not erase "You"// Real pairs from one Zoom recording. A user talking alone scores 0 at any length.