Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

test(meeting): opt-in acoustic probes and the measurements they produced

Shreeraman A Kcb3ac902c804
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj
index b1e3b9a2..616ebbc0 100644
--- a/Fluid.xcodeproj/project.pbxproj
+++ b/Fluid.xcodeproj/project.pbxproj
@@ -29,6 +29,16 @@
F1BD00000000000000000002 /* StripThinkingTagsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F1BD00000000000000000001 /* StripThinkingTagsTests.swift */; };
D1A600000000000000000204 /* MeetingWindowSelectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */; };
D1A600000000000000000206 /* MeetingRecoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A600000000000000000205 /* MeetingRecoveryTests.swift */; };
+ 7E57C8BE6F444F6A9AC9FCB0 /* MeetingEchoSignalScorerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4AD7433B35340CEB85EA268 /* MeetingEchoSignalScorerTests.swift */; };
+ E5EC00000000000000000002 /* MeetingEchoVetoTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000001 /* MeetingEchoVetoTests.swift */; };
+ E5EC00000000000000000012 /* MeetingFallbackEchoProbeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000011 /* MeetingFallbackEchoProbeTests.swift */; };
+ E5EC00000000000000000022 /* MeetingNearFieldGateTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000021 /* MeetingNearFieldGateTests.swift */; };
+ E5EC00000000000000000032 /* MeetingLiveSilenceEndpointTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000031 /* MeetingLiveSilenceEndpointTests.swift */; };
+ E5EC00000000000000000042 /* VoiceProcessingDuckingProbeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000041 /* VoiceProcessingDuckingProbeTests.swift */; };
+ E5EC00000000000000000052 /* VoiceProcessingSpeechDuckingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000051 /* VoiceProcessingSpeechDuckingTests.swift */; };
+ E5EC00000000000000000062 /* VoiceProcessingAsrQualityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000061 /* VoiceProcessingAsrQualityTests.swift */; };
+ A6E2246FA3744C7B3E2DB91C /* MeetingLiveTranscriptionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */; };
+ 2952403C87FFDE41C1BC85D5 /* MeetingLiveReplayTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */; };
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; };
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; };
DA7200020000000000000002 /* AudioHardwareRecoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */; };
@@ -82,6 +92,16 @@
F1BD00000000000000000001 /* StripThinkingTagsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StripThinkingTagsTests.swift; sourceTree = "<group>"; };
D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingWindowSelectorTests.swift; sourceTree = "<group>"; };
D1A600000000000000000205 /* MeetingRecoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingRecoveryTests.swift; sourceTree = "<group>"; };
+ B4AD7433B35340CEB85EA268 /* MeetingEchoSignalScorerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingEchoSignalScorerTests.swift; sourceTree = "<group>"; };
+ E5EC00000000000000000001 /* MeetingEchoVetoTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingEchoVetoTests.swift; sourceTree = "<group>"; };
+ E5EC00000000000000000011 /* MeetingFallbackEchoProbeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingFallbackEchoProbeTests.swift; sourceTree = "<group>"; };
+ E5EC00000000000000000021 /* MeetingNearFieldGateTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingNearFieldGateTests.swift; sourceTree = "<group>"; };
+ E5EC00000000000000000031 /* MeetingLiveSilenceEndpointTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveSilenceEndpointTests.swift; sourceTree = "<group>"; };
+ E5EC00000000000000000041 /* VoiceProcessingDuckingProbeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceProcessingDuckingProbeTests.swift; sourceTree = "<group>"; };
+ E5EC00000000000000000051 /* VoiceProcessingSpeechDuckingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceProcessingSpeechDuckingTests.swift; sourceTree = "<group>"; };
+ E5EC00000000000000000061 /* VoiceProcessingAsrQualityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceProcessingAsrQualityTests.swift; sourceTree = "<group>"; };
+ 355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveTranscriptionTests.swift; sourceTree = "<group>"; };
+ 7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveReplayTests.swift; sourceTree = "<group>"; };
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = "<group>"; };
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = "<group>"; };
DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioHardwareRecoveryTests.swift; sourceTree = "<group>"; };
@@ -183,6 +203,16 @@
F1BD00000000000000000001 /* StripThinkingTagsTests.swift */,
D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */,
D1A600000000000000000205 /* MeetingRecoveryTests.swift */,
+ B4AD7433B35340CEB85EA268 /* MeetingEchoSignalScorerTests.swift */,
+ E5EC00000000000000000001 /* MeetingEchoVetoTests.swift */,
+ E5EC00000000000000000011 /* MeetingFallbackEchoProbeTests.swift */,
+ E5EC00000000000000000021 /* MeetingNearFieldGateTests.swift */,
+ E5EC00000000000000000031 /* MeetingLiveSilenceEndpointTests.swift */,
+ E5EC00000000000000000041 /* VoiceProcessingDuckingProbeTests.swift */,
+ E5EC00000000000000000051 /* VoiceProcessingSpeechDuckingTests.swift */,
+ E5EC00000000000000000061 /* VoiceProcessingAsrQualityTests.swift */,
+ 355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */,
+ 7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */,
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */,
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,
DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */,
@@ -369,6 +399,16 @@
F1BD00000000000000000002 /* StripThinkingTagsTests.swift in Sources */,
D1A600000000000000000204 /* MeetingWindowSelectorTests.swift in Sources */,
D1A600000000000000000206 /* MeetingRecoveryTests.swift in Sources */,
+ 7E57C8BE6F444F6A9AC9FCB0 /* MeetingEchoSignalScorerTests.swift in Sources */,
+ E5EC00000000000000000002 /* MeetingEchoVetoTests.swift in Sources */,
+ E5EC00000000000000000012 /* MeetingFallbackEchoProbeTests.swift in Sources */,
+ E5EC00000000000000000022 /* MeetingNearFieldGateTests.swift in Sources */,
+ E5EC00000000000000000032 /* MeetingLiveSilenceEndpointTests.swift in Sources */,
+ E5EC00000000000000000042 /* VoiceProcessingDuckingProbeTests.swift in Sources */,
+ E5EC00000000000000000052 /* VoiceProcessingSpeechDuckingTests.swift in Sources */,
+ E5EC00000000000000000062 /* VoiceProcessingAsrQualityTests.swift in Sources */,
+ A6E2246FA3744C7B3E2DB91C /* MeetingLiveTranscriptionTests.swift in Sources */,
+ 2952403C87FFDE41C1BC85D5 /* MeetingLiveReplayTests.swift in Sources */,
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */,
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,
DA7200020000000000000002 /* AudioHardwareRecoveryTests.swift in Sources */,
diff --git a/MEETING_ECHO_AND_LIVE_PLAN.md b/MEETING_ECHO_AND_LIVE_PLAN.md
new file mode 100644
index 00000000..a8b48007
--- /dev/null
+++ b/MEETING_ECHO_AND_LIVE_PLAN.md
@@ -0,0 +1,257 @@
+# Meeting echo handling and live transcription — measured plan
+
+Status: **analysis complete, scorer built standalone, integration not started.** Measurements taken
+2026-08-14 against a real 3.2-minute Zoom session (both raw tracks on disk).
+
+---
+
+## 1. Corrected root causes
+
+Two separate defects were conflated earlier. They need different fixes.
+
+### 1a. Interjections swallowed by echo classification
+
+`MeetingEchoDetector` scores a whole diarized turn by word containment against the far end. A turn
+that is mostly speaker bleed but contains a short local interjection scores as echo, and the user's
+words are hidden with it. Observed: "I'm testing." and "Notetaker." absent from our transcript,
+present in a competitor's.
+
+### 1b. One utterance split across a chunk boundary, labelled two ways
+
+Observed: `119.09–120.12` → *Microphone / Unknown*, `120.06–127.53` → *You*. Same sentence, split at
+the 120 s chunk boundary.
+
+**An earlier diagnosis blaming per-chunk local-speaker resolution was wrong.**
+`processMicrophoneTrack` stages turns across all chunks and calls `finishMicrophoneTrack` once, so
+resolution is already session-wide. The real cause is cluster identity: the emit rule keys on
+`turn.clusterID == localClusterID`, and the 1.04 s fragment was assigned a different cluster.
+Diarization runs per chunk; a ~1 s fragment yields a weak embedding, and cross-chunk matching
+(max cosine distance 0.35, ambiguity margin 0.08) rejects it.
+
+Consequence: **signal-based echo scoring cannot fix 1b.** A turn in the wrong cluster is not labelled
+`You` regardless of its echo verdict. 1b needs its own fix, un-designed as of tonight.
+
+---
+
+## 2. Measurements
+
+All on the real session, raw tracks, `microphone` vs `applicationAudio`.
+
+### Alignment
+- Both tracks decode to identical duration (192.400 s) — one shared `SCStream` clock.
+- Delay by GCC-PHAT: mic content **leads** app audio by **31.9 ms**, std **0.3 ms** across the session.
+- Drift **+6.9 ppm** (~25 ms/hour) — real, above measurement resolution, slow enough to track.
+- Per chunk: −32.28, −31.97, −31.62, −31.38 ms. A smooth monotonic progression **through** chunk
+ boundaries (6.4 ppm), not discontinuities. AAC priming does **not** shift the tracks differentially.
+- The 31.9 ms is a capture-pipeline offset, not an acoustic path. It is hardware-specific, can be
+ either sign, and Bluetooth output can invert it by 100–300 ms. **Never hardcode it; always estimate.**
+
+### Cancellation potential
+- Per-2 s frequency-domain Wiener filter: **9–11 dB ERLE** in echo-dominant blocks.
+- A single session-wide filter achieves only 1.6 dB — drift defeats it. Adaptation is mandatory.
+- Overall 3.9 dB, dragged down by blocks where the far end is silent (nothing to cancel).
+
+### Turn-level separability (the decisive measurement)
+
+Per-frame explained fraction = `1 − residual_energy / mic_energy`, 1024-sample frames, 256 hop.
+
+| turn kind | n | mean | median | longest low-run (<0.35) |
+|---|---|---|---|---|
+| pure echo | 5 | 0.53–0.68 | 0.62–0.82 | **0.24–0.59 s** |
+| contains local speech | 1 | 0.43 | 0.38 | **0.98 s** |
+| local speech only | 3 | 0.02–0.03 | 0.00–0.02 | **0.96–10.27 s** |
+
+**Mean and median cannot separate the mixed turn** — it sits between the two populations. The
+**longest contiguous low-scoring run** does, with a clean gap: echo never exceeds 0.59 s, anything
+containing local speech never falls below 0.96 s. This is the rescue statistic the design needs.
+
+Caveat: one recording, one room, ten turns. The gap is real but the sample is small.
+
+### Operating envelope of the rescue rule (found in code review, measured)
+
+The statistic is `1 − residual/mic`. When local speech is **added on top of** echo — which is what
+real speakerphone double-talk is — the fraction is approximately `echo / (echo + local)`. To fall
+below `localSpeechFractionThreshold` (0.35) the local speech must therefore carry roughly
+**1.86× the echo energy at the microphone**.
+
+That holds comfortably for a laptop mic near the speaker's mouth with the far end coming from the
+same laptop's speakers — it was satisfied throughout the real recording. It does **not** hold for a
+quiet interjection over a loud far end, or a mic far from the user. That is the rule's operating
+envelope, and it is a genuine limitation rather than a tuning constant: a louder threshold trades
+directly against false rescues of pure echo.
+
+Consequence for calibration: the held-out recordings must include a quiet-interjection case, not
+just more of the easy one.
+
+---
+
+## 3. Decisions from adversarial review
+
+Reviewed by two independent external models. Both independently reached the same conclusions.
+
+| decision | reason |
+|---|---|
+| Signal acts as a **veto only** on text-positive turns | Making it primary creates a new false-positive class that hides user speech |
+| **Do not** feed signal evidence into `localSpeakerEvidence` | The purity rule `supporting > echoed` plus the 1.75× winner margin means a misclassification can elect the *far-end* cluster as "You" — a high-amplification failure |
+| Three-state verdict: echo / containsLocalSpeech / **unknown** | Absence of reference audio is not evidence of local speech. Headphone users, silent far end, and low delay confidence must abstain, not score 0 |
+| Turn aggregation must use the **low-run** statistic, not an average | Measured above; an average reproduces the exact bug being fixed |
+| Reference audio must be gathered by **time range across all app chunks**, not chunk N ↔ chunk N | The two tracks' `MeetingAudioChunkWriter`s rotate independently and diverge permanently after a discontinuity on either track |
+| Reference-read failure must be **total** (degrade to `.unknown`, never throw) | Today only the mic chunk's own `audioUnreadable` skips a chunk; a sick app track must not take down a healthy mic track |
+| Exclude discontinuity-adjacent windows | A shared clock does not make content contiguous after dropped samples |
+| Bump `pipelineVersion` when classification changes | Checkpoint reuse would otherwise mix classification rules mid-session |
+
+Open, unresolved: threshold portability across hardware/rooms/volumes (one recording); non-speech
+far-end audio (music, video) is a false-positive class today's text detector cannot produce; whether
+to persist the score, and if so versioned against the scorer.
+
+---
+
+## 4. Built
+
+**`MeetingEchoSignalScorer.swift`** — pure functions over PCM: GCC-PHAT delay estimation with a
+confidence measure, per-frame explained fractions with NaN for unscoreable frames, and a three-state
+turn verdict using the low-run rule. Unit tested on deterministic synthetic signals.
+
+**Integrated as a veto** (`pipelineVersion` 3 → 4). The design point both reviewers converged on is
+**two separate echo fields**:
+
+- `isLikelyEcho` stays **text-only** and remains the sole input to `localSpeakerEvidence`. Signal
+ evidence must never influence the local-speaker election — a misclassification there could elect
+ the *far-end* cluster as "You" through the 1.75× winner rule, which is a far worse failure than
+ any single mislabelled turn.
+- `effectiveEcho = isLikelyEcho && signalVerdict != .containsLocalSpeech` drives **both** loss paths:
+ the `isCleanLocalTurn` assignment and the emitted echo flag that the UI and exporter filter on.
+
+Supporting decisions:
+- Reference audio is gathered by **presentation timestamp** across all intersecting application
+ chunks. Never by chunk index: the two writers rotate independently and diverge permanently after a
+ discontinuity, so a mic chunk can straddle two app chunks.
+- Gaps are **zero-filled, not masked**. A zero-filled block has no reference energy, so the scorer's
+ own floor already forces those frames to NaN — an explicit mask would mean reopening a tested
+ module for no behavioural gain.
+- Delay falls back to a **rolling median within a capture epoch**, never a session-wide scalar; an
+ epoch ends at any discontinuity on either track.
+- Every reference failure degrades to `.unknown` and mutates nothing. A sick application track must
+ never skip a healthy microphone chunk.
+
+**Known limits, deliberately documented rather than hidden:**
+- Turns shorter than ~0.8 s can never be vetoed (`minimumLocalSpeechRunSeconds` is 0.75 s), so a
+ standalone backchannel — "yeah", "okay" — stays hidden. The measured failure case survives only
+ because those interjections sat inside a 25.9 s turn.
+- The 1.86× energy envelope means a quiet interjection over a loud far end is not rescued.
+
+Together these cap how much of the motivating bug this fixes. It is a real improvement, not a
+general solution.
+
+---
+
+## 5. Live transcription — shape only
+
+Capture stays on ScreenCaptureKit. Voice Processing I/O is disqualified: instantiating it ducks all
+other applications' audio system-wide, which would corrupt the very `applicationAudio` track we
+record. Core Audio process taps are a permission-UX decision, not an echo fix, and risk zero-filled
+buffers for WebRTC apps (Zoom, Meet, Teams).
+
+**Decision (2026-08-14): no diarization during the meeting.** Live shows only *You* and *Them*,
+derived from which track the audio arrived on — not from analysing voices. Speaker identity is
+resolved after the meeting by the existing batch pipeline.
+
+- **Live** — streaming ASR on both tracks. Mic = You, application audio = Them. Correct by
+ construction in any 1:1. **Depends on echo suppression**, or "You" transcribes the far end.
+- **On meeting end** — today's batch pipeline runs unchanged and replaces the live text with the
+ word-aligned, properly diarized final transcript, turning *Them* into Speaker 1/2/3.
+
+Why this ordering rather than a live diarizer:
+- The streaming diarizer measured **1,011 MB resident** (see below). Deferring it keeps that
+ entirely out of the meeting, which removes the model-contention question rather than answering it.
+- Streaming diarization is 10–15 % worse DER than the offline path, so deferring *improves* the
+ final transcript instead of compromising it.
+- Label churn disappears: nothing can retroactively relabel text the user already read, because no
+ speaker decision is made live.
+- Cost: in a 3+ person call, everyone else reads as *Them* until processing finishes. In a 1:1,
+ nothing is lost.
+
+### Measured memory (2026-08-14, this machine)
+
+| component | peak RSS | note |
+|---|---|---|
+| Streaming diarizer (Sortformer) | **1,011 MB** | real run, 56.7 s for 192 s audio (~3.4× realtime) |
+| Nemotron streaming ASR | 688 MB | models loaded, then **failed** before transcribing |
+
+The diarizer's 1 GB is far above its 21 MB on-disk footprint. Contention between the two was **not**
+measured: Nemotron aborts with a CoreML `zero shape error` during type inference on ordinary 16 kHz
+mono input, both full-length and on a 30 s clip with an explicit chunk size. That failure blocks the
+live path itself, not just the measurement, and is the first thing to resolve.
+
+Still open: whether one streaming ASR instance can serve both tracks, or whether two instances
+(~690 MB each) are required.
+
+Hard constraints, verified in FluidAudio at the pinned commit:
+- Streaming ASR produces **no word timestamps** (zero `TokenTiming` references in the streaming
+ path). Our word→turn assignment cannot run live; final quality must come from the batch pass.
+- Streaming "partials" only grow — greedy RNNT, never revised. `.provisional` would have meant
+ "growing", not "correctable". **As built, `MeetingTranscriptStatus.provisional` stays unused**:
+ live text is held in memory and published from the coordinator, never entering
+ `session.transcriptSegments`, so there is nothing to mark. That containment is deliberate — it
+ makes lowercase, unpunctuated live text unreachable by export and copy by construction rather
+ than by filtering, and means a crash or a failed batch pass cannot leave it posing as the
+ transcript.
+- Streaming diarization is 10–15 % worse DER than the offline VBx pipeline in use today.
+- Below ~560 ms chunks, WER degrades sharply (2.1 % → ~10 % at 160 ms).
+- Nothing in FluidAudio combines streaming ASR with streaming diarization; we would align them.
+
+---
+
+## 6. First real live run (2026-08-14, headphones, 2.5 min Zoom)
+
+Live transcription ran end to end against a real meeting. Both tracks transcribed; "You" and "Them"
+were correct by construction. Four defects found and fixed.
+
+| defect | evidence | fix |
+|---|---|---|
+| Utterance spans swallow leading silence | mic EOU #1 reported `119.6s`; wall clock shows speech `18:56:32→18:57:11`, EOU at `18:57:13` | anchor `utteranceStartPTS` to the first decoded token, backdated one chunk, clamped to the previous utterance's end |
+| ~5s of meeting-opening audio shed | 381 drops between `11.123` and `13.833`, then never again | run CoreML warmup on silence *before* `isModelReady`; queue 24 → 512 slots |
+| 255 decoder resets and 255 degraded notifications in one burst | one `handleDrop` per dropped sample | coalesce to one resync per burst, applied by the drain loop |
+| `handleEOU` reentrancy on `await manager.reset()` | `consume` could set a stale start during the suspension | dissolves — `consume` no longer touches utterance start |
+
+**The endpointer is not slow.** EOU fires ~1.5–2s after speech stops, matching the model's 1280ms
+debounce. An earlier reading of "78–120s latency" was a misinterpretation of the inflated spans.
+
+## 7. Whole-chunk microphone fallback: neither echo classifier can score it
+
+The same run put 60s of garbled far-end bleed in the transcript as `Microphone / Unknown`,
+`isLikelyEcho=false`. Root cause: the whole-chunk fallback appends straight to
+`accumulator.segments`, bypassing `stagedTurns` — and all echo classification runs over
+`stagedTurns`.
+
+Two candidate fixes were **measured and rejected**:
+
+- **Text detector.** The bleed block scores **0.31** containment against its own 60s window
+ (threshold 0.5) — it would not be flagged. Genuine user speech from the same session scores
+ 0.05–0.29 even against the whole-meeting corpus, so widening the window is not the answer either.
+- **Signal scorer.** A probe over the real chunks returns **no delay estimate and no verdict** for
+ the two bleed chunks (`verdict=nil, delay=none`); GCC-PHAT finds no confident echo path and
+ correctly abstains. Only chunk 2 scored, at `containsLocalSpeech`, delay `-50ms`.
+
+Reproduce with `MeetingFallbackEchoProbeTests` (`TEST_RUNNER_FLUIDVOICE_ECHO_PROBE=<session dir>`).
+
+**Actual fix:** `LabeledPassError.emptyTurn` no longer falls through to `transcribeWholeChunk`.
+Diarization succeeding while every turn transcribes empty is positive evidence that the microphone
+holds no intelligible speech; re-transcribing 60s of near-silence only produces hallucinated text
+that is unattributable and unscoreable. Genuine diarization failure — including
+`SpeakerDiarizationService.isSupported == false` — still gets the fallback. `pipelineVersion` 4 → 5.
+
+Still open: `wordAttachmentTolerance` is `0` for the microphone (`MeetingProcessingPipeline:1463`),
+which disables the nearest-neighbour rescue in `assignWords` and drops any word whose timing
+disagrees with a diarizer boundary (observed `29/67`, `8/30` unassigned). Loosening it trades
+directly against crediting bleed to the user; needs its own measurement.
+
+---
+
+## 8. Deferred
+
+Fluid One transcript cleanup (measured: ~648 ms/turn on a 1.5 B proxy model with prefix KV caching
+on, ~2 min/meeting-hour; Fluid One is 3.3 GB so estimate 5–8 min — a background job, not a blocking
+step). Vocabulary biasing for meetings is independent and cheap: the meeting pipeline does not use
+`ParakeetVocabularyStore` / `PronunciationDictionaryStore` at all today, which likely explains
+"Cloud Code" for "Claude Code".
diff --git a/MIC_CAPTURE_MIGRATION_PLAN.md b/MIC_CAPTURE_MIGRATION_PLAN.md
new file mode 100644
index 00000000..e6a434ca
--- /dev/null
+++ b/MIC_CAPTURE_MIGRATION_PLAN.md
@@ -0,0 +1,167 @@
+# Microphone capture migration — SCStream → AVAudioEngine with Voice Processing I/O
+
+Status: **v4, planned, not started.** v1–v3 were each rejected by two independent reviewers. v4 is
+smaller than v3, per the reviewers' closing instruction. Written 2026-08-15.
+
+## 1. What v4 changes from v3
+
+v3 was rejected mainly for three self-inflicted errors. Each is corrected by *narrowing a claim*, not
+by adding machinery:
+
+| v3 error | v4 |
+|---|---|
+| "No diarization, no clustering" | **Keep the diarizer.** It is the *segmenter*, not a labeller (`MeetingProcessingPipeline.swift:1385-1391`); every turn boundary, `transcribeTurns`, word alignment, `turnLoudness` and the empty-turn guard depend on it. Deleting it leaves only whole-chunk transcription — the hallucination path fixed earlier today. v4 keeps diarization and **merges its mic clusters into "You"**. |
+| "No fallback" | **Pre-flight capability decision, not mid-meeting fallback.** VPIO viability is decided *before* capture starts; the session then runs one way for its whole life. No mixed provenance, and no lost meeting. |
+| "Session surfaces a degraded microphone track" | False — capture start is atomic (`MeetingCaptureEngine.swift:88-95`) and a runtime failure stops all writers. v4 does not claim a state the code lacks. |
+
+## 2. Measurements
+
+| question | result |
+|---|---|
+| Does AEC cancel the far end? | raw mic −23.7 dBFS / 62 words of far-end text; VPIO mic −63.6 dBFS / **zero words** |
+| Cancellation vs ducking | ~40 dB total, ~7 dB of it ducking → **~33 dB genuine cancellation** |
+| Double-talk | raw: 8 words of mush; VPIO: **51 words of the user's passage, no far-end content** |
+| Effect on the user's own voice | ~14.5% WER either way; VPIO ran 12 dB hotter (AGC) |
+| Ducking cost at `.min` | −6.9 dB mean on **all** other audio, swing 9.2 → 16.9 dB |
+| Downlink-DSP faults | 2066 during the successful run — **not root-caused** |
+
+Shipping config (`VoiceProcessingAsrQualityTests.swift:54-57`). **One machine, one room, built-in mic
+and speakers.** Not measured: quiet interjections under a loud far end; any other hardware or macOS
+build.
+
+**Competitor precedent is weaker than v3 claimed.** The surveyed competitor ships a thin capture
+client with no local models; segmentation and diarization happen server-side, so their client never
+had an election to delete. What genuinely transfers: separate dictation/meeting audio configs, source
+attribution as the live default, no AEC fallback, logging the *settled* audio constraint rather than
+the requested one, and accepting the ducking with no mitigation.
+
+## 3. Activation conditions
+
+VPIO is used only when **all** hold, decided before capture starts:
+
+1. Mode is `.onlineCall` and the microphone role is `.personal`.
+2. **Output is not headphones.** With headphones there is no bleed to cancel, so VPIO delivers zero
+ benefit and the full ducking cost — pumping the audio the user is hearing the call on. v3 applied
+ VPIO unconditionally; the user's accepted trade (§4) was measured on speakers only.
+3. The selected microphone can be bound and verified (§6).
+
+Otherwise the session runs today's SCStream path unchanged, with full echo machinery. This is a
+**capability decision, not a fallback**: homogeneous for the session's lifetime.
+
+Mid-session output changes (speakers → AirPods) are a known limitation, not handled in v4.
+
+## 4. Product decision — accepted, with a stated boundary
+
+Enabling VPIO ducks **all** other audio by ~7 dB with ±8 dB of pumping that increases while the user
+speaks, audible live. **Accepted by the user, 2026-08-15**, after being shown the measured figures and
+told it affects live listening rather than only the recording.
+
+That acceptance was given for the **speakers** case and does not extend to headphones — hence §3.2.
+
+## 5. Attribution
+
+For a VPIO session, on the microphone track:
+
+- Diarization runs exactly as today (it is the segmenter), and **the election runs unchanged**.
+- VPIO turns are marked **scored clean** — `echoScored = true, isLikelyEcho = false` — because that is
+ what they are: examined, and known to contain no far-end audio. They therefore count as supporting
+ evidence in `localSpeakerEvidence` (`MeetingProcessingPipeline.swift:357`) exactly as a
+ text-examined clean turn does today.
+- **This is the whole change.** No branch on cluster count, no bypass. AEC does not make diarization
+ unnecessary — it makes it work, by removing the echo clusters that competed with the user's own
+ voice and defeated the 1.75× margin rule in the observed failure. A second voice in the room is then
+ handled by the election as it always was, which matters because misattribution is
+ **uncorrectable in-product**: `mergeSpeakers` refuses any speaker with `isLocalUser`
+ (`MeetingModels.swift:542`).
+- The text echo detector is not consulted; the signal veto is not applied.
+- `MeetingEchoSignalScorer` still runs as a **monitor**: its verdicts are recorded, never applied.
+ This is the only field evidence that AEC worked on hardware we have not measured; without it a
+ silently-degraded canceller yields bleed labelled "You", forever undetected.
+- `MeetingLiveEchoFilter` is disabled for VPIO sessions (`MeetingLiveTranscriptionCoordinator.swift:150`).
+- `pipelineVersion` bumps; attribution rules changed.
+
+Unchanged: `.inRoom` (`AVCaptureSession`, not SCStream — `MeetingCaptureEngine.swift:624`), non-VPIO
+sessions, and the near-field gate.
+
+## 6. Device selection
+
+`kAudioOutputUnitProperty_CurrentDevice` fails for Bluetooth and aggregate devices, and the existing
+binder falls back to the system default on **any** non-zero status (`ASRService.swift:2569-2584`).
+SCStream honours `microphoneCaptureDeviceID` exactly today, so this is a real regression.
+
+Order of attempts:
+1. Bind, then **read the device back and compare** after voice processing is enabled and the engine is
+ running — a successful status is not proof, since enabling VPIO can replace the I/O unit.
+2. If binding fails but the **system default input is already the requested device**, proceed unbound:
+ identity is correct even though binding is not. This covers most AirPods users, who selected the
+ device in System Settings precisely because they are on a call.
+3. Otherwise VPIO is not viable → §3, session runs today's path.
+
+Binding order relative to `setVoiceProcessingEnabled` is a **Phase 1 measurement**, not an assumption.
+Unresolved: `AVCaptureDevice.uniqueID` → `AudioObjectID` mapping is by display name and yields nil on
+collision (`MeetingCaptureEngine.swift:855-866`).
+
+## 7. Provenance
+
+No-fallback removes *mixed* provenance, not provenance itself. A **track-level** `captureMethod` is
+required, describing verified capture rather than intended configuration, and read by: batch
+attribution (§5), the live coordinator's echo-filter decision, retry after relaunch, checkpoint
+resume, and reprocessing of sessions recorded before this change (absence = legacy).
+
+## 8. Timeline
+
+Cross-track timing survives for transcript ordering (batch `MeetingProcessingPipeline.swift:1100-1104`
+and live `MeetingLiveModels.swift:39-44`), the `.overlapsOtherTrack` badge, and live origin
+establishment. Echo reference alignment is gone.
+
+- Microphone PTS is the tap's `AVAudioTime.hostTime` — acquisition, first frame — as dictation already
+ does (`ASRService.swift:4617-4620`). Never `mach_absolute_time()` at callback time.
+- Invalid `hostTime`: extrapolate from the last valid stamp plus output duration, **capped at 500 ms**;
+ beyond that, record a discontinuity and resync.
+- **The constant-offset model is contingent.** This repo measured ~6.4 ppm drift, ~23 ms/hour
+ (`MeetingProcessingPipeline.swift:722-724`); a fixed offset is ~46 ms wrong at two hours, consuming
+ the whole budget. Phase 0 decides: same clock domain ⇒ offset is identity and no model is needed;
+ different domains ⇒ a linear model with periodic re-measurement, not a constant.
+
+**Budget: ±50 ms**, justified by turn ordering alone. Stated as a claim to be tested, not derived —
+v3 wrongly cited the 0.15 s `overlapsRemote` threshold, which is a minimum-overlap predicate.
+
+## 9. Phases and gates
+
+**Phase 0 — clock domain (unattended).** Compare *rates*, not arrival deltas: regress SCStream PTS
+against wall time and microphone PTS against wall time over 30 minutes. *Gate:* report both slopes in
+ppm with confidence intervals. |slope difference| < 20 ppm ⇒ constant offset acceptable; ≥ 20 ppm ⇒
+§8's linear model is mandatory. Subtracting `mach_absolute_time()` at arrival is **not** the test — it
+conflates queueing jitter with clock domain.
+
+**Phase 1 — `MeetingMicrophoneCapture`, standalone.** *Gate*, over 10 minutes on the built-in mic
+**and** one Bluetooth device: valid `hostTime` on ≥99% of buffers; output Float32 LPCM (else the
+writer's telemetry silently dies, `MeetingAudioChunkWriter.swift:499-505`); buffers accepted by
+`MeetingLiveSampleCopy.copy` and by the writer with zero unexpected rotations
+(`MeetingAudioChunkWriter.swift:177-193`); read-back device equals requested, or §6.2/§6.3 fires;
+binding order recorded.
+
+**Phase 2 — timeline.** *Gate:* drift within Phase 0's bound over 30 minutes. **Absolute offset cannot
+be measured with an acoustic click** — AEC removes it, so a failed correlation would mean cancellation
+worked. Use a loud chirp and matched-filter the ~33 dB residual; if the residual is unrecoverable,
+record that absolute offset is unmeasured, accept ordering-tolerance validation on a real call, and
+say so rather than reporting a number.
+
+**Phase 3 — integrate behind a flag, default off.** *Gate:* a real call whose transcript ordering is
+compared turn-by-turn against a human-checked reference. Not a simultaneous old-path capture — two
+microphone clients with one enabling VPIO perturbs the system under test.
+
+**Phase 4 — attribution.** One change: mark VPIO turns scored-clean and let the existing election run.
+*Gate:* a single-voice VPIO session elects "You" with no `Microphone / Unknown`; a **two-voice**
+session still elects the owner and demotes the second speaker; a non-VPIO session produces a
+transcript identical to today's. The first alone proves nothing — the second is the real test, and it
+is only possible because the diarizer was kept.
+
+## 10. Not claimable unattended
+
+Quiet interjections under a loud far end (**a release blocker, not a footnote**); AirPods selection and
+mid-session profile switching; the ducking experience; permission prompts; real-call ordering under
+load; post-meeting dictation and Bluetooth restoration; AEC on any untested hardware or macOS build;
+the 2066 downlink faults, which remain un-root-caused and gate nothing today.
+
+Phases 0–2 are unattended-safe. **Phases 3–4 are not, and have no owner or schedule yet.**
diff --git a/Tests/FluidDictationIntegrationTests/MeetingFallbackEchoProbeTests.swift b/Tests/FluidDictationIntegrationTests/MeetingFallbackEchoProbeTests.swift
new file mode 100644
index 00000000..a8cc5b1e
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/MeetingFallbackEchoProbeTests.swift
@@ -0,0 +1,115 @@
+@testable import FluidVoice_Debug
+import Foundation
+import XCTest
+
+/// Opt-in probe against a real recorded session, used to decide whether the whole-chunk microphone
+/// fallback can be classified by the signal scorer. Set FLUIDVOICE_ECHO_PROBE to the session
+/// directory to run it.
+final class MeetingFallbackEchoProbeTests: XCTestCase {
+ func testWholeChunkFallbackVerdictOnRealSession() throws {
+ guard let root = ProcessInfo.processInfo.environment["FLUIDVOICE_ECHO_PROBE"] else {
+ throw XCTSkip("FLUIDVOICE_ECHO_PROBE not set")
+ }
+ let sessionDirectory = URL(fileURLWithPath: root, isDirectory: true)
+ let decoder = JSONDecoder()
+ decoder.dateDecodingStrategy = .iso8601
+ let micTrack = try decoder.decode(
+ MeetingAudioTrack.self,
+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/microphone/track.json"))
+ )
+ let appTrack = try decoder.decode(
+ MeetingAudioTrack.self,
+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/applicationAudio/track.json"))
+ )
+
+ let origin = min(
+ micTrack.chunks.first?.presentationStart.seconds ?? 0,
+ appTrack.chunks.first?.presentationStart.seconds ?? 0
+ )
+
+ for chunk in micTrack.chunks {
+ let chunkOffset = chunk.presentationStart.seconds - origin
+ let chunkDuration = chunk.presentationEnd.seconds - chunk.presentationStart.seconds
+ let url = sessionDirectory.appending(path: chunk.relativeFilePath)
+
+ // One turn spanning the whole chunk — exactly what the fallback would score.
+ let wholeChunk = [(index: 0, start: chunkOffset, end: chunkOffset + chunkDuration)]
+ let outcome = try MeetingProcessingPipeline.computeEchoVerdicts(
+ micChunk: chunk,
+ micChunkURL: url,
+ chunkOffset: chunkOffset,
+ chunkDuration: chunkDuration,
+ turns: wholeChunk,
+ applicationTrack: appTrack,
+ origin: origin,
+ sessionDirectory: sessionDirectory,
+ acceptedDelaysInEpoch: []
+ )
+ let verdict = outcome.verdicts[0].map { String(describing: $0) } ?? "nil"
+ print(String(
+ format: "[probe] chunk seq=%d span=[%.1f-%.1f] verdict=%@ delay=%@",
+ chunk.sequence, chunkOffset, chunkOffset + chunkDuration, verdict,
+ outcome.acceptedDelaySeconds.map { String(format: "%.4fs", $0) } ?? "none"
+ ))
+ }
+ }
+
+ /// Scores specific turns that text-echo detection missed, to see whether the signal scorer
+ /// could have caught them. FLUIDVOICE_ECHO_PROBE_TURNS is "label:start:end,label:start:end".
+ func testNamedTurnsAgainstSignalScorer() throws {
+ guard let root = ProcessInfo.processInfo.environment["FLUIDVOICE_ECHO_PROBE"],
+ let spec = ProcessInfo.processInfo.environment["FLUIDVOICE_ECHO_PROBE_TURNS"]
+ else { throw XCTSkip("probe env not set") }
+
+ let sessionDirectory = URL(fileURLWithPath: root, isDirectory: true)
+ let decoder = JSONDecoder()
+ decoder.dateDecodingStrategy = .iso8601
+ let micTrack = try decoder.decode(
+ MeetingAudioTrack.self,
+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/microphone/track.json"))
+ )
+ let appTrack = try decoder.decode(
+ MeetingAudioTrack.self,
+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/applicationAudio/track.json"))
+ )
+ let origin = min(
+ micTrack.chunks.first?.presentationStart.seconds ?? 0,
+ appTrack.chunks.first?.presentationStart.seconds ?? 0
+ )
+
+ for entry in spec.split(separator: ",") {
+ let parts = entry.split(separator: ":")
+ guard parts.count == 3, let start = Double(parts[1]), let end = Double(parts[2]) else { continue }
+ let label = String(parts[0])
+ guard let chunk = micTrack.chunks.first(where: {
+ let chunkStart = $0.presentationStart.seconds - origin
+ let chunkEnd = $0.presentationEnd.seconds - origin
+ return start >= chunkStart && start < chunkEnd
+ }) else {
+ print("[turn] \(label) no chunk covers \(start)")
+ continue
+ }
+ let chunkOffset = chunk.presentationStart.seconds - origin
+ let chunkDuration = chunk.presentationEnd.seconds - chunk.presentationStart.seconds
+ // Clamp: a turn may run past its chunk, and the scorer reads one chunk's audio.
+ let clampedEnd = min(end, chunkOffset + chunkDuration)
+ let outcome = try MeetingProcessingPipeline.computeEchoVerdicts(
+ micChunk: chunk,
+ micChunkURL: sessionDirectory.appending(path: chunk.relativeFilePath),
+ chunkOffset: chunkOffset,
+ chunkDuration: chunkDuration,
+ turns: [(index: 0, start: start, end: clampedEnd)],
+ applicationTrack: appTrack,
+ origin: origin,
+ sessionDirectory: sessionDirectory,
+ acceptedDelaysInEpoch: []
+ )
+ print(String(
+ format: "[turn] %-22@ [%.1f-%.1f] verdict=%@ delay=%@",
+ label as NSString, start, clampedEnd,
+ outcome.verdicts[0].map { String(describing: $0) } ?? "nil",
+ outcome.acceptedDelaySeconds.map { String(format: "%.4fs", $0) } ?? "none"
+ ))
+ }
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift b/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift
new file mode 100644
index 00000000..f142f843
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift
@@ -0,0 +1,341 @@
+@testable import FluidVoice_Debug
+import AVFoundation
+import CoreMedia
+import Foundation
+import XCTest
+
+/// Opt-in replay harness: drives the REAL `MeetingLiveTranscriptionCoordinator` with a real,
+/// previously-recorded Zoom meeting's decoded microphone/application-audio WAVs, to see what the
+/// live-transcription pipeline actually produces. Gated behind an env var so the normal suite
+/// stays deterministic — this test downloads/loads real CoreML models and takes real wall time.
+///
+/// Set `FLUIDVOICE_LIVE_REPLAY=1` (and optionally `FLUIDVOICE_LIVE_REPLAY_MIC_WAV` /
+/// `FLUIDVOICE_LIVE_REPLAY_APP_WAV` to override the fixture paths) to run it.
+final class MeetingLiveReplayTests: XCTestCase {
+ private static let chunkFrames = 320 // 20ms @ 16kHz
+ private static let sampleRate: Double = 16_000
+ private static let trailingSilenceSeconds: Double = 3
+
+ func testReplayRealMeetingThroughLiveCoordinator() async throws {
+ guard let replayFlag = ProcessInfo.processInfo.environment["FLUIDVOICE_LIVE_REPLAY"], !replayFlag.isEmpty else {
+ throw XCTSkip("Set FLUIDVOICE_LIVE_REPLAY=1 to run the live-transcription replay harness.")
+ }
+
+ let env = ProcessInfo.processInfo.environment
+ let micPath = env["FLUIDVOICE_LIVE_REPLAY_MIC_WAV"].flatMap { $0.isEmpty ? nil : $0 }
+ ?? "/private/tmp/claude-501/-Users-shreeram-Projects-FluidVoice/33323536-f2a9-4586-ad02-f71ced211c2a/scratchpad/microphone.wav"
+ let appPath = env["FLUIDVOICE_LIVE_REPLAY_APP_WAV"].flatMap { $0.isEmpty ? nil : $0 }
+ ?? "/private/tmp/claude-501/-Users-shreeram-Projects-FluidVoice/33323536-f2a9-4586-ad02-f71ced211c2a/scratchpad/applicationAudio.wav"
+
+ let micSamples = try Self.load16kMonoFloatSamples(path: micPath)
+ let appSamples = try Self.load16kMonoFloatSamples(path: appPath)
+ print("[replay] mic samples=\(micSamples.count) (\(Double(micSamples.count) / Self.sampleRate)s), app samples=\(appSamples.count) (\(Double(appSamples.count) / Self.sampleRate)s)")
+
+ guard let pcmFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.sampleRate, channels: 1, interleaved: false),
+ let formatDescription = Self.makeFormatDescription(format: pcmFormat)
+ else {
+ XCTFail("Could not build the 16kHz mono Float32 format used to feed the coordinator.")
+ return
+ }
+
+ // Interleave both tracks' 20ms chunks in true time order. Both files share a clock and
+ // (empirically) an identical sample count, so ties are broken mic-first — matching the
+ // typical ordering of the real capture tee.
+ var events: [(kind: MeetingAudioTrackKind, samples: ArraySlice<Float>, ptsSamples: Int)] = []
+ events.reserveCapacity(((micSamples.count + appSamples.count) / Self.chunkFrames) + 4)
+ Self.appendChunks(&events, kind: .microphone, samples: micSamples)
+ Self.appendChunks(&events, kind: .applicationAudio, samples: appSamples)
+ // Trailing silence on both tracks so any in-flight utterance at end-of-file can still hit
+ // its 1280ms EOU silence debounce and finalize, instead of being lost to `stop()`'s reset.
+ let silenceFrames = Int(Self.trailingSilenceSeconds * Self.sampleRate)
+ let silence = [Float](repeating: 0, count: silenceFrames)
+ let micTailStart = micSamples.count
+ let appTailStart = appSamples.count
+ Self.appendChunks(&events, kind: .microphone, samples: silence, ptsBaseSamples: micTailStart)
+ Self.appendChunks(&events, kind: .applicationAudio, samples: silence, ptsBaseSamples: appTailStart)
+ events.sort { lhs, rhs in
+ if lhs.ptsSamples != rhs.ptsSamples { return lhs.ptsSamples < rhs.ptsSamples }
+ return lhs.kind == .microphone && rhs.kind != .microphone
+ }
+ print("[replay] total chunks to feed: \(events.count)")
+
+ final class SnapshotBox: @unchecked Sendable {
+ let lock = NSLock()
+ var latest: MeetingLiveTranscriptSnapshot = .empty
+ var log: [(elapsed: TimeInterval, snapshot: MeetingLiveTranscriptSnapshot)] = []
+ let start = Date()
+ func record(_ snapshot: MeetingLiveTranscriptSnapshot) {
+ self.lock.lock()
+ defer { self.lock.unlock() }
+ self.latest = snapshot
+ self.log.append((Date().timeIntervalSince(self.start), snapshot))
+ }
+ }
+ let box = SnapshotBox()
+ let coordinator = MeetingLiveTranscriptionCoordinator { snapshot in
+ box.record(snapshot)
+ }
+
+ let harnessStart = Date()
+ coordinator.start(mode: .onlineCall)
+
+ // Wait for the pipeline to report ready before hammering it with non-real-time audio: a
+ // capacity-24 (~480ms) bounded queue means feeding faster than real time before the model
+ // is loaded would just drop nearly everything, which would test harness pacing, not the
+ // pipeline's actual transcription/echo-suppression behavior.
+ let readyTimeout: TimeInterval = 300
+ let readyDeadline = Date().addingTimeInterval(readyTimeout)
+ while Date() < readyDeadline {
+ let available = box.lock.withLock { box.latest.availability == .available }
+ if available { break }
+ try await Task.sleep(nanoseconds: 200_000_000)
+ }
+ let firstAvailableElapsed = Date().timeIntervalSince(harnessStart)
+ let becameAvailable = box.lock.withLock { box.latest.availability == .available }
+ print("[replay] first-track-ready after \(String(format: "%.1f", firstAvailableElapsed))s (available=\(becameAvailable))")
+ XCTAssertTrue(becameAvailable, "Live captions never became available within \(readyTimeout)s — model load likely failed.")
+ // Both engines load their own model instance in parallel; give the slower one a grace
+ // window since the snapshot only reports "available" on the FIRST track's readiness.
+ try await Task.sleep(nanoseconds: 5_000_000_000)
+
+ // The bounded queue holds only ~480ms per track (capacity 24 * 20ms). Blasting all ~19.5k
+ // chunks in a tight loop (sub-second) overwhelms it before the drain loop can ever get
+ // scheduled, which would test harness timing, not the pipeline. Cap the feed rate at
+ // `accelerationFactor`x real time instead — still far from a real 192s sleep, but paced
+ // enough for the actor's drain loop to keep up with genuine ASR inference.
+ let accelerationFactor: Double = 1.5
+ let feedStart = Date()
+ var bufferFailures = 0
+ var sampleBufferFailures = 0
+ var offeredCount = 0
+ var totalWaitSeconds: Double = 0
+ for (index, event) in events.enumerated() {
+ guard let buffer = Self.makeBuffer(format: pcmFormat, samples: event.samples) else {
+ bufferFailures += 1
+ continue
+ }
+ let pts = CMTime(value: CMTimeValue(event.ptsSamples), timescale: CMTimeScale(Self.sampleRate))
+ guard let sampleBuffer = Self.makeSampleBuffer(pcm: buffer, formatDescription: formatDescription, pts: pts) else {
+ sampleBufferFailures += 1
+ continue
+ }
+ coordinator.offer(kind: event.kind, sampleBuffer: sampleBuffer)
+ offeredCount += 1
+ let ptsSeconds = Double(event.ptsSamples) / Self.sampleRate
+ let targetElapsed = ptsSeconds / accelerationFactor
+ let actualElapsed = Date().timeIntervalSince(feedStart)
+ let waitSeconds = targetElapsed - actualElapsed
+ if waitSeconds > 0.001 {
+ totalWaitSeconds += waitSeconds
+ try await Task.sleep(nanoseconds: UInt64(waitSeconds * 1_000_000_000))
+ } else if index % 8 == 0 {
+ await Task.yield()
+ }
+ }
+ let feedElapsed = Date().timeIntervalSince(feedStart)
+ print("[replay] fed \(events.count) chunks in \(String(format: "%.2f", feedElapsed))s (offered=\(offeredCount), bufferFailures=\(bufferFailures), sampleBufferFailures=\(sampleBufferFailures), totalWaitSeconds=\(String(format: "%.2f", totalWaitSeconds)))")
+
+ // Let the drain loops work through whatever is still queued, until the transcript stops
+ // changing.
+ var lastSnapshot = box.lock.withLock { box.latest }
+ var stableCount = 0
+ let settleDeadline = Date().addingTimeInterval(60)
+ while Date() < settleDeadline, stableCount < 10 {
+ try await Task.sleep(nanoseconds: 300_000_000)
+ let current = box.lock.withLock { box.latest }
+ if current == lastSnapshot {
+ stableCount += 1
+ } else {
+ stableCount = 0
+ lastSnapshot = current
+ }
+ }
+
+ await coordinator.stop()
+ // stop() may itself publish nothing further, but give any last async callback a moment.
+ try await Task.sleep(nanoseconds: 300_000_000)
+ let final = box.lock.withLock { box.latest }
+
+ // ---- Report ----
+ print("=== LIVE REPLAY TRANSCRIPT (\(final.utterances.count) utterances) ===")
+ for utterance in final.utterances {
+ let speaker = utterance.speaker == .you ? "You " : "Them"
+ print(String(format: "[%7.2f - %7.2f] %@: %@", utterance.start, utterance.end, speaker, utterance.text))
+ }
+ if !final.partials.isEmpty {
+ print("--- Leftover partials at end ---")
+ for (speaker, text) in final.partials {
+ print(" \(speaker): \(text)")
+ }
+ }
+ print("Final availability: \(final.availability)")
+
+ let degradedTransitions = box.lock.withLock {
+ box.log.filter { if case .degraded = $0.snapshot.availability { return true } else { return false } }
+ }
+ print("Degraded-availability updates observed: \(degradedTransitions.count)")
+ for transition in degradedTransitions.prefix(20) {
+ print(" t+\(String(format: "%.1f", transition.elapsed))s: \(transition.snapshot.availability)")
+ }
+
+ let youUtterances = final.utterances.filter { $0.speaker == .you }
+ let themUtterances = final.utterances.filter { $0.speaker == .them }
+ print("You utterances: \(youUtterances.count), Them utterances: \(themUtterances.count)")
+
+ // This is an observational experiment, not a pass/fail gate: assert only that the harness
+ // itself ran the real pipeline end-to-end without erroring out.
+ XCTAssertTrue(true, "See console output above for the replayed transcript and analysis.")
+ }
+
+ // MARK: - Helpers
+
+ private static func appendChunks(
+ _ events: inout [(kind: MeetingAudioTrackKind, samples: ArraySlice<Float>, ptsSamples: Int)],
+ kind: MeetingAudioTrackKind,
+ samples: [Float],
+ ptsBaseSamples: Int = 0
+ ) {
+ var offset = 0
+ while offset < samples.count {
+ let end = min(offset + self.chunkFrames, samples.count)
+ events.append((kind: kind, samples: samples[offset..<end], ptsSamples: ptsBaseSamples + offset))
+ offset = end
+ }
+ }
+
+ private static func load16kMonoFloatSamples(path: String) throws -> [Float] {
+ let url = URL(fileURLWithPath: path)
+ guard FileManager.default.fileExists(atPath: path) else {
+ throw NSError(domain: "MeetingLiveReplayTests", code: 1, userInfo: [NSLocalizedDescriptionKey: "Fixture not found at \(path)"])
+ }
+ let inputFile = try AVAudioFile(forReading: url)
+ let inputFormat = inputFile.processingFormat
+ guard let desiredFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: self.sampleRate, channels: 1, interleaved: false) else {
+ throw NSError(domain: "MeetingLiveReplayTests", code: 2, userInfo: [NSLocalizedDescriptionKey: "Could not build target format"])
+ }
+
+ let inputBuffer: AVAudioPCMBuffer = try {
+ let frameCount = AVAudioFrameCount(inputFile.length)
+ guard let buffer = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: max(frameCount, 1)) else {
+ throw NSError(domain: "MeetingLiveReplayTests", code: 3, userInfo: [NSLocalizedDescriptionKey: "Could not allocate input buffer"])
+ }
+ try inputFile.read(into: buffer)
+ return buffer
+ }()
+
+ if inputFormat.sampleRate == desiredFormat.sampleRate,
+ inputFormat.channelCount == desiredFormat.channelCount,
+ inputFormat.commonFormat == desiredFormat.commonFormat
+ {
+ return Self.extractSamples(inputBuffer)
+ }
+
+ guard let converter = AVAudioConverter(from: inputFormat, to: desiredFormat) else {
+ throw NSError(domain: "MeetingLiveReplayTests", code: 4, userInfo: [NSLocalizedDescriptionKey: "Could not build converter"])
+ }
+ let ratio = desiredFormat.sampleRate / inputFormat.sampleRate
+ let estimatedFrames = max(1, AVAudioFrameCount(Double(inputBuffer.frameLength) * ratio) + 1)
+ guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: desiredFormat, frameCapacity: estimatedFrames) else {
+ throw NSError(domain: "MeetingLiveReplayTests", code: 5, userInfo: [NSLocalizedDescriptionKey: "Could not allocate output buffer"])
+ }
+ var providedInput = false
+ var conversionError: NSError?
+ converter.convert(to: outputBuffer, error: &conversionError) { _, outStatus in
+ if providedInput {
+ outStatus.pointee = .endOfStream
+ return nil
+ }
+ providedInput = true
+ outStatus.pointee = .haveData
+ return inputBuffer
+ }
+ if let conversionError { throw conversionError }
+ return Self.extractSamples(outputBuffer)
+ }
+
+ private static func extractSamples(_ buffer: AVAudioPCMBuffer) -> [Float] {
+ guard let channelData = buffer.floatChannelData else { return [] }
+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)))
+ }
+
+ private static func makeBuffer(format: AVAudioFormat, samples: ArraySlice<Float>) -> AVAudioPCMBuffer? {
+ guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count)),
+ let channelData = buffer.floatChannelData
+ else { return nil }
+ buffer.frameLength = AVAudioFrameCount(samples.count)
+ for (index, sample) in samples.enumerated() {
+ channelData[0][index] = sample
+ }
+ return buffer
+ }
+
+ private static func makeFormatDescription(format: AVAudioFormat) -> CMAudioFormatDescription? {
+ var formatDescription: CMAudioFormatDescription?
+ var asbd = format.streamDescription.pointee
+ let status = CMAudioFormatDescriptionCreate(
+ allocator: kCFAllocatorDefault,
+ asbd: &asbd,
+ layoutSize: 0,
+ layout: nil,
+ magicCookieSize: 0,
+ magicCookie: nil,
+ extensions: nil,
+ formatDescriptionOut: &formatDescription
+ )
+ guard status == noErr else { return nil }
+ return formatDescription
+ }
+
+ /// Builds a `dataReady` sample buffer directly from a copy of the PCM buffer's bytes, rather
+ /// than via `CMSampleBufferSetDataBufferFromAudioBufferList` (which reliably returned
+ /// `kCMSampleBufferError_RequiredParameterMissing` for this mono/non-interleaved Float32 case
+ /// in practice, despite matching Apple's documented parameters).
+ private static func makeSampleBuffer(pcm: AVAudioPCMBuffer, formatDescription: CMAudioFormatDescription, pts: CMTime) -> CMSampleBuffer? {
+ let frameCount = Int(pcm.frameLength)
+ guard frameCount > 0, let channelData = pcm.floatChannelData else { return nil }
+ let bytesPerFrame = Int(pcm.format.streamDescription.pointee.mBytesPerFrame)
+ let dataLength = frameCount * bytesPerFrame
+
+ var blockBuffer: CMBlockBuffer?
+ let blockStatus = CMBlockBufferCreateWithMemoryBlock(
+ allocator: kCFAllocatorDefault,
+ memoryBlock: nil,
+ blockLength: dataLength,
+ blockAllocator: kCFAllocatorDefault,
+ customBlockSource: nil,
+ offsetToData: 0,
+ dataLength: dataLength,
+ flags: 0,
+ blockBufferOut: &blockBuffer
+ )
+ guard blockStatus == kCMBlockBufferNoErr, let blockBuffer else { return nil }
+ let copyStatus = channelData[0].withMemoryRebound(to: UInt8.self, capacity: dataLength) { bytes in
+ CMBlockBufferReplaceDataBytes(with: bytes, blockBuffer: blockBuffer, offsetIntoDestination: 0, dataLength: dataLength)
+ }
+ guard copyStatus == kCMBlockBufferNoErr else { return nil }
+
+ var timing = CMSampleTimingInfo(
+ duration: CMTime(value: CMTimeValue(frameCount), timescale: CMTimeScale(self.sampleRate)),
+ presentationTimeStamp: pts,
+ decodeTimeStamp: .invalid
+ )
+ var sampleSize = bytesPerFrame
+ var sampleBuffer: CMSampleBuffer?
+ let createStatus = CMSampleBufferCreate(
+ allocator: kCFAllocatorDefault,
+ dataBuffer: blockBuffer,
+ dataReady: true,
+ makeDataReadyCallback: nil,
+ refcon: nil,
+ formatDescription: formatDescription,
+ sampleCount: frameCount,
+ sampleTimingEntryCount: 1,
+ sampleTimingArray: &timing,
+ sampleSizeEntryCount: 1,
+ sampleSizeArray: &sampleSize,
+ sampleBufferOut: &sampleBuffer
+ )
+ guard createStatus == noErr, let sampleBuffer else { return nil }
+ return sampleBuffer
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/MeetingLiveSilenceEndpointTests.swift b/Tests/FluidDictationIntegrationTests/MeetingLiveSilenceEndpointTests.swift
new file mode 100644
index 00000000..95d1ed1e
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/MeetingLiveSilenceEndpointTests.swift
@@ -0,0 +1,113 @@
+@testable import FluidVoice_Debug
+import AVFoundation
+import Foundation
+import XCTest
+
+#if arch(arm64)
+import FluidAudio
+
+/// Measurement: does `StreamingEouAsrManager` endpoint on sustained digital silence?
+///
+/// Option A for live echo suppression gates echo-dominated audio to zeros so the microphone regains
+/// the pauses its endpointer needs. Nothing in FluidAudio's source answers whether a learned EOU
+/// token head fires on pure zeros — the warmup pass feeds 1s of silence and never checks. If it does
+/// not fire, gating cleans the audio but utterances still never close, and the endpointing half of
+/// the plan has to be driven from gate state instead.
+///
+/// Run with FLUIDVOICE_SILENCE_EOU=<session dir>.
+final class MeetingLiveSilenceEndpointTests: XCTestCase {
+ func testEndpointerFiresOnSustainedDigitalSilence() async throws {
+ guard let root = ProcessInfo.processInfo.environment["FLUIDVOICE_SILENCE_EOU"] else {
+ throw XCTSkip("FLUIDVOICE_SILENCE_EOU not set")
+ }
+ let sessionDirectory = URL(fileURLWithPath: root, isDirectory: true)
+ let decoder = JSONDecoder()
+ decoder.dateDecodingStrategy = .iso8601
+ let micTrack = try decoder.decode(
+ MeetingAudioTrack.self,
+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/microphone/track.json"))
+ )
+ // Chunk 3 spans session 180-240s and contains known real user speech at 215.2-224.7.
+ guard let chunk = micTrack.chunks.first(where: { $0.sequence == 3 }) else {
+ throw XCTSkip("expected chunk sequence 3")
+ }
+ let speech = try MeetingProcessingPipeline.readSamples(
+ fileURL: sessionDirectory.appending(path: chunk.relativeFilePath),
+ startSeconds: 35.2,
+ endSeconds: 44.7
+ )
+ XCTAssertFalse(speech.isEmpty)
+
+ let manager = StreamingEouAsrManager(chunkSize: .ms160)
+ try await manager.loadModelsFromHuggingFace()
+
+ let events = EouEventLog()
+ await manager.setEouCallback { transcript in
+ Task { await events.record(transcript) }
+ }
+
+ let sampleRate = 16_000.0
+ let blockFrames = 2_560 // one 160ms chunk
+
+ func feed(_ samples: [Float]) async throws {
+ var index = 0
+ while index < samples.count {
+ let end = min(index + blockFrames, samples.count)
+ let slice = Array(samples[index..<end])
+ guard let buffer = Self.buffer(slice, sampleRate: sampleRate) else { break }
+ try await manager.appendAudio(buffer)
+ try await manager.processBufferedAudio()
+ index = end
+ }
+ }
+
+ try await feed(speech)
+ let afterSpeech = await events.count
+ print("[silence-eou] fed \(String(format: "%.1f", Double(speech.count) / sampleRate))s speech, EOUs so far: \(afterSpeech)")
+
+ // Then sustained digital silence, checked block by block so we can report WHEN it fires.
+ let silenceBlocks = Int(8.0 * sampleRate) / blockFrames
+ var firedAfterSeconds: Double?
+ for block in 0..<silenceBlocks {
+ guard let buffer = Self.buffer(Array(repeating: 0, count: blockFrames), sampleRate: sampleRate) else { break }
+ try await manager.appendAudio(buffer)
+ try await manager.processBufferedAudio()
+ await Task.yield()
+ if firedAfterSeconds == nil, await events.count > afterSpeech {
+ firedAfterSeconds = Double(block + 1) * Double(blockFrames) / sampleRate
+ }
+ }
+ // Callbacks hop through tasks; give them a moment to land.
+ try? await Task.sleep(nanoseconds: 300_000_000)
+ let total = await events.count
+
+ print("[silence-eou] total EOUs=\(total) (before silence: \(afterSpeech))")
+ if let firedAfterSeconds {
+ print(String(format: "[silence-eou] RESULT: EOU FIRED %.2fs into silence (debounce is 1.28s)", firedAfterSeconds))
+ } else {
+ print("[silence-eou] RESULT: NO EOU after 8.0s of digital silence")
+ }
+ for (index, text) in await events.transcripts.enumerated() {
+ print("[silence-eou] eou#\(index + 1) chars=\(text.count) text=\(text.prefix(90))")
+ }
+ await manager.reset()
+ }
+
+ private static func buffer(_ samples: [Float], sampleRate: Double) -> AVAudioPCMBuffer? {
+ guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false),
+ let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count))
+ else { return nil }
+ buffer.frameLength = AVAudioFrameCount(samples.count)
+ samples.withUnsafeBufferPointer { source in
+ buffer.floatChannelData?[0].update(from: source.baseAddress!, count: samples.count)
+ }
+ return buffer
+ }
+}
+
+private actor EouEventLog {
+ private(set) var transcripts: [String] = []
+ var count: Int { self.transcripts.count }
+ func record(_ text: String) { self.transcripts.append(text) }
+}
+#endif
diff --git a/Tests/FluidDictationIntegrationTests/VoiceProcessingAsrQualityTests.swift b/Tests/FluidDictationIntegrationTests/VoiceProcessingAsrQualityTests.swift
new file mode 100644
index 00000000..39e08d9c
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/VoiceProcessingAsrQualityTests.swift
@@ -0,0 +1,128 @@
+@testable import FluidVoice_Debug
+import AVFoundation
+import Foundation
+import XCTest
+
+#if arch(arm64)
+import FluidAudio
+
+/// Measurement: does Apple's Voice Processing I/O degrade transcription of the user's OWN voice?
+///
+/// macOS bundles noise suppression and automatic gain control with its echo canceller, and shipping
+/// dictation products disable all three for dictation while enabling them only for meetings — which
+/// implies the processing costs something. Two takes of the same passage, one raw and one
+/// VPIO-processed, each transcribed and compared against the known text.
+///
+/// Deliberately avoids ScreenCaptureKit: the test host loses its Screen Recording grant on every
+/// rebuild, and nothing here needs it.
+///
+/// Run with FLUIDVOICE_ASR_AB=1.
+final class VoiceProcessingAsrQualityTests: XCTestCase {
+ private static let captureSeconds = 22.0
+
+ func testVpioEffectOnOwnVoiceTranscription() async throws {
+ guard ProcessInfo.processInfo.environment["FLUIDVOICE_ASR_AB"] != nil else {
+ throw XCTSkip("FLUIDVOICE_ASR_AB not set")
+ }
+
+ // Lead-in so the operator can start speaking before the first take opens.
+ print("\n[ab] starting in 15 seconds — begin reading now and keep going\n")
+ try await Task.sleep(nanoseconds: 15_000_000_000)
+ let raw = try await Self.capture(voiceProcessing: false, label: "TAKE 1 of 2 — voice processing OFF")
+ try await Task.sleep(nanoseconds: 2_000_000_000)
+ let processed = try await Self.capture(voiceProcessing: true, label: "TAKE 2 of 2 — voice processing ON")
+
+ print(String(format: "\n[ab] raw %.1fs rms=%.5f (%.1f dBFS)",
+ Double(raw.count) / 16_000, Self.rms(raw), 20 * log10(max(Self.rms(raw), 1e-9))))
+ print(String(format: "[ab] vpio %.1fs rms=%.5f (%.1f dBFS)",
+ Double(processed.count) / 16_000, Self.rms(processed), 20 * log10(max(Self.rms(processed), 1e-9))))
+
+ let rawText = try await Self.transcribe(raw)
+ let vpioText = try await Self.transcribe(processed)
+ print("\n[ab] RAW : \(rawText)")
+ print("\n[ab] VPIO : \(vpioText)\n")
+ print("[ab] words raw=\(rawText.split(separator: " ").count) vpio=\(vpioText.split(separator: " ").count)")
+
+ XCTAssertFalse(raw.isEmpty, "no audio captured on take 1")
+ XCTAssertFalse(processed.isEmpty, "no audio captured on take 2")
+ }
+
+ private static func capture(voiceProcessing: Bool, label: String) async throws -> [Float] {
+ let engine = AVAudioEngine()
+ let input = engine.inputNode
+ if voiceProcessing {
+ try input.setVoiceProcessingEnabled(true)
+ var ducking = AVAudioVoiceProcessingOtherAudioDuckingConfiguration()
+ ducking.enableAdvancedDucking = false
+ ducking.duckingLevel = .min
+ input.voiceProcessingOtherAudioDuckingConfiguration = ducking
+ }
+ let format = input.outputFormat(forBus: 0)
+ let sink = SampleSink()
+ input.installTap(onBus: 0, bufferSize: 1024, format: format) { buffer, _ in
+ guard let channel = buffer.floatChannelData?[0] else { return }
+ let slice = Array(UnsafeBufferPointer(start: channel, count: Int(buffer.frameLength)))
+ Task { await sink.append(slice, sourceRate: format.sampleRate) }
+ }
+ engine.prepare()
+ try engine.start()
+ print("\n[ab] ==========================================================")
+ print("[ab] \(label)")
+ print("[ab] READ THE PASSAGE NOW — \(Int(Self.captureSeconds)) seconds")
+ print("[ab] ==========================================================\n")
+ try await Task.sleep(nanoseconds: UInt64(Self.captureSeconds * 1_000_000_000))
+ engine.stop()
+ input.removeTap(onBus: 0)
+ return await sink.samples()
+ }
+
+ private static func rms(_ samples: [Float]) -> Double {
+ guard !samples.isEmpty else { return 0 }
+ return (samples.reduce(0.0) { $0 + Double($1) * Double($1) } / Double(samples.count)).squareRoot()
+ }
+
+ private static func transcribe(_ samples: [Float]) async throws -> String {
+ guard !samples.isEmpty else { return "(no audio)" }
+ let manager = StreamingEouAsrManager(chunkSize: .ms160)
+ try await manager.loadModelsFromHuggingFace()
+ guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 16_000, channels: 1, interleaved: false)
+ else { return "(format error)" }
+ var index = 0
+ while index < samples.count {
+ let end = min(index + 2_560, samples.count)
+ guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(end - index)) else { break }
+ buffer.frameLength = AVAudioFrameCount(end - index)
+ samples[index..<end].withUnsafeBufferPointer { source in
+ buffer.floatChannelData?[0].update(from: source.baseAddress!, count: end - index)
+ }
+ try await manager.appendAudio(buffer)
+ try await manager.processBufferedAudio()
+ index = end
+ }
+ let text = try await manager.finish()
+ await manager.reset()
+ return text.trimmingCharacters(in: .whitespacesAndNewlines)
+ }
+}
+
+/// Accumulates mono 16 kHz float samples, decimating from the source rate.
+private actor SampleSink {
+ private var storage: [Float] = []
+
+ func append(_ incoming: [Float], sourceRate: Double) {
+ guard sourceRate > 0 else { return }
+ let ratio = sourceRate / 16_000
+ guard ratio > 1.01 else {
+ self.storage.append(contentsOf: incoming)
+ return
+ }
+ var position = 0.0
+ while Int(position) < incoming.count {
+ self.storage.append(incoming[Int(position)])
+ position += ratio
+ }
+ }
+
+ func samples() -> [Float] { self.storage }
+}
+#endif
diff --git a/Tests/FluidDictationIntegrationTests/VoiceProcessingDuckingProbeTests.swift b/Tests/FluidDictationIntegrationTests/VoiceProcessingDuckingProbeTests.swift
new file mode 100644
index 00000000..78664e1b
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/VoiceProcessingDuckingProbeTests.swift
@@ -0,0 +1,172 @@
+@testable import FluidVoice_Debug
+import AVFoundation
+import Foundation
+import ScreenCaptureKit
+import XCTest
+
+/// Measurement: does instantiating Voice Processing I/O attenuate what ScreenCaptureKit captures
+/// from another application?
+///
+/// VPIO is Apple's hardware-assisted AEC and would replace a hand-rolled echo gate entirely. It was
+/// ruled out earlier on the assumption that its system-wide ducking of "other audio" would corrupt
+/// the recorded `applicationAudio` track — an assumption never measured. SCStream taps per-process;
+/// ducking applies at output mixing, so they may not be the same path.
+///
+/// Plays a steady tone from a separate process (`afplay`), captures it via SCStream, and compares
+/// captured level across three phases: no VPIO, VPIO with default ducking, VPIO with ducking
+/// minimised. Run with FLUIDVOICE_VPIO_PROBE=1.
+final class VoiceProcessingDuckingProbeTests: XCTestCase {
+ func testVoiceProcessingDuckingEffectOnScreenCaptureAudio() async throws {
+ guard ProcessInfo.processInfo.environment["FLUIDVOICE_VPIO_PROBE"] != nil else {
+ throw XCTSkip("FLUIDVOICE_VPIO_PROBE not set")
+ }
+
+ // Audio source is an external windowed application (a browser tab playing a tone). A
+ // windowless helper such as `afplay` is not part of a display's shareable content, so
+ // SCStream delivers buffers of pure zeros for it — measured: 584 buffers, all silent.
+ try await Task.sleep(nanoseconds: 500_000_000)
+
+ let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+ guard let display = content.displays.first else { throw XCTSkip("no display") }
+ let filter = SCContentFilter(display: display, excludingApplications: [], exceptingWindows: [])
+
+ let configuration = SCStreamConfiguration()
+ configuration.width = 2
+ configuration.height = 2
+ configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
+ configuration.queueDepth = 3
+ configuration.showsCursor = false
+ configuration.capturesAudio = true
+ configuration.sampleRate = 48_000
+ configuration.channelCount = 2
+ configuration.excludesCurrentProcessAudio = true
+
+ let collector = AudioLevelCollector()
+ let output = LevelStreamOutput(collector: collector)
+ let errorDelegate = ProbeStreamDelegate()
+ let stream = SCStream(filter: filter, configuration: configuration, delegate: errorDelegate)
+ try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: DispatchQueue(label: "vpio.probe"))
+ try await stream.startCapture()
+ defer { Task { try? await stream.stopCapture() } }
+ print("[vpio] capture started")
+
+ func phase(_ label: String, seconds: Double) async -> Double {
+ await collector.mark(label)
+ try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))
+ return await collector.rms(for: label)
+ }
+
+ let baseline = await phase("1-no-vpio", seconds: 3.0)
+
+ // Instantiating and running an engine with voice processing on the IO nodes is what turns
+ // VPIO on process-wide; merely constructing AVAudioEngine does not.
+ let engine = AVAudioEngine()
+ let input = engine.inputNode
+ // Enabling on the input node switches the whole IO unit to VPIO; doing it on the output node
+ // as well fails initialisation with -10875.
+ try input.setVoiceProcessingEnabled(true)
+ input.installTap(onBus: 0, bufferSize: 1024, format: input.outputFormat(forBus: 0)) { _, _ in }
+
+ // Adding a render chain to satisfy VPIO's downlink fails initialisation with -10875 here, so
+ // the engine runs capture-only. VPIO is still instantiated and active — it logs its own
+ // downlink faults — which is what the ducking measurement needs.
+ engine.prepare()
+ try engine.start()
+ defer { engine.stop() }
+ try await Task.sleep(nanoseconds: 1_200_000_000)
+
+ let withDefaultDucking = await phase("2-vpio-default", seconds: 3.0)
+
+ var ducking = AVAudioVoiceProcessingOtherAudioDuckingConfiguration()
+ ducking.enableAdvancedDucking = false
+ ducking.duckingLevel = .min
+ input.voiceProcessingOtherAudioDuckingConfiguration = ducking
+ try await Task.sleep(nanoseconds: 700_000_000)
+
+ let withMinDucking = await phase("3-vpio-min-ducking", seconds: 3.0)
+
+ func dB(_ value: Double) -> String {
+ value > 0 ? String(format: "%.1f dBFS", 20 * log10(value)) : "silent"
+ }
+ print("[vpio] baseline (no VPIO) rms=\(String(format: "%.5f", baseline)) \(dB(baseline))")
+ print("[vpio] VPIO, default ducking rms=\(String(format: "%.5f", withDefaultDucking)) \(dB(withDefaultDucking))")
+ print("[vpio] VPIO, ducking = .min rms=\(String(format: "%.5f", withMinDucking)) \(dB(withMinDucking))")
+ if baseline > 0 {
+ print(String(format: "[vpio] default ducking attenuation: %.1f dB", 20 * log10(withDefaultDucking / baseline)))
+ print(String(format: "[vpio] min ducking attenuation: %.1f dB", 20 * log10(withMinDucking / baseline)))
+ }
+ let stats = await collector.diagnostics()
+ print("[vpio] buffers=\(stats.buffers) copyFailures=\(stats.copyFailures) framesTotal=\(stats.frames) streamError=\(errorDelegate.lastError ?? "none")")
+ XCTAssertGreaterThan(baseline, 0, "SCStream captured no audio; tone or permission problem")
+ }
+
+}
+
+private actor AudioLevelCollector {
+ private var current: String?
+ private var sums: [String: (total: Double, frames: Int)] = [:]
+ private var buffers = 0
+ private var copyFailures = 0
+ private var frames = 0
+
+ func mark(_ label: String) { self.current = label }
+
+ func diagnostics() -> (buffers: Int, copyFailures: Int, frames: Int) {
+ (self.buffers, self.copyFailures, self.frames)
+ }
+
+ func noteBuffer() { self.buffers += 1 }
+ func noteCopyFailure() { self.copyFailures += 1 }
+
+ func add(sumOfSquares: Double, frames: Int) {
+ self.frames += frames
+ guard let current else { return }
+ let existing = self.sums[current] ?? (0, 0)
+ self.sums[current] = (existing.total + sumOfSquares, existing.frames + frames)
+ }
+
+ func rms(for label: String) -> Double {
+ guard let entry = self.sums[label], entry.frames > 0 else { return 0 }
+ return (entry.total / Double(entry.frames)).squareRoot()
+ }
+}
+
+private final class LevelStreamOutput: NSObject, SCStreamOutput {
+ private let collector: AudioLevelCollector
+ init(collector: AudioLevelCollector) { self.collector = collector }
+
+ func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+ guard type == .audio else { return }
+ Task { [collector] in await collector.noteBuffer() }
+ guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),
+ let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription),
+ let format = AVAudioFormat(streamDescription: asbd)
+ else { return }
+ let frames = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
+ guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames) else { return }
+ buffer.frameLength = frames
+ guard CMSampleBufferCopyPCMDataIntoAudioBufferList(
+ sampleBuffer, at: 0, frameCount: Int32(frames), into: buffer.mutableAudioBufferList
+ ) == noErr, let channel = buffer.floatChannelData?[0] else {
+ Task { [collector] in await collector.noteCopyFailure() }
+ return
+ }
+ var sum = 0.0
+ for i in 0..<Int(frames) { sum += Double(channel[i]) * Double(channel[i]) }
+ Task { [collector] in await collector.add(sumOfSquares: sum, frames: Int(frames)) }
+ }
+}
+
+private final class ProbeStreamDelegate: NSObject, SCStreamDelegate {
+ private let lock = NSLock()
+ private var stored: String?
+ var lastError: String? {
+ self.lock.lock(); defer { self.lock.unlock() }
+ return self.stored
+ }
+
+ func stream(_ stream: SCStream, didStopWithError error: Error) {
+ self.lock.lock(); defer { self.lock.unlock() }
+ self.stored = String(describing: error)
+ }
+}
diff --git a/Tests/FluidDictationIntegrationTests/VoiceProcessingSpeechDuckingTests.swift b/Tests/FluidDictationIntegrationTests/VoiceProcessingSpeechDuckingTests.swift
new file mode 100644
index 00000000..a6cfad5d
--- /dev/null
+++ b/Tests/FluidDictationIntegrationTests/VoiceProcessingSpeechDuckingTests.swift
@@ -0,0 +1,176 @@
+@testable import FluidVoice_Debug
+import AVFoundation
+import Foundation
+import ScreenCaptureKit
+import XCTest
+
+/// Two measurements in one run, both needed before adopting Apple's AEC for the meeting microphone.
+///
+/// A. Ducking against REAL SPEECH rather than a steady tone. Ducking is voice-activity driven, so a
+/// tone understates both its depth and its variability. Reports a per-second level timeline.
+/// B. Relative clock drift between an `AVAudioEngine` microphone and SCStream application audio.
+/// Moving the microphone off SCStream costs the shared capture clock; this quantifies what that
+/// is worth in parts per million.
+///
+/// Play speech through a windowed app, then run with FLUIDVOICE_VPIO_SPEECH=1.
+final class VoiceProcessingSpeechDuckingTests: XCTestCase {
+ func testDuckingAgainstSpeechAndClockDrift() async throws {
+ guard ProcessInfo.processInfo.environment["FLUIDVOICE_VPIO_SPEECH"] != nil else {
+ throw XCTSkip("FLUIDVOICE_VPIO_SPEECH not set")
+ }
+
+ let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+ guard let display = content.displays.first else { throw XCTSkip("no display") }
+ let filter = SCContentFilter(display: display, excludingApplications: [], exceptingWindows: [])
+
+ let configuration = SCStreamConfiguration()
+ configuration.width = 2
+ configuration.height = 2
+ configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)
+ configuration.queueDepth = 3
+ configuration.showsCursor = false
+ configuration.capturesAudio = true
+ configuration.sampleRate = 48_000
+ configuration.channelCount = 2
+ configuration.excludesCurrentProcessAudio = true
+
+ let recorder = TimelineRecorder()
+ let output = TimelineStreamOutput(recorder: recorder)
+ let stream = SCStream(filter: filter, configuration: configuration, delegate: nil)
+ try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: DispatchQueue(label: "vpio.speech"))
+ try await stream.startCapture()
+ defer { Task { try? await stream.stopCapture() } }
+
+ await recorder.mark("baseline")
+ try await Task.sleep(nanoseconds: 6_000_000_000)
+
+ let engine = AVAudioEngine()
+ let input = engine.inputNode
+ try input.setVoiceProcessingEnabled(true)
+ var ducking = AVAudioVoiceProcessingOtherAudioDuckingConfiguration()
+ ducking.enableAdvancedDucking = false
+ ducking.duckingLevel = .min
+ input.voiceProcessingOtherAudioDuckingConfiguration = ducking
+
+ // Count microphone frames against host time to derive this clock's true rate.
+ let micClock = ClockCounter(nominalRate: input.outputFormat(forBus: 0).sampleRate)
+ input.installTap(onBus: 0, bufferSize: 1024, format: input.outputFormat(forBus: 0)) { buffer, _ in
+ Task { await micClock.add(frames: Int(buffer.frameLength)) }
+ }
+ engine.prepare()
+ try engine.start()
+ defer { engine.stop() }
+ try await Task.sleep(nanoseconds: 1_000_000_000)
+
+ await recorder.mark("vpio-min-ducking")
+ await micClock.begin()
+ try await Task.sleep(nanoseconds: 20_000_000_000)
+
+ let timeline = await recorder.timeline()
+ let baseline = timeline.filter { $0.phase == "baseline" }.map(\.rms)
+ let ducked = timeline.filter { $0.phase == "vpio-min-ducking" }.map(\.rms)
+
+ func dB(_ v: Double) -> Double { v > 0 ? 20 * log10(v) : -120 }
+ func summarise(_ label: String, _ values: [Double]) {
+ guard !values.isEmpty else { print("[speech] \(label): no data"); return }
+ let mean = values.reduce(0, +) / Double(values.count)
+ print(String(
+ format: "[speech] %-18@ seconds=%d mean=%.1f dBFS min=%.1f max=%.1f swing=%.1f dB",
+ label as NSString, values.count, dB(mean), dB(values.min() ?? 0), dB(values.max() ?? 0),
+ dB(values.max() ?? 0) - dB(values.min() ?? 0)
+ ))
+ }
+ summarise("baseline", baseline)
+ summarise("VPIO min ducking", ducked)
+ if let b = baseline.max(), let d = ducked.max(), b > 0, d > 0 {
+ print(String(format: "[speech] peak-to-peak attenuation: %.1f dB", dB(d) - dB(b)))
+ }
+ print("[speech] per-second dBFS (ducked): "
+ + ducked.map { String(format: "%.0f", dB($0)) }.joined(separator: " "))
+
+ let (micFrames, micSeconds) = await micClock.result()
+ let (appFrames, appSeconds) = await recorder.clockResult()
+ func ppm(_ frames: Int, _ seconds: Double, _ nominal: Double) -> Double {
+ guard seconds > 0, nominal > 0 else { return .nan }
+ return (Double(frames) / seconds / nominal - 1) * 1_000_000
+ }
+ let micPPM = ppm(micFrames, micSeconds, await micClock.nominalRate)
+ let appPPM = ppm(appFrames, appSeconds, 48_000)
+ print(String(format: "[drift] mic clock %.0f ppm vs nominal (%d frames / %.2fs)", micPPM, micFrames, micSeconds))
+ print(String(format: "[drift] app clock %.0f ppm vs nominal (%d frames / %.2fs)", appPPM, appFrames, appSeconds))
+ print(String(format: "[drift] RELATIVE %.0f ppm => %.0f ms of skew per hour", micPPM - appPPM, abs(micPPM - appPPM) * 3.6))
+
+ XCTAssertFalse(baseline.isEmpty, "no baseline audio captured — is speech playing?")
+ }
+}
+
+private actor ClockCounter {
+ let nominalRate: Double
+ private var frames = 0
+ private var start: CFAbsoluteTime?
+ init(nominalRate: Double) { self.nominalRate = nominalRate }
+ func begin() { self.start = CFAbsoluteTimeGetCurrent(); self.frames = 0 }
+ func add(frames count: Int) { if self.start != nil { self.frames += count } }
+ func result() -> (Int, Double) {
+ guard let start else { return (0, 0) }
+ return (self.frames, CFAbsoluteTimeGetCurrent() - start)
+ }
+}
+
+private actor TimelineRecorder {
+ struct Sample { let phase: String; let rms: Double }
+ private var phase = ""
+ private var bucketStart = CFAbsoluteTimeGetCurrent()
+ private var bucketSum = 0.0
+ private var bucketFrames = 0
+ private var samples: [Sample] = []
+ private var clockFrames = 0
+ private var clockStart: CFAbsoluteTime?
+
+ func mark(_ label: String) {
+ self.phase = label
+ if self.clockStart == nil { self.clockStart = CFAbsoluteTimeGetCurrent() }
+ }
+
+ func add(sumOfSquares: Double, frames: Int) {
+ guard !self.phase.isEmpty else { return }
+ self.clockFrames += frames
+ self.bucketSum += sumOfSquares
+ self.bucketFrames += frames
+ let now = CFAbsoluteTimeGetCurrent()
+ if now - self.bucketStart >= 1.0, self.bucketFrames > 0 {
+ self.samples.append(Sample(phase: self.phase, rms: (self.bucketSum / Double(self.bucketFrames)).squareRoot()))
+ self.bucketStart = now
+ self.bucketSum = 0
+ self.bucketFrames = 0
+ }
+ }
+
+ func timeline() -> [Sample] { self.samples }
+ func clockResult() -> (Int, Double) {
+ guard let clockStart else { return (0, 0) }
+ return (self.clockFrames, CFAbsoluteTimeGetCurrent() - clockStart)
+ }
+}
+
+private final class TimelineStreamOutput: NSObject, SCStreamOutput {
+ private let recorder: TimelineRecorder
+ init(recorder: TimelineRecorder) { self.recorder = recorder }
+
+ func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+ guard type == .audio,
+ let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),
+ let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription),
+ let format = AVAudioFormat(streamDescription: asbd)
+ else { return }
+ let frames = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))
+ guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames) else { return }
+ buffer.frameLength = frames
+ guard CMSampleBufferCopyPCMDataIntoAudioBufferList(
+ sampleBuffer, at: 0, frameCount: Int32(frames), into: buffer.mutableAudioBufferList
+ ) == noErr, let channel = buffer.floatChannelData?[0] else { return }
+ var sum = 0.0
+ for i in 0..<Int(frames) { sum += Double(channel[i]) * Double(channel[i]) }
+ Task { [recorder] in await recorder.add(sumOfSquares: sum, frames: Int(frames)) }
+ }
+}