Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
test(meeting): opt-in acoustic probes and the measurements they produced
Shreeraman A Kcb3ac902c804
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxprojindex b1e3b9a2..616ebbc0 100644--- a/Fluid.xcodeproj/project.pbxproj+++ b/Fluid.xcodeproj/project.pbxproj@@ -29,6 +29,16 @@F1BD00000000000000000002 /* StripThinkingTagsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F1BD00000000000000000001 /* StripThinkingTagsTests.swift */; };D1A600000000000000000204 /* MeetingWindowSelectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */; };D1A600000000000000000206 /* MeetingRecoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1A600000000000000000205 /* MeetingRecoveryTests.swift */; };+ 7E57C8BE6F444F6A9AC9FCB0 /* MeetingEchoSignalScorerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4AD7433B35340CEB85EA268 /* MeetingEchoSignalScorerTests.swift */; };+ E5EC00000000000000000002 /* MeetingEchoVetoTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000001 /* MeetingEchoVetoTests.swift */; };+ E5EC00000000000000000012 /* MeetingFallbackEchoProbeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000011 /* MeetingFallbackEchoProbeTests.swift */; };+ E5EC00000000000000000022 /* MeetingNearFieldGateTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000021 /* MeetingNearFieldGateTests.swift */; };+ E5EC00000000000000000032 /* MeetingLiveSilenceEndpointTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000031 /* MeetingLiveSilenceEndpointTests.swift */; };+ E5EC00000000000000000042 /* VoiceProcessingDuckingProbeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000041 /* VoiceProcessingDuckingProbeTests.swift */; };+ E5EC00000000000000000052 /* VoiceProcessingSpeechDuckingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000051 /* VoiceProcessingSpeechDuckingTests.swift */; };+ E5EC00000000000000000062 /* VoiceProcessingAsrQualityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E5EC00000000000000000061 /* VoiceProcessingAsrQualityTests.swift */; };+ A6E2246FA3744C7B3E2DB91C /* MeetingLiveTranscriptionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */; };+ 2952403C87FFDE41C1BC85D5 /* MeetingLiveReplayTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */; };C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; };DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; };DA7200020000000000000002 /* AudioHardwareRecoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */; };@@ -82,6 +92,16 @@F1BD00000000000000000001 /* StripThinkingTagsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StripThinkingTagsTests.swift; sourceTree = "<group>"; };D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingWindowSelectorTests.swift; sourceTree = "<group>"; };D1A600000000000000000205 /* MeetingRecoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingRecoveryTests.swift; sourceTree = "<group>"; };+ B4AD7433B35340CEB85EA268 /* MeetingEchoSignalScorerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingEchoSignalScorerTests.swift; sourceTree = "<group>"; };+ E5EC00000000000000000001 /* MeetingEchoVetoTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingEchoVetoTests.swift; sourceTree = "<group>"; };+ E5EC00000000000000000011 /* MeetingFallbackEchoProbeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingFallbackEchoProbeTests.swift; sourceTree = "<group>"; };+ E5EC00000000000000000021 /* MeetingNearFieldGateTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingNearFieldGateTests.swift; sourceTree = "<group>"; };+ E5EC00000000000000000031 /* MeetingLiveSilenceEndpointTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveSilenceEndpointTests.swift; sourceTree = "<group>"; };+ E5EC00000000000000000041 /* VoiceProcessingDuckingProbeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceProcessingDuckingProbeTests.swift; sourceTree = "<group>"; };+ E5EC00000000000000000051 /* VoiceProcessingSpeechDuckingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceProcessingSpeechDuckingTests.swift; sourceTree = "<group>"; };+ E5EC00000000000000000061 /* VoiceProcessingAsrQualityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceProcessingAsrQualityTests.swift; sourceTree = "<group>"; };+ 355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveTranscriptionTests.swift; sourceTree = "<group>"; };+ 7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingLiveReplayTests.swift; sourceTree = "<group>"; };C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = "<group>"; };DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = "<group>"; };DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioHardwareRecoveryTests.swift; sourceTree = "<group>"; };@@ -183,6 +203,16 @@F1BD00000000000000000001 /* StripThinkingTagsTests.swift */,D1A600000000000000000203 /* MeetingWindowSelectorTests.swift */,D1A600000000000000000205 /* MeetingRecoveryTests.swift */,+ B4AD7433B35340CEB85EA268 /* MeetingEchoSignalScorerTests.swift */,+ E5EC00000000000000000001 /* MeetingEchoVetoTests.swift */,+ E5EC00000000000000000011 /* MeetingFallbackEchoProbeTests.swift */,+ E5EC00000000000000000021 /* MeetingNearFieldGateTests.swift */,+ E5EC00000000000000000031 /* MeetingLiveSilenceEndpointTests.swift */,+ E5EC00000000000000000041 /* VoiceProcessingDuckingProbeTests.swift */,+ E5EC00000000000000000051 /* VoiceProcessingSpeechDuckingTests.swift */,+ E5EC00000000000000000061 /* VoiceProcessingAsrQualityTests.swift */,+ 355C1248D5386D1F815EB7D7 /* MeetingLiveTranscriptionTests.swift */,+ 7BE63EB94BB280F38F5CB028 /* MeetingLiveReplayTests.swift */,C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */,DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,DA7200010000000000000001 /* AudioHardwareRecoveryTests.swift */,@@ -369,6 +399,16 @@F1BD00000000000000000002 /* StripThinkingTagsTests.swift in Sources */,D1A600000000000000000204 /* MeetingWindowSelectorTests.swift in Sources */,D1A600000000000000000206 /* MeetingRecoveryTests.swift in Sources */,+ 7E57C8BE6F444F6A9AC9FCB0 /* MeetingEchoSignalScorerTests.swift in Sources */,+ E5EC00000000000000000002 /* MeetingEchoVetoTests.swift in Sources */,+ E5EC00000000000000000012 /* MeetingFallbackEchoProbeTests.swift in Sources */,+ E5EC00000000000000000022 /* MeetingNearFieldGateTests.swift in Sources */,+ E5EC00000000000000000032 /* MeetingLiveSilenceEndpointTests.swift in Sources */,+ E5EC00000000000000000042 /* VoiceProcessingDuckingProbeTests.swift in Sources */,+ E5EC00000000000000000052 /* VoiceProcessingSpeechDuckingTests.swift in Sources */,+ E5EC00000000000000000062 /* VoiceProcessingAsrQualityTests.swift in Sources */,+ A6E2246FA3744C7B3E2DB91C /* MeetingLiveTranscriptionTests.swift in Sources */,+ 2952403C87FFDE41C1BC85D5 /* MeetingLiveReplayTests.swift in Sources */,C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */,DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,DA7200020000000000000002 /* AudioHardwareRecoveryTests.swift in Sources */,diff --git a/MEETING_ECHO_AND_LIVE_PLAN.md b/MEETING_ECHO_AND_LIVE_PLAN.mdnew file mode 100644index 00000000..a8b48007--- /dev/null+++ b/MEETING_ECHO_AND_LIVE_PLAN.md@@ -0,0 +1,257 @@+# Meeting echo handling and live transcription — measured plan++Status: **analysis complete, scorer built standalone, integration not started.** Measurements taken+2026-08-14 against a real 3.2-minute Zoom session (both raw tracks on disk).++---++## 1. Corrected root causes++Two separate defects were conflated earlier. They need different fixes.++### 1a. Interjections swallowed by echo classification++`MeetingEchoDetector` scores a whole diarized turn by word containment against the far end. A turn+that is mostly speaker bleed but contains a short local interjection scores as echo, and the user's+words are hidden with it. Observed: "I'm testing." and "Notetaker." absent from our transcript,+present in a competitor's.++### 1b. One utterance split across a chunk boundary, labelled two ways++Observed: `119.09–120.12` → *Microphone / Unknown*, `120.06–127.53` → *You*. Same sentence, split at+the 120 s chunk boundary.++**An earlier diagnosis blaming per-chunk local-speaker resolution was wrong.**+`processMicrophoneTrack` stages turns across all chunks and calls `finishMicrophoneTrack` once, so+resolution is already session-wide. The real cause is cluster identity: the emit rule keys on+`turn.clusterID == localClusterID`, and the 1.04 s fragment was assigned a different cluster.+Diarization runs per chunk; a ~1 s fragment yields a weak embedding, and cross-chunk matching+(max cosine distance 0.35, ambiguity margin 0.08) rejects it.++Consequence: **signal-based echo scoring cannot fix 1b.** A turn in the wrong cluster is not labelled+`You` regardless of its echo verdict. 1b needs its own fix, un-designed as of tonight.++---++## 2. Measurements++All on the real session, raw tracks, `microphone` vs `applicationAudio`.++### Alignment+- Both tracks decode to identical duration (192.400 s) — one shared `SCStream` clock.+- Delay by GCC-PHAT: mic content **leads** app audio by **31.9 ms**, std **0.3 ms** across the session.+- Drift **+6.9 ppm** (~25 ms/hour) — real, above measurement resolution, slow enough to track.+- Per chunk: −32.28, −31.97, −31.62, −31.38 ms. A smooth monotonic progression **through** chunk+ boundaries (6.4 ppm), not discontinuities. AAC priming does **not** shift the tracks differentially.+- The 31.9 ms is a capture-pipeline offset, not an acoustic path. It is hardware-specific, can be+ either sign, and Bluetooth output can invert it by 100–300 ms. **Never hardcode it; always estimate.**++### Cancellation potential+- Per-2 s frequency-domain Wiener filter: **9–11 dB ERLE** in echo-dominant blocks.+- A single session-wide filter achieves only 1.6 dB — drift defeats it. Adaptation is mandatory.+- Overall 3.9 dB, dragged down by blocks where the far end is silent (nothing to cancel).++### Turn-level separability (the decisive measurement)++Per-frame explained fraction = `1 − residual_energy / mic_energy`, 1024-sample frames, 256 hop.++| turn kind | n | mean | median | longest low-run (<0.35) |+|---|---|---|---|---|+| pure echo | 5 | 0.53–0.68 | 0.62–0.82 | **0.24–0.59 s** |+| contains local speech | 1 | 0.43 | 0.38 | **0.98 s** |+| local speech only | 3 | 0.02–0.03 | 0.00–0.02 | **0.96–10.27 s** |++**Mean and median cannot separate the mixed turn** — it sits between the two populations. The+**longest contiguous low-scoring run** does, with a clean gap: echo never exceeds 0.59 s, anything+containing local speech never falls below 0.96 s. This is the rescue statistic the design needs.++Caveat: one recording, one room, ten turns. The gap is real but the sample is small.++### Operating envelope of the rescue rule (found in code review, measured)++The statistic is `1 − residual/mic`. When local speech is **added on top of** echo — which is what+real speakerphone double-talk is — the fraction is approximately `echo / (echo + local)`. To fall+below `localSpeechFractionThreshold` (0.35) the local speech must therefore carry roughly+**1.86× the echo energy at the microphone**.++That holds comfortably for a laptop mic near the speaker's mouth with the far end coming from the+same laptop's speakers — it was satisfied throughout the real recording. It does **not** hold for a+quiet interjection over a loud far end, or a mic far from the user. That is the rule's operating+envelope, and it is a genuine limitation rather than a tuning constant: a louder threshold trades+directly against false rescues of pure echo.++Consequence for calibration: the held-out recordings must include a quiet-interjection case, not+just more of the easy one.++---++## 3. Decisions from adversarial review++Reviewed by two independent external models. Both independently reached the same conclusions.++| decision | reason |+|---|---|+| Signal acts as a **veto only** on text-positive turns | Making it primary creates a new false-positive class that hides user speech |+| **Do not** feed signal evidence into `localSpeakerEvidence` | The purity rule `supporting > echoed` plus the 1.75× winner margin means a misclassification can elect the *far-end* cluster as "You" — a high-amplification failure |+| Three-state verdict: echo / containsLocalSpeech / **unknown** | Absence of reference audio is not evidence of local speech. Headphone users, silent far end, and low delay confidence must abstain, not score 0 |+| Turn aggregation must use the **low-run** statistic, not an average | Measured above; an average reproduces the exact bug being fixed |+| Reference audio must be gathered by **time range across all app chunks**, not chunk N ↔ chunk N | The two tracks' `MeetingAudioChunkWriter`s rotate independently and diverge permanently after a discontinuity on either track |+| Reference-read failure must be **total** (degrade to `.unknown`, never throw) | Today only the mic chunk's own `audioUnreadable` skips a chunk; a sick app track must not take down a healthy mic track |+| Exclude discontinuity-adjacent windows | A shared clock does not make content contiguous after dropped samples |+| Bump `pipelineVersion` when classification changes | Checkpoint reuse would otherwise mix classification rules mid-session |++Open, unresolved: threshold portability across hardware/rooms/volumes (one recording); non-speech+far-end audio (music, video) is a false-positive class today's text detector cannot produce; whether+to persist the score, and if so versioned against the scorer.++---++## 4. Built++**`MeetingEchoSignalScorer.swift`** — pure functions over PCM: GCC-PHAT delay estimation with a+confidence measure, per-frame explained fractions with NaN for unscoreable frames, and a three-state+turn verdict using the low-run rule. Unit tested on deterministic synthetic signals.++**Integrated as a veto** (`pipelineVersion` 3 → 4). The design point both reviewers converged on is+**two separate echo fields**:++- `isLikelyEcho` stays **text-only** and remains the sole input to `localSpeakerEvidence`. Signal+ evidence must never influence the local-speaker election — a misclassification there could elect+ the *far-end* cluster as "You" through the 1.75× winner rule, which is a far worse failure than+ any single mislabelled turn.+- `effectiveEcho = isLikelyEcho && signalVerdict != .containsLocalSpeech` drives **both** loss paths:+ the `isCleanLocalTurn` assignment and the emitted echo flag that the UI and exporter filter on.++Supporting decisions:+- Reference audio is gathered by **presentation timestamp** across all intersecting application+ chunks. Never by chunk index: the two writers rotate independently and diverge permanently after a+ discontinuity, so a mic chunk can straddle two app chunks.+- Gaps are **zero-filled, not masked**. A zero-filled block has no reference energy, so the scorer's+ own floor already forces those frames to NaN — an explicit mask would mean reopening a tested+ module for no behavioural gain.+- Delay falls back to a **rolling median within a capture epoch**, never a session-wide scalar; an+ epoch ends at any discontinuity on either track.+- Every reference failure degrades to `.unknown` and mutates nothing. A sick application track must+ never skip a healthy microphone chunk.++**Known limits, deliberately documented rather than hidden:**+- Turns shorter than ~0.8 s can never be vetoed (`minimumLocalSpeechRunSeconds` is 0.75 s), so a+ standalone backchannel — "yeah", "okay" — stays hidden. The measured failure case survives only+ because those interjections sat inside a 25.9 s turn.+- The 1.86× energy envelope means a quiet interjection over a loud far end is not rescued.++Together these cap how much of the motivating bug this fixes. It is a real improvement, not a+general solution.++---++## 5. Live transcription — shape only++Capture stays on ScreenCaptureKit. Voice Processing I/O is disqualified: instantiating it ducks all+other applications' audio system-wide, which would corrupt the very `applicationAudio` track we+record. Core Audio process taps are a permission-UX decision, not an echo fix, and risk zero-filled+buffers for WebRTC apps (Zoom, Meet, Teams).++**Decision (2026-08-14): no diarization during the meeting.** Live shows only *You* and *Them*,+derived from which track the audio arrived on — not from analysing voices. Speaker identity is+resolved after the meeting by the existing batch pipeline.++- **Live** — streaming ASR on both tracks. Mic = You, application audio = Them. Correct by+ construction in any 1:1. **Depends on echo suppression**, or "You" transcribes the far end.+- **On meeting end** — today's batch pipeline runs unchanged and replaces the live text with the+ word-aligned, properly diarized final transcript, turning *Them* into Speaker 1/2/3.++Why this ordering rather than a live diarizer:+- The streaming diarizer measured **1,011 MB resident** (see below). Deferring it keeps that+ entirely out of the meeting, which removes the model-contention question rather than answering it.+- Streaming diarization is 10–15 % worse DER than the offline path, so deferring *improves* the+ final transcript instead of compromising it.+- Label churn disappears: nothing can retroactively relabel text the user already read, because no+ speaker decision is made live.+- Cost: in a 3+ person call, everyone else reads as *Them* until processing finishes. In a 1:1,+ nothing is lost.++### Measured memory (2026-08-14, this machine)++| component | peak RSS | note |+|---|---|---|+| Streaming diarizer (Sortformer) | **1,011 MB** | real run, 56.7 s for 192 s audio (~3.4× realtime) |+| Nemotron streaming ASR | 688 MB | models loaded, then **failed** before transcribing |++The diarizer's 1 GB is far above its 21 MB on-disk footprint. Contention between the two was **not**+measured: Nemotron aborts with a CoreML `zero shape error` during type inference on ordinary 16 kHz+mono input, both full-length and on a 30 s clip with an explicit chunk size. That failure blocks the+live path itself, not just the measurement, and is the first thing to resolve.++Still open: whether one streaming ASR instance can serve both tracks, or whether two instances+(~690 MB each) are required.++Hard constraints, verified in FluidAudio at the pinned commit:+- Streaming ASR produces **no word timestamps** (zero `TokenTiming` references in the streaming+ path). Our word→turn assignment cannot run live; final quality must come from the batch pass.+- Streaming "partials" only grow — greedy RNNT, never revised. `.provisional` would have meant+ "growing", not "correctable". **As built, `MeetingTranscriptStatus.provisional` stays unused**:+ live text is held in memory and published from the coordinator, never entering+ `session.transcriptSegments`, so there is nothing to mark. That containment is deliberate — it+ makes lowercase, unpunctuated live text unreachable by export and copy by construction rather+ than by filtering, and means a crash or a failed batch pass cannot leave it posing as the+ transcript.+- Streaming diarization is 10–15 % worse DER than the offline VBx pipeline in use today.+- Below ~560 ms chunks, WER degrades sharply (2.1 % → ~10 % at 160 ms).+- Nothing in FluidAudio combines streaming ASR with streaming diarization; we would align them.++---++## 6. First real live run (2026-08-14, headphones, 2.5 min Zoom)++Live transcription ran end to end against a real meeting. Both tracks transcribed; "You" and "Them"+were correct by construction. Four defects found and fixed.++| defect | evidence | fix |+|---|---|---|+| Utterance spans swallow leading silence | mic EOU #1 reported `119.6s`; wall clock shows speech `18:56:32→18:57:11`, EOU at `18:57:13` | anchor `utteranceStartPTS` to the first decoded token, backdated one chunk, clamped to the previous utterance's end |+| ~5s of meeting-opening audio shed | 381 drops between `11.123` and `13.833`, then never again | run CoreML warmup on silence *before* `isModelReady`; queue 24 → 512 slots |+| 255 decoder resets and 255 degraded notifications in one burst | one `handleDrop` per dropped sample | coalesce to one resync per burst, applied by the drain loop |+| `handleEOU` reentrancy on `await manager.reset()` | `consume` could set a stale start during the suspension | dissolves — `consume` no longer touches utterance start |++**The endpointer is not slow.** EOU fires ~1.5–2s after speech stops, matching the model's 1280ms+debounce. An earlier reading of "78–120s latency" was a misinterpretation of the inflated spans.++## 7. Whole-chunk microphone fallback: neither echo classifier can score it++The same run put 60s of garbled far-end bleed in the transcript as `Microphone / Unknown`,+`isLikelyEcho=false`. Root cause: the whole-chunk fallback appends straight to+`accumulator.segments`, bypassing `stagedTurns` — and all echo classification runs over+`stagedTurns`.++Two candidate fixes were **measured and rejected**:++- **Text detector.** The bleed block scores **0.31** containment against its own 60s window+ (threshold 0.5) — it would not be flagged. Genuine user speech from the same session scores+ 0.05–0.29 even against the whole-meeting corpus, so widening the window is not the answer either.+- **Signal scorer.** A probe over the real chunks returns **no delay estimate and no verdict** for+ the two bleed chunks (`verdict=nil, delay=none`); GCC-PHAT finds no confident echo path and+ correctly abstains. Only chunk 2 scored, at `containsLocalSpeech`, delay `-50ms`.++Reproduce with `MeetingFallbackEchoProbeTests` (`TEST_RUNNER_FLUIDVOICE_ECHO_PROBE=<session dir>`).++**Actual fix:** `LabeledPassError.emptyTurn` no longer falls through to `transcribeWholeChunk`.+Diarization succeeding while every turn transcribes empty is positive evidence that the microphone+holds no intelligible speech; re-transcribing 60s of near-silence only produces hallucinated text+that is unattributable and unscoreable. Genuine diarization failure — including+`SpeakerDiarizationService.isSupported == false` — still gets the fallback. `pipelineVersion` 4 → 5.++Still open: `wordAttachmentTolerance` is `0` for the microphone (`MeetingProcessingPipeline:1463`),+which disables the nearest-neighbour rescue in `assignWords` and drops any word whose timing+disagrees with a diarizer boundary (observed `29/67`, `8/30` unassigned). Loosening it trades+directly against crediting bleed to the user; needs its own measurement.++---++## 8. Deferred++Fluid One transcript cleanup (measured: ~648 ms/turn on a 1.5 B proxy model with prefix KV caching+on, ~2 min/meeting-hour; Fluid One is 3.3 GB so estimate 5–8 min — a background job, not a blocking+step). Vocabulary biasing for meetings is independent and cheap: the meeting pipeline does not use+`ParakeetVocabularyStore` / `PronunciationDictionaryStore` at all today, which likely explains+"Cloud Code" for "Claude Code".diff --git a/MIC_CAPTURE_MIGRATION_PLAN.md b/MIC_CAPTURE_MIGRATION_PLAN.mdnew file mode 100644index 00000000..e6a434ca--- /dev/null+++ b/MIC_CAPTURE_MIGRATION_PLAN.md@@ -0,0 +1,167 @@+# Microphone capture migration — SCStream → AVAudioEngine with Voice Processing I/O++Status: **v4, planned, not started.** v1–v3 were each rejected by two independent reviewers. v4 is+smaller than v3, per the reviewers' closing instruction. Written 2026-08-15.++## 1. What v4 changes from v3++v3 was rejected mainly for three self-inflicted errors. Each is corrected by *narrowing a claim*, not+by adding machinery:++| v3 error | v4 |+|---|---|+| "No diarization, no clustering" | **Keep the diarizer.** It is the *segmenter*, not a labeller (`MeetingProcessingPipeline.swift:1385-1391`); every turn boundary, `transcribeTurns`, word alignment, `turnLoudness` and the empty-turn guard depend on it. Deleting it leaves only whole-chunk transcription — the hallucination path fixed earlier today. v4 keeps diarization and **merges its mic clusters into "You"**. |+| "No fallback" | **Pre-flight capability decision, not mid-meeting fallback.** VPIO viability is decided *before* capture starts; the session then runs one way for its whole life. No mixed provenance, and no lost meeting. |+| "Session surfaces a degraded microphone track" | False — capture start is atomic (`MeetingCaptureEngine.swift:88-95`) and a runtime failure stops all writers. v4 does not claim a state the code lacks. |++## 2. Measurements++| question | result |+|---|---|+| Does AEC cancel the far end? | raw mic −23.7 dBFS / 62 words of far-end text; VPIO mic −63.6 dBFS / **zero words** |+| Cancellation vs ducking | ~40 dB total, ~7 dB of it ducking → **~33 dB genuine cancellation** |+| Double-talk | raw: 8 words of mush; VPIO: **51 words of the user's passage, no far-end content** |+| Effect on the user's own voice | ~14.5% WER either way; VPIO ran 12 dB hotter (AGC) |+| Ducking cost at `.min` | −6.9 dB mean on **all** other audio, swing 9.2 → 16.9 dB |+| Downlink-DSP faults | 2066 during the successful run — **not root-caused** |++Shipping config (`VoiceProcessingAsrQualityTests.swift:54-57`). **One machine, one room, built-in mic+and speakers.** Not measured: quiet interjections under a loud far end; any other hardware or macOS+build.++**Competitor precedent is weaker than v3 claimed.** The surveyed competitor ships a thin capture+client with no local models; segmentation and diarization happen server-side, so their client never+had an election to delete. What genuinely transfers: separate dictation/meeting audio configs, source+attribution as the live default, no AEC fallback, logging the *settled* audio constraint rather than+the requested one, and accepting the ducking with no mitigation.++## 3. Activation conditions++VPIO is used only when **all** hold, decided before capture starts:++1. Mode is `.onlineCall` and the microphone role is `.personal`.+2. **Output is not headphones.** With headphones there is no bleed to cancel, so VPIO delivers zero+ benefit and the full ducking cost — pumping the audio the user is hearing the call on. v3 applied+ VPIO unconditionally; the user's accepted trade (§4) was measured on speakers only.+3. The selected microphone can be bound and verified (§6).++Otherwise the session runs today's SCStream path unchanged, with full echo machinery. This is a+**capability decision, not a fallback**: homogeneous for the session's lifetime.++Mid-session output changes (speakers → AirPods) are a known limitation, not handled in v4.++## 4. Product decision — accepted, with a stated boundary++Enabling VPIO ducks **all** other audio by ~7 dB with ±8 dB of pumping that increases while the user+speaks, audible live. **Accepted by the user, 2026-08-15**, after being shown the measured figures and+told it affects live listening rather than only the recording.++That acceptance was given for the **speakers** case and does not extend to headphones — hence §3.2.++## 5. Attribution++For a VPIO session, on the microphone track:++- Diarization runs exactly as today (it is the segmenter), and **the election runs unchanged**.+- VPIO turns are marked **scored clean** — `echoScored = true, isLikelyEcho = false` — because that is+ what they are: examined, and known to contain no far-end audio. They therefore count as supporting+ evidence in `localSpeakerEvidence` (`MeetingProcessingPipeline.swift:357`) exactly as a+ text-examined clean turn does today.+- **This is the whole change.** No branch on cluster count, no bypass. AEC does not make diarization+ unnecessary — it makes it work, by removing the echo clusters that competed with the user's own+ voice and defeated the 1.75× margin rule in the observed failure. A second voice in the room is then+ handled by the election as it always was, which matters because misattribution is+ **uncorrectable in-product**: `mergeSpeakers` refuses any speaker with `isLocalUser`+ (`MeetingModels.swift:542`).+- The text echo detector is not consulted; the signal veto is not applied.+- `MeetingEchoSignalScorer` still runs as a **monitor**: its verdicts are recorded, never applied.+ This is the only field evidence that AEC worked on hardware we have not measured; without it a+ silently-degraded canceller yields bleed labelled "You", forever undetected.+- `MeetingLiveEchoFilter` is disabled for VPIO sessions (`MeetingLiveTranscriptionCoordinator.swift:150`).+- `pipelineVersion` bumps; attribution rules changed.++Unchanged: `.inRoom` (`AVCaptureSession`, not SCStream — `MeetingCaptureEngine.swift:624`), non-VPIO+sessions, and the near-field gate.++## 6. Device selection++`kAudioOutputUnitProperty_CurrentDevice` fails for Bluetooth and aggregate devices, and the existing+binder falls back to the system default on **any** non-zero status (`ASRService.swift:2569-2584`).+SCStream honours `microphoneCaptureDeviceID` exactly today, so this is a real regression.++Order of attempts:+1. Bind, then **read the device back and compare** after voice processing is enabled and the engine is+ running — a successful status is not proof, since enabling VPIO can replace the I/O unit.+2. If binding fails but the **system default input is already the requested device**, proceed unbound:+ identity is correct even though binding is not. This covers most AirPods users, who selected the+ device in System Settings precisely because they are on a call.+3. Otherwise VPIO is not viable → §3, session runs today's path.++Binding order relative to `setVoiceProcessingEnabled` is a **Phase 1 measurement**, not an assumption.+Unresolved: `AVCaptureDevice.uniqueID` → `AudioObjectID` mapping is by display name and yields nil on+collision (`MeetingCaptureEngine.swift:855-866`).++## 7. Provenance++No-fallback removes *mixed* provenance, not provenance itself. A **track-level** `captureMethod` is+required, describing verified capture rather than intended configuration, and read by: batch+attribution (§5), the live coordinator's echo-filter decision, retry after relaunch, checkpoint+resume, and reprocessing of sessions recorded before this change (absence = legacy).++## 8. Timeline++Cross-track timing survives for transcript ordering (batch `MeetingProcessingPipeline.swift:1100-1104`+and live `MeetingLiveModels.swift:39-44`), the `.overlapsOtherTrack` badge, and live origin+establishment. Echo reference alignment is gone.++- Microphone PTS is the tap's `AVAudioTime.hostTime` — acquisition, first frame — as dictation already+ does (`ASRService.swift:4617-4620`). Never `mach_absolute_time()` at callback time.+- Invalid `hostTime`: extrapolate from the last valid stamp plus output duration, **capped at 500 ms**;+ beyond that, record a discontinuity and resync.+- **The constant-offset model is contingent.** This repo measured ~6.4 ppm drift, ~23 ms/hour+ (`MeetingProcessingPipeline.swift:722-724`); a fixed offset is ~46 ms wrong at two hours, consuming+ the whole budget. Phase 0 decides: same clock domain ⇒ offset is identity and no model is needed;+ different domains ⇒ a linear model with periodic re-measurement, not a constant.++**Budget: ±50 ms**, justified by turn ordering alone. Stated as a claim to be tested, not derived —+v3 wrongly cited the 0.15 s `overlapsRemote` threshold, which is a minimum-overlap predicate.++## 9. Phases and gates++**Phase 0 — clock domain (unattended).** Compare *rates*, not arrival deltas: regress SCStream PTS+against wall time and microphone PTS against wall time over 30 minutes. *Gate:* report both slopes in+ppm with confidence intervals. |slope difference| < 20 ppm ⇒ constant offset acceptable; ≥ 20 ppm ⇒+§8's linear model is mandatory. Subtracting `mach_absolute_time()` at arrival is **not** the test — it+conflates queueing jitter with clock domain.++**Phase 1 — `MeetingMicrophoneCapture`, standalone.** *Gate*, over 10 minutes on the built-in mic+**and** one Bluetooth device: valid `hostTime` on ≥99% of buffers; output Float32 LPCM (else the+writer's telemetry silently dies, `MeetingAudioChunkWriter.swift:499-505`); buffers accepted by+`MeetingLiveSampleCopy.copy` and by the writer with zero unexpected rotations+(`MeetingAudioChunkWriter.swift:177-193`); read-back device equals requested, or §6.2/§6.3 fires;+binding order recorded.++**Phase 2 — timeline.** *Gate:* drift within Phase 0's bound over 30 minutes. **Absolute offset cannot+be measured with an acoustic click** — AEC removes it, so a failed correlation would mean cancellation+worked. Use a loud chirp and matched-filter the ~33 dB residual; if the residual is unrecoverable,+record that absolute offset is unmeasured, accept ordering-tolerance validation on a real call, and+say so rather than reporting a number.++**Phase 3 — integrate behind a flag, default off.** *Gate:* a real call whose transcript ordering is+compared turn-by-turn against a human-checked reference. Not a simultaneous old-path capture — two+microphone clients with one enabling VPIO perturbs the system under test.++**Phase 4 — attribution.** One change: mark VPIO turns scored-clean and let the existing election run.+*Gate:* a single-voice VPIO session elects "You" with no `Microphone / Unknown`; a **two-voice**+session still elects the owner and demotes the second speaker; a non-VPIO session produces a+transcript identical to today's. The first alone proves nothing — the second is the real test, and it+is only possible because the diarizer was kept.++## 10. Not claimable unattended++Quiet interjections under a loud far end (**a release blocker, not a footnote**); AirPods selection and+mid-session profile switching; the ducking experience; permission prompts; real-call ordering under+load; post-meeting dictation and Bluetooth restoration; AEC on any untested hardware or macOS build;+the 2066 downlink faults, which remain un-root-caused and gate nothing today.++Phases 0–2 are unattended-safe. **Phases 3–4 are not, and have no owner or schedule yet.**diff --git a/Tests/FluidDictationIntegrationTests/MeetingFallbackEchoProbeTests.swift b/Tests/FluidDictationIntegrationTests/MeetingFallbackEchoProbeTests.swiftnew file mode 100644index 00000000..a8cc5b1e--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/MeetingFallbackEchoProbeTests.swift@@ -0,0 +1,115 @@+@testable import FluidVoice_Debug+import Foundation+import XCTest++/// Opt-in probe against a real recorded session, used to decide whether the whole-chunk microphone+/// fallback can be classified by the signal scorer. Set FLUIDVOICE_ECHO_PROBE to the session+/// directory to run it.+final class MeetingFallbackEchoProbeTests: XCTestCase {+ func testWholeChunkFallbackVerdictOnRealSession() throws {+ guard let root = ProcessInfo.processInfo.environment["FLUIDVOICE_ECHO_PROBE"] else {+ throw XCTSkip("FLUIDVOICE_ECHO_PROBE not set")+ }+ let sessionDirectory = URL(fileURLWithPath: root, isDirectory: true)+ let decoder = JSONDecoder()+ decoder.dateDecodingStrategy = .iso8601+ let micTrack = try decoder.decode(+ MeetingAudioTrack.self,+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/microphone/track.json"))+ )+ let appTrack = try decoder.decode(+ MeetingAudioTrack.self,+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/applicationAudio/track.json"))+ )++ let origin = min(+ micTrack.chunks.first?.presentationStart.seconds ?? 0,+ appTrack.chunks.first?.presentationStart.seconds ?? 0+ )++ for chunk in micTrack.chunks {+ let chunkOffset = chunk.presentationStart.seconds - origin+ let chunkDuration = chunk.presentationEnd.seconds - chunk.presentationStart.seconds+ let url = sessionDirectory.appending(path: chunk.relativeFilePath)++ // One turn spanning the whole chunk — exactly what the fallback would score.+ let wholeChunk = [(index: 0, start: chunkOffset, end: chunkOffset + chunkDuration)]+ let outcome = try MeetingProcessingPipeline.computeEchoVerdicts(+ micChunk: chunk,+ micChunkURL: url,+ chunkOffset: chunkOffset,+ chunkDuration: chunkDuration,+ turns: wholeChunk,+ applicationTrack: appTrack,+ origin: origin,+ sessionDirectory: sessionDirectory,+ acceptedDelaysInEpoch: []+ )+ let verdict = outcome.verdicts[0].map { String(describing: $0) } ?? "nil"+ print(String(+ format: "[probe] chunk seq=%d span=[%.1f-%.1f] verdict=%@ delay=%@",+ chunk.sequence, chunkOffset, chunkOffset + chunkDuration, verdict,+ outcome.acceptedDelaySeconds.map { String(format: "%.4fs", $0) } ?? "none"+ ))+ }+ }++ /// Scores specific turns that text-echo detection missed, to see whether the signal scorer+ /// could have caught them. FLUIDVOICE_ECHO_PROBE_TURNS is "label:start:end,label:start:end".+ func testNamedTurnsAgainstSignalScorer() throws {+ guard let root = ProcessInfo.processInfo.environment["FLUIDVOICE_ECHO_PROBE"],+ let spec = ProcessInfo.processInfo.environment["FLUIDVOICE_ECHO_PROBE_TURNS"]+ else { throw XCTSkip("probe env not set") }++ let sessionDirectory = URL(fileURLWithPath: root, isDirectory: true)+ let decoder = JSONDecoder()+ decoder.dateDecodingStrategy = .iso8601+ let micTrack = try decoder.decode(+ MeetingAudioTrack.self,+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/microphone/track.json"))+ )+ let appTrack = try decoder.decode(+ MeetingAudioTrack.self,+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/applicationAudio/track.json"))+ )+ let origin = min(+ micTrack.chunks.first?.presentationStart.seconds ?? 0,+ appTrack.chunks.first?.presentationStart.seconds ?? 0+ )++ for entry in spec.split(separator: ",") {+ let parts = entry.split(separator: ":")+ guard parts.count == 3, let start = Double(parts[1]), let end = Double(parts[2]) else { continue }+ let label = String(parts[0])+ guard let chunk = micTrack.chunks.first(where: {+ let chunkStart = $0.presentationStart.seconds - origin+ let chunkEnd = $0.presentationEnd.seconds - origin+ return start >= chunkStart && start < chunkEnd+ }) else {+ print("[turn] \(label) no chunk covers \(start)")+ continue+ }+ let chunkOffset = chunk.presentationStart.seconds - origin+ let chunkDuration = chunk.presentationEnd.seconds - chunk.presentationStart.seconds+ // Clamp: a turn may run past its chunk, and the scorer reads one chunk's audio.+ let clampedEnd = min(end, chunkOffset + chunkDuration)+ let outcome = try MeetingProcessingPipeline.computeEchoVerdicts(+ micChunk: chunk,+ micChunkURL: sessionDirectory.appending(path: chunk.relativeFilePath),+ chunkOffset: chunkOffset,+ chunkDuration: chunkDuration,+ turns: [(index: 0, start: start, end: clampedEnd)],+ applicationTrack: appTrack,+ origin: origin,+ sessionDirectory: sessionDirectory,+ acceptedDelaysInEpoch: []+ )+ print(String(+ format: "[turn] %-22@ [%.1f-%.1f] verdict=%@ delay=%@",+ label as NSString, start, clampedEnd,+ outcome.verdicts[0].map { String(describing: $0) } ?? "nil",+ outcome.acceptedDelaySeconds.map { String(format: "%.4fs", $0) } ?? "none"+ ))+ }+ }+}diff --git a/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift b/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swiftnew file mode 100644index 00000000..f142f843--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/MeetingLiveReplayTests.swift@@ -0,0 +1,341 @@+@testable import FluidVoice_Debug+import AVFoundation+import CoreMedia+import Foundation+import XCTest++/// Opt-in replay harness: drives the REAL `MeetingLiveTranscriptionCoordinator` with a real,+/// previously-recorded Zoom meeting's decoded microphone/application-audio WAVs, to see what the+/// live-transcription pipeline actually produces. Gated behind an env var so the normal suite+/// stays deterministic — this test downloads/loads real CoreML models and takes real wall time.+///+/// Set `FLUIDVOICE_LIVE_REPLAY=1` (and optionally `FLUIDVOICE_LIVE_REPLAY_MIC_WAV` /+/// `FLUIDVOICE_LIVE_REPLAY_APP_WAV` to override the fixture paths) to run it.+final class MeetingLiveReplayTests: XCTestCase {+ private static let chunkFrames = 320 // 20ms @ 16kHz+ private static let sampleRate: Double = 16_000+ private static let trailingSilenceSeconds: Double = 3++ func testReplayRealMeetingThroughLiveCoordinator() async throws {+ guard let replayFlag = ProcessInfo.processInfo.environment["FLUIDVOICE_LIVE_REPLAY"], !replayFlag.isEmpty else {+ throw XCTSkip("Set FLUIDVOICE_LIVE_REPLAY=1 to run the live-transcription replay harness.")+ }++ let env = ProcessInfo.processInfo.environment+ let micPath = env["FLUIDVOICE_LIVE_REPLAY_MIC_WAV"].flatMap { $0.isEmpty ? nil : $0 }+ ?? "/private/tmp/claude-501/-Users-shreeram-Projects-FluidVoice/33323536-f2a9-4586-ad02-f71ced211c2a/scratchpad/microphone.wav"+ let appPath = env["FLUIDVOICE_LIVE_REPLAY_APP_WAV"].flatMap { $0.isEmpty ? nil : $0 }+ ?? "/private/tmp/claude-501/-Users-shreeram-Projects-FluidVoice/33323536-f2a9-4586-ad02-f71ced211c2a/scratchpad/applicationAudio.wav"++ let micSamples = try Self.load16kMonoFloatSamples(path: micPath)+ let appSamples = try Self.load16kMonoFloatSamples(path: appPath)+ print("[replay] mic samples=\(micSamples.count) (\(Double(micSamples.count) / Self.sampleRate)s), app samples=\(appSamples.count) (\(Double(appSamples.count) / Self.sampleRate)s)")++ guard let pcmFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: Self.sampleRate, channels: 1, interleaved: false),+ let formatDescription = Self.makeFormatDescription(format: pcmFormat)+ else {+ XCTFail("Could not build the 16kHz mono Float32 format used to feed the coordinator.")+ return+ }++ // Interleave both tracks' 20ms chunks in true time order. Both files share a clock and+ // (empirically) an identical sample count, so ties are broken mic-first — matching the+ // typical ordering of the real capture tee.+ var events: [(kind: MeetingAudioTrackKind, samples: ArraySlice<Float>, ptsSamples: Int)] = []+ events.reserveCapacity(((micSamples.count + appSamples.count) / Self.chunkFrames) + 4)+ Self.appendChunks(&events, kind: .microphone, samples: micSamples)+ Self.appendChunks(&events, kind: .applicationAudio, samples: appSamples)+ // Trailing silence on both tracks so any in-flight utterance at end-of-file can still hit+ // its 1280ms EOU silence debounce and finalize, instead of being lost to `stop()`'s reset.+ let silenceFrames = Int(Self.trailingSilenceSeconds * Self.sampleRate)+ let silence = [Float](repeating: 0, count: silenceFrames)+ let micTailStart = micSamples.count+ let appTailStart = appSamples.count+ Self.appendChunks(&events, kind: .microphone, samples: silence, ptsBaseSamples: micTailStart)+ Self.appendChunks(&events, kind: .applicationAudio, samples: silence, ptsBaseSamples: appTailStart)+ events.sort { lhs, rhs in+ if lhs.ptsSamples != rhs.ptsSamples { return lhs.ptsSamples < rhs.ptsSamples }+ return lhs.kind == .microphone && rhs.kind != .microphone+ }+ print("[replay] total chunks to feed: \(events.count)")++ final class SnapshotBox: @unchecked Sendable {+ let lock = NSLock()+ var latest: MeetingLiveTranscriptSnapshot = .empty+ var log: [(elapsed: TimeInterval, snapshot: MeetingLiveTranscriptSnapshot)] = []+ let start = Date()+ func record(_ snapshot: MeetingLiveTranscriptSnapshot) {+ self.lock.lock()+ defer { self.lock.unlock() }+ self.latest = snapshot+ self.log.append((Date().timeIntervalSince(self.start), snapshot))+ }+ }+ let box = SnapshotBox()+ let coordinator = MeetingLiveTranscriptionCoordinator { snapshot in+ box.record(snapshot)+ }++ let harnessStart = Date()+ coordinator.start(mode: .onlineCall)++ // Wait for the pipeline to report ready before hammering it with non-real-time audio: a+ // capacity-24 (~480ms) bounded queue means feeding faster than real time before the model+ // is loaded would just drop nearly everything, which would test harness pacing, not the+ // pipeline's actual transcription/echo-suppression behavior.+ let readyTimeout: TimeInterval = 300+ let readyDeadline = Date().addingTimeInterval(readyTimeout)+ while Date() < readyDeadline {+ let available = box.lock.withLock { box.latest.availability == .available }+ if available { break }+ try await Task.sleep(nanoseconds: 200_000_000)+ }+ let firstAvailableElapsed = Date().timeIntervalSince(harnessStart)+ let becameAvailable = box.lock.withLock { box.latest.availability == .available }+ print("[replay] first-track-ready after \(String(format: "%.1f", firstAvailableElapsed))s (available=\(becameAvailable))")+ XCTAssertTrue(becameAvailable, "Live captions never became available within \(readyTimeout)s — model load likely failed.")+ // Both engines load their own model instance in parallel; give the slower one a grace+ // window since the snapshot only reports "available" on the FIRST track's readiness.+ try await Task.sleep(nanoseconds: 5_000_000_000)++ // The bounded queue holds only ~480ms per track (capacity 24 * 20ms). Blasting all ~19.5k+ // chunks in a tight loop (sub-second) overwhelms it before the drain loop can ever get+ // scheduled, which would test harness timing, not the pipeline. Cap the feed rate at+ // `accelerationFactor`x real time instead — still far from a real 192s sleep, but paced+ // enough for the actor's drain loop to keep up with genuine ASR inference.+ let accelerationFactor: Double = 1.5+ let feedStart = Date()+ var bufferFailures = 0+ var sampleBufferFailures = 0+ var offeredCount = 0+ var totalWaitSeconds: Double = 0+ for (index, event) in events.enumerated() {+ guard let buffer = Self.makeBuffer(format: pcmFormat, samples: event.samples) else {+ bufferFailures += 1+ continue+ }+ let pts = CMTime(value: CMTimeValue(event.ptsSamples), timescale: CMTimeScale(Self.sampleRate))+ guard let sampleBuffer = Self.makeSampleBuffer(pcm: buffer, formatDescription: formatDescription, pts: pts) else {+ sampleBufferFailures += 1+ continue+ }+ coordinator.offer(kind: event.kind, sampleBuffer: sampleBuffer)+ offeredCount += 1+ let ptsSeconds = Double(event.ptsSamples) / Self.sampleRate+ let targetElapsed = ptsSeconds / accelerationFactor+ let actualElapsed = Date().timeIntervalSince(feedStart)+ let waitSeconds = targetElapsed - actualElapsed+ if waitSeconds > 0.001 {+ totalWaitSeconds += waitSeconds+ try await Task.sleep(nanoseconds: UInt64(waitSeconds * 1_000_000_000))+ } else if index % 8 == 0 {+ await Task.yield()+ }+ }+ let feedElapsed = Date().timeIntervalSince(feedStart)+ print("[replay] fed \(events.count) chunks in \(String(format: "%.2f", feedElapsed))s (offered=\(offeredCount), bufferFailures=\(bufferFailures), sampleBufferFailures=\(sampleBufferFailures), totalWaitSeconds=\(String(format: "%.2f", totalWaitSeconds)))")++ // Let the drain loops work through whatever is still queued, until the transcript stops+ // changing.+ var lastSnapshot = box.lock.withLock { box.latest }+ var stableCount = 0+ let settleDeadline = Date().addingTimeInterval(60)+ while Date() < settleDeadline, stableCount < 10 {+ try await Task.sleep(nanoseconds: 300_000_000)+ let current = box.lock.withLock { box.latest }+ if current == lastSnapshot {+ stableCount += 1+ } else {+ stableCount = 0+ lastSnapshot = current+ }+ }++ await coordinator.stop()+ // stop() may itself publish nothing further, but give any last async callback a moment.+ try await Task.sleep(nanoseconds: 300_000_000)+ let final = box.lock.withLock { box.latest }++ // ---- Report ----+ print("=== LIVE REPLAY TRANSCRIPT (\(final.utterances.count) utterances) ===")+ for utterance in final.utterances {+ let speaker = utterance.speaker == .you ? "You " : "Them"+ print(String(format: "[%7.2f - %7.2f] %@: %@", utterance.start, utterance.end, speaker, utterance.text))+ }+ if !final.partials.isEmpty {+ print("--- Leftover partials at end ---")+ for (speaker, text) in final.partials {+ print(" \(speaker): \(text)")+ }+ }+ print("Final availability: \(final.availability)")++ let degradedTransitions = box.lock.withLock {+ box.log.filter { if case .degraded = $0.snapshot.availability { return true } else { return false } }+ }+ print("Degraded-availability updates observed: \(degradedTransitions.count)")+ for transition in degradedTransitions.prefix(20) {+ print(" t+\(String(format: "%.1f", transition.elapsed))s: \(transition.snapshot.availability)")+ }++ let youUtterances = final.utterances.filter { $0.speaker == .you }+ let themUtterances = final.utterances.filter { $0.speaker == .them }+ print("You utterances: \(youUtterances.count), Them utterances: \(themUtterances.count)")++ // This is an observational experiment, not a pass/fail gate: assert only that the harness+ // itself ran the real pipeline end-to-end without erroring out.+ XCTAssertTrue(true, "See console output above for the replayed transcript and analysis.")+ }++ // MARK: - Helpers++ private static func appendChunks(+ _ events: inout [(kind: MeetingAudioTrackKind, samples: ArraySlice<Float>, ptsSamples: Int)],+ kind: MeetingAudioTrackKind,+ samples: [Float],+ ptsBaseSamples: Int = 0+ ) {+ var offset = 0+ while offset < samples.count {+ let end = min(offset + self.chunkFrames, samples.count)+ events.append((kind: kind, samples: samples[offset..<end], ptsSamples: ptsBaseSamples + offset))+ offset = end+ }+ }++ private static func load16kMonoFloatSamples(path: String) throws -> [Float] {+ let url = URL(fileURLWithPath: path)+ guard FileManager.default.fileExists(atPath: path) else {+ throw NSError(domain: "MeetingLiveReplayTests", code: 1, userInfo: [NSLocalizedDescriptionKey: "Fixture not found at \(path)"])+ }+ let inputFile = try AVAudioFile(forReading: url)+ let inputFormat = inputFile.processingFormat+ guard let desiredFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: self.sampleRate, channels: 1, interleaved: false) else {+ throw NSError(domain: "MeetingLiveReplayTests", code: 2, userInfo: [NSLocalizedDescriptionKey: "Could not build target format"])+ }++ let inputBuffer: AVAudioPCMBuffer = try {+ let frameCount = AVAudioFrameCount(inputFile.length)+ guard let buffer = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: max(frameCount, 1)) else {+ throw NSError(domain: "MeetingLiveReplayTests", code: 3, userInfo: [NSLocalizedDescriptionKey: "Could not allocate input buffer"])+ }+ try inputFile.read(into: buffer)+ return buffer+ }()++ if inputFormat.sampleRate == desiredFormat.sampleRate,+ inputFormat.channelCount == desiredFormat.channelCount,+ inputFormat.commonFormat == desiredFormat.commonFormat+ {+ return Self.extractSamples(inputBuffer)+ }++ guard let converter = AVAudioConverter(from: inputFormat, to: desiredFormat) else {+ throw NSError(domain: "MeetingLiveReplayTests", code: 4, userInfo: [NSLocalizedDescriptionKey: "Could not build converter"])+ }+ let ratio = desiredFormat.sampleRate / inputFormat.sampleRate+ let estimatedFrames = max(1, AVAudioFrameCount(Double(inputBuffer.frameLength) * ratio) + 1)+ guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: desiredFormat, frameCapacity: estimatedFrames) else {+ throw NSError(domain: "MeetingLiveReplayTests", code: 5, userInfo: [NSLocalizedDescriptionKey: "Could not allocate output buffer"])+ }+ var providedInput = false+ var conversionError: NSError?+ converter.convert(to: outputBuffer, error: &conversionError) { _, outStatus in+ if providedInput {+ outStatus.pointee = .endOfStream+ return nil+ }+ providedInput = true+ outStatus.pointee = .haveData+ return inputBuffer+ }+ if let conversionError { throw conversionError }+ return Self.extractSamples(outputBuffer)+ }++ private static func extractSamples(_ buffer: AVAudioPCMBuffer) -> [Float] {+ guard let channelData = buffer.floatChannelData else { return [] }+ return Array(UnsafeBufferPointer(start: channelData[0], count: Int(buffer.frameLength)))+ }++ private static func makeBuffer(format: AVAudioFormat, samples: ArraySlice<Float>) -> AVAudioPCMBuffer? {+ guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count)),+ let channelData = buffer.floatChannelData+ else { return nil }+ buffer.frameLength = AVAudioFrameCount(samples.count)+ for (index, sample) in samples.enumerated() {+ channelData[0][index] = sample+ }+ return buffer+ }++ private static func makeFormatDescription(format: AVAudioFormat) -> CMAudioFormatDescription? {+ var formatDescription: CMAudioFormatDescription?+ var asbd = format.streamDescription.pointee+ let status = CMAudioFormatDescriptionCreate(+ allocator: kCFAllocatorDefault,+ asbd: &asbd,+ layoutSize: 0,+ layout: nil,+ magicCookieSize: 0,+ magicCookie: nil,+ extensions: nil,+ formatDescriptionOut: &formatDescription+ )+ guard status == noErr else { return nil }+ return formatDescription+ }++ /// Builds a `dataReady` sample buffer directly from a copy of the PCM buffer's bytes, rather+ /// than via `CMSampleBufferSetDataBufferFromAudioBufferList` (which reliably returned+ /// `kCMSampleBufferError_RequiredParameterMissing` for this mono/non-interleaved Float32 case+ /// in practice, despite matching Apple's documented parameters).+ private static func makeSampleBuffer(pcm: AVAudioPCMBuffer, formatDescription: CMAudioFormatDescription, pts: CMTime) -> CMSampleBuffer? {+ let frameCount = Int(pcm.frameLength)+ guard frameCount > 0, let channelData = pcm.floatChannelData else { return nil }+ let bytesPerFrame = Int(pcm.format.streamDescription.pointee.mBytesPerFrame)+ let dataLength = frameCount * bytesPerFrame++ var blockBuffer: CMBlockBuffer?+ let blockStatus = CMBlockBufferCreateWithMemoryBlock(+ allocator: kCFAllocatorDefault,+ memoryBlock: nil,+ blockLength: dataLength,+ blockAllocator: kCFAllocatorDefault,+ customBlockSource: nil,+ offsetToData: 0,+ dataLength: dataLength,+ flags: 0,+ blockBufferOut: &blockBuffer+ )+ guard blockStatus == kCMBlockBufferNoErr, let blockBuffer else { return nil }+ let copyStatus = channelData[0].withMemoryRebound(to: UInt8.self, capacity: dataLength) { bytes in+ CMBlockBufferReplaceDataBytes(with: bytes, blockBuffer: blockBuffer, offsetIntoDestination: 0, dataLength: dataLength)+ }+ guard copyStatus == kCMBlockBufferNoErr else { return nil }++ var timing = CMSampleTimingInfo(+ duration: CMTime(value: CMTimeValue(frameCount), timescale: CMTimeScale(self.sampleRate)),+ presentationTimeStamp: pts,+ decodeTimeStamp: .invalid+ )+ var sampleSize = bytesPerFrame+ var sampleBuffer: CMSampleBuffer?+ let createStatus = CMSampleBufferCreate(+ allocator: kCFAllocatorDefault,+ dataBuffer: blockBuffer,+ dataReady: true,+ makeDataReadyCallback: nil,+ refcon: nil,+ formatDescription: formatDescription,+ sampleCount: frameCount,+ sampleTimingEntryCount: 1,+ sampleTimingArray: &timing,+ sampleSizeEntryCount: 1,+ sampleSizeArray: &sampleSize,+ sampleBufferOut: &sampleBuffer+ )+ guard createStatus == noErr, let sampleBuffer else { return nil }+ return sampleBuffer+ }+}diff --git a/Tests/FluidDictationIntegrationTests/MeetingLiveSilenceEndpointTests.swift b/Tests/FluidDictationIntegrationTests/MeetingLiveSilenceEndpointTests.swiftnew file mode 100644index 00000000..95d1ed1e--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/MeetingLiveSilenceEndpointTests.swift@@ -0,0 +1,113 @@+@testable import FluidVoice_Debug+import AVFoundation+import Foundation+import XCTest++#if arch(arm64)+import FluidAudio++/// Measurement: does `StreamingEouAsrManager` endpoint on sustained digital silence?+///+/// Option A for live echo suppression gates echo-dominated audio to zeros so the microphone regains+/// the pauses its endpointer needs. Nothing in FluidAudio's source answers whether a learned EOU+/// token head fires on pure zeros — the warmup pass feeds 1s of silence and never checks. If it does+/// not fire, gating cleans the audio but utterances still never close, and the endpointing half of+/// the plan has to be driven from gate state instead.+///+/// Run with FLUIDVOICE_SILENCE_EOU=<session dir>.+final class MeetingLiveSilenceEndpointTests: XCTestCase {+ func testEndpointerFiresOnSustainedDigitalSilence() async throws {+ guard let root = ProcessInfo.processInfo.environment["FLUIDVOICE_SILENCE_EOU"] else {+ throw XCTSkip("FLUIDVOICE_SILENCE_EOU not set")+ }+ let sessionDirectory = URL(fileURLWithPath: root, isDirectory: true)+ let decoder = JSONDecoder()+ decoder.dateDecodingStrategy = .iso8601+ let micTrack = try decoder.decode(+ MeetingAudioTrack.self,+ from: Data(contentsOf: sessionDirectory.appending(path: "tracks/microphone/track.json"))+ )+ // Chunk 3 spans session 180-240s and contains known real user speech at 215.2-224.7.+ guard let chunk = micTrack.chunks.first(where: { $0.sequence == 3 }) else {+ throw XCTSkip("expected chunk sequence 3")+ }+ let speech = try MeetingProcessingPipeline.readSamples(+ fileURL: sessionDirectory.appending(path: chunk.relativeFilePath),+ startSeconds: 35.2,+ endSeconds: 44.7+ )+ XCTAssertFalse(speech.isEmpty)++ let manager = StreamingEouAsrManager(chunkSize: .ms160)+ try await manager.loadModelsFromHuggingFace()++ let events = EouEventLog()+ await manager.setEouCallback { transcript in+ Task { await events.record(transcript) }+ }++ let sampleRate = 16_000.0+ let blockFrames = 2_560 // one 160ms chunk++ func feed(_ samples: [Float]) async throws {+ var index = 0+ while index < samples.count {+ let end = min(index + blockFrames, samples.count)+ let slice = Array(samples[index..<end])+ guard let buffer = Self.buffer(slice, sampleRate: sampleRate) else { break }+ try await manager.appendAudio(buffer)+ try await manager.processBufferedAudio()+ index = end+ }+ }++ try await feed(speech)+ let afterSpeech = await events.count+ print("[silence-eou] fed \(String(format: "%.1f", Double(speech.count) / sampleRate))s speech, EOUs so far: \(afterSpeech)")++ // Then sustained digital silence, checked block by block so we can report WHEN it fires.+ let silenceBlocks = Int(8.0 * sampleRate) / blockFrames+ var firedAfterSeconds: Double?+ for block in 0..<silenceBlocks {+ guard let buffer = Self.buffer(Array(repeating: 0, count: blockFrames), sampleRate: sampleRate) else { break }+ try await manager.appendAudio(buffer)+ try await manager.processBufferedAudio()+ await Task.yield()+ if firedAfterSeconds == nil, await events.count > afterSpeech {+ firedAfterSeconds = Double(block + 1) * Double(blockFrames) / sampleRate+ }+ }+ // Callbacks hop through tasks; give them a moment to land.+ try? await Task.sleep(nanoseconds: 300_000_000)+ let total = await events.count++ print("[silence-eou] total EOUs=\(total) (before silence: \(afterSpeech))")+ if let firedAfterSeconds {+ print(String(format: "[silence-eou] RESULT: EOU FIRED %.2fs into silence (debounce is 1.28s)", firedAfterSeconds))+ } else {+ print("[silence-eou] RESULT: NO EOU after 8.0s of digital silence")+ }+ for (index, text) in await events.transcripts.enumerated() {+ print("[silence-eou] eou#\(index + 1) chars=\(text.count) text=\(text.prefix(90))")+ }+ await manager.reset()+ }++ private static func buffer(_ samples: [Float], sampleRate: Double) -> AVAudioPCMBuffer? {+ guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false),+ let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count))+ else { return nil }+ buffer.frameLength = AVAudioFrameCount(samples.count)+ samples.withUnsafeBufferPointer { source in+ buffer.floatChannelData?[0].update(from: source.baseAddress!, count: samples.count)+ }+ return buffer+ }+}++private actor EouEventLog {+ private(set) var transcripts: [String] = []+ var count: Int { self.transcripts.count }+ func record(_ text: String) { self.transcripts.append(text) }+}+#endifdiff --git a/Tests/FluidDictationIntegrationTests/VoiceProcessingAsrQualityTests.swift b/Tests/FluidDictationIntegrationTests/VoiceProcessingAsrQualityTests.swiftnew file mode 100644index 00000000..39e08d9c--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/VoiceProcessingAsrQualityTests.swift@@ -0,0 +1,128 @@+@testable import FluidVoice_Debug+import AVFoundation+import Foundation+import XCTest++#if arch(arm64)+import FluidAudio++/// Measurement: does Apple's Voice Processing I/O degrade transcription of the user's OWN voice?+///+/// macOS bundles noise suppression and automatic gain control with its echo canceller, and shipping+/// dictation products disable all three for dictation while enabling them only for meetings — which+/// implies the processing costs something. Two takes of the same passage, one raw and one+/// VPIO-processed, each transcribed and compared against the known text.+///+/// Deliberately avoids ScreenCaptureKit: the test host loses its Screen Recording grant on every+/// rebuild, and nothing here needs it.+///+/// Run with FLUIDVOICE_ASR_AB=1.+final class VoiceProcessingAsrQualityTests: XCTestCase {+ private static let captureSeconds = 22.0++ func testVpioEffectOnOwnVoiceTranscription() async throws {+ guard ProcessInfo.processInfo.environment["FLUIDVOICE_ASR_AB"] != nil else {+ throw XCTSkip("FLUIDVOICE_ASR_AB not set")+ }++ // Lead-in so the operator can start speaking before the first take opens.+ print("\n[ab] starting in 15 seconds — begin reading now and keep going\n")+ try await Task.sleep(nanoseconds: 15_000_000_000)+ let raw = try await Self.capture(voiceProcessing: false, label: "TAKE 1 of 2 — voice processing OFF")+ try await Task.sleep(nanoseconds: 2_000_000_000)+ let processed = try await Self.capture(voiceProcessing: true, label: "TAKE 2 of 2 — voice processing ON")++ print(String(format: "\n[ab] raw %.1fs rms=%.5f (%.1f dBFS)",+ Double(raw.count) / 16_000, Self.rms(raw), 20 * log10(max(Self.rms(raw), 1e-9))))+ print(String(format: "[ab] vpio %.1fs rms=%.5f (%.1f dBFS)",+ Double(processed.count) / 16_000, Self.rms(processed), 20 * log10(max(Self.rms(processed), 1e-9))))++ let rawText = try await Self.transcribe(raw)+ let vpioText = try await Self.transcribe(processed)+ print("\n[ab] RAW : \(rawText)")+ print("\n[ab] VPIO : \(vpioText)\n")+ print("[ab] words raw=\(rawText.split(separator: " ").count) vpio=\(vpioText.split(separator: " ").count)")++ XCTAssertFalse(raw.isEmpty, "no audio captured on take 1")+ XCTAssertFalse(processed.isEmpty, "no audio captured on take 2")+ }++ private static func capture(voiceProcessing: Bool, label: String) async throws -> [Float] {+ let engine = AVAudioEngine()+ let input = engine.inputNode+ if voiceProcessing {+ try input.setVoiceProcessingEnabled(true)+ var ducking = AVAudioVoiceProcessingOtherAudioDuckingConfiguration()+ ducking.enableAdvancedDucking = false+ ducking.duckingLevel = .min+ input.voiceProcessingOtherAudioDuckingConfiguration = ducking+ }+ let format = input.outputFormat(forBus: 0)+ let sink = SampleSink()+ input.installTap(onBus: 0, bufferSize: 1024, format: format) { buffer, _ in+ guard let channel = buffer.floatChannelData?[0] else { return }+ let slice = Array(UnsafeBufferPointer(start: channel, count: Int(buffer.frameLength)))+ Task { await sink.append(slice, sourceRate: format.sampleRate) }+ }+ engine.prepare()+ try engine.start()+ print("\n[ab] ==========================================================")+ print("[ab] \(label)")+ print("[ab] READ THE PASSAGE NOW — \(Int(Self.captureSeconds)) seconds")+ print("[ab] ==========================================================\n")+ try await Task.sleep(nanoseconds: UInt64(Self.captureSeconds * 1_000_000_000))+ engine.stop()+ input.removeTap(onBus: 0)+ return await sink.samples()+ }++ private static func rms(_ samples: [Float]) -> Double {+ guard !samples.isEmpty else { return 0 }+ return (samples.reduce(0.0) { $0 + Double($1) * Double($1) } / Double(samples.count)).squareRoot()+ }++ private static func transcribe(_ samples: [Float]) async throws -> String {+ guard !samples.isEmpty else { return "(no audio)" }+ let manager = StreamingEouAsrManager(chunkSize: .ms160)+ try await manager.loadModelsFromHuggingFace()+ guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 16_000, channels: 1, interleaved: false)+ else { return "(format error)" }+ var index = 0+ while index < samples.count {+ let end = min(index + 2_560, samples.count)+ guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(end - index)) else { break }+ buffer.frameLength = AVAudioFrameCount(end - index)+ samples[index..<end].withUnsafeBufferPointer { source in+ buffer.floatChannelData?[0].update(from: source.baseAddress!, count: end - index)+ }+ try await manager.appendAudio(buffer)+ try await manager.processBufferedAudio()+ index = end+ }+ let text = try await manager.finish()+ await manager.reset()+ return text.trimmingCharacters(in: .whitespacesAndNewlines)+ }+}++/// Accumulates mono 16 kHz float samples, decimating from the source rate.+private actor SampleSink {+ private var storage: [Float] = []++ func append(_ incoming: [Float], sourceRate: Double) {+ guard sourceRate > 0 else { return }+ let ratio = sourceRate / 16_000+ guard ratio > 1.01 else {+ self.storage.append(contentsOf: incoming)+ return+ }+ var position = 0.0+ while Int(position) < incoming.count {+ self.storage.append(incoming[Int(position)])+ position += ratio+ }+ }++ func samples() -> [Float] { self.storage }+}+#endifdiff --git a/Tests/FluidDictationIntegrationTests/VoiceProcessingDuckingProbeTests.swift b/Tests/FluidDictationIntegrationTests/VoiceProcessingDuckingProbeTests.swiftnew file mode 100644index 00000000..78664e1b--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/VoiceProcessingDuckingProbeTests.swift@@ -0,0 +1,172 @@+@testable import FluidVoice_Debug+import AVFoundation+import Foundation+import ScreenCaptureKit+import XCTest++/// Measurement: does instantiating Voice Processing I/O attenuate what ScreenCaptureKit captures+/// from another application?+///+/// VPIO is Apple's hardware-assisted AEC and would replace a hand-rolled echo gate entirely. It was+/// ruled out earlier on the assumption that its system-wide ducking of "other audio" would corrupt+/// the recorded `applicationAudio` track — an assumption never measured. SCStream taps per-process;+/// ducking applies at output mixing, so they may not be the same path.+///+/// Plays a steady tone from a separate process (`afplay`), captures it via SCStream, and compares+/// captured level across three phases: no VPIO, VPIO with default ducking, VPIO with ducking+/// minimised. Run with FLUIDVOICE_VPIO_PROBE=1.+final class VoiceProcessingDuckingProbeTests: XCTestCase {+ func testVoiceProcessingDuckingEffectOnScreenCaptureAudio() async throws {+ guard ProcessInfo.processInfo.environment["FLUIDVOICE_VPIO_PROBE"] != nil else {+ throw XCTSkip("FLUIDVOICE_VPIO_PROBE not set")+ }++ // Audio source is an external windowed application (a browser tab playing a tone). A+ // windowless helper such as `afplay` is not part of a display's shareable content, so+ // SCStream delivers buffers of pure zeros for it — measured: 584 buffers, all silent.+ try await Task.sleep(nanoseconds: 500_000_000)++ let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)+ guard let display = content.displays.first else { throw XCTSkip("no display") }+ let filter = SCContentFilter(display: display, excludingApplications: [], exceptingWindows: [])++ let configuration = SCStreamConfiguration()+ configuration.width = 2+ configuration.height = 2+ configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)+ configuration.queueDepth = 3+ configuration.showsCursor = false+ configuration.capturesAudio = true+ configuration.sampleRate = 48_000+ configuration.channelCount = 2+ configuration.excludesCurrentProcessAudio = true++ let collector = AudioLevelCollector()+ let output = LevelStreamOutput(collector: collector)+ let errorDelegate = ProbeStreamDelegate()+ let stream = SCStream(filter: filter, configuration: configuration, delegate: errorDelegate)+ try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: DispatchQueue(label: "vpio.probe"))+ try await stream.startCapture()+ defer { Task { try? await stream.stopCapture() } }+ print("[vpio] capture started")++ func phase(_ label: String, seconds: Double) async -> Double {+ await collector.mark(label)+ try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000))+ return await collector.rms(for: label)+ }++ let baseline = await phase("1-no-vpio", seconds: 3.0)++ // Instantiating and running an engine with voice processing on the IO nodes is what turns+ // VPIO on process-wide; merely constructing AVAudioEngine does not.+ let engine = AVAudioEngine()+ let input = engine.inputNode+ // Enabling on the input node switches the whole IO unit to VPIO; doing it on the output node+ // as well fails initialisation with -10875.+ try input.setVoiceProcessingEnabled(true)+ input.installTap(onBus: 0, bufferSize: 1024, format: input.outputFormat(forBus: 0)) { _, _ in }++ // Adding a render chain to satisfy VPIO's downlink fails initialisation with -10875 here, so+ // the engine runs capture-only. VPIO is still instantiated and active — it logs its own+ // downlink faults — which is what the ducking measurement needs.+ engine.prepare()+ try engine.start()+ defer { engine.stop() }+ try await Task.sleep(nanoseconds: 1_200_000_000)++ let withDefaultDucking = await phase("2-vpio-default", seconds: 3.0)++ var ducking = AVAudioVoiceProcessingOtherAudioDuckingConfiguration()+ ducking.enableAdvancedDucking = false+ ducking.duckingLevel = .min+ input.voiceProcessingOtherAudioDuckingConfiguration = ducking+ try await Task.sleep(nanoseconds: 700_000_000)++ let withMinDucking = await phase("3-vpio-min-ducking", seconds: 3.0)++ func dB(_ value: Double) -> String {+ value > 0 ? String(format: "%.1f dBFS", 20 * log10(value)) : "silent"+ }+ print("[vpio] baseline (no VPIO) rms=\(String(format: "%.5f", baseline)) \(dB(baseline))")+ print("[vpio] VPIO, default ducking rms=\(String(format: "%.5f", withDefaultDucking)) \(dB(withDefaultDucking))")+ print("[vpio] VPIO, ducking = .min rms=\(String(format: "%.5f", withMinDucking)) \(dB(withMinDucking))")+ if baseline > 0 {+ print(String(format: "[vpio] default ducking attenuation: %.1f dB", 20 * log10(withDefaultDucking / baseline)))+ print(String(format: "[vpio] min ducking attenuation: %.1f dB", 20 * log10(withMinDucking / baseline)))+ }+ let stats = await collector.diagnostics()+ print("[vpio] buffers=\(stats.buffers) copyFailures=\(stats.copyFailures) framesTotal=\(stats.frames) streamError=\(errorDelegate.lastError ?? "none")")+ XCTAssertGreaterThan(baseline, 0, "SCStream captured no audio; tone or permission problem")+ }++}++private actor AudioLevelCollector {+ private var current: String?+ private var sums: [String: (total: Double, frames: Int)] = [:]+ private var buffers = 0+ private var copyFailures = 0+ private var frames = 0++ func mark(_ label: String) { self.current = label }++ func diagnostics() -> (buffers: Int, copyFailures: Int, frames: Int) {+ (self.buffers, self.copyFailures, self.frames)+ }++ func noteBuffer() { self.buffers += 1 }+ func noteCopyFailure() { self.copyFailures += 1 }++ func add(sumOfSquares: Double, frames: Int) {+ self.frames += frames+ guard let current else { return }+ let existing = self.sums[current] ?? (0, 0)+ self.sums[current] = (existing.total + sumOfSquares, existing.frames + frames)+ }++ func rms(for label: String) -> Double {+ guard let entry = self.sums[label], entry.frames > 0 else { return 0 }+ return (entry.total / Double(entry.frames)).squareRoot()+ }+}++private final class LevelStreamOutput: NSObject, SCStreamOutput {+ private let collector: AudioLevelCollector+ init(collector: AudioLevelCollector) { self.collector = collector }++ func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {+ guard type == .audio else { return }+ Task { [collector] in await collector.noteBuffer() }+ guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),+ let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription),+ let format = AVAudioFormat(streamDescription: asbd)+ else { return }+ let frames = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))+ guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames) else { return }+ buffer.frameLength = frames+ guard CMSampleBufferCopyPCMDataIntoAudioBufferList(+ sampleBuffer, at: 0, frameCount: Int32(frames), into: buffer.mutableAudioBufferList+ ) == noErr, let channel = buffer.floatChannelData?[0] else {+ Task { [collector] in await collector.noteCopyFailure() }+ return+ }+ var sum = 0.0+ for i in 0..<Int(frames) { sum += Double(channel[i]) * Double(channel[i]) }+ Task { [collector] in await collector.add(sumOfSquares: sum, frames: Int(frames)) }+ }+}++private final class ProbeStreamDelegate: NSObject, SCStreamDelegate {+ private let lock = NSLock()+ private var stored: String?+ var lastError: String? {+ self.lock.lock(); defer { self.lock.unlock() }+ return self.stored+ }++ func stream(_ stream: SCStream, didStopWithError error: Error) {+ self.lock.lock(); defer { self.lock.unlock() }+ self.stored = String(describing: error)+ }+}diff --git a/Tests/FluidDictationIntegrationTests/VoiceProcessingSpeechDuckingTests.swift b/Tests/FluidDictationIntegrationTests/VoiceProcessingSpeechDuckingTests.swiftnew file mode 100644index 00000000..a6cfad5d--- /dev/null+++ b/Tests/FluidDictationIntegrationTests/VoiceProcessingSpeechDuckingTests.swift@@ -0,0 +1,176 @@+@testable import FluidVoice_Debug+import AVFoundation+import Foundation+import ScreenCaptureKit+import XCTest++/// Two measurements in one run, both needed before adopting Apple's AEC for the meeting microphone.+///+/// A. Ducking against REAL SPEECH rather than a steady tone. Ducking is voice-activity driven, so a+/// tone understates both its depth and its variability. Reports a per-second level timeline.+/// B. Relative clock drift between an `AVAudioEngine` microphone and SCStream application audio.+/// Moving the microphone off SCStream costs the shared capture clock; this quantifies what that+/// is worth in parts per million.+///+/// Play speech through a windowed app, then run with FLUIDVOICE_VPIO_SPEECH=1.+final class VoiceProcessingSpeechDuckingTests: XCTestCase {+ func testDuckingAgainstSpeechAndClockDrift() async throws {+ guard ProcessInfo.processInfo.environment["FLUIDVOICE_VPIO_SPEECH"] != nil else {+ throw XCTSkip("FLUIDVOICE_VPIO_SPEECH not set")+ }++ let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)+ guard let display = content.displays.first else { throw XCTSkip("no display") }+ let filter = SCContentFilter(display: display, excludingApplications: [], exceptingWindows: [])++ let configuration = SCStreamConfiguration()+ configuration.width = 2+ configuration.height = 2+ configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1)+ configuration.queueDepth = 3+ configuration.showsCursor = false+ configuration.capturesAudio = true+ configuration.sampleRate = 48_000+ configuration.channelCount = 2+ configuration.excludesCurrentProcessAudio = true++ let recorder = TimelineRecorder()+ let output = TimelineStreamOutput(recorder: recorder)+ let stream = SCStream(filter: filter, configuration: configuration, delegate: nil)+ try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: DispatchQueue(label: "vpio.speech"))+ try await stream.startCapture()+ defer { Task { try? await stream.stopCapture() } }++ await recorder.mark("baseline")+ try await Task.sleep(nanoseconds: 6_000_000_000)++ let engine = AVAudioEngine()+ let input = engine.inputNode+ try input.setVoiceProcessingEnabled(true)+ var ducking = AVAudioVoiceProcessingOtherAudioDuckingConfiguration()+ ducking.enableAdvancedDucking = false+ ducking.duckingLevel = .min+ input.voiceProcessingOtherAudioDuckingConfiguration = ducking++ // Count microphone frames against host time to derive this clock's true rate.+ let micClock = ClockCounter(nominalRate: input.outputFormat(forBus: 0).sampleRate)+ input.installTap(onBus: 0, bufferSize: 1024, format: input.outputFormat(forBus: 0)) { buffer, _ in+ Task { await micClock.add(frames: Int(buffer.frameLength)) }+ }+ engine.prepare()+ try engine.start()+ defer { engine.stop() }+ try await Task.sleep(nanoseconds: 1_000_000_000)++ await recorder.mark("vpio-min-ducking")+ await micClock.begin()+ try await Task.sleep(nanoseconds: 20_000_000_000)++ let timeline = await recorder.timeline()+ let baseline = timeline.filter { $0.phase == "baseline" }.map(\.rms)+ let ducked = timeline.filter { $0.phase == "vpio-min-ducking" }.map(\.rms)++ func dB(_ v: Double) -> Double { v > 0 ? 20 * log10(v) : -120 }+ func summarise(_ label: String, _ values: [Double]) {+ guard !values.isEmpty else { print("[speech] \(label): no data"); return }+ let mean = values.reduce(0, +) / Double(values.count)+ print(String(+ format: "[speech] %-18@ seconds=%d mean=%.1f dBFS min=%.1f max=%.1f swing=%.1f dB",+ label as NSString, values.count, dB(mean), dB(values.min() ?? 0), dB(values.max() ?? 0),+ dB(values.max() ?? 0) - dB(values.min() ?? 0)+ ))+ }+ summarise("baseline", baseline)+ summarise("VPIO min ducking", ducked)+ if let b = baseline.max(), let d = ducked.max(), b > 0, d > 0 {+ print(String(format: "[speech] peak-to-peak attenuation: %.1f dB", dB(d) - dB(b)))+ }+ print("[speech] per-second dBFS (ducked): "+ + ducked.map { String(format: "%.0f", dB($0)) }.joined(separator: " "))++ let (micFrames, micSeconds) = await micClock.result()+ let (appFrames, appSeconds) = await recorder.clockResult()+ func ppm(_ frames: Int, _ seconds: Double, _ nominal: Double) -> Double {+ guard seconds > 0, nominal > 0 else { return .nan }+ return (Double(frames) / seconds / nominal - 1) * 1_000_000+ }+ let micPPM = ppm(micFrames, micSeconds, await micClock.nominalRate)+ let appPPM = ppm(appFrames, appSeconds, 48_000)+ print(String(format: "[drift] mic clock %.0f ppm vs nominal (%d frames / %.2fs)", micPPM, micFrames, micSeconds))+ print(String(format: "[drift] app clock %.0f ppm vs nominal (%d frames / %.2fs)", appPPM, appFrames, appSeconds))+ print(String(format: "[drift] RELATIVE %.0f ppm => %.0f ms of skew per hour", micPPM - appPPM, abs(micPPM - appPPM) * 3.6))++ XCTAssertFalse(baseline.isEmpty, "no baseline audio captured — is speech playing?")+ }+}++private actor ClockCounter {+ let nominalRate: Double+ private var frames = 0+ private var start: CFAbsoluteTime?+ init(nominalRate: Double) { self.nominalRate = nominalRate }+ func begin() { self.start = CFAbsoluteTimeGetCurrent(); self.frames = 0 }+ func add(frames count: Int) { if self.start != nil { self.frames += count } }+ func result() -> (Int, Double) {+ guard let start else { return (0, 0) }+ return (self.frames, CFAbsoluteTimeGetCurrent() - start)+ }+}++private actor TimelineRecorder {+ struct Sample { let phase: String; let rms: Double }+ private var phase = ""+ private var bucketStart = CFAbsoluteTimeGetCurrent()+ private var bucketSum = 0.0+ private var bucketFrames = 0+ private var samples: [Sample] = []+ private var clockFrames = 0+ private var clockStart: CFAbsoluteTime?++ func mark(_ label: String) {+ self.phase = label+ if self.clockStart == nil { self.clockStart = CFAbsoluteTimeGetCurrent() }+ }++ func add(sumOfSquares: Double, frames: Int) {+ guard !self.phase.isEmpty else { return }+ self.clockFrames += frames+ self.bucketSum += sumOfSquares+ self.bucketFrames += frames+ let now = CFAbsoluteTimeGetCurrent()+ if now - self.bucketStart >= 1.0, self.bucketFrames > 0 {+ self.samples.append(Sample(phase: self.phase, rms: (self.bucketSum / Double(self.bucketFrames)).squareRoot()))+ self.bucketStart = now+ self.bucketSum = 0+ self.bucketFrames = 0+ }+ }++ func timeline() -> [Sample] { self.samples }+ func clockResult() -> (Int, Double) {+ guard let clockStart else { return (0, 0) }+ return (self.clockFrames, CFAbsoluteTimeGetCurrent() - clockStart)+ }+}++private final class TimelineStreamOutput: NSObject, SCStreamOutput {+ private let recorder: TimelineRecorder+ init(recorder: TimelineRecorder) { self.recorder = recorder }++ func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {+ guard type == .audio,+ let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer),+ let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription),+ let format = AVAudioFormat(streamDescription: asbd)+ else { return }+ let frames = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer))+ guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames) else { return }+ buffer.frameLength = frames+ guard CMSampleBufferCopyPCMDataIntoAudioBufferList(+ sampleBuffer, at: 0, frameCount: Int32(frames), into: buffer.mutableAudioBufferList+ ) == noErr, let channel = buffer.floatChannelData?[0] else { return }+ var sum = 0.0+ for i in 0..<Int(frames) { sum += Double(channel[i]) * Double(channel[i]) }+ Task { [recorder] in await recorder.add(sumOfSquares: sum, frames: Int(frames)) }+ }+}