Skip to content

Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more

perf: optimize audio capture startup

altic-devb75c23f8cf2a
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj
index 1155efef..0725e76d 100644
--- a/Fluid.xcodeproj/project.pbxproj
+++ b/Fluid.xcodeproj/project.pbxproj
@@ -447,6 +447,7 @@
STRING_CATALOG_GENERATE_SYMBOLS = YES;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited) ARCH_ARM64";
SWIFT_APPROACHABLE_CONCURRENCY = YES;
+ SWIFT_OBJC_BRIDGING_HEADER = "Sources/Fluid/Fluid-Bridging-Header.h";
SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
@@ -501,6 +502,7 @@
STRING_CATALOG_GENERATE_SYMBOLS = YES;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited) ARCH_ARM64";
SWIFT_APPROACHABLE_CONCURRENCY = YES;
+ SWIFT_OBJC_BRIDGING_HEADER = "Sources/Fluid/Fluid-Bridging-Header.h";
SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
diff --git a/Package.swift b/Package.swift
index 836ecaa1..740ac232 100644
--- a/Package.swift
+++ b/Package.swift
@@ -17,10 +17,18 @@ let package = Package(
.package(url: "https://github.com/PostHog/posthog-ios.git", from: "3.0.0"),
],
targets: [
+ .target(
+ name: "CoreAudioCaptureSupport",
+ path: "Sources/CoreAudioCaptureSupport",
+ linkerSettings: [
+ .linkedFramework("CoreAudio"),
+ ]
+ ),
.executableTarget(
name: "FluidVoice",
dependencies: [
"AppUpdater",
+ "CoreAudioCaptureSupport",
"FluidAudio",
"PromiseKit",
"DynamicNotchKit",
diff --git a/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c b/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c
new file mode 100644
index 00000000..07f631fd
--- /dev/null
+++ b/Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c
@@ -0,0 +1,498 @@
+#include "include/CoreAudioCaptureSupport.h"
+
+#include <dispatch/dispatch.h>
+#include <limits.h>
+#include <mach/mach_time.h>
+#include <stdatomic.h>
+#include <stdlib.h>
+#include <string.h>
+
+// 64 hardware cycles provides ample scheduling headroom while keeping the
+// realtime producer strictly allocation-free.
+#define FV_RING_CAPACITY 64u
+#define FV_MAX_FRAMES_PER_PACKET 8192u
+
+typedef struct {
+ float samples[FV_MAX_FRAMES_PER_PACKET];
+ uint32_t frameCount;
+ uint64_t inputHostTime;
+ int64_t inputSampleTime;
+ uint64_t sequence;
+} FVPacketSlot;
+
+typedef struct {
+ AudioObjectID deviceID;
+ AudioDeviceIOProcID ioProcID;
+ AudioStreamBasicDescription format;
+ uint32_t bufferFrameSize;
+ uint32_t bytesPerSample;
+ dispatch_semaphore_t packetSemaphore;
+ _Atomic uint64_t writeIndex;
+ _Atomic uint64_t readIndex;
+ _Atomic uint64_t droppedPackets;
+ _Atomic bool running;
+ FVPacketSlot slots[FV_RING_CAPACITY];
+} FVCapture;
+
+static OSStatus fv_get_input_stream_format(
+ AudioObjectID deviceID,
+ AudioStreamBasicDescription *format
+) {
+ AudioObjectPropertyAddress streamsAddress = {
+ kAudioDevicePropertyStreams,
+ kAudioObjectPropertyScopeInput,
+ kAudioObjectPropertyElementMain,
+ };
+ UInt32 streamsSize = 0;
+ OSStatus status = AudioObjectGetPropertyDataSize(
+ deviceID,
+ &streamsAddress,
+ 0,
+ NULL,
+ &streamsSize
+ );
+ if (status != noErr || streamsSize != sizeof(AudioStreamID)) {
+ // Multiple independent input streams can have different formats. Let
+ // AVAudioEngine handle those uncommon devices instead of guessing.
+ return status != noErr ? status : kAudioHardwareUnsupportedOperationError;
+ }
+
+ AudioStreamID streamID = kAudioObjectUnknown;
+ status = AudioObjectGetPropertyData(
+ deviceID,
+ &streamsAddress,
+ 0,
+ NULL,
+ &streamsSize,
+ &streamID
+ );
+ if (status != noErr || streamID == kAudioObjectUnknown) {
+ return status != noErr ? status : kAudioHardwareBadObjectError;
+ }
+
+ AudioObjectPropertyAddress formatAddress = {
+ kAudioStreamPropertyVirtualFormat,
+ kAudioObjectPropertyScopeGlobal,
+ kAudioObjectPropertyElementMain,
+ };
+ UInt32 formatSize = sizeof(*format);
+ return AudioObjectGetPropertyData(
+ streamID,
+ &formatAddress,
+ 0,
+ NULL,
+ &formatSize,
+ format
+ );
+}
+
+static OSStatus fv_get_buffer_frame_size(AudioObjectID deviceID, uint32_t *frameSize) {
+ AudioObjectPropertyAddress address = {
+ kAudioDevicePropertyBufferFrameSize,
+ kAudioObjectPropertyScopeGlobal,
+ kAudioObjectPropertyElementMain,
+ };
+ UInt32 size = sizeof(*frameSize);
+ return AudioObjectGetPropertyData(deviceID, &address, 0, NULL, &size, frameSize);
+}
+
+static bool fv_format_is_supported(
+ const AudioStreamBasicDescription *format,
+ uint32_t *bytesPerSample
+) {
+ if (format->mFormatID != kAudioFormatLinearPCM ||
+ format->mSampleRate <= 0.0 ||
+ format->mChannelsPerFrame == 0 ||
+ format->mBytesPerFrame == 0 ||
+ (format->mFormatFlags & kAudioFormatFlagIsBigEndian) != 0) {
+ return false;
+ }
+
+ const bool nonInterleaved =
+ (format->mFormatFlags & kAudioFormatFlagIsNonInterleaved) != 0;
+ const uint32_t channelDivisor = nonInterleaved ? 1u : format->mChannelsPerFrame;
+ if (channelDivisor == 0 || format->mBytesPerFrame % channelDivisor != 0) {
+ return false;
+ }
+
+ const uint32_t sampleBytes = format->mBytesPerFrame / channelDivisor;
+ const bool isFloat = (format->mFormatFlags & kAudioFormatFlagIsFloat) != 0;
+ const bool isSignedInteger =
+ (format->mFormatFlags & kAudioFormatFlagIsSignedInteger) != 0;
+ const bool isPacked = (format->mFormatFlags & kAudioFormatFlagIsPacked) != 0;
+
+ if (isFloat && format->mBitsPerChannel == 32 && sampleBytes == sizeof(float)) {
+ *bytesPerSample = sampleBytes;
+ return true;
+ }
+ if (isSignedInteger && isPacked &&
+ ((format->mBitsPerChannel == 16 && sampleBytes == 2) ||
+ (format->mBitsPerChannel == 24 && sampleBytes == 3) ||
+ (format->mBitsPerChannel == 32 && sampleBytes == 4))) {
+ *bytesPerSample = sampleBytes;
+ return true;
+ }
+ return false;
+}
+
+static inline float fv_read_sample(
+ const uint8_t *bytes,
+ const AudioStreamBasicDescription *format
+) {
+ if ((format->mFormatFlags & kAudioFormatFlagIsFloat) != 0) {
+ float sample;
+ memcpy(&sample, bytes, sizeof(sample));
+ return sample;
+ }
+
+ switch (format->mBitsPerChannel) {
+ case 16: {
+ int16_t sample;
+ memcpy(&sample, bytes, sizeof(sample));
+ return (float) sample / 32768.0f;
+ }
+ case 24: {
+ int32_t sample =
+ ((int32_t) bytes[0]) |
+ ((int32_t) bytes[1] << 8) |
+ ((int32_t) bytes[2] << 16);
+ if ((sample & 0x00800000) != 0) {
+ sample |= (int32_t) 0xFF000000;
+ }
+ return (float) sample / 8388608.0f;
+ }
+ case 32: {
+ int32_t sample;
+ memcpy(&sample, bytes, sizeof(sample));
+ return (float) ((double) sample / 2147483648.0);
+ }
+ default:
+ return 0.0f;
+ }
+}
+
+static uint32_t fv_frame_count(
+ const FVCapture *capture,
+ const AudioBufferList *inputData
+) {
+ uint32_t frameCount = UINT_MAX;
+ const AudioBuffer *buffers = inputData->mBuffers;
+ for (UInt32 index = 0; index < inputData->mNumberBuffers; ++index) {
+ const AudioBuffer *buffer = &buffers[index];
+ if (buffer->mData == NULL || buffer->mDataByteSize == 0) {
+ continue;
+ }
+ const uint32_t frames = buffer->mDataByteSize / capture->format.mBytesPerFrame;
+ if (frames < frameCount) {
+ frameCount = frames;
+ }
+ }
+ return frameCount == UINT_MAX ? 0u : frameCount;
+}
+
+static OSStatus fv_io_proc(
+ AudioObjectID inDevice,
+ const AudioTimeStamp *inNow,
+ const AudioBufferList *inInputData,
+ const AudioTimeStamp *inInputTime,
+ AudioBufferList *outOutputData,
+ const AudioTimeStamp *inOutputTime,
+ void *inClientData
+) {
+ (void) inDevice;
+ (void) inNow;
+ (void) outOutputData;
+ (void) inOutputTime;
+
+ FVCapture *capture = (FVCapture *) inClientData;
+ if (capture == NULL || inInputData == NULL ||
+ !atomic_load_explicit(&capture->running, memory_order_relaxed)) {
+ return noErr;
+ }
+
+ uint32_t frameCount = fv_frame_count(capture, inInputData);
+ if (frameCount == 0) {
+ return noErr;
+ }
+ if (frameCount > FV_MAX_FRAMES_PER_PACKET) {
+ atomic_fetch_add_explicit(&capture->droppedPackets, 1, memory_order_relaxed);
+ return noErr;
+ }
+
+ const uint64_t writeIndex =
+ atomic_load_explicit(&capture->writeIndex, memory_order_relaxed);
+ const uint64_t readIndex =
+ atomic_load_explicit(&capture->readIndex, memory_order_acquire);
+ if (writeIndex - readIndex >= FV_RING_CAPACITY) {
+ atomic_fetch_add_explicit(&capture->droppedPackets, 1, memory_order_relaxed);
+ return noErr;
+ }
+
+ FVPacketSlot *slot = &capture->slots[writeIndex % FV_RING_CAPACITY];
+ memset(slot->samples, 0, frameCount * sizeof(float));
+
+ uint32_t availableChannels = 0;
+ const AudioBuffer *buffers = inInputData->mBuffers;
+ for (UInt32 bufferIndex = 0; bufferIndex < inInputData->mNumberBuffers; ++bufferIndex) {
+ const AudioBuffer *buffer = &buffers[bufferIndex];
+ if (buffer->mData == NULL || buffer->mDataByteSize == 0 ||
+ buffer->mNumberChannels == 0) {
+ continue;
+ }
+
+ const uint8_t *data = (const uint8_t *) buffer->mData;
+ const uint32_t channelCount = buffer->mNumberChannels;
+ for (uint32_t frame = 0; frame < frameCount; ++frame) {
+ const uint8_t *frameData = data + frame * capture->format.mBytesPerFrame;
+ float sum = 0.0f;
+ for (uint32_t channel = 0; channel < channelCount; ++channel) {
+ sum += fv_read_sample(
+ frameData + channel * capture->bytesPerSample,
+ &capture->format
+ );
+ }
+ slot->samples[frame] += sum;
+ }
+ availableChannels += channelCount;
+ }
+
+ if (availableChannels == 0) {
+ return noErr;
+ }
+ const float channelScale = 1.0f / (float) availableChannels;
+ for (uint32_t frame = 0; frame < frameCount; ++frame) {
+ slot->samples[frame] *= channelScale;
+ }
+
+ slot->frameCount = frameCount;
+ slot->inputHostTime =
+ inInputTime != NULL &&
+ (inInputTime->mFlags & kAudioTimeStampHostTimeValid) != 0
+ ? inInputTime->mHostTime
+ : mach_absolute_time();
+ slot->inputSampleTime =
+ inInputTime != NULL &&
+ (inInputTime->mFlags & kAudioTimeStampSampleTimeValid) != 0
+ ? (int64_t) inInputTime->mSampleTime
+ : -1;
+ slot->sequence = writeIndex;
+
+ atomic_store_explicit(&capture->writeIndex, writeIndex + 1, memory_order_release);
+ dispatch_semaphore_signal(capture->packetSemaphore);
+ return noErr;
+}
+
+int32_t fv_core_audio_capture_create(
+ AudioObjectID deviceID,
+ FVCoreAudioCaptureRef *outCapture
+) {
+ if (outCapture == NULL || deviceID == kAudioObjectUnknown) {
+ return kAudioHardwareBadObjectError;
+ }
+ *outCapture = NULL;
+
+ FVCapture *capture = (FVCapture *) calloc(1, sizeof(FVCapture));
+ if (capture == NULL) {
+ return kAudioHardwareUnspecifiedError;
+ }
+ capture->deviceID = deviceID;
+
+ OSStatus status = fv_get_input_stream_format(deviceID, &capture->format);
+ if (status == noErr &&
+ !fv_format_is_supported(&capture->format, &capture->bytesPerSample)) {
+ status = kAudioHardwareUnsupportedOperationError;
+ }
+ if (status == noErr) {
+ status = fv_get_buffer_frame_size(deviceID, &capture->bufferFrameSize);
+ }
+ if (status == noErr &&
+ (capture->bufferFrameSize == 0 ||
+ capture->bufferFrameSize > FV_MAX_FRAMES_PER_PACKET)) {
+ status = kAudioHardwareUnsupportedOperationError;
+ }
+ if (status != noErr) {
+ free(capture);
+ return status;
+ }
+
+ capture->packetSemaphore = dispatch_semaphore_create(0);
+ if (capture->packetSemaphore == NULL) {
+ free(capture);
+ return kAudioHardwareUnspecifiedError;
+ }
+ atomic_init(&capture->writeIndex, 0);
+ atomic_init(&capture->readIndex, 0);
+ atomic_init(&capture->droppedPackets, 0);
+ atomic_init(&capture->running, false);
+
+ status = AudioDeviceCreateIOProcID(
+ deviceID,
+ fv_io_proc,
+ capture,
+ &capture->ioProcID
+ );
+ if (status != noErr) {
+#if !OS_OBJECT_USE_OBJC
+ dispatch_release(capture->packetSemaphore);
+#endif
+ free(capture);
+ return status;
+ }
+
+ *outCapture = (FVCoreAudioCaptureRef) capture;
+ return noErr;
+}
+
+int32_t fv_core_audio_capture_start(FVCoreAudioCaptureRef captureRef) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture == NULL || capture->ioProcID == NULL) {
+ return kAudioHardwareBadObjectError;
+ }
+ if (atomic_load_explicit(&capture->running, memory_order_acquire)) {
+ return noErr;
+ }
+
+ atomic_store_explicit(&capture->running, true, memory_order_release);
+ OSStatus status = AudioDeviceStart(capture->deviceID, capture->ioProcID);
+ if (status != noErr) {
+ atomic_store_explicit(&capture->running, false, memory_order_release);
+ dispatch_semaphore_signal(capture->packetSemaphore);
+ }
+ return status;
+}
+
+int32_t fv_core_audio_capture_stop(FVCoreAudioCaptureRef captureRef) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture == NULL || capture->ioProcID == NULL) {
+ return kAudioHardwareBadObjectError;
+ }
+ if (!atomic_load_explicit(&capture->running, memory_order_acquire)) {
+ dispatch_semaphore_signal(capture->packetSemaphore);
+ return noErr;
+ }
+
+ // Keep accepting callbacks until AudioDeviceStop has synchronized with the
+ // IOProc. Packets acquired before the caller's stop boundary can then be
+ // timestamp-trimmed by the consumer instead of being dropped here.
+ OSStatus status = AudioDeviceStop(capture->deviceID, capture->ioProcID);
+ atomic_store_explicit(&capture->running, false, memory_order_release);
+ dispatch_semaphore_signal(capture->packetSemaphore);
+ return status;
+}
+
+void fv_core_audio_capture_destroy(FVCoreAudioCaptureRef captureRef) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture == NULL) {
+ return;
+ }
+ if (atomic_load_explicit(&capture->running, memory_order_acquire)) {
+ (void) fv_core_audio_capture_stop(captureRef);
+ }
+ if (capture->ioProcID != NULL) {
+ (void) AudioDeviceDestroyIOProcID(capture->deviceID, capture->ioProcID);
+ capture->ioProcID = NULL;
+ }
+#if !OS_OBJECT_USE_OBJC
+ dispatch_release(capture->packetSemaphore);
+#endif
+ free(capture);
+}
+
+bool fv_core_audio_capture_wait(
+ FVCoreAudioCaptureRef captureRef,
+ uint32_t timeoutMilliseconds
+) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture == NULL) {
+ return false;
+ }
+ dispatch_time_t timeout = dispatch_time(
+ DISPATCH_TIME_NOW,
+ (int64_t) timeoutMilliseconds * NSEC_PER_MSEC
+ );
+ return dispatch_semaphore_wait(capture->packetSemaphore, timeout) == 0;
+}
+
+bool fv_core_audio_capture_peek(
+ FVCoreAudioCaptureRef captureRef,
+ FVCoreAudioPacket *packet
+) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture == NULL || packet == NULL) {
+ return false;
+ }
+ const uint64_t readIndex =
+ atomic_load_explicit(&capture->readIndex, memory_order_relaxed);
+ const uint64_t writeIndex =
+ atomic_load_explicit(&capture->writeIndex, memory_order_acquire);
+ if (readIndex == writeIndex) {
+ return false;
+ }
+
+ const FVPacketSlot *slot = &capture->slots[readIndex % FV_RING_CAPACITY];
+ packet->samples = slot->samples;
+ packet->frameCount = slot->frameCount;
+ packet->sampleRate = capture->format.mSampleRate;
+ packet->inputHostTime = slot->inputHostTime;
+ packet->inputSampleTime = slot->inputSampleTime;
+ packet->sequence = slot->sequence;
+ return true;
+}
+
+void fv_core_audio_capture_consume(FVCoreAudioCaptureRef captureRef) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture == NULL) {
+ return;
+ }
+ const uint64_t readIndex =
+ atomic_load_explicit(&capture->readIndex, memory_order_relaxed);
+ const uint64_t writeIndex =
+ atomic_load_explicit(&capture->writeIndex, memory_order_acquire);
+ if (readIndex != writeIndex) {
+ atomic_store_explicit(&capture->readIndex, readIndex + 1, memory_order_release);
+ }
+}
+
+void fv_core_audio_capture_clear(FVCoreAudioCaptureRef captureRef) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture == NULL || atomic_load_explicit(&capture->running, memory_order_acquire)) {
+ return;
+ }
+ const uint64_t writeIndex =
+ atomic_load_explicit(&capture->writeIndex, memory_order_acquire);
+ atomic_store_explicit(&capture->readIndex, writeIndex, memory_order_release);
+ atomic_store_explicit(&capture->droppedPackets, 0, memory_order_relaxed);
+ while (dispatch_semaphore_wait(capture->packetSemaphore, DISPATCH_TIME_NOW) == 0) {
+ // Remove stale per-packet wakeups from the previous stopped session.
+ }
+}
+
+void fv_core_audio_capture_wake(FVCoreAudioCaptureRef captureRef) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ if (capture != NULL) {
+ dispatch_semaphore_signal(capture->packetSemaphore);
+ }
+}
+
+bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef captureRef) {
+ FVCapture *capture = (FVCapture *) captureRef;
+ return capture != NULL &&
+ atomic_load_explicit(&capture->running, memory_order_acquire);
+}
+
+double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef captureRef) {
+ const FVCapture *capture = (const FVCapture *) captureRef;
+ return capture == NULL ? 0.0 : capture->format.mSampleRate;
+}
+
+uint32_t fv_core_audio_capture_buffer_frame_size(FVCoreAudioCaptureRef captureRef) {
+ const FVCapture *capture = (const FVCapture *) captureRef;
+ return capture == NULL ? 0u : capture->bufferFrameSize;
+}
+
+uint64_t fv_core_audio_capture_dropped_packet_count(FVCoreAudioCaptureRef captureRef) {
+ const FVCapture *capture = (const FVCapture *) captureRef;
+ return capture == NULL
+ ? 0u
+ : atomic_load_explicit(&capture->droppedPackets, memory_order_relaxed);
+}
diff --git a/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h b/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h
new file mode 100644
index 00000000..f733297a
--- /dev/null
+++ b/Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h
@@ -0,0 +1,54 @@
+#ifndef CORE_AUDIO_CAPTURE_SUPPORT_H
+#define CORE_AUDIO_CAPTURE_SUPPORT_H
+
+#include <CoreAudio/CoreAudio.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void *FVCoreAudioCaptureRef;
+
+typedef struct {
+ const float *samples;
+ uint32_t frameCount;
+ double sampleRate;
+ uint64_t inputHostTime;
+ int64_t inputSampleTime;
+ uint64_t sequence;
+} FVCoreAudioPacket;
+
+/// Creates a prepared, input-only capture for one physical Core Audio device.
+/// The device is not started and the microphone is not active until start.
+int32_t fv_core_audio_capture_create(
+ AudioObjectID deviceID,
+ FVCoreAudioCaptureRef *outCapture
+);
+
+int32_t fv_core_audio_capture_start(FVCoreAudioCaptureRef capture);
+int32_t fv_core_audio_capture_stop(FVCoreAudioCaptureRef capture);
+void fv_core_audio_capture_destroy(FVCoreAudioCaptureRef capture);
+
+/// Waits until the realtime producer publishes a packet or capture stops.
+/// Returns true when the consumer should attempt to drain the ring.
+bool fv_core_audio_capture_wait(FVCoreAudioCaptureRef capture, uint32_t timeoutMilliseconds);
+
+/// Returns a view into the next immutable ring slot. The pointer remains valid
+/// until consume is called and must never escape the consumer callback.
+bool fv_core_audio_capture_peek(FVCoreAudioCaptureRef capture, FVCoreAudioPacket *packet);
+void fv_core_audio_capture_consume(FVCoreAudioCaptureRef capture);
+void fv_core_audio_capture_clear(FVCoreAudioCaptureRef capture);
+void fv_core_audio_capture_wake(FVCoreAudioCaptureRef capture);
+
+bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef capture);
+double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef capture);
+uint32_t fv_core_audio_capture_buffer_frame_size(FVCoreAudioCaptureRef capture);
+uint64_t fv_core_audio_capture_dropped_packet_count(FVCoreAudioCaptureRef capture);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift
index 31784b45..36ef6763 100644
--- a/Sources/Fluid/ContentView.swift
+++ b/Sources/Fluid/ContentView.swift
@@ -250,6 +250,7 @@ struct ContentView: View {
@State private var accessibilityGuideMonitorTask: Task<Void, Never>?
@State private var accessibilityGuideRequestID: UUID?
@State private var prewarmDictationTask: Task<Void, Never>?
+ @State private var overlayLifecycleID: UInt64 = 0
private var isRecordingAnyShortcutCapture: Bool {
self.activeShortcutRecordingTarget != nil
@@ -1602,7 +1603,7 @@ struct ContentView: View {
return nil
}
- private func captureRecordingContext() {
+ private func captureRecordingTargetContext() {
// Capture the focused target PID BEFORE any overlay/UI changes.
// Used to restore focus when the user interacts with overlay dropdowns.
let focusedPID = TypingService.captureSystemFocusedPID()
@@ -1616,7 +1617,9 @@ struct ContentView: View {
"Captured recording app context: app=\(info.name), bundleId=\(info.bundleId), title=\(info.windowTitle)",
source: "ContentView"
)
+ }
+ private func captureRecordingFormattingContextIfNeeded() {
// Capture text before the caret only when formatting needs focused-field context.
if SettingsStore.shared.needsDictationFormattingContext {
self.recordingPrecedingText = TypingService.textBeforeCursorInFocusedField()
@@ -1629,6 +1632,11 @@ struct ContentView: View {
}
}
+ private func captureRecordingContext() {
+ self.captureRecordingTargetContext()
+ self.captureRecordingFormattingContextIfNeeded()
+ }
+
private func resolveTypingTargetPID() -> (pid: pid_t?, shouldRestoreOriginalFocus: Bool) {
let originalPID = NotchContentState.shared.recordingTargetPID
let currentFocusedPID = TypingService.captureSystemFocusedPID()
@@ -2056,6 +2064,16 @@ struct ContentView: View {
let wasCommandMode = modeAtStop == .command || self.isRecordingForCommand
let activeDictationSlot = self.currentDictationShortcutSlot(for: modeAtStop)
let promptOverride = self.promptModeOverrideText
+ let promptTest = DictationPromptTestCoordinator.shared
+ let shouldUseAIOnStop = activeDictationSlot.map {
+ DictationAIPostProcessingGate.isConfigured(for: $0, appBundleID: self.recordingAppInfo?.bundleId)
+ } ?? DictationAIPostProcessingGate.isConfigured(for: .primary, appBundleID: self.recordingAppInfo?.bundleId)
+ let shouldHideOverlayOnStop = route == .normal &&
+ !wasRewriteMode &&
+ !wasCommandMode &&
+ !promptTest.isActive &&
+ !shouldUseAIOnStop
+ var didRequestOverlayHideOnStop = false
DebugLogger.shared.info(
"Routing decision snapshot | activeMode=\(modeAtStop.rawValue) | rewrite=\(wasRewriteMode) | command=\(wasCommandMode) | overlay=\(NotchContentState.shared.mode.rawValue)",
source: "ContentView"
@@ -2063,18 +2081,22 @@ struct ContentView: View {
self.clearActiveRecordingMode()
- // Show "Transcribing" state before calling stop() to keep overlay visible.
- // The asr.stop() call performs the final transcription which can take a moment
- // (especially for slower models like Whisper Medium/Large).
- DebugLogger.shared.debug("Showing transcription processing state", source: "ContentView")
- self.appBench("processing_ui_request status=Transcribing")
- self.menuBarManager.setProcessing(true)
- NotchOverlayManager.shared.updateTranscriptionText("Transcribing")
- self.appBench("processing_ui_requested status=Transcribing")
-
- // Give SwiftUI a chance to render the processing state before we do heavier work
- // (ASR finalization + optional AI post-processing).
- await Task.yield()
+ if shouldHideOverlayOnStop {
+ didRequestOverlayHideOnStop = true
+ DebugLogger.shared.debug("Hiding dictation overlay at stop path", source: "ContentView")
+ self.hideOverlayAsync(reason: "stop_path")
+ } else {
+ // Show "Transcribing" state before calling stop() when the overlay needs
+ // to remain available for prompt, command, rewrite, or AI feedback.
+ DebugLogger.shared.debug("Showing transcription processing state", source: "ContentView")
+ self.appBench("processing_ui_request status=Transcribing")
+ self.menuBarManager.setProcessing(true)
+ NotchOverlayManager.shared.updateTranscriptionText("Transcribing")
+ self.appBench("processing_ui_requested status=Transcribing")
+
+ // Give SwiftUI a chance to render the processing state before heavier work.
+ await Task.yield()
+ }
// Stop the ASR service and wait for transcription to complete
// The processing indicator will stay visible during this phase
@@ -2098,14 +2120,14 @@ struct ContentView: View {
guard transcribedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else {
DebugLogger.shared.debug("Transcription returned empty text", source: "ContentView")
- // Hide processing state when returning early
- self.menuBarManager.setProcessing(false)
- NotchOverlayManager.shared.hide()
+ // Finish the same short exit transition even when no text is emitted.
+ if !didRequestOverlayHideOnStop {
+ await self.menuBarManager.finishProcessingAndHideOverlay()
+ }
return
}
// Prompt Test Mode: reroute dictation hotkey output into the prompt editor (no typing/clipboard/history).
- let promptTest = DictationPromptTestCoordinator.shared
if promptTest.isActive {
promptTest.lastTranscriptionText = transcribedText
promptTest.lastOutputText = ""
@@ -2238,12 +2260,8 @@ struct ContentView: View {
// a brief non-shimmer "Refining..." preview flash.
NotchOverlayManager.shared.updateTranscriptionText("")
- // Hide processing animation
- self.menuBarManager.setProcessing(false)
} else {
finalText = transcribedText
- // No AI processing, hide the processing state
- self.menuBarManager.setProcessing(false)
}
// Apply GAAV formatting as the FINAL step (after AI post-processing)
@@ -2288,6 +2306,15 @@ struct ContentView: View {
)
}
+ let shouldShowAIProcessingFailure = shouldPersistOutputs && aiFallbackReason != nil
+ if shouldShowAIProcessingFailure {
+ self.pendingAIReprocessText = transcribedText
+ NotchContentState.shared.showAIProcessingFailure()
+ self.menuBarManager.finishProcessingKeepingOverlayVisible()
+ } else {
+ self.pendingAIReprocessText = nil
+ }
+
let frontmostApp = NSWorkspace.shared.frontmostApplication
let frontmostName = frontmostApp?.localizedName ?? "Unknown"
let isFluidFrontmost = frontmostApp?.bundleIdentifier == Bundle.main.bundleIdentifier
@@ -2314,14 +2341,6 @@ struct ContentView: View {
model: transcriptionModelInfo.model
)
}
- let shouldShowAIProcessingFailure = shouldPersistOutputs && aiFallbackReason != nil
- if shouldShowAIProcessingFailure {
- self.pendingAIReprocessText = transcribedText
- NotchContentState.shared.showAIProcessingFailure()
- } else {
- self.pendingAIReprocessText = nil
- }
-
// When FluidVoice itself is frontmost, the bound editor already receives `finalText`.
// Avoid re-inserting or overwriting the clipboard in that self-target case.
let shouldCopyToClipboard = shouldPersistOutputs &&
@@ -2349,8 +2368,8 @@ struct ContentView: View {
if shouldTypeExternally {
let typingTarget = self.resolveTypingTargetPID()
- // Await typing completion before proceeding to edit tracker
- // This ensures the tracker window opens after text has been typed
+ // Dispatch insertion as soon as the destination app is ready; the
+ // overlay hides asynchronously after output so it cannot delay paste.
if typingTarget.shouldRestoreOriginalFocus {
await self.restoreFocusToRecordingTarget()
}
@@ -2363,6 +2382,9 @@ struct ContentView: View {
textReadyAt: finalTextReadyAt
)
didTypeExternally = true
+ if !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop {
+ self.hideOverlayAfterOutput()
+ }
}
if didTypeExternally {
@@ -2374,7 +2396,7 @@ struct ContentView: View {
]
)
- // Now that typing is complete, start the edit tracker
+ // Register the post-transcription edit observation after insertion is dispatched.
let wordsBucket = AnalyticsBuckets.bucketWords(AnalyticsBuckets.wordCount(in: finalText))
let modelInfo = self.currentDictationAIModelInfo()
await PostTranscriptionEditTracker.shared.markTranscriptionCompleted(
@@ -2385,10 +2407,6 @@ struct ContentView: View {
aiModel: modelInfo.model,
aiProvider: modelInfo.provider
)
-
- if !shouldShowAIProcessingFailure {
- NotchOverlayManager.shared.hide()
- }
} else if shouldPersistOutputs,
SettingsStore.shared.copyTranscriptionToClipboard == false,
SettingsStore.shared.saveTranscriptionHistory
@@ -2402,8 +2420,35 @@ struct ContentView: View {
)
}
- if !didTypeExternally, !shouldShowAIProcessingFailure {
- NotchOverlayManager.shared.hide()
+ if !didTypeExternally, !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop {
+ self.hideOverlayAfterOutput()
+ }
+ }
+
+ private func hideOverlayAfterOutput() {
+ self.hideOverlayAsync(reason: "after_output")
+ }
+
+ private func advanceOverlayLifecycle() {
+ self.overlayLifecycleID &+= 1
+ }
+
+ private func hideOverlayAsync(reason: String) {
+ let expectedOverlayLifecycleID = self.overlayLifecycleID
+ self.appBench("overlay_hide_request reason=\(reason) lifecycle=\(expectedOverlayLifecycleID)")
+ Task { @MainActor in
+ guard self.overlayLifecycleID == expectedOverlayLifecycleID else {
+ self.appBench(
+ "overlay_hide_skipped reason=\(reason) staleLifecycle=\(expectedOverlayLifecycleID) currentLifecycle=\(self.overlayLifecycleID)"
+ )
+ return
+ }
+
+ let overlayHideStartedAt = ProcessInfo.processInfo.systemUptime
+ await self.menuBarManager.finishProcessingAndHideOverlay()
+ self.appBench(
+ "overlay_hidden reason=\(reason) elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - overlayHideStartedAt) * 1000).rounded()))"
+ )
}
}
@@ -2710,7 +2755,6 @@ struct ContentView: View {
}
NotchOverlayManager.shared.updateTranscriptionText("")
- self.menuBarManager.setProcessing(false)
finalText = ASRService.applyGAAVFormatting(finalText)
let precedingText = SettingsStore.shared.needsDictationFormattingContext
@@ -2733,6 +2777,7 @@ struct ContentView: View {
if aiFallbackReason != nil {
self.pendingAIReprocessText = transcribedText
NotchContentState.shared.showAIProcessingFailure()
+ self.menuBarManager.finishProcessingKeepingOverlayVisible()
} else {
self.pendingAIReprocessText = nil
}
@@ -2759,6 +2804,10 @@ struct ContentView: View {
)
}
+ if aiFallbackReason == nil {
+ self.hideOverlayAfterOutput()
+ }
+
self.clearActiveRecordingMode()
}
@@ -2780,9 +2829,6 @@ struct ContentView: View {
// - Without originalText: improves/refines the spoken text
await self.rewriteModeService.processRewriteRequest(instruction)
- // Hide processing animation
- self.menuBarManager.setProcessing(false)
-
// If rewrite was successful, type the result
if !self.rewriteModeService.rewrittenText.isEmpty {
DebugLogger.shared.info("Rewrite successful, typing result (chars: \(self.rewriteModeService.rewrittenText.count))", source: "ContentView")
@@ -2818,11 +2864,9 @@ struct ContentView: View {
// Clear the rewrite service state for next use
self.rewriteModeService.clearState()
-
- Task { @MainActor in
- NotchOverlayManager.shared.hide()
- }
+ self.hideOverlayAfterOutput()
} else {
+ await self.menuBarManager.finishProcessingAndHideOverlay()
DebugLogger.shared.error("Rewrite failed - no result", source: "ContentView")
AnalyticsService.shared.capture(
.errorOccurred,
@@ -2942,21 +2986,29 @@ struct ContentView: View {
source: "ContentView"
)
- self.captureRecordingContext()
+ self.advanceOverlayLifecycle()
self.setActiveRecordingMode(.dictate)
+ let shouldShowDictationOverlay = !self.isRecordingForCommand && !self.isRecordingForRewrite
+ let shouldPlayStartSound = !self.isRecordingForCommand
+ && !self.isRecordingForRewrite
+ && self.asr.micStatus == .authorized
// Ensure normal dictation mode is set (command/rewrite modes set their own)
- if !self.isRecordingForCommand, !self.isRecordingForRewrite {
+ if shouldShowDictationOverlay {
self.menuBarManager.setOverlayMode(.dictation)
- self.menuBarManager.showRecordingOverlayImmediately()
- }
-
- if !self.isRecordingForCommand, !self.isRecordingForRewrite {
- TranscriptionSoundPlayer.shared.playStartSound()
}
Task {
- await self.asr.start()
+ await self.asr.start(onCaptureStarted: {
+ if shouldPlayStartSound {
+ TranscriptionSoundPlayer.shared.playStartSound()
+ }
+ self.captureRecordingContext()
+ self.prewarmPrivateAIDictationIfNeeded(for: .primary)
+ if shouldShowDictationOverlay {
+ self.menuBarManager.showRecordingOverlayImmediately()
+ }
+ })
if !self.asr.isRunning {
self.menuBarManager.hideRecordingOverlayImmediately(reason: "asr_start_failed")
}
@@ -2972,8 +3024,6 @@ struct ContentView: View {
DebugLogger.shared.error("Failed to pre-load model: \(error)", source: "ContentView")
}
}
-
- self.prewarmPrivateAIDictationIfNeeded(for: .primary)
}
private func prewarmPrivateAIDictationIfNeeded(for slot: SettingsStore.DictationShortcutSlot) {
@@ -3196,6 +3246,8 @@ struct ContentView: View {
guard !self.asr.isRunning else { return }
+ self.advanceOverlayLifecycle()
+
// Start recording immediately for the command
DebugLogger.shared.info(
"Starting voice recording for command",
@@ -3234,6 +3286,8 @@ struct ContentView: View {
guard !self.asr.isRunning else { return }
+ self.advanceOverlayLifecycle()
+
// Start recording immediately for the edit instruction
DebugLogger.shared.info("Starting voice recording for edit mode", source: "ContentView")
TranscriptionSoundPlayer.shared.playStartSound()
@@ -3544,28 +3598,30 @@ extension ContentView {
self.settings.playgroundUsed = false
self.playgroundUsed = false
}
- self.captureRecordingContext()
- self.applyDictationPromptConfiguration(for: SettingsStore.shared.dictationPromptSelection(for: slot))
self.applyDictationShortcutSelectionContext(for: slot)
self.setActiveRecordingMode(mode)
self.rewriteModeService.clearState()
- self.appBench("overlay_mode_request mode=Dictation")
- self.menuBarManager.setOverlayMode(.dictation)
- self.menuBarManager.showRecordingOverlayImmediately()
- self.appBench("overlay_mode_requested mode=Dictation")
- self.prewarmPrivateAIDictationIfNeeded(for: slot)
guard !self.asr.isRunning else {
self.appBench("asr_start_skipped reason=already_running")
return
}
- if SettingsStore.shared.enableTranscriptionSounds {
- TranscriptionSoundPlayer.shared.playStartSound()
- }
+ self.advanceOverlayLifecycle()
Task {
let asrStartStartedAt = ProcessInfo.processInfo.systemUptime
DebugLogger.shared.benchmark("APP_BENCH", message: "asr_start_call", source: "AppBenchmark")
- await self.asr.start()
+ await self.asr.start(onCaptureStarted: {
+ if SettingsStore.shared.enableTranscriptionSounds {
+ TranscriptionSoundPlayer.shared.playStartSound()
+ }
+ self.captureRecordingContext()
+ self.applyDictationPromptConfiguration(for: SettingsStore.shared.dictationPromptSelection(for: slot))
+ self.appBench("overlay_mode_request mode=Dictation")
+ self.menuBarManager.setOverlayMode(.dictation)
+ self.menuBarManager.showRecordingOverlayImmediately()
+ self.appBench("overlay_mode_requested mode=Dictation")
+ self.prewarmPrivateAIDictationIfNeeded(for: slot)
+ })
if !self.asr.isRunning {
self.menuBarManager.hideRecordingOverlayImmediately(reason: "asr_start_failed")
}
diff --git a/Sources/Fluid/CoreAudioCaptureSupportBridge.c b/Sources/Fluid/CoreAudioCaptureSupportBridge.c
new file mode 100644
index 00000000..99bf2004
--- /dev/null
+++ b/Sources/Fluid/CoreAudioCaptureSupportBridge.c
@@ -0,0 +1,3 @@
+// Fluid.xcodeproj builds the file-synchronized Sources/Fluid tree directly.
+// Keep the realtime implementation single-sourced in its SwiftPM C target.
+#include "../CoreAudioCaptureSupport/CoreAudioCaptureSupport.c"
diff --git a/Sources/Fluid/CoreAudioCaptureSupportBridge.h b/Sources/Fluid/CoreAudioCaptureSupportBridge.h
new file mode 100644
index 00000000..d459973a
--- /dev/null
+++ b/Sources/Fluid/CoreAudioCaptureSupportBridge.h
@@ -0,0 +1,6 @@
+#ifndef CORE_AUDIO_CAPTURE_SUPPORT_BRIDGE_H
+#define CORE_AUDIO_CAPTURE_SUPPORT_BRIDGE_H
+
+#include "../CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h"
+
+#endif
diff --git a/Sources/Fluid/Fluid-Bridging-Header.h b/Sources/Fluid/Fluid-Bridging-Header.h
new file mode 100644
index 00000000..94f6c2e8
--- /dev/null
+++ b/Sources/Fluid/Fluid-Bridging-Header.h
@@ -0,0 +1,6 @@
+#ifndef FLUID_BRIDGING_HEADER_H
+#define FLUID_BRIDGING_HEADER_H
+
+#include "CoreAudioCaptureSupportBridge.h"
+
+#endif
diff --git a/Sources/Fluid/Persistence/ParakeetFinalizationMode.swift b/Sources/Fluid/Persistence/ParakeetFinalizationMode.swift
deleted file mode 100644
index 0dce28e5..00000000
--- a/Sources/Fluid/Persistence/ParakeetFinalizationMode.swift
+++ /dev/null
@@ -1,26 +0,0 @@
-import Foundation
-
-enum ParakeetFinalizationMode: String, CaseIterable, Codable, Identifiable {
- case stableFullFinal
- case tokenTimedChunkMerge
-
- var id: String { self.rawValue }
-
- var displayName: String {
- switch self {
- case .stableFullFinal:
- return "Standard"
- case .tokenTimedChunkMerge:
- return "Fast"
- }
- }
-
- var detailText: String {
- switch self {
- case .stableFullFinal:
- return "Most reliable."
- case .tokenTimedChunkMerge:
- return "Faster, but maybe inaccurate."
- }
- }
-}
diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift
index a4501277..807333e6 100644
--- a/Sources/Fluid/Persistence/SettingsStore.swift
+++ b/Sources/Fluid/Persistence/SettingsStore.swift
@@ -1635,13 +1635,13 @@ final class SettingsStore: ObservableObject {
}
}
- var parakeetFinalizationMode: ParakeetFinalizationMode {
- get {
- self.defaults.string(forKey: Keys.parakeetFinalizationMode).flatMap(ParakeetFinalizationMode.init(rawValue:)) ?? .stableFullFinal
- }
+ /// Experimental direct Core Audio capture. Default is deliberately off
+ /// until the low-latency backend has broader device coverage validation.
+ var experimentalDirectAudioCaptureEnabled: Bool {
+ get { self.defaults.bool(forKey: Keys.experimentalDirectAudioCaptureEnabled) }
set {
objectWillChange.send()
- self.defaults.set(newValue.rawValue, forKey: Keys.parakeetFinalizationMode)
+ self.defaults.set(newValue, forKey: Keys.experimentalDirectAudioCaptureEnabled)
}
}
@@ -4128,15 +4128,6 @@ final class SettingsStore: ObservableObject {
}
}
- var supportsFastDictationProcessing: Bool {
- switch self {
- case .parakeetTDT, .parakeetTDTv2:
- return true
- default:
- return false
- }
- }
-
/// Whether this model supports real-time streaming/chunk processing.
/// Large Whisper models are too slow for streaming, so they only do final transcription on stop.
var supportsStreaming: Bool {
@@ -4461,7 +4452,7 @@ private extension SettingsStore {
static let hotkeyMode = "HotkeyMode"
static let enableStreamingPreview = "EnableStreamingPreview"
static let enableAIStreaming = "EnableAIStreaming"
- static let parakeetFinalizationMode = "ParakeetFinalizationMode"
+ static let experimentalDirectAudioCaptureEnabled = "ExperimentalDirectAudioCaptureEnabled"
static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"
static let textInsertionMode = "TextInsertionMode"
static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled"
diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift
index f8a244fa..294f8bc5 100644
--- a/Sources/Fluid/Services/ASRService.swift
+++ b/Sources/Fluid/Services/ASRService.swift
@@ -1,6 +1,7 @@
import Accelerate
import AVFoundation
import Combine
+import Darwin
import Foundation
#if arch(arm64)
import FluidAudio
@@ -540,6 +541,274 @@ final class ASRService: ObservableObject {
return created
}
+ private var hasWarmAudioEngine: Bool {
+ self.engineStorage is AVAudioEngine
+ }
+
+ private enum AudioCaptureBackend {
+ case none
+ case directCoreAudio
+ case audioEngine
+ }
+
+ private var directAudioInput: DirectCoreAudioInput?
+ private var activeAudioCaptureBackend: AudioCaptureBackend = .none
+
+ private var hasPreparedAudioCapture: Bool {
+ self.directAudioInput != nil || self.hasWarmAudioEngine
+ }
+
+ private func retireAudioEngine(reason: String) {
+ self.audioEngineStandbyTask?.cancel()
+ self.audioEngineStandbyTask = nil
+
+ if let directAudioInput = self.directAudioInput {
+ directAudioInput.invalidate()
+ self.directAudioInput = nil
+ }
+ self.activeAudioCaptureBackend = .none
+
+ if self.isEngineTapInstalled {
+ if let engine = self.engineStorage as? AVAudioEngine {
+ engine.inputNode.removeTap(onBus: 0)
+ }
+ self.isEngineTapInstalled = false
+ }
+ if let engine = self.engineStorage as? AVAudioEngine, engine.isRunning {
+ engine.stop()
+ }
+ self.audioCapturePipeline.clearPreroll()
+
+ let oldEngine = self.engineStorage
+ self.engineStorage = nil
+ if let oldEngine {
+ DispatchQueue.global(qos: .utility).async { _ = oldEngine }
+ }
+ DebugLogger.shared.debug("Audio engine retired (\(reason))", source: "ASRService")
+ }
+
+ private func scheduleAudioEngineStandbyRetirement() {
+ self.audioEngineStandbyTask?.cancel()
+ let delay = self.audioEngineStandbyNanoseconds
+ self.audioEngineStandbyTask = Task { [weak self] in
+ do {
+ try await Task.sleep(nanoseconds: delay)
+ } catch {
+ return
+ }
+ self?.retireWarmAudioEngineIfIdle()
+ }
+ }
+
+ private func retireWarmAudioEngineIfIdle() {
+ guard self.isRunning == false, self.isStarting == false else { return }
+ self.coolDownAudioEngineStandby(reason: "standby_timeout")
+ }
+
+ private func coolDownAudioEngineStandby(reason: String) {
+ self.audioEngineStandbyTask?.cancel()
+ self.audioEngineStandbyTask = nil
+
+ if let directAudioInput = self.directAudioInput {
+ directAudioInput.invalidate()
+ self.directAudioInput = nil
+ }
+ self.activeAudioCaptureBackend = .none
+
+ if self.isEngineTapInstalled {
+ if let engine = self.engineStorage as? AVAudioEngine {
+ engine.inputNode.removeTap(onBus: 0)
+ }
+ self.isEngineTapInstalled = false
+ }
+ if let engine = self.engineStorage as? AVAudioEngine, engine.isRunning {
+ engine.stop()
+ }
+ self.audioCapturePipeline.clearPreroll()
+ self.benchmarkLog("audio_engine_standby cooled=true reason=\(reason)")
+ DebugLogger.shared.debug("Audio engine cooled to stopped warm state (\(reason))", source: "ASRService")
+ }
+
+ private func prewarmAudioEngineIfPossible(reason: String) {
+ guard self.micStatus == .authorized else {
+ DebugLogger.shared.debug("Audio engine prewarm skipped - mic not authorized", source: "ASRService")
+ return
+ }
+ guard self.isRunning == false, self.isStarting == false else {
+ DebugLogger.shared.debug("Audio engine prewarm skipped - capture active", source: "ASRService")
+ return
+ }
+ guard self.hasPreparedAudioCapture == false else {
+ DebugLogger.shared.debug("Audio capture prewarm skipped - backend already prepared", source: "ASRService")
+ return
+ }
+
+ let startedAt = Date().timeIntervalSince1970
+ if SettingsStore.shared.experimentalDirectAudioCaptureEnabled,
+ self.prepareDirectAudioInputIfPossible(reason: reason)
+ {
+ self.benchmarkLog("direct_audio_prewarm reason=\(reason) elapsedMs=\(self.elapsedMilliseconds(since: startedAt))")
+ return
+ }
+
+ do {
+ try self.configureSession()
+ self.benchmarkLog("audio_engine_prewarm reason=\(reason) elapsedMs=\(self.elapsedMilliseconds(since: startedAt))")
+ } catch {
+ self.retireAudioEngine(reason: "prewarm_failed")
+ DebugLogger.shared.warning("Audio engine prewarm failed: \(error.localizedDescription)", source: "ASRService")
+ }
+ }
+
+ private func resolvedInputDeviceForCapture() -> AudioDevice.Device? {
+ if SettingsStore.shared.syncAudioDevicesWithSystem == false,
+ let preferredUID = SettingsStore.shared.preferredInputDeviceUID,
+ preferredUID.isEmpty == false,
+ let preferredDevice = AudioDevice.getInputDevice(byUID: preferredUID)
+ {
+ return preferredDevice
+ }
+ return AudioDevice.getDefaultInputDevice()
+ }
+
+ /// Prepares the direct device callback without starting hardware IO. This
+ /// keeps the default idle state privacy-friendly while removing device and
+ /// ring allocation from the hotkey path.
+ @discardableResult
+ private func prepareDirectAudioInputIfPossible(reason: String) -> Bool {
+ guard self.micStatus == .authorized else { return false }
+ guard let device = self.resolvedInputDeviceForCapture() else {
+ DebugLogger.shared.warning("No input device is available for direct capture", source: "ASRService")
+ return false
+ }
+
+ if let directAudioInput = self.directAudioInput,
+ directAudioInput.deviceID == device.id
+ {
+ return true
+ }
+
+ self.directAudioInput?.invalidate()
+ self.directAudioInput = nil
+
+ let pipeline = self.audioCapturePipeline
+ do {
+ let directAudioInput = try DirectCoreAudioInput(deviceID: device.id) { samples, frameCount, sampleRate, inputHostTime, inputSampleTime in
+ pipeline.handle(
+ samples: samples,
+ frameCount: frameCount,
+ sampleRate: sampleRate,
+ inputHostTime: inputHostTime,
+ inputSampleTime: inputSampleTime
+ )
+ }
+ self.directAudioInput = directAudioInput
+ DebugLogger.shared.info(
+ "Prepared direct Core Audio input '\(device.name)' " +
+ "(\(Int(directAudioInput.sampleRate.rounded()))Hz, " +
+ "\(directAudioInput.hardwareBufferFrameSize) frames, reason=\(reason))",
+ source: "ASRService"
+ )
+ return true
+ } catch {
+ DebugLogger.shared.warning(
+ "Direct Core Audio input unavailable for '\(device.name)': \(error.localizedDescription). " +
+ "Falling back to AVAudioEngine.",
+ source: "ASRService"
+ )
+ return false
+ }
+ }
+
+ private func startPreferredAudioCapture() throws {
+ let directCaptureEnabled = SettingsStore.shared.experimentalDirectAudioCaptureEnabled
+ if directCaptureEnabled,
+ self.prepareDirectAudioInputIfPossible(reason: "recording_start"),
+ let directAudioInput = self.directAudioInput
+ {
+ do {
+ try directAudioInput.start()
+ self.activeAudioCaptureBackend = .directCoreAudio
+ let callbackMs = Int(
+ (Double(directAudioInput.hardwareBufferFrameSize) /
+ directAudioInput.sampleRate * 1000).rounded()
+ )
+ self.benchmarkLog(
+ "audio_backend kind=direct_core_audio device=\(directAudioInput.deviceID) " +
+ "frames=\(directAudioInput.hardwareBufferFrameSize) callbackMs=\(callbackMs)"
+ )
+ return
+ } catch {
+ DebugLogger.shared.warning(
+ "Direct Core Audio start failed: \(error.localizedDescription). Falling back to AVAudioEngine.",
+ source: "ASRService"
+ )
+ directAudioInput.invalidate()
+ self.directAudioInput = nil
+ }
+ }
+
+ if directCaptureEnabled == false, let directAudioInput = self.directAudioInput {
+ directAudioInput.invalidate()
+ self.directAudioInput = nil
+ }
+
+ self.benchmarkLog(
+ "audio_backend kind=av_audio_engine_fallback reason=" +
+ (directCaptureEnabled ? "direct_unavailable" : "experimental_disabled")
+ )
+ try self.configureSession()
+ try self.startEngine()
+ try self.setupEngineTap()
+ self.activeAudioCaptureBackend = .audioEngine
+ }
+
+ private func stopActiveAudioCapture() {
+ switch self.activeAudioCaptureBackend {
+ case .directCoreAudio:
+ if let directAudioInput = self.directAudioInput {
+ let status = directAudioInput.stop()
+ if status != noErr {
+ DebugLogger.shared.warning(
+ "Direct Core Audio stop returned OSStatus \(status)",
+ source: "ASRService"
+ )
+ }
+ let droppedPackets = directAudioInput.droppedPacketCount
+ if droppedPackets > 0 {
+ DebugLogger.shared.warning(
+ "Direct Core Audio dropped \(droppedPackets) packet(s)",
+ source: "ASRService"
+ )
+ }
+ }
+ case .audioEngine:
+ self.removeEngineTap()
+ if let engine = self.engineStorage as? AVAudioEngine, engine.isRunning {
+ engine.stop()
+ }
+ case .none:
+ break
+ }
+ self.activeAudioCaptureBackend = .none
+ }
+
+ /// Applies the experimental capture preference immediately when idle.
+ /// If a recording transition is already underway, start() reads the latest
+ /// persisted value and the following session will use it.
+ func refreshAudioCaptureBackendPreference() {
+ guard self.isRunning == false, self.isStarting == false else {
+ DebugLogger.shared.debug(
+ "Audio capture preference changed during a recording transition; deferring backend refresh",
+ source: "ASRService"
+ )
+ return
+ }
+
+ self.retireAudioEngine(reason: "capture_preference_changed")
+ self.prewarmAudioEngineIfPossible(reason: "capture_preference_changed")
+ }
+
private var inputFormat: AVAudioFormat?
private var micPermissionGranted = false
@@ -577,13 +846,10 @@ final class ASRService: ObservableObject {
private var engineConfigurationChangeObserver: NSObjectProtocol?
private var audioRouteRecoveryTask: Task<Void, Never>?
private let audioRouteRecoveryDelayNanoseconds: UInt64 = 1_000_000_000
+ private var audioEngineStandbyTask: Task<Void, Never>?
+ private let audioEngineStandbyNanoseconds: UInt64 = 8_000_000_000
+ private var isEngineTapInstalled = false
private var isRecoveringAudioRoute = false
- private let fastPreviewStopGraceNanoseconds: UInt64 = 200_000_000
- private let fastPreviewSampleRate = 16_000
- private let fastPreviewMinimumSamples = 32_000
- private let fastPreviewTailAudioToleranceMs = 300
- private let fastPreviewStopGraceMinimumCoverage = 0.72
- private let fastPreviewStopGraceTargetCoverage = 0.88
/// Tracks whether we paused system media for this recording session.
/// Used to resume playback only if we were the ones who paused it.
@@ -601,11 +867,6 @@ final class ASRService: ObservableObject {
private var streamingChunkDurationSeconds: Double {
let selectedModel = SettingsStore.shared.selectedSpeechModel
- if selectedModel == .parakeetTDT || selectedModel == .parakeetTDTv2,
- SettingsStore.shared.parakeetFinalizationMode == .tokenTimedChunkMerge
- {
- return 0.4
- }
return selectedModel.streamingPreviewIntervalSeconds
}
@@ -617,6 +878,16 @@ final class ASRService: ObservableObject {
/// from CoreAudio's realtime callback thread.
private lazy var audioCapturePipeline: AudioCapturePipeline = .init(
audioBuffer: self.audioBuffer,
+ onFirstAudio: { sessionID, sampleCount, frameLength, sampleRate, acquisitionMs, elapsedMs in
+ DispatchQueue.main.async {
+ let bufferMs = Int((Double(frameLength) / sampleRate * 1000).rounded())
+ DebugLogger.shared.benchmark(
+ "ASR_BENCH",
+ message: "session=\(sessionID) first_audio sampleCount=\(sampleCount) frameLength=\(frameLength) sampleRate=\(Int(sampleRate.rounded())) bufferMs=\(bufferMs) acquisitionMs=\(acquisitionMs) elapsedMs=\(elapsedMs)",
+ source: "ASRBenchmark"
+ )
+ }
+ },
onLevel: { [weak self] level in
// Keep Combine sends on the main queue.
DispatchQueue.main.async { [weak self] in
@@ -650,6 +921,7 @@ final class ASRService: ObservableObject {
}
deinit {
+ self.directAudioInput?.invalidate()
if let observer = self.vocabularyChangeObserver {
NotificationCenter.default.removeObserver(observer)
}
@@ -741,6 +1013,10 @@ final class ASRService: ObservableObject {
// Initialize device list cache
self.cacheCurrentDeviceList(AudioDevice.listInputDevices())
+ // Register the input callback and allocate its fixed ring now. This
+ // does not start the device or show the microphone privacy indicator.
+ self.prewarmAudioEngineIfPossible(reason: "startup")
+
// Check if models exist on disk and auto-load if present
// This is done in a Task to support async model detection (e.g., AppleSpeechAnalyzerProvider)
Task { [weak self] in
@@ -755,6 +1031,7 @@ final class ASRService: ObservableObject {
do {
try await self.ensureAsrReady()
DebugLogger.shared.info("Models auto-loaded successfully on startup", source: "ASRService")
+ self.prewarmAudioEngineIfPossible(reason: "startup")
} catch {
DebugLogger.shared.error("Failed to auto-load models on startup: \(error)", source: "ASRService")
}
@@ -810,6 +1087,9 @@ final class ASRService: ObservableObject {
Task { @MainActor in
self.micPermissionGranted = granted
self.micStatus = granted ? .authorized : .denied
+ if granted {
+ self.prewarmAudioEngineIfPossible(reason: "permission_granted")
+ }
}
}
}
@@ -840,7 +1120,10 @@ final class ASRService: ObservableObject {
/// ## Errors
/// If audio session configuration fails, the method will silently fail
/// and `isRunning` will remain `false`. Check the debug logs for details.
- func start(forDictionaryTraining: Bool = false) async {
+ func start(
+ forDictionaryTraining: Bool = false,
+ onCaptureStarted: (@MainActor () -> Void)? = nil
+ ) async {
DebugLogger.shared.info("🎤 START() called - beginning recording session", source: "ASRService")
guard self.micStatus == .authorized else {
@@ -854,6 +1137,8 @@ final class ASRService: ObservableObject {
// Reset media pause state for this session
self.didPauseMediaForThisSession = false
+ self.audioEngineStandbyTask?.cancel()
+ self.audioEngineStandbyTask = nil
self.audioRouteRecoveryTask?.cancel()
self.audioRouteRecoveryTask = nil
self.isRecoveringAudioRoute = false
@@ -875,7 +1160,11 @@ final class ASRService: ObservableObject {
self.streamingChunkAnalyticsSuccessCount = 0
self.lastStreamingChunkFailureAnalyticsAt = nil
(self.transcriptionProvider as? FluidAudioProvider)?.resetStreamingPreviewCache()
- self.audioCapturePipeline.setRecordingEnabled(true)
+ self.audioCapturePipeline.setRecordingEnabled(
+ true,
+ sessionID: self.benchmarkSessionID,
+ startHostTime: mach_absolute_time()
+ )
self.refreshWordBoostStatus()
let dims = self.currentTranscriptionAnalyticsDimensions()
self.benchmarkLog("recording_start model=\(dims.model) provider=\(dims.provider) supportsStreaming=\(SettingsStore.shared.selectedSpeechModel.supportsStreaming)")
@@ -886,32 +1175,29 @@ final class ASRService: ObservableObject {
self.isDictionaryTrainingCaptureActive = false
do {
- DebugLogger.shared.debug("⚙️ Calling configureSession()...", source: "ASRService")
- try self.configureSession()
- DebugLogger.shared.debug("✅ configureSession() completed", source: "ASRService")
-
- DebugLogger.shared.debug("🚀 Calling startEngine()...", source: "ASRService")
- try self.startEngine()
- DebugLogger.shared.debug("✅ startEngine() completed", source: "ASRService")
-
- DebugLogger.shared.debug("🎧 Setting up engine tap...", source: "ASRService")
- try self.setupEngineTap()
- DebugLogger.shared.debug("✅ Engine tap setup complete", source: "ASRService")
+ try self.startPreferredAudioCapture()
+ self.isRunning = true
+ self.isDictionaryTrainingCaptureActive = forDictionaryTraining
+ DebugLogger.shared.info("✅ Audio capture running", source: "ASRService")
+ onCaptureStarted?()
- // Pause system media AFTER successful audio setup but BEFORE setting isRunning
- // This ensures we only pause media when we know recording will succeed
+ // Pause only after capture is live so media control cannot delay the
+ // first PCM packet. A quick stop while this await is in flight is
+ // handled explicitly below.
if SettingsStore.shared.pauseMediaDuringTranscription {
let didPause = await MediaPlaybackService.shared.pauseIfPlaying()
+ guard self.isRunning else {
+ if didPause {
+ await MediaPlaybackService.shared.resumeIfWePaused(true)
+ }
+ return
+ }
self.didPauseMediaForThisSession = didPause
if didPause {
DebugLogger.shared.info("🎵 Paused system media for transcription", source: "ASRService")
}
}
- self.isRunning = true
- self.isDictionaryTrainingCaptureActive = forDictionaryTraining
- DebugLogger.shared.info("✅ isRunning set to TRUE", source: "ASRService")
-
// Start monitoring the currently bound device for disconnection
if let currentDevice = getCurrentlyBoundInputDevice() {
DebugLogger.shared.debug("👀 Starting device monitoring for: \(currentDevice.name)", source: "ASRService")
@@ -934,6 +1220,10 @@ final class ASRService: ObservableObject {
DebugLogger.shared.info("✅ START() completed successfully", source: "ASRService")
} catch {
self.isDictionaryTrainingCaptureActive = false
+ self.audioCapturePipeline.setRecordingEnabled(false)
+ self.isRunning = false
+ self.stopActiveAudioCapture()
+ self.retireAudioEngine(reason: "start_failed")
DebugLogger.shared.error("Failed to start ASR session: \(error)", source: "ASRService")
// Resume media if we paused it before the failure
@@ -1031,15 +1321,13 @@ final class ASRService: ObservableObject {
DebugLogger.shared.debug("📍 Preparing final transcription", source: "ASRService")
- DebugLogger.shared.debug("🚫 Setting audioCapturePipeline recording = false...", source: "ASRService")
- self.audioCapturePipeline.setRecordingEnabled(false)
- DebugLogger.shared.debug("✅ Capture pipeline disabled", source: "ASRService")
+ // Freeze an exact acquisition boundary before stopping hardware. The
+ // direct IOProc is synchronously drained and the pipeline trims the
+ // final hardware packet to this host time, preserving the last phoneme
+ // without appending audio from the next session.
+ self.audioCapturePipeline.markRecordingEnd(atHostTime: mach_absolute_time())
- if !useDictionaryTrainingPath {
- await self.runFastPreviewStopGraceIfNeeded()
- }
-
- // CRITICAL: Set isRunning to false before teardown so in-flight chunks stop safely.
+ // Set isRunning to false before teardown so in-flight ASR chunks stop safely.
DebugLogger.shared.debug("🚫 Setting isRunning = false...", source: "ASRService")
self.isRunning = false
DebugLogger.shared.debug("✅ isRunning disabled", source: "ASRService")
@@ -1049,26 +1337,27 @@ final class ASRService: ObservableObject {
self.stopMonitoringDevice()
DebugLogger.shared.debug("✅ Device monitoring stopped", source: "ASRService")
- // Stop the audio engine to stop new audio from coming in
- DebugLogger.shared.debug("🎧 Removing engine tap...", source: "ASRService")
- self.removeEngineTap()
- DebugLogger.shared.debug("✅ Engine tap removed", source: "ASRService")
-
- DebugLogger.shared.debug("🛑 Calling engine.stop()...", source: "ASRService")
- self.engine.stop()
- DebugLogger.shared.debug("✅ Engine stopped", source: "ASRService")
+ self.stopActiveAudioCapture()
+ self.audioCapturePipeline.finishRecording()
+
+ // A prepared direct IOProc owns only fixed memory and registration; it
+ // does not run hardware, show the mic indicator, or hold Bluetooth in
+ // headset mode. Keep it prepared across idle periods. The heavier
+ // AVAudioEngine fallback retains its existing bounded timeout.
+ if self.directAudioInput != nil {
+ self.audioEngineStandbyTask?.cancel()
+ self.audioEngineStandbyTask = nil
+ DebugLogger.shared.debug("♻️ Direct audio capture remains prepared", source: "ASRService")
+ } else {
+ self.scheduleAudioEngineStandbyRetirement()
+ }
// Capture has fully ended — invoke the callback so callers can play a
// stop cue or release capture-dependent UI without waiting on the
// (potentially slow) final transcription pass.
await MainActor.run { onCaptureStopped?() }
- // Recreate the engine instance instead of calling reset() to prevent format corruption
- // VoiceInk approach: tearing down and rebuilding ensures fresh, valid audio format on restart
- DebugLogger.shared.debug("🗑️ Deallocating old engine and creating fresh instance...", source: "ASRService")
- self.engineStorage = nil // Explicitly release old engine
- // New engine will be lazily created on next access via computed property
- DebugLogger.shared.debug("✅ Engine instance recreated", source: "ASRService")
+ self.benchmarkLog("audio_capture_prepared retained=\(self.directAudioInput != nil)")
// CRITICAL FIX: Await completion of streaming task AND any pending transcriptions
// This prevents use-after-free crashes (EXC_BAD_ACCESS) when clearing buffer
@@ -1163,12 +1452,6 @@ final class ASRService: ObservableObject {
}
}
finalSource = "dictionaryTraining"
- } else if let fluidProvider = provider as? FluidAudioProvider,
- let cachedResult = await fluidProvider.transcribeCachedStreamingPreviewIfAvailable(pcm)
- {
- result = cachedResult
- finalSource = "livePreview"
- self.benchmarkLog("final_fast_preview_bypass hit=true")
} else {
result = try await self.transcriptionExecutor.run { [provider] in
try await provider.transcribeFinal(pcm)
@@ -1362,20 +1645,11 @@ final class ASRService: ObservableObject {
// Stop monitoring device
self.stopMonitoringDevice()
- self.removeEngineTap()
- DebugLogger.shared.debug("Engine tap removed", source: "ASRService")
+ self.stopActiveAudioCapture()
+ DebugLogger.shared.debug("Audio capture stopped", source: "ASRService")
- self.engine.stop()
- DebugLogger.shared.debug("Engine stopped", source: "ASRService")
-
- // Release old engine on a background thread — if the underlying device just died,
- // AVAudioEngine deallocation can block in CoreAudio's internal teardown.
- // No new engine is created here (it's lazy on next start()), so no overlap risk.
- let oldEngine = self.engineStorage
- self.engineStorage = nil
- if let oldEngine {
- DispatchQueue.global(qos: .utility).async { _ = oldEngine }
- }
+ // Cancel/no-transcription paths stay conservative and retire the engine.
+ self.retireAudioEngine(reason: "stop_without_transcription")
// CRITICAL FIX: Await completion of streaming task AND any pending transcriptions
// This prevents use-after-free crashes (EXC_BAD_ACCESS) when clearing buffer
@@ -1403,25 +1677,27 @@ final class ASRService: ObservableObject {
private func configureSession() throws {
DebugLogger.shared.debug("🔧 configureSession() - ENTERED", source: "ASRService")
- if self.engine.isRunning {
+ let wasWarm = self.hasWarmAudioEngine
+ let engine = self.engine
+ DebugLogger.shared.debug(
+ wasWarm ? "♻️ Reusing warm audio engine" : "ℹ️ Creating audio engine lazily",
+ source: "ASRService"
+ )
+
+ if engine.isRunning {
DebugLogger.shared.debug("⚠️ Engine is running, stopping before configuration", source: "ASRService")
- self.engine.stop()
+ engine.stop()
DebugLogger.shared.debug("✅ Engine stopped", source: "ASRService")
}
- // No need to call engine.reset() here - we created a fresh engine in stop()
- // Accessing the engine property will either return the existing fresh engine,
- // or create a new one if this is the first start
- DebugLogger.shared.debug("ℹ️ Using fresh engine instance (created lazily)", source: "ASRService")
-
// Force input node instantiation (ensures the underlying AUHAL AudioUnit exists)
DebugLogger.shared.debug("📍 Forcing input node instantiation...", source: "ASRService")
- _ = self.engine.inputNode
+ _ = engine.inputNode
DebugLogger.shared.debug("Input node instantiated", source: "ASRService")
// Force output node instantiation for output device binding
DebugLogger.shared.debug("📍 Forcing output node instantiation...", source: "ASRService")
- _ = self.engine.outputNode
+ _ = engine.outputNode
DebugLogger.shared.debug("✅ Output node instantiated", source: "ASRService")
// NOTE: Device binding occurs in startEngine() BEFORE engine.prepare()
@@ -1798,7 +2074,7 @@ final class ASRService: ObservableObject {
// If this isn't the last attempt, recreate engine and reconfigure
if attempts < 3 {
DebugLogger.shared.debug("⚠️ Start failed, recreating engine for retry...", source: "ASRService")
- self.engineStorage = nil // Deallocate failed engine
+ self.retireAudioEngine(reason: "start_retry")
// Need to reconfigure the new engine
try? self.configureSession()
DebugLogger.shared.debug("✅ Engine recreated and reconfigured, will retry", source: "ASRService")
@@ -1826,7 +2102,11 @@ final class ASRService: ObservableObject {
}
private func removeEngineTap() {
- self.engine.inputNode.removeTap(onBus: 0)
+ guard self.isEngineTapInstalled else { return }
+ if let engine = self.engineStorage as? AVAudioEngine {
+ engine.inputNode.removeTap(onBus: 0)
+ }
+ self.isEngineTapInstalled = false
}
private func setupEngineTap() throws {
@@ -1883,16 +2163,25 @@ final class ASRService: ObservableObject {
self.inputFormat = inFormat
let pipeline = self.audioCapturePipeline
+ if self.isEngineTapInstalled {
+ input.removeTap(onBus: 0)
+ self.isEngineTapInstalled = false
+ }
DebugLogger.shared.debug("🎧 Installing tap on bus 0...", source: "ASRService")
- input.installTap(onBus: 0, bufferSize: 4096, format: inFormat) { buffer, _ in
- pipeline.handle(buffer: buffer)
+ input.installTap(onBus: 0, bufferSize: 4096, format: inFormat) { buffer, time in
+ pipeline.handle(buffer: buffer, time: time)
}
+ self.isEngineTapInstalled = true
DebugLogger.shared.debug("✅ setupEngineTap() - COMPLETED", source: "ASRService")
}
private func scheduleAudioRouteRecovery(reason: String) {
guard self.isRunning else {
self.audioLevelSubject.send(0.0)
+ if self.hasPreparedAudioCapture {
+ self.retireAudioEngine(reason: "idle_route_change:\(reason)")
+ self.prewarmAudioEngineIfPossible(reason: "idle_route_change")
+ }
return
}
guard self.isRecoveringAudioRoute == false else {
@@ -1931,20 +2220,16 @@ final class ASRService: ObservableObject {
self.audioCapturePipeline.setRecordingEnabled(false)
self.stopMonitoringDevice()
- self.removeEngineTap()
- self.engine.stop()
-
- let oldEngine = self.engineStorage
- self.engineStorage = nil
- if let oldEngine {
- DispatchQueue.global(qos: .utility).async { _ = oldEngine }
- }
+ self.stopActiveAudioCapture()
+ self.retireAudioEngine(reason: "audio_route_recovery")
do {
- try self.configureSession()
- try self.startEngine()
- try self.setupEngineTap()
- self.audioCapturePipeline.setRecordingEnabled(true)
+ self.audioCapturePipeline.setRecordingEnabled(
+ true,
+ sessionID: self.benchmarkSessionID,
+ startHostTime: mach_absolute_time()
+ )
+ try self.startPreferredAudioCapture()
if let currentDevice = self.getCurrentlyBoundInputDevice() {
self.startMonitoringDevice(currentDevice.id)
@@ -1952,6 +2237,8 @@ final class ASRService: ObservableObject {
DebugLogger.shared.info("Audio route recovery succeeded", source: "ASRService")
} catch {
+ self.audioCapturePipeline.setRecordingEnabled(false)
+ self.stopActiveAudioCapture()
DebugLogger.shared.error("Audio route recovery failed: \(error)", source: "ASRService")
await self.stopWithoutTranscription()
NotificationCenter.default.post(
@@ -1979,6 +2266,11 @@ final class ASRService: ObservableObject {
return
}
+ // Input-only direct capture has no output device dependency.
+ if self.directAudioInput != nil {
+ return
+ }
+
self.scheduleAudioRouteRecovery(reason: "default output changed")
}
@@ -2274,6 +2566,10 @@ final class ASRService: ObservableObject {
/// Gets the currently bound input device (if determinable)
private func getCurrentlyBoundInputDevice() -> AudioDevice.Device? {
+ if let directAudioInput = self.directAudioInput {
+ return AudioDevice.listInputDevices().first { $0.id == directAudioInput.deviceID }
+ }
+
// Check if engine exists before accessing inputNode
guard self.engineStorage != nil else { return nil }
guard let audioUnit = self.engine.inputNode.audioUnit else { return nil }
@@ -3170,61 +3466,32 @@ private extension ASRService {
self.streamingTask?.cancel()
self.streamingTask = nil
}
-
- func runFastPreviewStopGraceIfNeeded() async {
- guard SettingsStore.shared.parakeetFinalizationMode == .tokenTimedChunkMerge else { return }
- guard SettingsStore.shared.selectedSpeechModel.supportsFastDictationProcessing else { return }
- guard SettingsStore.shared.selectedSpeechModel.supportsStreaming else { return }
- guard self.transcriptionProvider is FluidAudioProvider else { return }
-
- let currentSampleCount = self.audioBuffer.count
- guard currentSampleCount >= self.fastPreviewMinimumSamples else {
- self.benchmarkLog("fast_preview_stop_grace skipped=true reason=duration samples=\(currentSampleCount)")
- return
- }
-
- let processedSampleCount = min(self.lastProcessedSampleCount, currentSampleCount)
- let coverage = currentSampleCount > 0 ? Double(processedSampleCount) / Double(currentSampleCount) : 0
- let tailSamples = max(0, currentSampleCount - processedSampleCount)
- let tailMs = Int((Double(tailSamples) / Double(self.fastPreviewSampleRate) * 1000).rounded())
- guard coverage < self.fastPreviewStopGraceTargetCoverage || tailMs > self.fastPreviewTailAudioToleranceMs else {
- self.benchmarkLog(
- "fast_preview_stop_grace skipped=true reason=already_covered coverage=\(String(format: "%.3f", coverage)) tailMs=\(tailMs)"
- )
- return
- }
-
- if self.isProcessingChunk {
- self.benchmarkLog("fast_preview_stop_grace wait=in_flight coverage=\(String(format: "%.3f", coverage)) tailMs=\(tailMs)")
- try? await Task.sleep(nanoseconds: self.fastPreviewStopGraceNanoseconds)
- return
- }
-
- guard processedSampleCount > 0, coverage >= self.fastPreviewStopGraceMinimumCoverage else {
- self.benchmarkLog("fast_preview_stop_grace skipped=true reason=not_close coverage=\(String(format: "%.3f", coverage)) tailMs=\(tailMs)")
- return
- }
-
- let startedAt = Date().timeIntervalSince1970
- self.benchmarkLog("fast_preview_stop_grace forced_chunk=true coverage=\(String(format: "%.3f", coverage)) tailMs=\(tailMs) samples=\(currentSampleCount)")
- await self.processStreamingChunk()
- self.benchmarkLog("fast_preview_stop_grace done elapsedMs=\(self.elapsedMilliseconds(since: startedAt)) samples=\(self.audioBuffer.count)")
- }
}
// MARK: - Audio capture pipeline
//
-// AVAudioEngine's tap runs on a realtime audio thread. ASRService is @MainActor, so we must NOT
-// touch its state directly inside the tap callback. This pipeline keeps all tap-side state
-// thread-safe and only calls back with derived values (audio level + captured samples).
+// Audio callbacks are not main-actor isolated. Direct Core Audio arrives through
+// a lock-free C ring; AVAudioEngine remains the compatibility fallback. This
+// pipeline owns timestamp trimming, 16 kHz conversion, levels, and session-safe
+// delivery without touching ASRService from a realtime callback.
-private final class AudioCapturePipeline {
+private final nonisolated class AudioCapturePipeline: @unchecked Sendable {
private let audioBuffer: ThreadSafeAudioBuffer
+ private let onFirstAudio: (Int, Int, Int, Double, Int, Int) -> Void
private let onLevel: (CGFloat) -> Void
private let lock = NSLock()
private var recordingEnabled: Bool = false
+ private var firstAudioReported: Bool = false
+ private var recordingSessionID: Int = 0
+ private var recordingStartHostTime: UInt64 = 0
+ private var recordingStopHostTime: UInt64?
+ private var resampleSourceRate: Double = 0
+ private var resampleSourceFrameCursor: Int64 = 0
+ private var resampleNextSourcePosition: Double = 0
+ private var resamplePreviousSample: Float?
+ private var lastInputSampleEnd: Int64?
// Smoothing state (kept off ASRService/@MainActor)
private var levelHistory: [CGFloat] = []
@@ -3232,42 +3499,316 @@ private final class AudioCapturePipeline {
private let historySize: Int = 2
private let silenceThreshold: CGFloat = 0.04
- init(audioBuffer: ThreadSafeAudioBuffer, onLevel: @escaping (CGFloat) -> Void) {
+ private static let hostTicksPerSecond: Double = {
+ var info = mach_timebase_info_data_t()
+ mach_timebase_info(&info)
+ guard info.numer != 0 else { return 1_000_000_000 }
+ return 1_000_000_000.0 * Double(info.denom) / Double(info.numer)
+ }()
+
+ init(
+ audioBuffer: ThreadSafeAudioBuffer,
+ onFirstAudio: @escaping (Int, Int, Int, Double, Int, Int) -> Void,
+ onLevel: @escaping (CGFloat) -> Void
+ ) {
self.audioBuffer = audioBuffer
+ self.onFirstAudio = onFirstAudio
self.onLevel = onLevel
}
- func setRecordingEnabled(_ enabled: Bool) {
+ func setRecordingEnabled(
+ _ enabled: Bool,
+ sessionID: Int = 0,
+ startHostTime: UInt64 = 0
+ ) {
self.lock.lock()
- defer { self.lock.unlock() }
- self.recordingEnabled = enabled
+ if enabled {
+ self.firstAudioReported = false
+ self.recordingSessionID = sessionID
+ self.recordingStartHostTime = startHostTime == 0 ? mach_absolute_time() : startHostTime
+ self.recordingStopHostTime = nil
+ self.resetResamplerLocked()
+ self.lastInputSampleEnd = nil
+ self.recordingEnabled = true
+ }
if enabled == false {
+ self.recordingEnabled = false
+ self.recordingSessionID = 0
+ self.recordingStartHostTime = 0
+ self.recordingStopHostTime = nil
+ self.resetResamplerLocked()
+ self.lastInputSampleEnd = nil
self.levelHistory.removeAll(keepingCapacity: true)
self.smoothedLevel = 0.0
}
+ self.lock.unlock()
}
- func handle(buffer: AVAudioPCMBuffer) {
+ /// Sets the exact last acquisition time accepted for the current session.
+ /// Capture remains enabled until the backend has stopped and drained.
+ func markRecordingEnd(atHostTime hostTime: UInt64) {
self.lock.lock()
- let enabled = self.recordingEnabled
+ if self.recordingEnabled {
+ self.recordingStopHostTime = hostTime
+ }
self.lock.unlock()
+ }
+
+ func finishRecording() {
+ self.setRecordingEnabled(false)
+ self.onLevel(0.0)
+ }
- guard enabled else {
+ /// Compatibility for capture teardown paths. Session-scoped timestamps
+ /// replace the old cross-session preroll buffer, so there is nothing to clear.
+ func clearPreroll() {
+ // Intentionally empty.
+ }
+
+ func handle(buffer: AVAudioPCMBuffer, time: AVAudioTime) {
+ let mono = Self.downmixToMono(buffer)
+ guard mono.isEmpty == false else {
self.onLevel(0.0)
return
}
+ self.handleMonoSamples(
+ mono,
+ sampleRate: buffer.format.sampleRate,
+ inputHostTime: time.isHostTimeValid ? time.hostTime : 0,
+ inputSampleTime: time.isSampleTimeValid ? time.sampleTime : -1,
+ originalFrameCount: Int(buffer.frameLength)
+ )
+ }
- let mono16k = Self.toMono16k(floatBuffer: buffer)
- guard mono16k.isEmpty == false else {
+ func handle(
+ samples: UnsafePointer<Float>,
+ frameCount: Int,
+ sampleRate: Double,
+ inputHostTime: UInt64,
+ inputSampleTime: Int64
+ ) {
+ guard frameCount > 0 else { return }
+ self.handleMonoSamples(
+ Array(UnsafeBufferPointer(start: samples, count: frameCount)),
+ sampleRate: sampleRate,
+ inputHostTime: inputHostTime,
+ inputSampleTime: inputSampleTime,
+ originalFrameCount: frameCount
+ )
+ }
+
+ private func handleMonoSamples(
+ _ samples: [Float],
+ sampleRate: Double,
+ inputHostTime: UInt64,
+ inputSampleTime: Int64,
+ originalFrameCount: Int
+ ) {
+ guard samples.isEmpty == false, sampleRate > 0 else {
self.onLevel(0.0)
return
}
+ self.lock.lock()
+ guard self.recordingEnabled else {
+ self.lock.unlock()
+ return
+ }
+ let startHostTime = self.recordingStartHostTime
+ let stopHostTime = self.recordingStopHostTime
+ let recordingSessionID = self.recordingSessionID
+ self.lock.unlock()
+
+ guard let acceptedRange = Self.acceptedFrameRange(
+ frameCount: samples.count,
+ sampleRate: sampleRate,
+ packetHostTime: inputHostTime,
+ startHostTime: startHostTime,
+ stopHostTime: stopHostTime
+ ) else {
+ return
+ }
+
+ let acceptedSamples: [Float]
+ if acceptedRange.lowerBound == 0, acceptedRange.upperBound == samples.count {
+ acceptedSamples = samples
+ } else {
+ acceptedSamples = Array(samples[acceptedRange])
+ }
+ self.lock.lock()
+ guard self.recordingEnabled, self.recordingSessionID == recordingSessionID else {
+ self.lock.unlock()
+ return
+ }
+ if inputSampleTime >= 0 {
+ let acceptedSampleStart = inputSampleTime + Int64(acceptedRange.lowerBound)
+ if let lastInputSampleEnd = self.lastInputSampleEnd,
+ lastInputSampleEnd != acceptedSampleStart
+ {
+ // Do not interpolate across a hardware discontinuity or a
+ // packet dropped under extreme consumer backpressure.
+ self.resetResamplerLocked()
+ }
+ self.lastInputSampleEnd = inputSampleTime + Int64(acceptedRange.upperBound)
+ }
+ let mono16k = self.resampleTo16kLocked(
+ acceptedSamples,
+ sourceSampleRate: sampleRate
+ )
+ guard mono16k.isEmpty == false else {
+ self.lock.unlock()
+ return
+ }
+ let shouldReportFirstAudio = self.firstAudioReported == false
+ if shouldReportFirstAudio {
+ self.firstAudioReported = true
+ }
+ self.lock.unlock()
+
self.audioBuffer.append(mono16k)
+ if shouldReportFirstAudio {
+ let acceptedHostTime = Self.hostTime(
+ inputHostTime,
+ advancedByFrames: acceptedRange.lowerBound,
+ sampleRate: sampleRate
+ )
+ let acquisitionMs = Self.elapsedMilliseconds(
+ from: startHostTime,
+ to: acceptedHostTime
+ )
+ let deliveryMs = Self.elapsedMilliseconds(
+ from: startHostTime,
+ to: mach_absolute_time()
+ )
+ self.onFirstAudio(
+ recordingSessionID,
+ Int(mono16k.count),
+ originalFrameCount,
+ sampleRate,
+ acquisitionMs,
+ deliveryMs
+ )
+ }
let level = self.calculateAudioLevel(mono16k)
self.onLevel(level)
}
+ private static func acceptedFrameRange(
+ frameCount: Int,
+ sampleRate: Double,
+ packetHostTime: UInt64,
+ startHostTime: UInt64,
+ stopHostTime: UInt64?
+ ) -> Range<Int>? {
+ guard frameCount > 0 else { return nil }
+ // AVAudioEngine can occasionally omit host time. It remains the
+ // conservative fallback and accepts the whole callback in that case.
+ guard packetHostTime > 0, startHostTime > 0 else { return 0..<frameCount }
+
+ var lowerBound = 0
+ if packetHostTime < startHostTime {
+ let framesBeforeStart = Int(ceil(
+ Double(startHostTime - packetHostTime) /
+ Self.hostTicksPerSecond * sampleRate
+ ))
+ lowerBound = min(max(framesBeforeStart, 0), frameCount)
+ }
+
+ var upperBound = frameCount
+ if let stopHostTime {
+ if stopHostTime <= packetHostTime {
+ return nil
+ }
+ let framesBeforeStop = Int(floor(
+ Double(stopHostTime - packetHostTime) /
+ Self.hostTicksPerSecond * sampleRate
+ ))
+ upperBound = min(max(framesBeforeStop, 0), frameCount)
+ }
+
+ guard lowerBound < upperBound else { return nil }
+ return lowerBound..<upperBound
+ }
+
+ private static func hostTime(
+ _ hostTime: UInt64,
+ advancedByFrames frames: Int,
+ sampleRate: Double
+ ) -> UInt64 {
+ guard hostTime > 0, frames > 0, sampleRate > 0 else { return hostTime }
+ let ticks = Double(frames) / sampleRate * Self.hostTicksPerSecond
+ return hostTime &+ UInt64(max(ticks.rounded(), 0))
+ }
+
+ private static func elapsedMilliseconds(from start: UInt64, to end: UInt64) -> Int {
+ guard start > 0, end >= start else { return 0 }
+ return Int((Double(end - start) / self.hostTicksPerSecond * 1000).rounded())
+ }
+
+ private func resetResamplerLocked() {
+ self.resampleSourceRate = 0
+ self.resampleSourceFrameCursor = 0
+ self.resampleNextSourcePosition = 0
+ self.resamplePreviousSample = nil
+ }
+
+ /// Stateful linear resampling keeps fractional phase across small hardware
+ /// callbacks. Stateless per-packet conversion silently shortens 44.1 kHz
+ /// recordings and introduces a discontinuity at every device cycle.
+ private func resampleTo16kLocked(
+ _ samples: [Float],
+ sourceSampleRate: Double
+ ) -> [Float] {
+ guard samples.isEmpty == false else { return [] }
+ if sourceSampleRate == 16_000.0 {
+ return samples
+ }
+
+ if abs(self.resampleSourceRate - sourceSampleRate) > 0.5 {
+ self.resetResamplerLocked()
+ self.resampleSourceRate = sourceSampleRate
+ }
+
+ let chunkStart = Double(self.resampleSourceFrameCursor)
+ let chunkEnd = chunkStart + Double(samples.count)
+ let step = sourceSampleRate / 16_000.0
+ var output: [Float] = []
+ output.reserveCapacity(Int(ceil(Double(samples.count) / step)) + 1)
+
+ while self.resampleNextSourcePosition < chunkEnd {
+ let lowerFrame = Int64(floor(self.resampleNextSourcePosition))
+ let fraction = Float(self.resampleNextSourcePosition - Double(lowerFrame))
+ let localLower = lowerFrame - self.resampleSourceFrameCursor
+
+ let lowerSample: Float
+ let upperSample: Float
+ if localLower < 0 {
+ guard localLower == -1,
+ let previousSample = self.resamplePreviousSample
+ else { break }
+ lowerSample = previousSample
+ upperSample = samples[0]
+ } else {
+ let index = Int(localLower)
+ guard index < samples.count else { break }
+ lowerSample = samples[index]
+ if fraction == 0 {
+ upperSample = lowerSample
+ } else {
+ guard index + 1 < samples.count else { break }
+ upperSample = samples[index + 1]
+ }
+ }
+
+ output.append(lowerSample + (upperSample - lowerSample) * fraction)
+ self.resampleNextSourcePosition += step
+ }
+
+ self.resampleSourceFrameCursor += Int64(samples.count)
+ self.resamplePreviousSample = samples.last
+ return output
+ }
+
private func calculateAudioLevel(_ samples: [Float]) -> CGFloat {
guard samples.isEmpty == false else { return 0.0 }
@@ -3307,20 +3848,6 @@ private final class AudioCapturePipeline {
return self.smoothedLevel
}
- private static func toMono16k(floatBuffer: AVAudioPCMBuffer) -> [Float] {
- if let format = floatBuffer.format as AVAudioFormat?,
- format.sampleRate == 16_000.0,
- format.commonFormat == .pcmFormatFloat32,
- format.channelCount == 1,
- let channelData = floatBuffer.floatChannelData
- {
- let frameCount = Int(floatBuffer.frameLength)
- return Array(UnsafeBufferPointer(start: channelData[0], count: frameCount))
- }
- let mono = self.downmixToMono(floatBuffer)
- return self.resampleTo16k(mono, sourceSampleRate: floatBuffer.format.sampleRate)
- }
-
private static func downmixToMono(_ buffer: AVAudioPCMBuffer) -> [Float] {
guard let channelData = buffer.floatChannelData else { return [] }
let frameCount = Int(buffer.frameLength)
@@ -3337,26 +3864,4 @@ private final class AudioCapturePipeline {
vDSP_vsdiv(mono, 1, &div, &mono, 1, vDSP_Length(frameCount))
return mono
}
-
- private static func resampleTo16k(_ samples: [Float], sourceSampleRate: Double) -> [Float] {
- guard samples.isEmpty == false else { return [] }
- if sourceSampleRate == 16_000.0 { return samples }
- let ratio = 16_000.0 / sourceSampleRate
- let outCount = Int(Double(samples.count) * ratio)
- var output = [Float](repeating: 0, count: max(outCount, 0))
- if output.isEmpty { return [] }
- for i in 0..<outCount {
- let srcPos = Double(i) / ratio
- let idx = Int(srcPos)
- let frac = Float(srcPos - Double(idx))
- if idx + 1 < samples.count {
- let a = samples[idx]
- let b = samples[idx + 1]
- output[i] = a + (b - a) * frac
- } else if idx < samples.count {
- output[i] = samples[idx]
- }
- }
- return output
- }
}
diff --git a/Sources/Fluid/Services/DirectCoreAudioInput.swift b/Sources/Fluid/Services/DirectCoreAudioInput.swift
new file mode 100644
index 00000000..3e94e4ed
--- /dev/null
+++ b/Sources/Fluid/Services/DirectCoreAudioInput.swift
@@ -0,0 +1,143 @@
+import CoreAudio
+#if SWIFT_PACKAGE
+import CoreAudioCaptureSupport
+#endif
+import Foundation
+
+/// Prepared, input-only Core Audio capture.
+///
+/// `AudioDeviceIOProc` publishes native hardware-cycle PCM into a fixed SPSC
+/// ring in C. This Swift wrapper drains that ring away from Core Audio's
+/// realtime thread, so resampling, level calculation, logging, and ASR buffer
+/// mutation never happen in the device callback.
+final class DirectCoreAudioInput {
+ typealias PacketHandler = @Sendable (
+ _ samples: UnsafePointer<Float>,
+ _ frameCount: Int,
+ _ sampleRate: Double,
+ _ inputHostTime: UInt64,
+ _ inputSampleTime: Int64
+ ) -> Void
+
+ private struct SendableCaptureHandle: @unchecked Sendable {
+ let rawValue: FVCoreAudioCaptureRef
+ }
+
+ let deviceID: AudioObjectID
+ let sampleRate: Double
+ let hardwareBufferFrameSize: UInt32
+
+ private var capture: FVCoreAudioCaptureRef?
+ private let packetHandler: PacketHandler
+ private let workerQueue = DispatchQueue(
+ label: "com.fluidvoice.audio.direct-input-consumer",
+ qos: .userInteractive
+ )
+ private let workerGroup = DispatchGroup()
+
+ init(deviceID: AudioObjectID, packetHandler: @escaping PacketHandler) throws {
+ var capture: FVCoreAudioCaptureRef?
+ let status = fv_core_audio_capture_create(deviceID, &capture)
+ guard status == noErr, let capture else {
+ throw Self.error(status: status, operation: "prepare direct Core Audio input")
+ }
+
+ self.deviceID = deviceID
+ self.capture = capture
+ self.sampleRate = fv_core_audio_capture_sample_rate(capture)
+ self.hardwareBufferFrameSize = fv_core_audio_capture_buffer_frame_size(capture)
+ self.packetHandler = packetHandler
+ }
+
+ deinit {
+ self.invalidate()
+ }
+
+ var isRunning: Bool {
+ guard let capture else { return false }
+ return fv_core_audio_capture_is_running(capture)
+ }
+
+ var droppedPacketCount: UInt64 {
+ guard let capture else { return 0 }
+ return fv_core_audio_capture_dropped_packet_count(capture)
+ }
+
+ func start() throws {
+ guard let capture else {
+ throw Self.error(status: kAudioHardwareBadObjectError, operation: "start direct Core Audio input")
+ }
+ guard fv_core_audio_capture_is_running(capture) == false else { return }
+
+ fv_core_audio_capture_clear(capture)
+ let status = fv_core_audio_capture_start(capture)
+ guard status == noErr else {
+ throw Self.error(status: status, operation: "start direct Core Audio input")
+ }
+
+ let packetHandler = self.packetHandler
+ let workerGroup = self.workerGroup
+ let workerHandle = SendableCaptureHandle(rawValue: capture)
+ workerGroup.enter()
+ self.workerQueue.async {
+ defer { workerGroup.leave() }
+ Self.consumePackets(capture: workerHandle.rawValue, packetHandler: packetHandler)
+ }
+ }
+
+ /// Stops hardware IO and synchronously drains every packet already published
+ /// by the realtime callback before returning.
+ @discardableResult
+ func stop() -> OSStatus {
+ guard let capture else { return noErr }
+ let status = fv_core_audio_capture_stop(capture)
+ fv_core_audio_capture_wake(capture)
+ self.workerGroup.wait()
+ return status
+ }
+
+ func invalidate() {
+ guard let capture else { return }
+ _ = self.stop()
+ fv_core_audio_capture_destroy(capture)
+ self.capture = nil
+ }
+
+ private nonisolated static func consumePackets(
+ capture: FVCoreAudioCaptureRef,
+ packetHandler: PacketHandler
+ ) {
+ while true {
+ var packet = FVCoreAudioPacket()
+ while fv_core_audio_capture_peek(capture, &packet) {
+ if let samples = packet.samples, packet.frameCount > 0 {
+ packetHandler(
+ samples,
+ Int(packet.frameCount),
+ packet.sampleRate,
+ packet.inputHostTime,
+ packet.inputSampleTime
+ )
+ }
+ fv_core_audio_capture_consume(capture)
+ }
+
+ guard fv_core_audio_capture_is_running(capture) else {
+ // AudioDeviceStop waits for the IOProc to leave. One final
+ // acquire/drain above therefore captures the complete tail.
+ return
+ }
+ _ = fv_core_audio_capture_wait(capture, 100)
+ }
+ }
+
+ private static func error(status: OSStatus, operation: String) -> NSError {
+ NSError(
+ domain: NSOSStatusErrorDomain,
+ code: Int(status),
+ userInfo: [
+ NSLocalizedDescriptionKey: "Failed to \(operation) (OSStatus \(status)).",
+ ]
+ )
+ }
+}
diff --git a/Sources/Fluid/Services/FluidAudioProvider.swift b/Sources/Fluid/Services/FluidAudioProvider.swift
index 1304e351..aada1486 100644
--- a/Sources/Fluid/Services/FluidAudioProvider.swift
+++ b/Sources/Fluid/Services/FluidAudioProvider.swift
@@ -18,9 +18,6 @@ final class FluidAudioProvider: TranscriptionProvider {
private var latestStreamingPreviewText: String = ""
private var latestStreamingPreviewSampleCount: Int = 0
private var latestStreamingPreviewFinishedAt: TimeInterval?
- private let fastPreviewMinimumSamples = 32_000
- private let fastPreviewTailSilenceRMS: Float = 0.002
- private let fastPreviewTailAudioToleranceMs = 300
private(set) var isReady: Bool = false
private(set) var isWordBoostingActive: Bool = false
private(set) var boostedVocabularyTermsCount: Int = 0
@@ -189,12 +186,11 @@ final class FluidAudioProvider: TranscriptionProvider {
let elapsedMs = Int(((Date().timeIntervalSince1970 - startedAt) * 1000).rounded())
let audioMs = Int((Double(samples.count) / 16_000.0 * 1000).rounded())
let rtf = audioMs > 0 ? Double(elapsedMs) / Double(audioMs) : 0
- let finalizationMode = SettingsStore.shared.parakeetFinalizationMode.rawValue
DebugLogger.shared.info(
"""
ASR_BENCH provider_streaming_done samples=\(samples.count) audioMs=\(audioMs) \
elapsedMs=\(elapsedMs) textChars=\(text.trimmingCharacters(in: .whitespacesAndNewlines).count) \
- rtf=\(String(format: "%.3f", rtf)) finalizationMode=\(finalizationMode)
+ rtf=\(String(format: "%.3f", rtf))
""",
source: "ASRBenchmark"
)
@@ -214,13 +210,6 @@ final class FluidAudioProvider: TranscriptionProvider {
)
}
- let startedAt = Date().timeIntervalSince1970
- if SettingsStore.shared.parakeetFinalizationMode == .tokenTimedChunkMerge {
- if let previewResult = await self.cachedStreamingPreviewResult(for: samples, startedAt: startedAt) {
- return previewResult
- }
- }
-
// If the boosted final manager fails, fall back to the unboosted streaming
// manager so the user still gets a transcription (just without CTC rescoring).
do {
@@ -243,100 +232,6 @@ final class FluidAudioProvider: TranscriptionProvider {
}
}
- func transcribeCachedStreamingPreviewIfAvailable(_ samples: [Float]) async -> ASRTranscriptionResult? {
- guard SettingsStore.shared.parakeetFinalizationMode == .tokenTimedChunkMerge else {
- return nil
- }
- let startedAt = Date().timeIntervalSince1970
- return await self.cachedStreamingPreviewResult(for: samples, startedAt: startedAt)
- }
-
- private func cachedStreamingPreviewResult(for samples: [Float], startedAt: TimeInterval) async -> ASRTranscriptionResult? {
- let text = self.latestStreamingPreviewText.trimmingCharacters(in: .whitespacesAndNewlines)
- let finalSampleCount = samples.count
- let previewSampleCount = min(self.latestStreamingPreviewSampleCount, finalSampleCount)
- let tailSamples = max(0, finalSampleCount - previewSampleCount)
- let tailMs = Int((Double(tailSamples) / 16_000.0 * 1000).rounded())
- let tailRMS = self.rms(samples: samples, startIndex: previewSampleCount)
- let coverage = finalSampleCount > 0 ? Double(previewSampleCount) / Double(finalSampleCount) : 0
- let ageMs: Int
- if let latestStreamingPreviewFinishedAt {
- ageMs = Int(((Date().timeIntervalSince1970 - latestStreamingPreviewFinishedAt) * 1000).rounded())
- } else {
- ageMs = Int.max
- }
-
- DebugLogger.shared.info(
- """
- ASR_BENCH provider_fast_preview_check finalSamples=\(finalSampleCount) previewSamples=\(previewSampleCount) \
- tailMs=\(tailMs) coverage=\(String(format: "%.3f", coverage)) ageMs=\(ageMs) \
- tailRMS=\(String(format: "%.5f", tailRMS)) textChars=\(text.count) wordBoosting=\(self.isWordBoostingActive)
- """,
- source: "ASRBenchmark"
- )
-
- guard !text.isEmpty else {
- self.logFastPreviewMiss(reason: "empty", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
- return nil
- }
- guard self.latestStreamingPreviewFinishedAt != nil, self.latestStreamingPreviewSampleCount > 0 else {
- self.logFastPreviewMiss(reason: "missing_preview", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
- return nil
- }
- guard ageMs <= 3000 else {
- self.logFastPreviewMiss(reason: "stale", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
- return nil
- }
- guard finalSampleCount >= self.fastPreviewMinimumSamples else {
- self.logFastPreviewMiss(reason: "short_recording", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
- return nil
- }
- guard coverage >= 0.88 else {
- self.logFastPreviewMiss(reason: "low_coverage", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
- return nil
- }
- guard tailMs <= 1800 else {
- self.logFastPreviewMiss(reason: "large_tail", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
- return nil
- }
- guard tailSamples == 0 || tailMs <= self.fastPreviewTailAudioToleranceMs || tailRMS <= self.fastPreviewTailSilenceRMS else {
- self.logFastPreviewMiss(reason: "tail_has_audio", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
- return nil
- }
-
- DebugLogger.shared.info(
- """
- ASR_BENCH provider_fast_preview_hit tailMs=\(tailMs) coverage=\(String(format: "%.3f", coverage)) \
- ageMs=\(ageMs) tailRMS=\(String(format: "%.5f", tailRMS)) textChars=\(text.count)
- """,
- source: "ASRBenchmark"
- )
- self.logFinalBenchmark(samples: samples, text: text, startedAt: startedAt, usedFallback: false, source: "livePreview")
- return ASRTranscriptionResult(text: text, confidence: 0.95)
- }
-
- private func rms(samples: [Float], startIndex: Int) -> Float {
- guard startIndex < samples.count else { return 0 }
- var sum: Float = 0
- var count: Float = 0
- for sample in samples[startIndex...] {
- sum += sample * sample
- count += 1
- }
- guard count > 0 else { return 0 }
- return sqrt(sum / count)
- }
-
- private func logFastPreviewMiss(reason: String, tailMs: Int, coverage: Double, ageMs: Int, tailRMS: Float) {
- DebugLogger.shared.info(
- """
- ASR_BENCH provider_fast_preview_miss reason=\(reason) tailMs=\(tailMs) \
- coverage=\(String(format: "%.3f", coverage)) ageMs=\(ageMs) tailRMS=\(String(format: "%.5f", tailRMS))
- """,
- source: "ASRBenchmark"
- )
- }
-
private func logFinalBenchmark(
samples: [Float],
text: String,
@@ -347,12 +242,11 @@ final class FluidAudioProvider: TranscriptionProvider {
let elapsedMs = Int(((Date().timeIntervalSince1970 - startedAt) * 1000).rounded())
let audioMs = Int((Double(samples.count) / 16_000.0 * 1000).rounded())
let rtf = audioMs > 0 ? Double(elapsedMs) / Double(audioMs) : 0
- let finalizationMode = SettingsStore.shared.parakeetFinalizationMode.rawValue
DebugLogger.shared.info(
"""
ASR_BENCH provider_final_done samples=\(samples.count) audioMs=\(audioMs) \
elapsedMs=\(elapsedMs) textChars=\(text.trimmingCharacters(in: .whitespacesAndNewlines).count) \
- rtf=\(String(format: "%.3f", rtf)) fallback=\(usedFallback) finalizationMode=\(finalizationMode) source=\(source)
+ rtf=\(String(format: "%.3f", rtf)) fallback=\(usedFallback) source=\(source)
""",
source: "ASRBenchmark"
)
@@ -493,9 +387,5 @@ final class FluidAudioProvider: TranscriptionProvider {
}
func resetStreamingPreviewCache() {}
-
- func transcribeCachedStreamingPreviewIfAvailable(_ samples: [Float]) async -> ASRTranscriptionResult? {
- nil
- }
}
#endif
diff --git a/Sources/Fluid/Services/GlobalHotkeyManager.swift b/Sources/Fluid/Services/GlobalHotkeyManager.swift
index 7c9b271d..1d283e84 100644
--- a/Sources/Fluid/Services/GlobalHotkeyManager.swift
+++ b/Sources/Fluid/Services/GlobalHotkeyManager.swift
@@ -25,8 +25,6 @@ private final nonisolated class HotkeyState: @unchecked Sendable {
var modifierOnlyKeyDown = false
var otherKeyPressedDuringModifier = false
var modifierPressStartTime: Date?
- var pendingHoldModeStart: Task<Void, Never>?
- var pendingHoldModeType: HotkeyHoldModeType?
var holdModeStartTriggeredTypes: Set<HotkeyHoldModeType> = []
var pendingReleaseStopTasks: [HotkeyHoldModeType: Task<Void, Never>] = [:]
var pendingReleaseStopTokens: [HotkeyHoldModeType: UUID] = [:]
@@ -77,7 +75,6 @@ final class GlobalHotkeyManager: NSObject {
let shortcut: HotkeyShortcut
let isEnabled: Bool
let holdModeType: HotkeyHoldModeType
- let holdStartCancelledMessage: String
let holdStartMessage: String
let holdReleaseMessage: String
let toggleIgnoredMessage: String
@@ -146,17 +143,6 @@ final class GlobalHotkeyManager: NSObject {
set { self.state.withLock { self.state.modifierPressStartTime = newValue } }
}
- private nonisolated var pendingHoldModeStart: Task<Void, Never>? {
- get { self.state.withLock { self.state.pendingHoldModeStart } }
- set { self.state.withLock { self.state.pendingHoldModeStart = newValue } }
- }
-
- /// Tracks which mode's pending start is active (for cancellation on key combos)
- private nonisolated var pendingHoldModeType: HotkeyHoldModeType? {
- get { self.state.withLock { self.state.pendingHoldModeType } }
- set { self.state.withLock { self.state.pendingHoldModeType = newValue } }
- }
-
private func cancelPendingReleaseStop(for type: HotkeyHoldModeType) {
let task = self.state.withLock { () -> Task<Void, Never>? in
_ = self.state.pendingReleaseStopTokens.removeValue(forKey: type)
@@ -534,12 +520,6 @@ final class GlobalHotkeyManager: NSObject {
private func markOtherInputDuringModifierOnly() {
guard self.modifierOnlyKeyDown else { return }
self.otherKeyPressedDuringModifier = true
- if let pending = self.pendingHoldModeStart {
- pending.cancel()
- self.pendingHoldModeStart = nil
- self.pendingHoldModeType = nil
- DebugLogger.shared.info("Another input pressed - cancelled pending hold mode start", source: "GlobalHotkeyManager")
- }
}
private func mouseButton(from event: CGEvent) -> Int {
@@ -571,8 +551,7 @@ final class GlobalHotkeyManager: NSObject {
shortcut: shortcut,
isEnabled: true,
holdModeType: .transcription,
- holdStartCancelledMessage: "Transcription hold start cancelled - key combo detected",
- holdStartMessage: "Transcription modifier held (hold mode) - starting after delay",
+ holdStartMessage: "Transcription modifier held (hold mode) - starting",
holdReleaseMessage: "Transcription modifier released (hold mode) - stopping",
toggleIgnoredMessage: "Transcription modifier released but another key was pressed - ignoring",
isModeKeyPressed: { self.isKeyPressed },
@@ -929,8 +908,7 @@ final class GlobalHotkeyManager: NSObject {
shortcut: commandModeShortcut,
isEnabled: self.commandModeShortcutEnabled,
holdModeType: .commandMode,
- holdStartCancelledMessage: "Command mode hold start cancelled - key combo detected",
- holdStartMessage: "Command mode modifier held (hold mode) - starting after delay",
+ holdStartMessage: "Command mode modifier held (hold mode) - starting",
holdReleaseMessage: "Command mode modifier released (hold mode) - stopping",
toggleIgnoredMessage: "Command mode modifier released but another key was pressed - ignoring",
isModeKeyPressed: { self.isCommandModeKeyPressed },
@@ -962,8 +940,7 @@ final class GlobalHotkeyManager: NSObject {
shortcut: self.rewriteModeShortcut,
isEnabled: self.rewriteModeShortcutEnabled,
holdModeType: .rewriteMode,
- holdStartCancelledMessage: "Rewrite mode hold start cancelled - key combo detected",
- holdStartMessage: "Rewrite mode modifier held (hold mode) - starting after delay",
+ holdStartMessage: "Rewrite mode modifier held (hold mode) - starting",
holdReleaseMessage: "Rewrite mode modifier released (hold mode) - stopping",
toggleIgnoredMessage: "Rewrite mode modifier released but another key was pressed - ignoring",
isModeKeyPressed: { self.isRewriteKeyPressed },
@@ -1067,15 +1044,8 @@ final class GlobalHotkeyManager: NSObject {
return synchronizedKeyCodes
}
- private func cancelPendingModifierOnlyHoldStart(
- for behavior: ModifierOnlyShortcutBehavior,
- message: String
- ) {
- guard self.pendingHoldModeType == behavior.holdModeType else { return }
+ private func markModifierOnlyPressInterrupted(message: String) {
self.otherKeyPressedDuringModifier = true
- self.pendingHoldModeStart?.cancel()
- self.pendingHoldModeStart = nil
- self.pendingHoldModeType = nil
DebugLogger.shared.info(message, source: "GlobalHotkeyManager")
}
@@ -1286,30 +1256,20 @@ final class GlobalHotkeyManager: NSObject {
self.cancelPendingReleaseStop(for: behavior.holdModeType)
self.clearHoldModeStartTriggered(for: behavior.holdModeType)
behavior.setModeKeyPressed(true)
- self.pendingHoldModeStart?.cancel()
- self.pendingHoldModeType = behavior.holdModeType
let wasTargetActive = self.asrService.isRunning && behavior.isTargetModeActive()
if self.hotkeyMode == .automatic {
self.beginAutomaticPress(for: behavior.holdModeType, wasTargetActive: wasTargetActive)
}
- self.pendingHoldModeStart = Task { @MainActor [weak self] in
- try? await Task.sleep(nanoseconds: 150_000_000)
- guard let self = self, !Task.isCancelled else { return }
- guard behavior.isModeKeyPressed(), !self.otherKeyPressedDuringModifier else {
- DebugLogger.shared.debug(behavior.holdStartCancelledMessage, source: "GlobalHotkeyManager")
- return
- }
- guard self.hotkeyMode != .automatic || !wasTargetActive else { return }
- DebugLogger.shared.info(behavior.holdStartMessage, source: "GlobalHotkeyManager")
- if self.hotkeyMode == .hold {
- self.markHoldModeStartTriggered(for: behavior.holdModeType)
- }
- behavior.onHoldStart()
- if self.hotkeyMode == .automatic {
- self.markAutomaticPressStarted(for: behavior.holdModeType)
- }
+ guard self.hotkeyMode != .automatic || !wasTargetActive else { return }
+ DebugLogger.shared.info(behavior.holdStartMessage, source: "GlobalHotkeyManager")
+ if self.hotkeyMode == .hold {
+ self.markHoldModeStartTriggered(for: behavior.holdModeType)
+ }
+ behavior.onHoldStart()
+ if self.hotkeyMode == .automatic {
+ self.markAutomaticPressStarted(for: behavior.holdModeType)
}
}
@@ -1317,10 +1277,6 @@ final class GlobalHotkeyManager: NSObject {
for behavior: ModifierOnlyShortcutBehavior,
wasCleanPress: Bool
) {
- self.pendingHoldModeStart?.cancel()
- self.pendingHoldModeStart = nil
- self.pendingHoldModeType = nil
-
switch self.hotkeyMode {
case .hold:
if behavior.isModeKeyPressed() {
@@ -1368,9 +1324,6 @@ final class GlobalHotkeyManager: NSObject {
self.modifierOnlyKeyDown = false
self.otherKeyPressedDuringModifier = false
self.modifierPressStartTime = nil
- self.pendingHoldModeStart?.cancel()
- self.pendingHoldModeStart = nil
- self.pendingHoldModeType = nil
self.clearAutomaticPressTracking()
self.isKeyPressed = false
self.isPromptModeKeyPressed = false
@@ -1464,8 +1417,7 @@ final class GlobalHotkeyManager: NSObject {
shortcut: self.promptModeShortcut,
isEnabled: self.promptModeShortcutEnabled,
holdModeType: .promptMode,
- holdStartCancelledMessage: "Prompt mode hold start cancelled - key combo detected",
- holdStartMessage: "Prompt mode modifier held (hold mode) - starting after delay",
+ holdStartMessage: "Prompt mode modifier held (hold mode) - starting",
holdReleaseMessage: "Prompt mode modifier released (hold mode) - stopping",
toggleIgnoredMessage: "Prompt mode modifier released but another key was pressed - ignoring",
isModeKeyPressed: { self.isPromptModeKeyPressed },
@@ -1499,8 +1451,7 @@ final class GlobalHotkeyManager: NSObject {
shortcut: assignment.shortcut,
isEnabled: true,
holdModeType: .promptMode,
- holdStartCancelledMessage: "Prompt shortcut hold start cancelled - key combo detected",
- holdStartMessage: "Prompt shortcut modifier held (hold mode) - starting after delay",
+ holdStartMessage: "Prompt shortcut modifier held (hold mode) - starting",
holdReleaseMessage: "Prompt shortcut modifier released (hold mode) - stopping",
toggleIgnoredMessage: "Prompt shortcut modifier released but another key was pressed - ignoring",
isModeKeyPressed: { self.isPromptModeKeyPressed },
@@ -1557,9 +1508,8 @@ final class GlobalHotkeyManager: NSObject {
if self.modifierOnlyKeyDown || behavior.isModeKeyPressed() {
let extraModifierKeyCodes = pressedModifierKeyCodes.filter { !expectedModifierKeyCodes.contains($0) }
if !extraModifierKeyCodes.isEmpty {
- self.cancelPendingModifierOnlyHoldStart(
- for: behavior,
- message: "\(behavior.holdStartCancelledMessage) - extra modifier pressed"
+ self.markModifierOnlyPressInterrupted(
+ message: "\(self.label(for: behavior.holdModeType)) modifier-only press interrupted - extra modifier pressed"
)
}
}
@@ -1598,9 +1548,8 @@ final class GlobalHotkeyManager: NSObject {
if self.modifierOnlyKeyDown || behavior.isModeKeyPressed() {
let unexpectedModifiers = relevantModifiers.subtracting(expectedPressedModifiers)
if !unexpectedModifiers.isEmpty {
- self.cancelPendingModifierOnlyHoldStart(
- for: behavior,
- message: "\(behavior.holdStartCancelledMessage) - extra modifier pressed"
+ self.markModifierOnlyPressInterrupted(
+ message: "\(self.label(for: behavior.holdModeType)) modifier-only press interrupted - extra modifier pressed"
)
}
}
@@ -1760,9 +1709,6 @@ final class GlobalHotkeyManager: NSObject {
&& (self.isKeyPressed || self.isPromptModeKeyPressed || self.isCommandModeKeyPressed || self.isRewriteKeyPressed || self.isPromptAssignmentKeyPressed)
self.hotkeyMode = mode
- self.pendingHoldModeStart?.cancel()
- self.pendingHoldModeStart = nil
- self.pendingHoldModeType = nil
self.clearAutomaticPressTracking()
self.isKeyPressed = false
self.isPromptModeKeyPressed = false
diff --git a/Sources/Fluid/Services/MenuBarManager.swift b/Sources/Fluid/Services/MenuBarManager.swift
index 35c207ef..78b3fe40 100644
--- a/Sources/Fluid/Services/MenuBarManager.swift
+++ b/Sources/Fluid/Services/MenuBarManager.swift
@@ -48,9 +48,8 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
private var pendingProcessingShowOperation: DispatchWorkItem?
/// Show immediately so users see the processing state right away.
private let processingVisualDelay: DispatchTimeInterval = .milliseconds(0)
- /// Debounce the hide so a fast transcription doesn't flash the processing
- /// overlay for a single frame. 80ms is under the perception threshold but
- /// long enough to coalesce a quick show->hide cycle.
+ /// Legacy debounce used by generic processing callers. Successful dictation
+ /// completion dispatches output first, then hides the overlay asynchronously.
private let processingHideDelay: DispatchTimeInterval = .milliseconds(80)
/// Subscription for forwarding audio levels to expanded command notch
@@ -356,6 +355,44 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate {
}
}
+ /// Ends processing and waits for the recording overlay's exit transition.
+ /// Output paths normally call this asynchronously after insertion dispatch
+ /// so the exit animation cannot delay text delivery.
+ func finishProcessingAndHideOverlay() async {
+ let startedAt = ProcessInfo.processInfo.systemUptime
+ self.cancelPendingProcessingCompletionOperations()
+ self.isProcessingActive = false
+ self.overlayVisible = false
+
+ NotchOverlayManager.shared.setProcessing(false)
+ self.overlayBench("finish_hide_request")
+ await NotchOverlayManager.shared.hideAndWait()
+ self.overlayBench(
+ "finish_hide_complete elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()))"
+ )
+ }
+
+ /// Ends processing without dismissing an actionable overlay, such as the
+ /// AI fallback state that offers reprocessing and settings actions.
+ func finishProcessingKeepingOverlayVisible() {
+ self.cancelPendingProcessingCompletionOperations()
+ self.isProcessingActive = false
+ // Keep the physical overlay visible, but release recording/processing
+ // ownership so the next recording can establish a fresh lifecycle.
+ self.overlayVisible = false
+ NotchOverlayManager.shared.setProcessing(false)
+ self.overlayBench("finish_keep_visible")
+ }
+
+ private func cancelPendingProcessingCompletionOperations() {
+ self.pendingProcessingShowOperation?.cancel()
+ self.pendingProcessingShowOperation = nil
+ self.pendingHideOperation?.cancel()
+ self.pendingHideOperation = nil
+ self.pendingShowOperation?.cancel()
+ self.pendingShowOperation = nil
+ }
+
private func overlayBench(_ message: String) {
DebugLogger.shared.benchmark("OVERLAY_BENCH", message: "manager \(message)", source: "OverlayBenchmark")
}
diff --git a/Sources/Fluid/Services/NotchOverlayManager.swift b/Sources/Fluid/Services/NotchOverlayManager.swift
index b6bd753f..0ffa9365 100644
--- a/Sources/Fluid/Services/NotchOverlayManager.swift
+++ b/Sources/Fluid/Services/NotchOverlayManager.swift
@@ -81,6 +81,8 @@ final class NotchOverlayManager {
// Uses UInt64 to avoid overflow concerns in long-running sessions
private var generation: UInt64 = 0
private var commandOutputGeneration: UInt64 = 0
+ private var isHideInProgress = false
+ private var hideWaiters: [CheckedContinuation<Void, Never>] = []
/// Track pending retry task for cancellation
private var pendingRetryTask: Task<Void, Never>?
@@ -213,6 +215,7 @@ final class NotchOverlayManager {
private func showBottomOverlay(audioLevelPublisher: AnyPublisher<CGFloat, Never>, mode: OverlayMode) {
let startedAt = ProcessInfo.processInfo.systemUptime
Self.overlayBench("bottom_route_start mode=\(mode.rawValue)")
+ self.generation &+= 1
// Hide any existing notch first
if self.notch != nil {
@@ -291,15 +294,52 @@ final class NotchOverlayManager {
}
func hide() {
+ guard !self.isHideInProgress else { return }
+ self.isHideInProgress = true
+ self.generation &+= 1
+ let currentGeneration = self.generation
+ Task { [weak self] in
+ guard let self else { return }
+ await self.performHideAndWait(generation: currentGeneration)
+ self.completeHideOperation()
+ }
+ }
+
+ /// Dismisses the active recording overlay and returns only after its visual
+ /// exit is complete. This gives output delivery an explicit ordering point.
+ func hideAndWait() async {
+ if self.isHideInProgress {
+ await withCheckedContinuation { continuation in
+ self.hideWaiters.append(continuation)
+ }
+ return
+ }
+
+ self.isHideInProgress = true
+ self.generation &+= 1
+ let currentGeneration = self.generation
+ await self.performHideAndWait(generation: currentGeneration)
+ self.completeHideOperation()
+ }
+
+ private func performHideAndWait(generation currentGeneration: UInt64) async {
let startedAt = ProcessInfo.processInfo.systemUptime
Self.overlayBench("hide_called state=\(self.state) bottomVisible=\(self.isBottomOverlayVisible)")
+ guard self.generation == currentGeneration else {
+ Self.overlayBench("hide_return reason=stale_generation")
+ return
+ }
// Stop monitoring active app changes
ActiveAppMonitor.shared.stopMonitoring()
// Hide bottom overlay if visible
if self.isBottomOverlayVisible {
- BottomOverlayWindowController.shared.hide()
+ await BottomOverlayWindowController.shared.hideAndWait()
+ guard self.generation == currentGeneration else {
+ Self.overlayBench("hide_return reason=stale_generation")
+ return
+ }
self.isBottomOverlayVisible = false
}
@@ -310,30 +350,37 @@ final class NotchOverlayManager {
// Safety: reset processing state when hiding
NotchContentState.shared.setProcessing(false)
- // Increment generation to invalidate any pending show tasks
- self.generation &+= 1
- let currentGeneration = self.generation
-
// Handle visible or showing states (can hide while still expanding)
guard self.state == .visible || self.state == .showing, let currentNotch = notch else {
- // Force cleanup if stuck or in inconsistent state
+ // A bottom overlay has already completed its dismissal. Clean up
+ // any inconsistent notch state without scheduling another task.
Self.overlayBench("hide_return reason=not_visible state=\(self.state) notchExists=\(self.notch != nil)")
- Task { [weak self] in await self?.performCleanup() }
+ if let existingNotch = self.notch {
+ await existingNotch.hide()
+ }
+ guard self.generation == currentGeneration else { return }
+ self.notch = nil
+ self.state = .idle
+ Self.overlayBench("hide_complete target=none elapsedMs=\(Self.elapsedMs(since: startedAt))")
return
}
self.state = .hiding
+ Self.overlayBench("hide_animation_start")
+ await currentNotch.hide()
+ Self.overlayBench("hide_animation_complete elapsedMs=\(Self.elapsedMs(since: startedAt))")
+ // Only clear if we're still the active operation
+ guard self.generation == currentGeneration else { return }
+ self.notch = nil
+ self.state = .idle
+ Self.overlayBench("state_idle target=notch")
+ }
- Task { [weak self] in
- Self.overlayBench("hide_animation_start")
- await currentNotch.hide()
- Self.overlayBench("hide_animation_complete elapsedMs=\(Self.elapsedMs(since: startedAt))")
- // Only clear if we're still the active operation
- guard let self = self, self.generation == currentGeneration else { return }
- self.notch = nil
- self.state = .idle
- Self.overlayBench("state_idle target=notch")
- }
+ private func completeHideOperation() {
+ self.isHideInProgress = false
+ let waiters = self.hideWaiters
+ self.hideWaiters.removeAll(keepingCapacity: true)
+ waiters.forEach { $0.resume() }
}
/// Async cleanup that properly waits for hide to complete
diff --git a/Sources/Fluid/Services/TranscriptionSoundPlayer.swift b/Sources/Fluid/Services/TranscriptionSoundPlayer.swift
index 5c7ced21..977b3f65 100644
--- a/Sources/Fluid/Services/TranscriptionSoundPlayer.swift
+++ b/Sources/Fluid/Services/TranscriptionSoundPlayer.swift
@@ -2,52 +2,97 @@ import AVFoundation
import CoreAudio
import Foundation
-@MainActor
final class TranscriptionSoundPlayer {
static let shared = TranscriptionSoundPlayer()
+ private let playbackQueue = DispatchQueue(label: "app.fluidvoice.transcription-sounds", qos: .userInteractive)
private var players: [String: AVAudioPlayer] = [:]
private var savedSystemVolume: Float?
private init() {}
func playStartSound() {
- guard SettingsStore.shared.enableTranscriptionSounds else { return }
- let selected = SettingsStore.shared.transcriptionStartSound
+ let settings = SettingsStore.shared
+ guard settings.enableTranscriptionSounds else { return }
+ let selected = settings.transcriptionStartSound
guard let soundName = selected.startSoundFileName else { return }
- self.play(soundName: soundName)
+ self.play(
+ soundName: soundName,
+ desiredVolume: settings.transcriptionSoundVolume,
+ independentVolume: settings.transcriptionSoundIndependentVolume
+ )
}
func playStopSound() {
- guard SettingsStore.shared.enableTranscriptionSounds else { return }
- let selected = SettingsStore.shared.transcriptionStartSound
+ let settings = SettingsStore.shared
+ guard settings.enableTranscriptionSounds else { return }
+ let selected = settings.transcriptionStartSound
guard let soundName = selected.stopSoundFileName else { return }
- self.play(soundName: soundName)
+ self.play(
+ soundName: soundName,
+ desiredVolume: settings.transcriptionSoundVolume,
+ independentVolume: settings.transcriptionSoundIndependentVolume
+ )
}
/// Preview a specific sound at the current volume setting (used in Settings UI).
func playPreview(sound: SettingsStore.TranscriptionStartSound) {
guard let soundName = sound.startSoundFileName else { return }
- self.play(soundName: soundName)
+ let settings = SettingsStore.shared
+ self.play(
+ soundName: soundName,
+ desiredVolume: settings.transcriptionSoundVolume,
+ independentVolume: settings.transcriptionSoundIndependentVolume
+ )
}
/// Preview current sound at a specific volume (used when slider is released).
func playPreviewAtVolume(_ volume: Float) {
let selected = SettingsStore.shared.transcriptionStartSound
guard let soundName = selected.startSoundFileName else { return }
- self.play(soundName: soundName, overrideVolume: volume)
+ self.play(
+ soundName: soundName,
+ desiredVolume: volume,
+ independentVolume: SettingsStore.shared.transcriptionSoundIndependentVolume
+ )
}
- private func play(soundName: String, overrideVolume: Float? = nil) {
+ private func play(
+ soundName: String,
+ desiredVolume: Float,
+ independentVolume: Bool
+ ) {
+ let startedAt = ProcessInfo.processInfo.systemUptime
+ DebugLogger.shared.benchmark(
+ "APP_BENCH",
+ message: "sound_play_request sound=\(soundName)",
+ source: "AppBenchmark"
+ )
+
guard let url = Bundle.main.url(forResource: soundName, withExtension: "m4a") else {
DebugLogger.shared.error("Missing sound resource: \(soundName).m4a", source: "TranscriptionSoundPlayer")
return
}
- let settings = SettingsStore.shared
- let desiredVolume = overrideVolume ?? settings.transcriptionSoundVolume
+ self.playbackQueue.async { [weak self] in
+ self?.playOnPlaybackQueue(
+ soundName: soundName,
+ url: url,
+ desiredVolume: desiredVolume,
+ independentVolume: independentVolume,
+ startedAt: startedAt
+ )
+ }
+ }
- if settings.transcriptionSoundIndependentVolume {
+ private func playOnPlaybackQueue(
+ soundName: String,
+ url: URL,
+ desiredVolume: Float,
+ independentVolume: Bool,
+ startedAt: TimeInterval
+ ) {
+ if independentVolume {
let currentSystemVol = Self.getSystemVolume()
guard currentSystemVol > 0.001 else { return }
// Save current system volume and temporarily set it to desired level
@@ -66,17 +111,22 @@ final class TranscriptionSoundPlayer {
}
player.currentTime = 0
- if settings.transcriptionSoundIndependentVolume {
+ if independentVolume {
player.volume = 1.0
} else {
player.volume = desiredVolume
}
player.play()
+ DebugLogger.shared.benchmark(
+ "APP_BENCH",
+ message: "sound_play_dispatched sound=\(soundName) elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()))",
+ source: "AppBenchmark"
+ )
// Restore system volume after the sound finishes
- if settings.transcriptionSoundIndependentVolume, let saved = self.savedSystemVolume {
+ if independentVolume, let saved = self.savedSystemVolume {
let duration = player.duration
- DispatchQueue.main.asyncAfter(deadline: .now() + duration + 0.05) { [weak self] in
+ self.playbackQueue.asyncAfter(deadline: .now() + duration + 0.05) { [weak self] in
Self.setSystemVolume(saved)
self?.savedSystemVolume = nil
}
diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift
index aab5624d..b6ab6fad 100644
--- a/Sources/Fluid/UI/SettingsView.swift
+++ b/Sources/Fluid/UI/SettingsView.swift
@@ -1511,39 +1511,7 @@ struct SettingsView: View {
.foregroundStyle(.primary)
VStack(alignment: .leading, spacing: 8) {
- HStack(alignment: .center) {
- VStack(alignment: .leading, spacing: 2) {
- Text("Dictation Processing Speed")
- .font(self.theme.typography.bodyStrong)
- .foregroundStyle(self.settingsTitleText)
- }
-
- Spacer()
-
- Picker("", selection: Binding(
- get: {
- self.settings.selectedSpeechModel.supportsFastDictationProcessing
- ? self.settings.parakeetFinalizationMode
- : .stableFullFinal
- },
- set: { mode in
- if self.settings.selectedSpeechModel.supportsFastDictationProcessing {
- self.settings.parakeetFinalizationMode = mode
- }
- }
- )) {
- ForEach(ParakeetFinalizationMode.allCases) { mode in
- Text(mode.displayName).tag(mode)
- }
- }
- .pickerStyle(.menu)
- .frame(width: 170, alignment: .trailing)
- .disabled(self.asr.isRunning || !self.settings.selectedSpeechModel.supportsFastDictationProcessing)
- }
-
- Text(self.settings.selectedSpeechModel.supportsFastDictationProcessing ? "Standard: most reliable. Fast: faster, but maybe inaccurate." : "Fast processing is available for Parakeet TDT v2 and v3.")
- .font(self.theme.typography.bodySmall)
- .foregroundStyle(self.settingsSecondaryText)
+ self.lowLatencyAudioCaptureToggle
if self.asr.isRunning {
Text("Settings are disabled during active recording")
@@ -2541,6 +2509,28 @@ struct FlowLayout: Layout {
}
}
+private extension SettingsView {
+ var lowLatencyAudioCaptureToggle: some View {
+ Group {
+ self.settingsToggleRow(
+ title: "Low-Latency Audio Capture",
+ description: "Use the experimental direct Core Audio input path for faster recording startup. Turn this off if a microphone behaves unexpectedly.",
+ footnote: "Off by default while device compatibility is being validated.",
+ isOn: Binding(
+ get: { self.settings.experimentalDirectAudioCaptureEnabled },
+ set: { enabled in
+ self.settings.experimentalDirectAudioCaptureEnabled = enabled
+ self.asr.refreshAudioCaptureBackendPreference()
+ }
+ )
+ )
+ .disabled(self.asr.isRunning)
+
+ Divider().padding(.vertical, 8)
+ }
+ }
+}
+
// MARK: - Analytics modal confirmation
struct AnalyticsConfirmationView: View {
diff --git a/Sources/Fluid/Views/BottomOverlayView.swift b/Sources/Fluid/Views/BottomOverlayView.swift
index ee0211d3..a731946c 100644
--- a/Sources/Fluid/Views/BottomOverlayView.swift
+++ b/Sources/Fluid/Views/BottomOverlayView.swift
@@ -37,6 +37,11 @@ final class BottomOverlayWindowController {
private var targetScreen: NSScreen?
private var releaseTransitionActiveUntil: Date?
private var deferredResizePending = false
+ private var presentationGeneration: UInt64 = 0
+ private let presentationDuration: TimeInterval = 0.05
+ private let dismissalDuration: TimeInterval = 0.02
+ private var isHideInProgress = false
+ private var hideWaiters: [CheckedContinuation<Void, Never>] = []
private init() {
NotificationCenter.default.addObserver(forName: NSNotification.Name("OverlayOffsetChanged"), object: nil, queue: .main) { [weak self] _ in
@@ -54,6 +59,8 @@ final class BottomOverlayWindowController {
func show(audioPublisher: AnyPublisher<CGFloat, Never>, mode: OverlayMode) {
let startedAt = ProcessInfo.processInfo.systemUptime
Self.overlayBench("bottom_show_start mode=\(mode.rawValue) windowExists=\(self.window != nil)")
+ self.presentationGeneration &+= 1
+ let currentGeneration = self.presentationGeneration
self.endReleaseTransition(flushDeferredUpdate: false)
self.pendingResizeWorkItem?.cancel()
@@ -91,52 +98,129 @@ final class BottomOverlayWindowController {
self.targetScreen = OverlayScreenResolver.screenForCurrentPointer()
self.positionWindow()
- // Show immediately; ASR startup can delay AppKit animation completions.
- self.window?.alphaValue = 1
+ // Order the panel immediately, then use a very short fade so the first
+ // response is instant without the overlay visually popping into place.
+ self.window?.alphaValue = 0
self.window?.orderFrontRegardless()
Self.overlayBench("bottom_order_front elapsedMs=\(Self.elapsedMs(since: startedAt))")
- Self.overlayBench("bottom_fade_complete elapsedMs=\(Self.elapsedMs(since: startedAt))")
+ NSAnimationContext.runAnimationGroup { context in
+ context.duration = self.presentationDuration
+ context.allowsImplicitAnimation = true
+ self.window?.animator().alphaValue = 1
+ }
+ DispatchQueue.main.asyncAfter(deadline: .now() + self.presentationDuration) { [weak self] in
+ guard let self, self.presentationGeneration == currentGeneration else { return }
+ Self.overlayBench("bottom_fade_complete elapsedMs=\(Self.elapsedMs(since: startedAt))")
+ }
}
func hide() {
- let startedAt = ProcessInfo.processInfo.systemUptime
- Self.overlayBench("bottom_hide_start windowExists=\(self.window != nil)")
+ guard !self.isHideInProgress else { return }
+ self.isHideInProgress = true
+ self.presentationGeneration &+= 1
+ let currentGeneration = self.presentationGeneration
+ Task { [weak self] in
+ guard let self else { return }
+ await self.performHideAndWait(generation: currentGeneration)
+ self.completeHideOperation()
+ }
+ }
- // Cancel audio subscription
- self.audioSubscription?.cancel()
- self.audioSubscription = nil
- self.pendingResizeWorkItem?.cancel()
- self.pendingResizeWorkItem = nil
- self.pendingReleaseTransitionResetWorkItem?.cancel()
- self.targetScreen = nil
- self.removeMouseDownMonitors()
- BottomOverlayPromptMenuController.shared.hide()
- BottomOverlayModeMenuController.shared.hide()
- BottomOverlayActionsMenuController.shared.hide()
+ /// Performs the brief exit animation and returns after the panel is fully
+ /// ordered out, allowing callers to deliver text only after it disappears.
+ func hideAndWait() async {
+ if self.isHideInProgress {
+ await withCheckedContinuation { continuation in
+ self.hideWaiters.append(continuation)
+ }
+ return
+ }
- // Reset state
- NotchContentState.shared.setProcessing(false)
- NotchContentState.shared.bottomOverlayAudioLevel = 0
- NotchContentState.shared.targetAppIcon = nil
+ self.isHideInProgress = true
+ self.presentationGeneration &+= 1
+ let currentGeneration = self.presentationGeneration
+ await self.performHideAndWait(generation: currentGeneration)
+ self.completeHideOperation()
+ }
- guard let window = window else {
+ private func completeHideOperation() {
+ self.isHideInProgress = false
+ let waiters = self.hideWaiters
+ self.hideWaiters.removeAll(keepingCapacity: true)
+ waiters.forEach { $0.resume() }
+ }
+
+ private func performHideAndWait(generation currentGeneration: UInt64) async {
+ let startedAt = ProcessInfo.processInfo.systemUptime
+ Self.overlayBench("bottom_hide_start windowExists=\(self.window != nil)")
+ guard self.presentationGeneration == currentGeneration else {
+ Self.overlayBench("bottom_hide_return reason=stale_generation")
+ return
+ }
+
+ guard let window = self.window, window.isVisible else {
+ self.clearPresentationResources()
self.endReleaseTransition(flushDeferredUpdate: false)
NotchContentState.shared.setBottomOverlayDismissing(false)
+ NotchContentState.shared.targetAppIcon = nil
Self.overlayBench("bottom_hide_return reason=no_window")
return
}
NotchContentState.shared.setBottomOverlayReleaseTransitioning(true)
- NotchContentState.shared.setBottomOverlayDismissOffsetY(28)
+ NotchContentState.shared.setBottomOverlayDismissOffsetY(8)
NotchContentState.shared.setBottomOverlayDismissing(true)
- window.alphaValue = 0
+ // Start the visual response first. Resource cleanup then happens inside
+ // this same short budget instead of delaying the beginning of the fade.
+ let animationStartedAt = ProcessInfo.processInfo.systemUptime
+ Self.overlayBench("bottom_hide_animation_start")
+ await NSAnimationContext.runAnimationGroup { context in
+ context.duration = self.dismissalDuration
+ context.allowsImplicitAnimation = true
+ window.animator().alphaValue = 0
+ }
+ await Task.yield()
+ guard self.presentationGeneration == currentGeneration else {
+ Self.overlayBench("bottom_hide_return reason=stale_generation")
+ return
+ }
+ self.clearPresentationResources()
+
+ let elapsed = ProcessInfo.processInfo.systemUptime - animationStartedAt
+ let remainingDuration = max(0, self.dismissalDuration - elapsed)
+ if remainingDuration > 0 {
+ try? await Task.sleep(nanoseconds: UInt64(remainingDuration * 1_000_000_000))
+ }
+
+ guard self.presentationGeneration == currentGeneration else {
+ Self.overlayBench("bottom_hide_return reason=stale_generation")
+ return
+ }
+
window.orderOut(nil)
+ window.alphaValue = 1
self.endReleaseTransition(flushDeferredUpdate: false)
NotchContentState.shared.setBottomOverlayDismissing(false)
+ NotchContentState.shared.targetAppIcon = nil
Self.overlayBench("bottom_hide_complete elapsedMs=\(Self.elapsedMs(since: startedAt))")
}
+ private func clearPresentationResources() {
+ self.audioSubscription?.cancel()
+ self.audioSubscription = nil
+ self.pendingResizeWorkItem?.cancel()
+ self.pendingResizeWorkItem = nil
+ self.pendingReleaseTransitionResetWorkItem?.cancel()
+ self.targetScreen = nil
+ self.removeMouseDownMonitors()
+ BottomOverlayPromptMenuController.shared.hide()
+ BottomOverlayModeMenuController.shared.hide()
+ BottomOverlayActionsMenuController.shared.hide()
+ NotchContentState.shared.setProcessing(false)
+ NotchContentState.shared.bottomOverlayAudioLevel = 0
+ }
+
func setProcessing(_ processing: Bool) {
Self.overlayBench("bottom_set_processing processing=\(processing)")
NotchContentState.shared.setProcessing(processing)
@@ -2244,7 +2328,7 @@ struct BottomOverlayView: View {
}
private var overlayAnimatedScale: CGFloat {
- self.contentState.isBottomOverlayDismissing ? 0.97 : 1.0
+ self.contentState.isBottomOverlayDismissing ? 0.985 : 1.0
}
private var overlayAnimatedOpacity: Double {
@@ -2905,7 +2989,7 @@ struct BottomOverlayView: View {
.scaleEffect(self.overlayAnimatedScale, anchor: .center)
.offset(y: self.overlayAnimatedOffsetY)
.opacity(self.overlayAnimatedOpacity)
- .animation(.timingCurve(0.22, 0.0, 0.2, 1.0, duration: 0.28), value: self.contentState.isBottomOverlayDismissing)
+ .animation(.timingCurve(0.22, 0.0, 0.2, 1.0, duration: 0.02), value: self.contentState.isBottomOverlayDismissing)
.onChange(of: self.settings.overlaySize) { _, _ in
self.dynamicPreviewResizeBucket = self.previewResizeBucket(for: self.currentPreviewSizingText)
self.frozenDynamicPreviewHeight = nil