Fastest and only macOS Dictation app with on-device STT and custom trained AI enhancement model. Windows, iOS testflight! Android, Linux waitlist open! A local Wispr Flow alternative. Follow https://x.com/fluidvoiceapp for more
fix: limit local api changes to upload caps
altic-dev5a372cff99fc
diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxprojindex 4ca5c60b..2074994f 100644--- a/Fluid.xcodeproj/project.pbxproj+++ b/Fluid.xcodeproj/project.pbxproj@@ -28,7 +28,6 @@7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */; };803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 803000000000000000000001 /* MediaPlaybackServiceTests.swift */; };0B05F11E000000000000A002 /* SupportedFileExtensionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */; };- A87600000000000000000002 /* InferenceAPIControllerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A87600000000000000000001 /* InferenceAPIControllerTests.swift */; };803000000000000000000003 /* MediaRemoteAdapter in Frameworks */ = {isa = PBXBuildFile; productRef = 803000000000000000000004 /* MediaRemoteAdapter */; };B51800000000000000000002 /* SpokenSendTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51800000000000000000001 /* SpokenSendTests.swift */; };B51900000000000000000002 /* PrivateAIProviderPromptFormatTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51900000000000000000001 /* PrivateAIProviderPromptFormatTests.swift */; };@@ -84,7 +83,6 @@7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TypingServiceTransientPasteboardTests.swift; sourceTree = "<group>"; };803000000000000000000001 /* MediaPlaybackServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPlaybackServiceTests.swift; sourceTree = "<group>"; };0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SupportedFileExtensionsTests.swift; sourceTree = "<group>"; };- A87600000000000000000001 /* InferenceAPIControllerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InferenceAPIControllerTests.swift; sourceTree = "<group>"; };7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };@@ -172,7 +170,6 @@B52000000000000000000001 /* PrivateAIDictationTokenBudgetTests.swift */,C0DE63700000000000000001 /* WhisperLanguageSelectionTests.swift */,0B05F11E000000000000A001 /* SupportedFileExtensionsTests.swift */,- A87600000000000000000001 /* InferenceAPIControllerTests.swift */,);path = FluidDictationIntegrationTests;sourceTree = "<group>";@@ -347,7 +344,6 @@B51900000000000000000002 /* PrivateAIProviderPromptFormatTests.swift in Sources */,C0DE63700000000000000002 /* WhisperLanguageSelectionTests.swift in Sources */,0B05F11E000000000000A002 /* SupportedFileExtensionsTests.swift in Sources */,- A87600000000000000000002 /* InferenceAPIControllerTests.swift in Sources */,);runOnlyForDeploymentPostprocessing = 0;};diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swiftindex 14903cc8..3cb021aa 100644--- a/Sources/Fluid/Services/ASRService.swift+++ b/Sources/Fluid/Services/ASRService.swift@@ -2631,6 +2631,87 @@ final class ASRService: ObservableObject {}}+ func transcribeSamplesForAPI(_ inputSamples: [Float]) async throws -> ASRTranscriptionResult {+ var samples = inputSamples+ guard !samples.isEmpty else {+ return ASRTranscriptionResult(text: "", confidence: 0)+ }++ let minSamples = 16_000+ if samples.count < minSamples {+ samples.append(contentsOf: repeatElement(0.0, count: minSamples - samples.count))+ }++ try await self.ensureAsrReady()+ guard self.transcriptionProvider.isReady else {+ throw NSError(+ domain: "ASRService",+ code: -2,+ userInfo: [NSLocalizedDescriptionKey: "Transcription provider is not ready."]+ )+ }++ let result = try await transcriptionExecutor.run { [provider = self.transcriptionProvider] in+ try await provider.transcribeFinal(samples)+ }++ if !self.hasCompletedFirstTranscription {+ self.hasCompletedFirstTranscription = true+ self.isLoadingModel = false+ self.modelPreparationPhase = nil+ }++ let cleanedText = ASRService.applySpokenPunctuationFormatting(+ ASRService.applyCustomDictionary(ASRService.removeFillerWords(result.text))+ )+ self.recordWordBoostHitIfAny(transcribedText: cleanedText)+ return ASRTranscriptionResult(text: cleanedText, confidence: result.confidence)+ }++ func transcribeFileForAPI(_ fileURL: URL) async throws -> (result: ASRTranscriptionResult, sampleCount: Int) {+ guard FileManager.default.isReadableFile(atPath: fileURL.path) else {+ throw NSError(+ domain: "ASRService",+ code: -3,+ userInfo: [NSLocalizedDescriptionKey: "Audio file is not readable."]+ )+ }++ let estimatedSamples = try LocalAPIAudioDecoder.estimatedSampleCount(for: fileURL)++ try await self.ensureAsrReady()+ let provider = self.transcriptionProvider+ guard provider.isReady else {+ throw NSError(+ domain: "ASRService",+ code: -2,+ userInfo: [NSLocalizedDescriptionKey: "Transcription provider is not ready."]+ )+ }++ guard provider.prefersNativeFileTranscription else {+ let samples = try LocalAPIAudioDecoder.samples(from: fileURL)+ let result = try await self.transcribeSamplesForAPI(samples)+ return (result, samples.count)+ }++ let result = try await transcriptionExecutor.run { [provider] in+ try await provider.transcribeFile(at: fileURL)+ }++ if !self.hasCompletedFirstTranscription {+ self.hasCompletedFirstTranscription = true+ self.isLoadingModel = false+ self.modelPreparationPhase = nil+ }++ let cleanedText = ASRService.applySpokenPunctuationFormatting(+ ASRService.applyCustomDictionary(ASRService.removeFillerWords(result.text))+ )+ self.recordWordBoostHitIfAny(transcribedText: cleanedText)+ return (ASRTranscriptionResult(text: cleanedText, confidence: result.confidence), estimatedSamples)+ }+func stopWithoutTranscription() async {if self.isStarting, self.isRunning == false {await self.cancelPendingAudioCaptureStart(reason: "stop_without_transcription")diff --git a/Sources/Fluid/Services/LocalAPI/InferenceAPIController.swift b/Sources/Fluid/Services/LocalAPI/InferenceAPIController.swiftindex 3841ce04..c74a9f5f 100644--- a/Sources/Fluid/Services/LocalAPI/InferenceAPIController.swift+++ b/Sources/Fluid/Services/LocalAPI/InferenceAPIController.swift@@ -2,8 +2,6 @@ import Foundation@MainActorfinal class InferenceAPIController: LocalAPIRouteHandler {- typealias FileTranscriptionOperation = @MainActor (URL) async throws -> TranscriptionResult-struct TranscribeJSONRequest: Decodable {let path: String?let audioBase64: String?@@ -27,15 +25,6 @@ final class InferenceAPIController: LocalAPIRouteHandler {let model: String}- private let fileTranscriptionOperation: FileTranscriptionOperation-- init(fileTranscriptionOperation: FileTranscriptionOperation? = nil) {- self.fileTranscriptionOperation = fileTranscriptionOperation ?? { fileURL in- let service = MeetingTranscriptionService(asrService: AppServices.shared.asr)- return try await service.transcribeFile(fileURL)- }- }-func handle(_ request: LocalAPI.Request) async -> LocalAPI.Response {guard request.method == "POST" else {return LocalAPI.error("Method not allowed.", status: 405)@@ -54,42 +43,32 @@ final class InferenceAPIController: LocalAPIRouteHandler {private func transcribe(_ request: LocalAPI.Request) async -> LocalAPI.Response {do {if let fileURL = try self.decodeFilePath(from: request) {- return try await self.transcribeFile(fileURL)+ let apiResult = try await AppServices.shared.asr.transcribeFileForAPI(fileURL)+ return LocalAPI.json(+ TranscribeResponse(+ text: apiResult.result.text,+ confidence: apiResult.result.confidence,+ sampleCount: apiResult.sampleCount,+ provider: SettingsStore.shared.selectedSpeechModel.displayName+ )+ )}- let temporaryFile = try await self.writeUploadedAudioToTemporaryFile(from: request)- do {- let response = try await self.transcribeFile(temporaryFile.fileURL)- await Self.removeTemporaryAudioFile(at: temporaryFile.cleanupURL)- return response- } catch {- await Self.removeTemporaryAudioFile(at: temporaryFile.cleanupURL)- throw error- }+ let samples = try self.decodeAudioSamples(from: request)+ let result = try await AppServices.shared.asr.transcribeSamplesForAPI(samples)+ return LocalAPI.json(+ TranscribeResponse(+ text: result.text,+ confidence: result.confidence,+ sampleCount: samples.count,+ provider: SettingsStore.shared.selectedSpeechModel.displayName+ )+ )} catch {return LocalAPI.error(error.localizedDescription, status: 400)}}- private func transcribeFile(_ fileURL: URL) async throws -> LocalAPI.Response {- let result = try await self.fileTranscriptionOperation(fileURL)- return LocalAPI.json(- TranscribeResponse(- text: result.text,- confidence: result.confidence,- sampleCount: Self.sampleCount(forDuration: result.duration),- provider: SettingsStore.shared.selectedSpeechModel.displayName- )- )- }-- static func sampleCount(forDuration duration: TimeInterval) -> Int {- guard duration.isFinite, duration > 0 else { return 0 }- let estimatedCount = duration * 16_000- guard estimatedCount < Double(Int.max) else { return Int.max }- return Int(estimatedCount.rounded())- }-private func decodeFilePath(from request: LocalAPI.Request) throws -> URL? {guard self.isJSON(request) else { return nil }let payload: TranscribeJSONRequest@@ -119,14 +98,7 @@ final class InferenceAPIController: LocalAPIRouteHandler {}}- private nonisolated struct TemporaryAudioFile: Sendable {- let fileURL: URL- let cleanupURL: URL- }-- private func writeUploadedAudioToTemporaryFile(from request: LocalAPI.Request) async throws -> TemporaryAudioFile {- let data: Data- let filename: String+ private func decodeAudioSamples(from request: LocalAPI.Request) throws -> [Float] {if self.isJSON(request) {let payload: TranscribeJSONRequestdo {@@ -135,43 +107,31 @@ final class InferenceAPIController: LocalAPIRouteHandler {throw NSError(domain: "InferenceAPIController", code: -3, userInfo: [NSLocalizedDescriptionKey: "Invalid JSON audio payload."])}+ if let path = payload.path, !path.isEmpty {+ return try LocalAPIAudioDecoder.samples(from: URL(fileURLWithPath: path))+ }+if let audioBase64 = payload.audioBase64,- let decodedData = Data(base64Encoded: audioBase64)+ let data = Data(base64Encoded: audioBase64){- data = decodedData- filename = payload.filename ?? "audio.wav"- } else {- throw NSError(domain: "InferenceAPIController", code: -1, userInfo: [NSLocalizedDescriptionKey: "Missing audio path or audioBase64."])- }- } else {- guard !request.body.isEmpty else {- throw NSError(domain: "InferenceAPIController", code: -1, userInfo: [NSLocalizedDescriptionKey: "Missing audio body."])+ return try LocalAPIAudioDecoder.samples(+ fromAudioData: data,+ suggestedExtension: payload.filename.flatMap { URL(fileURLWithPath: $0).pathExtension } ?? "wav"+ )}- data = request.body- filename = request.headers["x-filename"] ?? "audio.wav"++ throw NSError(domain: "InferenceAPIController", code: -1, userInfo: [NSLocalizedDescriptionKey: "Missing audio path or audioBase64."])}- return try await Task.detached(priority: .utility) {- let suppliedName = URL(fileURLWithPath: filename).lastPathComponent- let safeName = suppliedName.isEmpty || suppliedName == "." || suppliedName == ".." ? "audio.wav" : suppliedName- let temporaryDirectoryURL = FileManager.default.temporaryDirectory- .appendingPathComponent("fluidvoice-api-\(UUID().uuidString)", isDirectory: true)- try FileManager.default.createDirectory(at: temporaryDirectoryURL, withIntermediateDirectories: false)- let temporaryFileURL = temporaryDirectoryURL.appendingPathComponent(safeName)- do {- try data.write(to: temporaryFileURL, options: .atomic)- } catch {- try? FileManager.default.removeItem(at: temporaryDirectoryURL)- throw error- }- return TemporaryAudioFile(fileURL: temporaryFileURL, cleanupURL: temporaryDirectoryURL)- }.value- }+ guard !request.body.isEmpty else {+ throw NSError(domain: "InferenceAPIController", code: -1, userInfo: [NSLocalizedDescriptionKey: "Missing audio body."])+ }- private nonisolated static func removeTemporaryAudioFile(at cleanupURL: URL) async {- await Task.detached(priority: .utility) {- try? FileManager.default.removeItem(at: cleanupURL)- }.value+ let filename = request.headers["x-filename"] ?? "audio.wav"+ return try LocalAPIAudioDecoder.samples(+ fromAudioData: request.body,+ suggestedExtension: URL(fileURLWithPath: filename).pathExtension+ )}private func decodeText(from request: LocalAPI.Request) throws -> String {diff --git a/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift b/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swiftnew file mode 100644index 00000000..a553b9e0--- /dev/null+++ b/Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift@@ -0,0 +1,48 @@+import AVFoundation+import Foundation++enum LocalAPIAudioDecoder {+ static let sampleRate: Double = 16_000++ static func samples(from fileURL: URL) throws -> [Float] {+ let file = try AVAudioFile(forReading: fileURL)+ let sourceFormat = file.processingFormat+ let framesToRead = file.length+ guard framesToRead > 0 else { return [] }++ guard let sourceBuffer = AVAudioPCMBuffer(+ pcmFormat: sourceFormat,+ frameCapacity: AVAudioFrameCount(framesToRead)+ ) else {+ throw NSError(domain: "LocalAPIAudioDecoder", code: -1, userInfo: [NSLocalizedDescriptionKey: "Unable to allocate audio buffer."])+ }++ try file.read(into: sourceBuffer, frameCount: AVAudioFrameCount(framesToRead))+ return try AudioBufferConverter.monoSamples(+ from: sourceBuffer,+ targetSampleRate: self.sampleRate+ )+ }++ static func samples(fromAudioData data: Data, suggestedExtension: String) throws -> [Float] {+ let ext = suggestedExtension.trimmingCharacters(in: CharacterSet(charactersIn: ". \n\t")).isEmpty+ ? "wav"+ : suggestedExtension.trimmingCharacters(in: CharacterSet(charactersIn: ". \n\t"))+ let url = FileManager.default.temporaryDirectory+ .appendingPathComponent("fluidvoice-api-\(UUID().uuidString)")+ .appendingPathExtension(ext)+ try data.write(to: url, options: .atomic)+ defer { try? FileManager.default.removeItem(at: url) }+ return try self.samples(from: url)+ }++ static func estimatedSampleCount(for fileURL: URL) throws -> Int {+ let file = try AVAudioFile(forReading: fileURL)+ let sourceFormat = file.processingFormat+ guard sourceFormat.sampleRate > 0 else {+ throw NSError(domain: "LocalAPIAudioDecoder", code: -6, userInfo: [NSLocalizedDescriptionKey: "Audio file has an invalid sample rate."])+ }++ return Int((Double(file.length) * self.sampleRate / sourceFormat.sampleRate).rounded())+ }+}diff --git a/Sources/Fluid/Services/MeetingTranscriptionService.swift b/Sources/Fluid/Services/MeetingTranscriptionService.swiftindex b915e65d..d153935d 100644--- a/Sources/Fluid/Services/MeetingTranscriptionService.swift+++ b/Sources/Fluid/Services/MeetingTranscriptionService.swift@@ -4,36 +4,6 @@ import CoreMediaimport Foundationimport UniformTypeIdentifiers-@MainActor-final class FileTranscriptionGate {- private var isRunning = false- private var waiters: [CheckedContinuation<Void, Never>] = []-- func run<T>(_ operation: () async throws -> T) async rethrows -> T {- await self.acquire()- defer { self.release() }- return try await operation()- }-- private func acquire() async {- guard self.isRunning else {- self.isRunning = true- return- }- await withCheckedContinuation { continuation in- self.waiters.append(continuation)- }- }-- private func release() {- guard !self.waiters.isEmpty else {- self.isRunning = false- return- }- self.waiters.removeFirst().resume()- }-}-/// One speaker-attributed portion of a file transcription.nonisolated struct SpeakerTranscriptSegment: Identifiable, Sendable, Codable, Equatable {let speaker: String@@ -353,8 +323,6 @@ nonisolated struct TranscriptionResult: Identifiable, Sendable, Codable {/// NOTE: This service shares the ASR models with ASRService to avoid duplicate memory usage@MainActorfinal class MeetingTranscriptionService: ObservableObject {- private static let transcriptionGate = FileTranscriptionGate()-@Published var isTranscribing: Bool = false@Published var progress: Double = 0.0@Published var currentStatus: String = ""@@ -435,12 +403,6 @@ final class MeetingTranscriptionService: ObservableObject {/// - Parameters:/// - fileURL: URL to the audio/video filefunc transcribeFile(_ fileURL: URL) async throws -> TranscriptionResult {- try await Self.transcriptionGate.run {- try await self.transcribeFileSerially(fileURL)- }- }-- private func transcribeFileSerially(_ fileURL: URL) async throws -> TranscriptionResult {self.isTranscribing = trueerror = nilself.fallbackNotice = nildiff --git a/Tests/FluidDictationIntegrationTests/InferenceAPIControllerTests.swift b/Tests/FluidDictationIntegrationTests/InferenceAPIControllerTests.swiftdeleted file mode 100644index 38952862..00000000--- a/Tests/FluidDictationIntegrationTests/InferenceAPIControllerTests.swift+++ /dev/null@@ -1,125 +0,0 @@-@testable import FluidVoice_Debug-import Foundation-import XCTest--@MainActor-final class InferenceAPIControllerTests: XCTestCase {- func testLongPathRequestUsesMeetingFileTranscription() async throws {- let expectedURL = FileManager.default.temporaryDirectory- .appendingPathComponent("one-hour-meeting-\(UUID().uuidString).wav")- try Data([0, 1, 2, 3]).write(to: expectedURL)- defer { try? FileManager.default.removeItem(at: expectedURL) }- var receivedURL: URL?- let controller = InferenceAPIController { fileURL in- receivedURL = fileURL- return Self.result(duration: 60 * 60, fileName: fileURL.lastPathComponent)- }- let body = try JSONSerialization.data(withJSONObject: ["path": expectedURL.path])- let request = LocalAPI.Request(- method: "POST",- path: "/v1/transcribe",- query: [:],- headers: ["content-type": "application/json"],- body: body- )-- let response = await controller.handle(request)-- XCTAssertEqual(response.status, 200)- XCTAssertEqual(receivedURL, expectedURL)- XCTAssertTrue(FileManager.default.fileExists(atPath: expectedURL.path))- XCTAssertEqual(try Self.responseJSON(response)["text"] as? String, "Test transcript")- XCTAssertEqual(try Self.responseJSON(response)["sampleCount"] as? Int, 57_600_000)- }-- func testRawUploadPreservesFilenameAndCleansUpTemporaryFile() async throws {- var receivedURL: URL?- var existedDuringTranscription = false- let controller = InferenceAPIController { fileURL in- receivedURL = fileURL- existedDuringTranscription = FileManager.default.fileExists(atPath: fileURL.path)- return Self.result(duration: 1, fileName: fileURL.lastPathComponent)- }- let request = LocalAPI.Request(- method: "POST",- path: "/v1/transcribe",- query: [:],- headers: ["x-filename": "meeting.m4a"],- body: Data([0, 1, 2, 3])- )-- let response = await controller.handle(request)-- XCTAssertEqual(response.status, 200)- XCTAssertEqual(receivedURL?.lastPathComponent, "meeting.m4a")- XCTAssertTrue(existedDuringTranscription)- XCTAssertFalse(receivedURL.map { FileManager.default.fileExists(atPath: $0.path) } ?? true)- }-- func testFailedRawUploadStillCleansUpTemporaryFile() async {- var receivedURL: URL?- let controller = InferenceAPIController { fileURL in- receivedURL = fileURL- throw NSError(domain: "InferenceAPIControllerTests", code: 1)- }- let request = LocalAPI.Request(- method: "POST",- path: "/v1/transcribe",- query: [:],- headers: ["x-filename": "meeting.wav"],- body: Data([0, 1, 2, 3])- )-- let response = await controller.handle(request)-- XCTAssertEqual(response.status, 400)- XCTAssertFalse(receivedURL.map { FileManager.default.fileExists(atPath: $0.path) } ?? true)- }-- func testHTTPBodySafetyLimitRemainsBounded() {- XCTAssertEqual(LocalAPI.maxRequestBytes, 500 * 1024 * 1024)- }-- func testInvalidDurationsDoNotOverflowSampleCount() {- XCTAssertEqual(InferenceAPIController.sampleCount(forDuration: .nan), 0)- XCTAssertEqual(InferenceAPIController.sampleCount(forDuration: -.infinity), 0)- XCTAssertEqual(InferenceAPIController.sampleCount(forDuration: -1), 0)- XCTAssertEqual(InferenceAPIController.sampleCount(forDuration: .greatestFiniteMagnitude), Int.max)- }-- func testFileTranscriptionGateSerializesConcurrentCallers() async {- let gate = FileTranscriptionGate()- var activeOperations = 0- var maximumActiveOperations = 0-- async let first: Void = gate.run {- activeOperations += 1- maximumActiveOperations = max(maximumActiveOperations, activeOperations)- await Task.yield()- activeOperations -= 1- }- async let second: Void = gate.run {- activeOperations += 1- maximumActiveOperations = max(maximumActiveOperations, activeOperations)- await Task.yield()- activeOperations -= 1- }-- _ = await (first, second)- XCTAssertEqual(maximumActiveOperations, 1)- }-- private static func result(duration: TimeInterval, fileName: String) -> TranscriptionResult {- TranscriptionResult(- text: "Test transcript",- confidence: 1,- duration: duration,- processingTime: 0.1,- fileName: fileName- )- }-- private static func responseJSON(_ response: LocalAPI.Response) throws -> [String: Any] {- try XCTUnwrap(JSONSerialization.jsonObject(with: response.body) as? [String: Any])- }-}