Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.
Rust acpagent-client-protocolai-agentsai-assistantclaudeclaude-codecodexcoding-agentscursorgeminigemini-cligrok-bot-alternativeiosiphonelinuxmacosmcpopen-sourcerustself-hosted
test(ios): cover dictating with a provider from record to transcript
leepokaie5e56faeb271
diff --git a/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swift b/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swiftindex 8dedf68b..3e9f375f 100644--- a/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swift+++ b/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swift@@ -57,12 +57,12 @@ final class CloudSpeechEngine: VoiceEngine {private var speakGeneration = 0private static let threshold: Float = 0.3- private static let prerollBytes = 16000 * 2 * 3 / 10+ private static let prerollBytes = MicTap.bytesPerSecond * 3 / 10/// An utterance has to fit one channel message once base64-encoded.- private static let maxBytes = 16000 * 2 * 20+ private static let maxBytes = MicTap.bytesPerSecond * 20/// Under 0.4 s of voice is a cough or noise, not something to send (and speech-to-text models/// invent words for noise).- private static let minVoicedBytes = 16000 * 2 * 4 / 10+ private static let minVoicedBytes = MicTap.bytesPerSecond * 4 / 10private var endOfUtterance: TimeInterval {let seconds = UserDefaults.standard.double(forKey: OnDeviceEngine.pauseKey)diff --git a/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swift b/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swiftindex aefc1c1a..f0457470 100644--- a/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swift+++ b/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swift@@ -139,6 +139,9 @@ enum VoiceSettings {}enum MicTap {+ /// Bytes per second of what `pcm16k` hands out: 16 kHz mono PCM16.+ static let bytesPerSecond = 16000 * 2+struct Unavailable: LocalizedError {var errorDescription: String? { "no microphone is available right now" }}diff --git a/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift b/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swiftindex de64dbd9..5aacf058 100644--- a/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift+++ b/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift@@ -3,6 +3,36 @@ import CodyncKitimport Speechimport SwiftUI+/// What dictating with a provider needs from outside `Dictation`: the provider picked beside the+/// mic, whether the computer has the user's key for it, the microphone, and a recording's transcript.+struct DictationService {+ var picked: @MainActor () -> VoiceProvider?+ var hasKey: @MainActor (VoiceProvider, BotStore) async throws -> Bool+ /// Whether the user lets the app record.+ var allowMic: @MainActor () async -> Bool+ /// Starts `engine`'s microphone, handing `heard` 16 kHz mono PCM16 and its level, 0…1; false when it can't be used.+ var openMic: @MainActor (_ engine: AVAudioEngine, _ heard: @escaping @Sendable (Data, Float) -> Void) -> Bool+ var transcribe: @MainActor (Data, VoiceProvider, BotStore) async throws -> String++ /// The user's setting, this phone's microphone, and their computer on their own key.+ static var live: DictationService {+ DictationService(+ picked: { VoiceSettings.dictationProvider },+ hasKey: { provider, model in+ let status = try await model.ready().voiceStatus()+ VoiceSettings.remember(status)+ return status.configured(provider)+ },+ allowMic: { await AVAudioApplication.requestRecordPermission() },+ openMic: { engine, heard in Dictation.openMic(engine, bufferSize: 2048) { MicTap.pcm16k(from: $0, out: heard) } },+ transcribe: { pcm, provider, model in+ let said = try await Dictation.transcribe(pcm, provider: provider, on: model)+ VoiceUsage.add(seconds: pcm.count / MicTap.bytesPerSecond, for: provider)+ return said+ })+ }+}+/// Dictation into the message box. With on-device speech the words appear as they're recognized,/// after what was already typed, and the recording pill's stop, or the recognizer finishing on a/// pause, stops it; it uses the call's language setting. With a provider picked beside the mic, it@@ -33,12 +63,17 @@ final class Dictation {/// Bumped per start and stop, so a start still waiting (key check, permission) or a late/// result from an earlier recognition is dropped.@ObservationIgnored private var generation = 0+ @ObservationIgnored private let service: DictationService++ init(service: DictationService = .live) {+ self.service = service+ }/// A transcription request has to fit one channel message once base64-encoded: 20 s of audio/// (20.5 s with a short tail still fits).- static let pieceBytes = 16000 * 2 * 20+ static let pieceBytes = MicTap.bytesPerSecond * 20/// Under half a second is a tap of the mic, not speech (and speech-to-text models invent words for silence).- static let minBytes = 16000 * 2 / 2+ static let minBytes = MicTap.bytesPerSecond / 2/// Transcription requests in flight at once, under the channel's and the providers' limits.private static let inFlight = 3@@ -56,7 +91,7 @@ final class Dictation {failure = nilgeneration += 1before = draft- if let provider = VoiceSettings.dictationProvider {+ if let provider = service.picked() {await record(provider, model: model, generation: generation, edit: edit)} else {await recognize(generation: generation, edit: edit)@@ -73,8 +108,7 @@ final class Dictation {transcribing = truetranscription = Task {do {- let said = try await Self.transcribe(recording.pcm, provider: recording.provider, on: recording.model)- VoiceUsage.add(seconds: recording.pcm.count / (16000 * 2), for: recording.provider)+ let said = try await service.transcribe(recording.pcm, recording.provider, recording.model)guard generation == self.generation else { return }edit? { Self.join($0, said) }} catch {@@ -113,7 +147,7 @@ final class Dictation {request.shouldReportPartialResults = truerequest.addsPunctuation = trueif recognizer.supportsOnDeviceRecognition { request.requiresOnDeviceRecognition = true }- guard openMic(bufferSize: 1024, tap: { _ in OnDeviceEngine.tap(request) { [weak self] level in+ guard Self.openMic(engine, bufferSize: 1024, tap: { _ in OnDeviceEngine.tap(request) { [weak self] level inTask { @MainActor in if self?.listening == true { self?.level = level } }} }) else { return }self.edit = edit@@ -126,10 +160,9 @@ final class Dictation {/// Records for `provider` once the computer has its key, so nothing said is lost to a missing one.private func record(_ provider: VoiceProvider, model: BotStore, generation: Int, edit: @escaping ((String) -> String) -> Void) async {do {- let status = try await model.ready().voiceStatus()- VoiceSettings.remember(status)+ let hasKey = try await service.hasKey(provider, model)guard generation == self.generation else { return }- guard status.configured(provider) else {+ guard hasKey else {failure = "Add your \(provider.name) key in Voice chat settings to dictate with it."return}@@ -137,15 +170,15 @@ final class Dictation {if generation == self.generation { failure = error.localizedDescription }return}- let allowed = await AVAudioApplication.requestRecordPermission()+ let allowed = await service.allowMic()guard generation == self.generation else { return }guard allowed else {failure = "Allow microphone access in Settings to dictate."return}- guard openMic(bufferSize: 2048, tap: { format in MicTap.pcm16k(from: format) { [weak self] pcm, level in+ guard service.openMic(engine, { [weak self] pcm, level inTask { @MainActor in self?.recorded(pcm, level: level, generation: generation) }- } }) else {+ }) else {failure = "The microphone isn't available."return}@@ -162,8 +195,8 @@ final class Dictation {}/// The call's audio session (a recording tap on a `.record` session fails on some routes) and- /// `tap` on the microphone. False, with everything let go, when the microphone can't be used.- private func openMic(bufferSize: AVAudioFrameCount, tap: (AVAudioFormat) -> AVAudioNodeTapBlock) -> Bool {+ /// `tap` on `engine`'s microphone. False, with everything let go, when the microphone can't be used.+ fileprivate static func openMic(_ engine: AVAudioEngine, bufferSize: AVAudioFrameCount, tap: (AVAudioFormat) -> AVAudioNodeTapBlock) -> Bool {let audio = AVAudioSession.sharedInstance()do {try audio.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetoothHFP, .duckOthers])@@ -191,7 +224,7 @@ final class Dictation {}/// The recording in pieces, a few transcribed at a time and joined in order.- private static func transcribe(_ pcm: Data, provider: VoiceProvider, on store: BotStore) async throws -> String {+ fileprivate static func transcribe(_ pcm: Data, provider: VoiceProvider, on store: BotStore) async throws -> String {// Asked for now, not at the start: a reconnect while recording replaces the client.let client = try await store.ready()let model = VoiceSettings.transcribeModel(provider)diff --git a/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swift b/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swiftindex b56c1fb8..4e828c2c 100644--- a/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swift+++ b/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swift@@ -1,4 +1,6 @@+import Foundationimport Testing+@testable import CodyncKit@testable import CodyncUI/// A provider's recording goes out in 20 s pieces that cover every byte in order; a tail under@@ -18,3 +20,186 @@ import Testing#expect(Dictation.join("Fix the bug\n", "now") == "Fix the bug\nnow")#expect(Dictation.join("Fix the bug", "") == "Fix the bug")}++/// A provider's network call that the test answers when it chooses.+@MainActor private final class Pending<Value: Sendable> {+ private var waiting: CheckedContinuation<Value, any Error>?+ private(set) var calls = 0+ /// Calls that have had their answer.+ private(set) var returned = 0++ func call() async throws -> Value {+ calls += 1+ defer { returned += 1 }+ return try await withCheckedThrowingContinuation { waiting = $0 }+ }++ /// Whether the call is waiting for its answer (2 s at most).+ func asked() async -> Bool {+ await until { self.waiting != nil }+ }++ func answer(_ value: Value) {+ waiting?.resume(returning: value)+ waiting = nil+ }++ func fail(_ error: any Error) {+ waiting?.resume(throwing: error)+ waiting = nil+ }+}++/// The phone's microphone, hearing what the test plays it.+@MainActor private final class Mic {+ /// Where the open microphone's audio goes; nil until it's opened.+ var heard: (@Sendable (Data, Float) -> Void)?+}++private struct Offline: LocalizedError {+ var errorDescription: String? { "The computer is offline." }+}++/// Dictation with OpenAI picked, the computer's answers and the microphone in the test's hands, and a store to dictate through.+@MainActor private func dictating(hasKey: @escaping @MainActor () async throws -> Bool = { true },+ transcript: Pending<String> = Pending(), mic: Mic = Mic(),+ _ body: (Dictation, BotStore) async -> Void) async {+ let suite = "DictationTests.\(UUID().uuidString)"+ let storage = SharedStore.Context(accountID: UUID().uuidString, suite: suite)+ defer { UserDefaults(suiteName: suite)?.removePersistentDomain(forName: suite) }+ let store = BotStore(computer: Computer(id: "dictation", name: "Mac", signKey: ""), storage: storage) { _ in+ throw HostError.unreachable+ }+ defer { store.retire() }+ let service = DictationService(picked: { .openAI },+ hasKey: { _, _ in try await hasKey() },+ allowMic: { true },+ openMic: { _, heard in+ mic.heard = heard+ return true+ },+ transcribe: { _, _, _ in try await transcript.call() })+ await body(Dictation(service: service), store)+}++private extension Dictation {+ /// Records `bytes` of what `mic` hears and taps the pill's stop; the transcript changes the box through `edit`.+ func dictate(_ bytes: Int, from mic: Mic, on store: BotStore, edit: @escaping ((String) -> String) -> Void) async {+ await start(after: "", model: store, edit: edit)+ mic.heard?(Data(count: bytes), 0.5)+ #expect(await until { self.level > 0 })+ finish()+ }+}++/// The pill's stop has the recording transcribed, and the words go after what's in the box when+/// they arrive, words typed while waiting included.+@MainActor @Test func aRecordingIsTranscribedAfterWhatTheBoxHoldsThen() async {+ let transcript = Pending<String>(), mic = Mic()+ await dictating(transcript: transcript, mic: mic) { dictation, store in+ var box = "Fix the bug"+ await dictation.dictate(Dictation.minBytes, from: mic, on: store) { box = $0(box) }+ #expect(dictation.transcribing && !dictation.listening)+ #expect(await transcript.asked())+ box += " now"+ transcript.answer("in the parser")+ #expect(await until { !dictation.transcribing })+ #expect(box == "Fix the bug now in the parser")+ #expect(dictation.failure == nil)+ }+}++/// A stop (a send, another chat) while the transcript is on its way drops it when it comes.+@MainActor @Test func aTranscriptArrivingAfterAStopIsDropped() async {+ let transcript = Pending<String>(), mic = Mic()+ await dictating(transcript: transcript, mic: mic) { dictation, store in+ var box = "Fix the bug"+ await dictation.dictate(Dictation.minBytes, from: mic, on: store) { box = $0(box) }+ #expect(await transcript.asked())+ dictation.stop()+ #expect(!dictation.transcribing)+ transcript.answer("too late")+ #expect(await until { transcript.returned == 1 })+ #expect(box == "Fix the bug")+ #expect(dictation.failure == nil && !dictation.transcribing)+ }+}++/// A failed transcription says why and leaves the box as it was.+@MainActor @Test func aFailedTranscriptionSaysWhy() async {+ let transcript = Pending<String>(), mic = Mic()+ await dictating(transcript: transcript, mic: mic) { dictation, store in+ var box = "Fix the bug"+ await dictation.dictate(Dictation.minBytes, from: mic, on: store) { box = $0(box) }+ #expect(await transcript.asked())+ transcript.fail(Offline())+ #expect(await until { !dictation.transcribing })+ #expect(dictation.failure == "The computer is offline.")+ #expect(box == "Fix the bug")+ }+}++/// Under half a second is a tap of the mic: nothing goes out on the user's key.+@MainActor @Test func aTapOfTheMicSendsNothing() async {+ let transcript = Pending<String>(), mic = Mic()+ await dictating(transcript: transcript, mic: mic) { dictation, store in+ await dictation.dictate(Dictation.minBytes - 1, from: mic, on: store) { _ in }+ #expect(!dictation.transcribing)+ #expect(transcript.calls == 0)+ }+}++/// What the microphone heard before a stop never joins the next recording.+@MainActor @Test func audioHeardBeforeAStopIsDropped() async {+ let mic = Mic()+ await dictating(mic: mic) { dictation, store in+ await dictation.start(after: "", model: store) { _ in }+ let earlier = mic.heard+ dictation.stop()+ await dictation.start(after: "", model: store) { _ in }+ earlier?(Data(count: Dictation.minBytes), 0.9)+ mic.heard?(Data(), 0.5)+ #expect(await until { dictation.level == 0.5 })+ dictation.finish()+ #expect(!dictation.transcribing)+ }+}++/// Without the provider's key on the computer it says where to add one, and never opens the microphone.+@MainActor @Test func dictatingWithoutAKeySaysWhereToAddOne() async {+ let key = Pending<Bool>(), mic = Mic()+ await dictating(hasKey: { try await key.call() }, mic: mic) { dictation, store in+ let starting = Task { await dictation.start(after: "", model: store) { _ in } }+ #expect(await key.asked())+ key.answer(false)+ await starting.value+ #expect(dictation.failure == "Add your OpenAI key in Voice chat settings to dictate with it.")+ #expect(!dictation.listening && mic.heard == nil)+ }+}++/// With the computer out of reach the key check says why, and no recording starts.+@MainActor @Test func anUnreachableComputerSaysWhy() async {+ let key = Pending<Bool>(), mic = Mic()+ await dictating(hasKey: { try await key.call() }, mic: mic) { dictation, store in+ let starting = Task { await dictation.start(after: "", model: store) { _ in } }+ #expect(await key.asked())+ key.fail(Offline())+ await starting.value+ #expect(dictation.failure == "The computer is offline.")+ #expect(!dictation.listening && mic.heard == nil)+ }+}++/// A stop while the key check is on its way wins, whatever the check says: no recording starts and nothing is reported.+@MainActor @Test(arguments: [true, false]) func aStopWhileTheKeyIsCheckedWins(checkAnswers: Bool) async {+ let key = Pending<Bool>(), mic = Mic()+ await dictating(hasKey: { try await key.call() }, mic: mic) { dictation, store in+ let starting = Task { await dictation.start(after: "", model: store) { _ in } }+ #expect(await key.asked())+ dictation.stop()+ if checkAnswers { key.answer(true) } else { key.fail(Offline()) }+ await starting.value+ #expect(!dictation.listening && mic.heard == nil && dictation.failure == nil)+ }+}diff --git a/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swift b/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swiftindex 707d647e..3d8d84ae 100644--- a/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swift+++ b/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swift@@ -35,22 +35,13 @@ private func message(_ id: String, kind: String, final: Bool? = nil, text: Strinlet store = BotStore(computer: Computer(id: "reply", name: "Mac", signKey: ""), storage: storage) { _ in fake }defer { store.retire() }store.setActive(true)- #expect(await eventually { await fake.subscribed })+ #expect(await until { await fake.subscribed })await fake.emit(#"{"type":"bot","bot":{"id":"b1","name":"Bot","rev":1,"lastAt":1}}"#)- #expect(await eventually { store.connection == .online })+ #expect(await until { store.connection == .online })store.setComposerDraft("Why?", for: "b1")#expect(store.sendComposerDraft(to: "b1", replyTo: message("a", kind: "agent", final: true)))#expect(store.chat("b1").last?.data.replyTo?.id == "a")#expect(store.composerDraft(for: "b1").isEmpty)- #expect(await eventually { await fake.replyTos == ["a"] })-}--/// Polls until `condition` holds (2 s at most).-@MainActor private func eventually(_ condition: @MainActor () async -> Bool) async -> Bool {- for _ in 0..<200 {- if await condition() { return true }- try? await Task.sleep(for: .milliseconds(10))- }- return false+ #expect(await until { await fake.replyTos == ["a"] })}diff --git a/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swift b/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swiftindex e65bd3ca..065efb82 100644--- a/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swift+++ b/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swift@@ -135,7 +135,7 @@ private func context() -> (SharedStore.Context, String) {/// Polls until `condition` holds (2 s at most).@MainActor-private func until(_ condition: @MainActor () async -> Bool) async -> Bool {+func until(_ condition: @MainActor () async -> Bool) async -> Bool {for _ in 0..<200 {if await condition() { return true }try? await Task.sleep(for: .milliseconds(10))