Skip to content

Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.

test(ios): cover dictating with a provider from record to transcript

leepokaie5e56faeb271
diff --git a/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swift b/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swift
index 8dedf68b..3e9f375f 100644
--- a/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swift
+++ b/apps/ios/Kit/Sources/CodyncUI/Call/CloudSpeechEngine.swift
@@ -57,12 +57,12 @@ final class CloudSpeechEngine: VoiceEngine {
private var speakGeneration = 0
private static let threshold: Float = 0.3
- private static let prerollBytes = 16000 * 2 * 3 / 10
+ private static let prerollBytes = MicTap.bytesPerSecond * 3 / 10
/// An utterance has to fit one channel message once base64-encoded.
- private static let maxBytes = 16000 * 2 * 20
+ private static let maxBytes = MicTap.bytesPerSecond * 20
/// Under 0.4 s of voice is a cough or noise, not something to send (and speech-to-text models
/// invent words for noise).
- private static let minVoicedBytes = 16000 * 2 * 4 / 10
+ private static let minVoicedBytes = MicTap.bytesPerSecond * 4 / 10
private var endOfUtterance: TimeInterval {
let seconds = UserDefaults.standard.double(forKey: OnDeviceEngine.pauseKey)
diff --git a/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swift b/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swift
index aefc1c1a..f0457470 100644
--- a/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swift
+++ b/apps/ios/Kit/Sources/CodyncUI/Call/VoiceEngine.swift
@@ -139,6 +139,9 @@ enum VoiceSettings {
}
enum MicTap {
+ /// Bytes per second of what `pcm16k` hands out: 16 kHz mono PCM16.
+ static let bytesPerSecond = 16000 * 2
+
struct Unavailable: LocalizedError {
var errorDescription: String? { "no microphone is available right now" }
}
diff --git a/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift b/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift
index de64dbd9..5aacf058 100644
--- a/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift
+++ b/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift
@@ -3,6 +3,36 @@ import CodyncKit
import Speech
import SwiftUI
+/// What dictating with a provider needs from outside `Dictation`: the provider picked beside the
+/// mic, whether the computer has the user's key for it, the microphone, and a recording's transcript.
+struct DictationService {
+ var picked: @MainActor () -> VoiceProvider?
+ var hasKey: @MainActor (VoiceProvider, BotStore) async throws -> Bool
+ /// Whether the user lets the app record.
+ var allowMic: @MainActor () async -> Bool
+ /// Starts `engine`'s microphone, handing `heard` 16 kHz mono PCM16 and its level, 0…1; false when it can't be used.
+ var openMic: @MainActor (_ engine: AVAudioEngine, _ heard: @escaping @Sendable (Data, Float) -> Void) -> Bool
+ var transcribe: @MainActor (Data, VoiceProvider, BotStore) async throws -> String
+
+ /// The user's setting, this phone's microphone, and their computer on their own key.
+ static var live: DictationService {
+ DictationService(
+ picked: { VoiceSettings.dictationProvider },
+ hasKey: { provider, model in
+ let status = try await model.ready().voiceStatus()
+ VoiceSettings.remember(status)
+ return status.configured(provider)
+ },
+ allowMic: { await AVAudioApplication.requestRecordPermission() },
+ openMic: { engine, heard in Dictation.openMic(engine, bufferSize: 2048) { MicTap.pcm16k(from: $0, out: heard) } },
+ transcribe: { pcm, provider, model in
+ let said = try await Dictation.transcribe(pcm, provider: provider, on: model)
+ VoiceUsage.add(seconds: pcm.count / MicTap.bytesPerSecond, for: provider)
+ return said
+ })
+ }
+}
+
/// Dictation into the message box. With on-device speech the words appear as they're recognized,
/// after what was already typed, and the recording pill's stop, or the recognizer finishing on a
/// pause, stops it; it uses the call's language setting. With a provider picked beside the mic, it
@@ -33,12 +63,17 @@ final class Dictation {
/// Bumped per start and stop, so a start still waiting (key check, permission) or a late
/// result from an earlier recognition is dropped.
@ObservationIgnored private var generation = 0
+ @ObservationIgnored private let service: DictationService
+
+ init(service: DictationService = .live) {
+ self.service = service
+ }
/// A transcription request has to fit one channel message once base64-encoded: 20 s of audio
/// (20.5 s with a short tail still fits).
- static let pieceBytes = 16000 * 2 * 20
+ static let pieceBytes = MicTap.bytesPerSecond * 20
/// Under half a second is a tap of the mic, not speech (and speech-to-text models invent words for silence).
- static let minBytes = 16000 * 2 / 2
+ static let minBytes = MicTap.bytesPerSecond / 2
/// Transcription requests in flight at once, under the channel's and the providers' limits.
private static let inFlight = 3
@@ -56,7 +91,7 @@ final class Dictation {
failure = nil
generation += 1
before = draft
- if let provider = VoiceSettings.dictationProvider {
+ if let provider = service.picked() {
await record(provider, model: model, generation: generation, edit: edit)
} else {
await recognize(generation: generation, edit: edit)
@@ -73,8 +108,7 @@ final class Dictation {
transcribing = true
transcription = Task {
do {
- let said = try await Self.transcribe(recording.pcm, provider: recording.provider, on: recording.model)
- VoiceUsage.add(seconds: recording.pcm.count / (16000 * 2), for: recording.provider)
+ let said = try await service.transcribe(recording.pcm, recording.provider, recording.model)
guard generation == self.generation else { return }
edit? { Self.join($0, said) }
} catch {
@@ -113,7 +147,7 @@ final class Dictation {
request.shouldReportPartialResults = true
request.addsPunctuation = true
if recognizer.supportsOnDeviceRecognition { request.requiresOnDeviceRecognition = true }
- guard openMic(bufferSize: 1024, tap: { _ in OnDeviceEngine.tap(request) { [weak self] level in
+ guard Self.openMic(engine, bufferSize: 1024, tap: { _ in OnDeviceEngine.tap(request) { [weak self] level in
Task { @MainActor in if self?.listening == true { self?.level = level } }
} }) else { return }
self.edit = edit
@@ -126,10 +160,9 @@ final class Dictation {
/// Records for `provider` once the computer has its key, so nothing said is lost to a missing one.
private func record(_ provider: VoiceProvider, model: BotStore, generation: Int, edit: @escaping ((String) -> String) -> Void) async {
do {
- let status = try await model.ready().voiceStatus()
- VoiceSettings.remember(status)
+ let hasKey = try await service.hasKey(provider, model)
guard generation == self.generation else { return }
- guard status.configured(provider) else {
+ guard hasKey else {
failure = "Add your \(provider.name) key in Voice chat settings to dictate with it."
return
}
@@ -137,15 +170,15 @@ final class Dictation {
if generation == self.generation { failure = error.localizedDescription }
return
}
- let allowed = await AVAudioApplication.requestRecordPermission()
+ let allowed = await service.allowMic()
guard generation == self.generation else { return }
guard allowed else {
failure = "Allow microphone access in Settings to dictate."
return
}
- guard openMic(bufferSize: 2048, tap: { format in MicTap.pcm16k(from: format) { [weak self] pcm, level in
+ guard service.openMic(engine, { [weak self] pcm, level in
Task { @MainActor in self?.recorded(pcm, level: level, generation: generation) }
- } }) else {
+ }) else {
failure = "The microphone isn't available."
return
}
@@ -162,8 +195,8 @@ final class Dictation {
}
/// The call's audio session (a recording tap on a `.record` session fails on some routes) and
- /// `tap` on the microphone. False, with everything let go, when the microphone can't be used.
- private func openMic(bufferSize: AVAudioFrameCount, tap: (AVAudioFormat) -> AVAudioNodeTapBlock) -> Bool {
+ /// `tap` on `engine`'s microphone. False, with everything let go, when the microphone can't be used.
+ fileprivate static func openMic(_ engine: AVAudioEngine, bufferSize: AVAudioFrameCount, tap: (AVAudioFormat) -> AVAudioNodeTapBlock) -> Bool {
let audio = AVAudioSession.sharedInstance()
do {
try audio.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetoothHFP, .duckOthers])
@@ -191,7 +224,7 @@ final class Dictation {
}
/// The recording in pieces, a few transcribed at a time and joined in order.
- private static func transcribe(_ pcm: Data, provider: VoiceProvider, on store: BotStore) async throws -> String {
+ fileprivate static func transcribe(_ pcm: Data, provider: VoiceProvider, on store: BotStore) async throws -> String {
// Asked for now, not at the start: a reconnect while recording replaces the client.
let client = try await store.ready()
let model = VoiceSettings.transcribeModel(provider)
diff --git a/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swift b/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swift
index b56c1fb8..4e828c2c 100644
--- a/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swift
+++ b/apps/ios/Kit/Tests/CodyncUITests/DictationTests.swift
@@ -1,4 +1,6 @@
+import Foundation
import Testing
+@testable import CodyncKit
@testable import CodyncUI
/// A provider's recording goes out in 20 s pieces that cover every byte in order; a tail under
@@ -18,3 +20,186 @@ import Testing
#expect(Dictation.join("Fix the bug\n", "now") == "Fix the bug\nnow")
#expect(Dictation.join("Fix the bug", "") == "Fix the bug")
}
+
+/// A provider's network call that the test answers when it chooses.
+@MainActor private final class Pending<Value: Sendable> {
+ private var waiting: CheckedContinuation<Value, any Error>?
+ private(set) var calls = 0
+ /// Calls that have had their answer.
+ private(set) var returned = 0
+
+ func call() async throws -> Value {
+ calls += 1
+ defer { returned += 1 }
+ return try await withCheckedThrowingContinuation { waiting = $0 }
+ }
+
+ /// Whether the call is waiting for its answer (2 s at most).
+ func asked() async -> Bool {
+ await until { self.waiting != nil }
+ }
+
+ func answer(_ value: Value) {
+ waiting?.resume(returning: value)
+ waiting = nil
+ }
+
+ func fail(_ error: any Error) {
+ waiting?.resume(throwing: error)
+ waiting = nil
+ }
+}
+
+/// The phone's microphone, hearing what the test plays it.
+@MainActor private final class Mic {
+ /// Where the open microphone's audio goes; nil until it's opened.
+ var heard: (@Sendable (Data, Float) -> Void)?
+}
+
+private struct Offline: LocalizedError {
+ var errorDescription: String? { "The computer is offline." }
+}
+
+/// Dictation with OpenAI picked, the computer's answers and the microphone in the test's hands, and a store to dictate through.
+@MainActor private func dictating(hasKey: @escaping @MainActor () async throws -> Bool = { true },
+ transcript: Pending<String> = Pending(), mic: Mic = Mic(),
+ _ body: (Dictation, BotStore) async -> Void) async {
+ let suite = "DictationTests.\(UUID().uuidString)"
+ let storage = SharedStore.Context(accountID: UUID().uuidString, suite: suite)
+ defer { UserDefaults(suiteName: suite)?.removePersistentDomain(forName: suite) }
+ let store = BotStore(computer: Computer(id: "dictation", name: "Mac", signKey: ""), storage: storage) { _ in
+ throw HostError.unreachable
+ }
+ defer { store.retire() }
+ let service = DictationService(picked: { .openAI },
+ hasKey: { _, _ in try await hasKey() },
+ allowMic: { true },
+ openMic: { _, heard in
+ mic.heard = heard
+ return true
+ },
+ transcribe: { _, _, _ in try await transcript.call() })
+ await body(Dictation(service: service), store)
+}
+
+private extension Dictation {
+ /// Records `bytes` of what `mic` hears and taps the pill's stop; the transcript changes the box through `edit`.
+ func dictate(_ bytes: Int, from mic: Mic, on store: BotStore, edit: @escaping ((String) -> String) -> Void) async {
+ await start(after: "", model: store, edit: edit)
+ mic.heard?(Data(count: bytes), 0.5)
+ #expect(await until { self.level > 0 })
+ finish()
+ }
+}
+
+/// The pill's stop has the recording transcribed, and the words go after what's in the box when
+/// they arrive, words typed while waiting included.
+@MainActor @Test func aRecordingIsTranscribedAfterWhatTheBoxHoldsThen() async {
+ let transcript = Pending<String>(), mic = Mic()
+ await dictating(transcript: transcript, mic: mic) { dictation, store in
+ var box = "Fix the bug"
+ await dictation.dictate(Dictation.minBytes, from: mic, on: store) { box = $0(box) }
+ #expect(dictation.transcribing && !dictation.listening)
+ #expect(await transcript.asked())
+ box += " now"
+ transcript.answer("in the parser")
+ #expect(await until { !dictation.transcribing })
+ #expect(box == "Fix the bug now in the parser")
+ #expect(dictation.failure == nil)
+ }
+}
+
+/// A stop (a send, another chat) while the transcript is on its way drops it when it comes.
+@MainActor @Test func aTranscriptArrivingAfterAStopIsDropped() async {
+ let transcript = Pending<String>(), mic = Mic()
+ await dictating(transcript: transcript, mic: mic) { dictation, store in
+ var box = "Fix the bug"
+ await dictation.dictate(Dictation.minBytes, from: mic, on: store) { box = $0(box) }
+ #expect(await transcript.asked())
+ dictation.stop()
+ #expect(!dictation.transcribing)
+ transcript.answer("too late")
+ #expect(await until { transcript.returned == 1 })
+ #expect(box == "Fix the bug")
+ #expect(dictation.failure == nil && !dictation.transcribing)
+ }
+}
+
+/// A failed transcription says why and leaves the box as it was.
+@MainActor @Test func aFailedTranscriptionSaysWhy() async {
+ let transcript = Pending<String>(), mic = Mic()
+ await dictating(transcript: transcript, mic: mic) { dictation, store in
+ var box = "Fix the bug"
+ await dictation.dictate(Dictation.minBytes, from: mic, on: store) { box = $0(box) }
+ #expect(await transcript.asked())
+ transcript.fail(Offline())
+ #expect(await until { !dictation.transcribing })
+ #expect(dictation.failure == "The computer is offline.")
+ #expect(box == "Fix the bug")
+ }
+}
+
+/// Under half a second is a tap of the mic: nothing goes out on the user's key.
+@MainActor @Test func aTapOfTheMicSendsNothing() async {
+ let transcript = Pending<String>(), mic = Mic()
+ await dictating(transcript: transcript, mic: mic) { dictation, store in
+ await dictation.dictate(Dictation.minBytes - 1, from: mic, on: store) { _ in }
+ #expect(!dictation.transcribing)
+ #expect(transcript.calls == 0)
+ }
+}
+
+/// What the microphone heard before a stop never joins the next recording.
+@MainActor @Test func audioHeardBeforeAStopIsDropped() async {
+ let mic = Mic()
+ await dictating(mic: mic) { dictation, store in
+ await dictation.start(after: "", model: store) { _ in }
+ let earlier = mic.heard
+ dictation.stop()
+ await dictation.start(after: "", model: store) { _ in }
+ earlier?(Data(count: Dictation.minBytes), 0.9)
+ mic.heard?(Data(), 0.5)
+ #expect(await until { dictation.level == 0.5 })
+ dictation.finish()
+ #expect(!dictation.transcribing)
+ }
+}
+
+/// Without the provider's key on the computer it says where to add one, and never opens the microphone.
+@MainActor @Test func dictatingWithoutAKeySaysWhereToAddOne() async {
+ let key = Pending<Bool>(), mic = Mic()
+ await dictating(hasKey: { try await key.call() }, mic: mic) { dictation, store in
+ let starting = Task { await dictation.start(after: "", model: store) { _ in } }
+ #expect(await key.asked())
+ key.answer(false)
+ await starting.value
+ #expect(dictation.failure == "Add your OpenAI key in Voice chat settings to dictate with it.")
+ #expect(!dictation.listening && mic.heard == nil)
+ }
+}
+
+/// With the computer out of reach the key check says why, and no recording starts.
+@MainActor @Test func anUnreachableComputerSaysWhy() async {
+ let key = Pending<Bool>(), mic = Mic()
+ await dictating(hasKey: { try await key.call() }, mic: mic) { dictation, store in
+ let starting = Task { await dictation.start(after: "", model: store) { _ in } }
+ #expect(await key.asked())
+ key.fail(Offline())
+ await starting.value
+ #expect(dictation.failure == "The computer is offline.")
+ #expect(!dictation.listening && mic.heard == nil)
+ }
+}
+
+/// A stop while the key check is on its way wins, whatever the check says: no recording starts and nothing is reported.
+@MainActor @Test(arguments: [true, false]) func aStopWhileTheKeyIsCheckedWins(checkAnswers: Bool) async {
+ let key = Pending<Bool>(), mic = Mic()
+ await dictating(hasKey: { try await key.call() }, mic: mic) { dictation, store in
+ let starting = Task { await dictation.start(after: "", model: store) { _ in } }
+ #expect(await key.asked())
+ dictation.stop()
+ if checkAnswers { key.answer(true) } else { key.fail(Offline()) }
+ await starting.value
+ #expect(!dictation.listening && mic.heard == nil && dictation.failure == nil)
+ }
+}
diff --git a/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swift b/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swift
index 707d647e..3d8d84ae 100644
--- a/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swift
+++ b/apps/ios/Kit/Tests/CodyncUITests/ReplyTests.swift
@@ -35,22 +35,13 @@ private func message(_ id: String, kind: String, final: Bool? = nil, text: Strin
let store = BotStore(computer: Computer(id: "reply", name: "Mac", signKey: ""), storage: storage) { _ in fake }
defer { store.retire() }
store.setActive(true)
- #expect(await eventually { await fake.subscribed })
+ #expect(await until { await fake.subscribed })
await fake.emit(#"{"type":"bot","bot":{"id":"b1","name":"Bot","rev":1,"lastAt":1}}"#)
- #expect(await eventually { store.connection == .online })
+ #expect(await until { store.connection == .online })
store.setComposerDraft("Why?", for: "b1")
#expect(store.sendComposerDraft(to: "b1", replyTo: message("a", kind: "agent", final: true)))
#expect(store.chat("b1").last?.data.replyTo?.id == "a")
#expect(store.composerDraft(for: "b1").isEmpty)
- #expect(await eventually { await fake.replyTos == ["a"] })
-}
-
-/// Polls until `condition` holds (2 s at most).
-@MainActor private func eventually(_ condition: @MainActor () async -> Bool) async -> Bool {
- for _ in 0..<200 {
- if await condition() { return true }
- try? await Task.sleep(for: .milliseconds(10))
- }
- return false
+ #expect(await until { await fake.replyTos == ["a"] })
}
diff --git a/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swift b/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swift
index e65bd3ca..065efb82 100644
--- a/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swift
+++ b/apps/ios/Kit/Tests/CodyncUITests/StoreTests.swift
@@ -135,7 +135,7 @@ private func context() -> (SharedStore.Context, String) {
/// Polls until `condition` holds (2 s at most).
@MainActor
-private func until(_ condition: @MainActor () async -> Bool) async -> Bool {
+func until(_ condition: @MainActor () async -> Bool) async -> Bool {
for _ in 0..<200 {
if await condition() { return true }
try? await Task.sleep(for: .milliseconds(10))