Skip to content

Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.

feat: dictate into the message box with a mic beside send

leepokai6818a721412e
diff --git a/apps/desktop/src/renderer/views/call/OnDeviceEngine.ts b/apps/desktop/src/renderer/views/call/OnDeviceEngine.ts
index b7e5c34c..20bba939 100644
--- a/apps/desktop/src/renderer/views/call/OnDeviceEngine.ts
+++ b/apps/desktop/src/renderer/views/call/OnDeviceEngine.ts
@@ -1,4 +1,5 @@
import type { SpeechEvent } from '@shared/ipc'
+import { stopDictation } from '../thread/dictation'
import { onDeviceUnavailableReason, spokenText, VoiceEngine, VoiceSettings } from './voice'
/**
@@ -54,6 +55,8 @@ export class OnDeviceEngine extends VoiceEngine {
async start() {
if (!window.codync.speech.available) return this.fail(onDeviceUnavailableReason())
+ // The call takes the recognizer from any dictation in progress.
+ stopDictation()
speechSynthesis.getVoices()
this.offEvents = window.codync.speech.onEvent((e) => this.event(e))
this.listen()
diff --git a/apps/desktop/src/renderer/views/thread/Composer.tsx b/apps/desktop/src/renderer/views/thread/Composer.tsx
index 73a3c684..09436bcb 100644
--- a/apps/desktop/src/renderer/views/thread/Composer.tsx
+++ b/apps/desktop/src/renderer/views/thread/Composer.tsx
@@ -12,6 +12,7 @@ import { pluginsFor } from '../marketplace/plugins'
import { composerEnter } from './composer-keyboard'
import { CommandMenu, useCommandMenu } from './CommandMenu'
import { ComposerAttachment } from './ComposerAttachment'
+import { useDictation } from './dictation'
import { builtInCommand, SKILLS_HOST, slashCommands, slashQuery, type SlashCommand } from './slash-commands'
type Trailing = 'interrupt' | 'stop' | 'call' | 'send'
@@ -44,6 +45,7 @@ function DestinationComposer({ botId, thread, onCall, onInterrupt }: {
const draft = useSyncExternalStore(drafts.subscribe, () => drafts.get(botId, thread))
const setDraft = (text: string) => drafts.set(botId, thread, text)
const [files, setFiles] = useState<OutgoingFile[]>([])
+ const dictation = useDictation(draft, setDraft)
const field = useRef<HTMLTextAreaElement>(null)
const measure = useRef<HTMLDivElement>(null)
const composing = useRef(false)
@@ -245,6 +247,17 @@ function DestinationComposer({ botId, thread, onCall, onInterrupt }: {
}}
/>
</div>
+ {dictation.available ? (
+ <button
+ className={`press composer-mic ${dictation.listening ? 'on' : ''}`}
+ aria-label={dictation.listening ? 'Stop dictating' : 'Dictate'}
+ title={dictation.listening ? 'Stop dictating' : 'Dictate'}
+ aria-pressed={dictation.listening}
+ onClick={dictation.toggle}
+ >
+ <Icon name={dictation.listening ? 'mic.fill' : 'mic'} size={13} weight="medium" />
+ </button>
+ ) : null}
<button
className="press composer-send"
disabled={!enabled}
diff --git a/apps/desktop/src/renderer/views/thread/composer.css b/apps/desktop/src/renderer/views/thread/composer.css
index ad67972d..405337b2 100644
--- a/apps/desktop/src/renderer/views/thread/composer.css
+++ b/apps/desktop/src/renderer/views/thread/composer.css
@@ -74,6 +74,26 @@
transition: opacity var(--morph), transform var(--morph);
}
.composer-symbol.active { opacity: 1; transform: scale(1) rotate(0); }
+/* Dictation, beside send / call (Grok Bot's): a light circle; filled while it listens. */
+.composer-mic {
+ width: 25px;
+ height: 25px;
+ border-radius: 50%;
+ display: grid;
+ place-items: center;
+ flex: none;
+ color: var(--secondary);
+ background: color-mix(in srgb, var(--text) 7%, var(--composer-fill));
+ margin-bottom: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));
+ transition: background-color var(--morph), color var(--morph), transform var(--press);
+}
+.composer-mic:hover {
+ color: var(--text);
+}
+.composer-mic.on {
+ background: var(--danger);
+ color: #fff;
+}
.composer-send {
width: 25px;
height: 25px;
diff --git a/apps/desktop/src/renderer/views/thread/dictation.ts b/apps/desktop/src/renderer/views/thread/dictation.ts
new file mode 100644
index 00000000..ff8dd23a
--- /dev/null
+++ b/apps/desktop/src/renderer/views/thread/dictation.ts
@@ -0,0 +1,56 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { VoiceSettings } from '../call/voice'
+
+/** The running dictation's stop: one recognizer at a time, and a call stops dictation first. */
+let stopActive: (() => void) | null = null
+
+export function stopDictation() {
+ stopActive?.()
+}
+
+/** `said` after what's already in the box, with one space between. */
+export function joinDictated(before: string, said: string) {
+ if (!before.trim()) return said
+ if (!said) return before
+ return /\s$/.test(before) ? before + said : `${before} ${said}`
+}
+
+/**
+ * Dictation into the composer with on-device speech (the speech helper; macOS). Words appear in
+ * the box as they're recognized, after what was already typed; the mic again, or the recognizer
+ * ending on a pause, stops it.
+ */
+export function useDictation(draft: string, setDraft: (text: string) => void) {
+ const [listening, setListening] = useState(false)
+ /** The box up to the words still being recognized. */
+ const before = useRef('')
+ const off = useRef<(() => void) | null>(null)
+ const write = useRef(setDraft)
+ write.current = setDraft
+
+ const finish = useCallback(() => {
+ off.current?.()
+ off.current = null
+ window.codync.speech.stop()
+ setListening(false)
+ if (stopActive === finish) stopActive = null
+ }, [])
+
+ const start = () => {
+ stopActive?.()
+ before.current = draft
+ off.current = window.codync.speech.onEvent((e) => {
+ if (e.type === 'partial' || e.type === 'final') write.current(joinDictated(before.current, e.text))
+ if (e.type === 'final') before.current = joinDictated(before.current, e.text)
+ if (e.type === 'stopped' || e.type === 'error') finish()
+ })
+ window.codync.speech.start(VoiceSettings.language() || null)
+ stopActive = finish
+ setListening(true)
+ }
+
+ // A composer that goes away (another chat) stops listening.
+ useEffect(() => () => { if (off.current) finish() }, [finish])
+
+ return { available: window.codync.speech.available, listening, toggle: () => (listening ? finish() : start()) }
+}
diff --git a/apps/ios/Kit/Sources/CodyncUI/Thread/Composer.swift b/apps/ios/Kit/Sources/CodyncUI/Thread/Composer.swift
index 90efcb3a..fcbe6aea 100644
--- a/apps/ios/Kit/Sources/CodyncUI/Thread/Composer.swift
+++ b/apps/ios/Kit/Sources/CodyncUI/Thread/Composer.swift
@@ -25,6 +25,9 @@ struct Composer: View {
@State private var pickingPhotos = false
@State private var photoItems: [PhotosPickerItem] = []
@FocusState private var focused: Bool
+ @State private var dictation = Dictation()
+ /// The box when dictation started; the words heard go after it.
+ @State private var dictatedAfter = ""
/// One line of the field's text, following Dynamic Type: the send button centers on the last line.
@ScaledMetric(relativeTo: .body) private var lineHeight: CGFloat = 22
@@ -175,7 +178,10 @@ struct Composer: View {
switch state {
case .interrupt: onInterrupt?()
case .stop: model.stop(botId)
- case .call: onCall?()
+ case .call:
+ // The call takes the microphone.
+ dictation.stop()
+ onCall?()
case .send: submit()
}
} label: {
@@ -195,6 +201,31 @@ struct Composer: View {
.animation(Motion.reduced(Motion.morph, reduceMotion), value: canSend)
}
+ /// Dictation, beside send / call (Grok Bot's): a light circle; red while it listens.
+ private var micButton: some View {
+ let on = dictation.listening
+ return Button {
+ if on {
+ dictation.stop()
+ } else {
+ dictatedAfter = draft
+ Task { await dictation.start { said in draft = Dictation.join(dictatedAfter, said) } }
+ }
+ } label: {
+ Image(systemName: on ? "mic.fill" : "mic")
+ .font(.system(size: 15, weight: .medium))
+ .contentTransition(.symbolEffect(.replace))
+ .foregroundStyle(on ? .white : Palette.secondary)
+ .frame(width: buttonSize, height: buttonSize)
+ .background(on ? Palette.danger : Palette.bubbleAgent, in: Circle())
+ .contentShape(Circle())
+ }
+ .buttonStyle(PressScale())
+ .accessibilityLabel(on ? "Stop dictating" : "Dictate")
+ .help(on ? "Stop dictating" : "Dictate")
+ .animation(Motion.reduced(Motion.morph, reduceMotion), value: on)
+ }
+
private var box: some View {
HStack(alignment: .bottom, spacing: 8) {
TextField(placeholder, text: $draft, axis: .vertical)
@@ -203,6 +234,8 @@ struct Composer: View {
.textFieldStyle(.plain)
.focused($focused)
.padding(.vertical, fieldPadding)
+ micButton
+ .padding(.bottom, max(0, (textLineHeight + 2 * fieldPadding - buttonSize) / 2))
trailingButton
.padding(.bottom, max(0, (textLineHeight + 2 * fieldPadding - buttonSize) / 2))
.animation(Motion.layout, value: isEmpty)
@@ -211,8 +244,10 @@ struct Composer: View {
.padding(.trailing, 6)
.padding(.vertical, boxPadding)
.onChange(of: [botId, thread], initial: true) {
+ dictation.stop()
draft = model.composerDraft(for: botId, thread: thread)
}
+ .onDisappear { dictation.stop() }
.onChange(of: draft) { _, value in
model.setComposerDraft(value, for: botId, thread: thread)
// Starting a command: list the computer's skills again.
diff --git a/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift b/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift
new file mode 100644
index 00000000..1974223c
--- /dev/null
+++ b/apps/ios/Kit/Sources/CodyncUI/Thread/Dictation.swift
@@ -0,0 +1,104 @@
+import AVFoundation
+import Speech
+import SwiftUI
+
+/// Dictation into the message box with on-device speech: the words appear as they're recognized,
+/// after what was already typed; the mic again, or the recognizer finishing on a pause, stops it.
+/// It uses the call's language setting.
+@MainActor @Observable
+final class Dictation {
+ private(set) var listening = false
+ @ObservationIgnored private let engine = AVAudioEngine()
+ @ObservationIgnored private var request: SFSpeechAudioBufferRecognitionRequest?
+ @ObservationIgnored private var task: SFSpeechRecognitionTask?
+ @ObservationIgnored private var heard: ((String) -> Void)?
+ /// Bumped per start and stop, so a late result from an earlier recognition is dropped.
+ @ObservationIgnored private var generation = 0
+
+ /// `said` after what's already in the box, with one space between.
+ static func join(_ before: String, _ said: String) -> String {
+ if before.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return said }
+ if said.isEmpty { return before }
+ return before.last?.isWhitespace == true ? before + said : before + " " + said
+ }
+
+ func start(heard: @escaping (String) -> Void) async {
+ guard !listening, await AVAudioApplication.requestRecordPermission(), await Self.authorized() else { return }
+ let recognizer = UserDefaults.standard.string(forKey: OnDeviceEngine.languageKey)
+ .flatMap { $0.isEmpty ? nil : SFSpeechRecognizer(locale: Locale(identifier: $0)) } ?? SFSpeechRecognizer()
+ guard let recognizer, recognizer.isAvailable else { return }
+ do {
+ let audio = AVAudioSession.sharedInstance()
+ // The call's session: a recording tap on a `.record` session fails on some routes.
+ try audio.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetoothHFP, .duckOthers])
+ try audio.setActive(true)
+ } catch {
+ return
+ }
+ // No input route (no microphone): installing a tap would throw.
+ let format = engine.inputNode.outputFormat(forBus: 0)
+ guard format.sampleRate > 0, format.channelCount > 0 else {
+ try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
+ return
+ }
+ generation += 1
+ self.heard = heard
+ let request = SFSpeechAudioBufferRecognitionRequest()
+ request.shouldReportPartialResults = true
+ request.addsPunctuation = true
+ if recognizer.supportsOnDeviceRecognition { request.requiresOnDeviceRecognition = true }
+ self.request = request
+ let input = engine.inputNode
+ input.removeTap(onBus: 0)
+ input.installTap(onBus: 0, bufferSize: 1024, format: format, block: Self.tap(request))
+ engine.prepare()
+ do {
+ try engine.start()
+ } catch {
+ return stop()
+ }
+ listening = true
+ task = recognizer.recognitionTask(with: request, resultHandler: Self.results(self, generation: generation))
+ }
+
+ func stop() {
+ generation += 1
+ task?.cancel()
+ task = nil
+ request?.endAudio()
+ request = nil
+ engine.inputNode.removeTap(onBus: 0)
+ if engine.isRunning { engine.stop() }
+ heard = nil
+ if listening { try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) }
+ listening = false
+ }
+
+ private func recognized(_ text: String, final: Bool, generation: Int) {
+ guard generation == self.generation else { return }
+ heard?(text)
+ if final { stop() }
+ }
+
+ private nonisolated static func tap(_ request: SFSpeechAudioBufferRecognitionRequest) -> AVAudioNodeTapBlock {
+ { buffer, _ in request.append(buffer) }
+ }
+
+ private nonisolated static func results(_ dictation: Dictation, generation: Int) -> @Sendable (SFSpeechRecognitionResult?, (any Error)?) -> Void {
+ { result, error in
+ let text = result?.bestTranscription.formattedString
+ let final = result?.isFinal ?? false
+ let failed = error != nil
+ Task { @MainActor in
+ if let text { dictation.recognized(text, final: final, generation: generation) }
+ if failed, generation == dictation.generation { dictation.stop() }
+ }
+ }
+ }
+
+ private nonisolated static func authorized() async -> Bool {
+ await withCheckedContinuation { done in
+ SFSpeechRecognizer.requestAuthorization { done.resume(returning: $0 == .authorized) }
+ }
+ }
+}
diff --git a/apps/ios/WhatsNew.md b/apps/ios/WhatsNew.md
index 9ede2b30..e1ffb14a 100644
--- a/apps/ios/WhatsNew.md
+++ b/apps/ios/WhatsNew.md
@@ -16,6 +16,7 @@ a version without one gets a generic "Bug fixes and improvements" line. Delete o
- 對話太長時,bot 會先摘要較早的內容,摘要可以在聊天裡展開;摘要也不夠用時,會改用一張卡片開新對話,你可以先編輯它要帶過去的內容,或選擇保留原本的對話。
- bot 可以把較長的細節寫成筆記(每個主題一份),在記憶裡就能閱讀、編輯、刪除或還原任何一個版本;bot 改了筆記時,聊天裡會顯示改了哪些地方,一鍵就能復原。
- bot 的詳細資料重新整理成一頁:上方是 bot 的頭像和名字,下面分成「詳細資料」、「記憶」、「設定」三個分頁,設定改了就自動儲存;也能從這裡把 bot 分享成範本。
+- 訊息框多了麥克風:用說的就能打字,在手機上辨識,按送出前都不會傳出去。
### en-US
@@ -26,6 +27,7 @@ a version without one gets a generic "Bug fixes and improvements" line. Delete o
- When a chat runs long, the bot summarizes the earlier part, and you can open the summary in the chat. When that's no longer enough, a card offers a fresh conversation: edit what it carries over, or keep the current one.
- Bots can keep longer notes, one topic each. Read, edit, delete or bring back any version from memory, and when a bot changes a note, the chat shows what changed with a one-tap undo.
- A bot's details are one page now: its avatar and name on top, then Details, Memory and Settings tabs. Settings save as you edit, and you can share the bot as a template from here.
+- A mic in the message box: say it instead of typing it. It's recognized on the phone, and nothing goes out until you send it.
## 2.17.5
diff --git a/docs/features/voice-call.md b/docs/features/voice-call.md
index d7a638fa..bed1a370 100644
--- a/docs/features/voice-call.md
+++ b/docs/features/voice-call.md
@@ -40,6 +40,20 @@ The bot is the same agent on the computer; the call only changes how you talk to
directly to the audio session, rather than waiting for a SwiftUI view update.
Replayed final replies, other bots and thread replies are not read into this call.
+## Dictation
+
+The composer's mic (beside send / call, as in Grok Bot) types what you say into the box with the
+same on-device recognizer and the call's language setting; nothing is sent until you send it.
+Words appear as they're recognized, after what's already typed; tapping the mic again, or the
+recognizer finishing on a pause, stops it, and a call takes the microphone from it.
+
+- Desktop (`views/thread/dictation.ts`): the speech helper, macOS only, so Linux and Windows show
+ no mic. An unpackaged `npm run dev` can't use it: macOS stops the helper because the dev Electron
+ has no speech usage description, so test it in a packaged build (`dist:mac:dev`), like calls.
+- iPhone (`Thread/Dictation.swift`): `SFSpeechRecognizer` on device; with no microphone route
+ (some simulators) the mic does nothing.
+- The terminal UI has no microphone, so no mic.
+
## Background verification
Store regression tests cover sending a recognized utterance after backgrounding,