Skip to content

Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.

feat(desktop): talk to a bot from any app with a double tap of ⌘

leepokai89b749257595
diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts
index e34638e1..22c23868 100644
--- a/apps/desktop/src/main/index.ts
+++ b/apps/desktop/src/main/index.ts
@@ -14,6 +14,7 @@ import { AccountService } from './account'
import { Updates, yieldsToInstall } from './updates'
import { handleURL, registerAuthIPC, registerSchemes, urlFromArgv } from './auth'
import { registerSpeech } from './speech'
+import { VoiceShortcutKeys } from './voice-shortcut'
import { registerScreenIPC } from './screen'
import { registerCloud } from './cloud'
import { registerSSH } from './ssh'
@@ -227,6 +228,8 @@ if (yieldsToInstall()) {
updates.on('change', () => tray.hostChanged())
registerIPC(tray)
island.register()
+ // Voice input goes to the island while it's on, else to the chat window's composer.
+ new VoiceShortcutKeys((key) => (island.enabled ? island.voiceKey(key) : send({ kind: 'voiceKey', key }, key === 'double'))).register()
registerAuthIPC()
registerSpeech()
registerScreenIPC()
diff --git a/apps/desktop/src/main/island.ts b/apps/desktop/src/main/island.ts
index 3720de1a..f0b730f0 100644
--- a/apps/desktop/src/main/island.ts
+++ b/apps/desktop/src/main/island.ts
@@ -1,6 +1,7 @@
import { execFile } from 'node:child_process'
import { BrowserWindow, globalShortcut, ipcMain, screen } from 'electron'
import type { WindowCommand } from '../shared/ipc'
+import type { VoiceKey } from '../shared/voice-shortcut'
import { hitRect, hits, notchOf, PANEL, type IslandRect, type IslandScreen, type IslandSummary, type NotchScreen } from '../shared/island'
import { prefs } from './host-controller'
@@ -202,6 +203,11 @@ export class Island {
}
}
+ /** The voice input shortcut's taps, while the island is on. */
+ voiceKey(key: VoiceKey) {
+ this.window?.webContents.send('island:voiceKey', key)
+ }
+
/** Top center of the display with the notch, else the main display. */
private async place() {
if (!this.window) return
diff --git a/apps/desktop/src/main/speech.ts b/apps/desktop/src/main/speech.ts
index ace32d5d..35a9ad68 100644
--- a/apps/desktop/src/main/speech.ts
+++ b/apps/desktop/src/main/speech.ts
@@ -16,6 +16,9 @@ function helperPath() {
return candidates.find((c): c is string => !!c && existsSync(c)) ?? null
}
+/** Whether this build can transcribe on device. */
+export const speechAvailable = () => process.platform === 'darwin' && helperPath() !== null
+
let child: ChildProcessWithoutNullStreams | null = null
let listener: WebContents | null = null
/** Bumped per start: a stopping process may still deliver its final result, an older one may not. */
@@ -27,6 +30,8 @@ function emit(event: SpeechEvent) {
function start(sender: WebContents, locale: string | null) {
stop()
+ // Another window's dictation or call loses the helper, and its exit (an older generation) won't say so.
+ if (listener && listener !== sender) emit({ type: 'stopped' })
listener = sender
const path = helperPath()
if (!path) return emit({ type: 'error', message: "On-device speech isn't available in this build." })
@@ -73,9 +78,12 @@ function locales() {
}
export function registerSpeech() {
- ipcMain.on('speech:available', (e) => (e.returnValue = process.platform === 'darwin' && helperPath() !== null))
+ ipcMain.on('speech:available', (e) => (e.returnValue = speechAvailable()))
ipcMain.on('speech:locales', (e) => (e.returnValue = locales()))
ipcMain.on('speech:start', (e, locale: string | null) => start(e.sender, locale))
- ipcMain.on('speech:stop', () => stop())
+ // Only the window using the helper stops it: one that lost it to another window has nothing to stop.
+ ipcMain.on('speech:stop', (e) => {
+ if (e.sender === listener) stop()
+ })
app.on('before-quit', stop)
}
diff --git a/apps/desktop/src/main/voice-shortcut.ts b/apps/desktop/src/main/voice-shortcut.ts
new file mode 100644
index 00000000..df01d3d6
--- /dev/null
+++ b/apps/desktop/src/main/voice-shortcut.ts
@@ -0,0 +1,83 @@
+import { spawn } from 'node:child_process'
+import { createInterface } from 'node:readline'
+import { app, ipcMain, type WebContents } from 'electron'
+import { CommandTaps, DEFAULT_VOICE_SHORTCUT, VOICE_SHORTCUTS, type VoiceKey, type VoiceShortcut } from '../shared/voice-shortcut'
+import { prefs } from './host-controller'
+import { speechAvailable } from './speech'
+
+// Every modifier change in any app, from an NSEvent global monitor, with the seconds since the
+// last key press or click. Neither needs the Accessibility or Input Monitoring grant (other keys
+// would), so the app learns only that a modifier moved, never what was typed. JXA reaches AppKit
+// without a helper binary.
+const MONITOR = `ObjC.import('AppKit'); ObjC.import('CoreGraphics')
+const out = $.NSFileHandle.fileHandleWithStandardOutput
+const since = (type) => $.CGEventSourceSecondsSinceLastEventType($.kCGEventSourceStateCombinedSessionState, type)
+$.NSApplication.sharedApplication.setActivationPolicy($.NSApplicationActivationPolicyProhibited)
+$.NSEvent.addGlobalMonitorForEventsMatchingMaskHandler($.NSEventMaskFlagsChanged, (e) => {
+ const sinceInput = Math.min(since($.kCGEventKeyDown), since($.kCGEventLeftMouseDown), since($.kCGEventRightMouseDown), since($.kCGEventOtherMouseDown))
+ out.writeData($(e.keyCode + ' ' + e.modifierFlags + ' ' + e.timestamp + ' ' + sinceInput + '\\n').dataUsingEncoding($.NSUTF8StringEncoding))
+})
+$.NSApp.run`
+
+/** Reports the shortcut key's taps from any app until the returned stop. */
+function watchTaps(shortcut: Exclude<VoiceShortcut, 'off'>, onKey: (key: VoiceKey) => void): () => void {
+ const taps = new CommandTaps(shortcut)
+ const proc = spawn('/usr/bin/osascript', ['-l', 'JavaScript', '-e', MONITOR], { stdio: ['ignore', 'pipe', 'pipe'] })
+ createInterface({ input: proc.stdout }).on('line', (line) => {
+ const [key = 0, flags = 0, at = 0, sinceInput = 0] = line.split(' ').map(Number)
+ const tap = taps.feed({ key, flags, at, sinceInput })
+ if (tap) onKey(tap)
+ })
+ proc.stderr.on('data', (chunk: Buffer) => console.error(`voice shortcut: ${chunk.toString('utf8').trim()}`))
+ proc.on('error', (error) => console.error(`voice shortcut: couldn't watch the keyboard: ${error.message}`))
+ let stopping = false
+ // ponytail: logged only, not restarted; the setting or a relaunch starts it again.
+ proc.on('exit', (code, signal) => {
+ if (!stopping) console.error(`voice shortcut: the keyboard watcher exited (${code ?? signal})`)
+ })
+ return () => {
+ stopping = true
+ proc.kill()
+ }
+}
+
+/**
+ * The voice input shortcut (macOS, with on-device speech): watches the chosen ⌘ key while it's on
+ * and hands its taps to `onKey`. The setting belongs to the desktop installation (`voiceShortcut`
+ * in `host-prefs.json`).
+ */
+export class VoiceShortcutKeys {
+ private stop: (() => void) | null = null
+ /** The window with a call on: the call has the microphone, so taps do nothing meanwhile. */
+ private call: WebContents | null = null
+
+ constructor(private onKey: (key: VoiceKey) => void) {}
+
+ get shortcut(): VoiceShortcut {
+ if (!speechAvailable()) return 'off'
+ const saved = prefs.get('voiceShortcut')
+ return VOICE_SHORTCUTS.find((s) => s.id === saved)?.id ?? DEFAULT_VOICE_SHORTCUT
+ }
+
+ register() {
+ ipcMain.handle('voice:shortcut', () => this.shortcut)
+ ipcMain.handle('voice:setShortcut', (_e, shortcut: VoiceShortcut) => {
+ prefs.set('voiceShortcut', shortcut)
+ this.sync()
+ return this.shortcut
+ })
+ ipcMain.on('voice:calling', (e, on: boolean) => (this.call = on ? e.sender : null))
+ app.on('will-quit', () => this.stop?.())
+ this.sync()
+ }
+
+ private sync() {
+ this.stop?.()
+ this.stop = null
+ const shortcut = this.shortcut
+ if (shortcut === 'off') return
+ this.stop = watchTaps(shortcut, (key) => {
+ if (!this.call || this.call.isDestroyed()) this.onKey(key)
+ })
+ }
+}
diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts
index 453d7d0f..48939706 100644
--- a/apps/desktop/src/preload/index.ts
+++ b/apps/desktop/src/preload/index.ts
@@ -62,6 +62,9 @@ const bridge: CodyncBridge = {
start: (locale) => ipcRenderer.send('speech:start', locale),
stop: () => ipcRenderer.send('speech:stop'),
onEvent: (cb) => listen('speech:event', cb),
+ shortcut: () => ipcRenderer.invoke('voice:shortcut'),
+ setShortcut: (shortcut) => ipcRenderer.invoke('voice:setShortcut', shortcut),
+ setCalling: (on) => ipcRenderer.send('voice:calling', on),
},
ssh: sshBridge,
island: {
@@ -77,6 +80,7 @@ const bridge: CodyncBridge = {
setFocusable: (on) => ipcRenderer.send('island:focusable', on),
setKeys: (on) => ipcRenderer.send('island:keys', on),
onKey: (cb) => listen('island:key', cb),
+ onVoiceKey: (cb) => listen('island:voiceKey', cb),
respond: (ref, entryId, optionId) => ipcRenderer.send('island:respond', ref, entryId, optionId),
send: (ref, text) => ipcRenderer.send('island:send', ref, text),
openBot: (ref) => ipcRenderer.send('island:openBot', ref),
diff --git a/apps/desktop/src/renderer/components/LevelDots.tsx b/apps/desktop/src/renderer/components/LevelDots.tsx
new file mode 100644
index 00000000..0f69f410
--- /dev/null
+++ b/apps/desktop/src/renderer/components/LevelDots.tsx
@@ -0,0 +1,67 @@
+import { useEffect, useRef } from 'react'
+import { useDark, usePixelRatio, useReduceMotion, withAlpha } from '../lib/theme'
+
+/** What the dots show this frame: your voice's level (0…1), the other side talking, or work going on. */
+export interface LevelSample {
+ level: number
+ speaking?: boolean
+ working?: boolean
+}
+
+/** The gap between dots, in points. */
+const SPACING = 8
+
+/**
+ * A dotted line that follows your voice while listening, ripples while the other side talks,
+ * breathes while work goes on, and rests as faint dots otherwise. `sample` is read every frame.
+ * `color` null is the text color.
+ */
+export function LevelDots({ sample, color, width, height = 16, label, className }: {
+ sample: () => LevelSample
+ color: string | null
+ width: number
+ height?: number
+ label: string
+ className?: string
+}) {
+ const canvas = useRef<HTMLCanvasElement>(null)
+ const dark = useDark()
+ const reduceMotion = useReduceMotion()
+ const scale = usePixelRatio()
+ const live = useRef({ sample, color, dark, reduceMotion })
+ live.current = { sample, color, dark, reduceMotion }
+
+ useEffect(() => {
+ const el = canvas.current
+ const ctx = el?.getContext('2d')
+ if (!el || !ctx) return
+ el.width = width * scale
+ el.height = height * scale
+ let frame = 0
+ const draw = (now: number) => {
+ const { sample, color, dark, reduceMotion } = live.current
+ const { level, speaking = false, working = false } = sample()
+ const t = reduceMotion ? 0 : now / 1000
+ const accent = dark ? '#ffffff' : '#000000'
+ const tint = speaking ? accent : (color ?? accent)
+ ctx.setTransform(scale, 0, 0, scale, 0, 0)
+ ctx.clearRect(0, 0, width, height)
+ const count = Math.max(1, Math.floor(width / SPACING))
+ for (let i = 0; i < count; i++) {
+ const x = i * SPACING + SPACING / 2
+ const wave = (Math.sin(i * 0.55 - t * 6) + 1) / 2
+ const amount = speaking ? 0.5 + 0.5 * wave : level > 0.05 ? Math.max(0.2, level) * (0.45 + 0.55 * wave) : working ? 0.25 * ((Math.sin(t * 2.4) + 1) / 2) : 0.08
+ const bar = 4 + amount * (height - 4)
+ ctx.fillStyle = withAlpha(tint, 0.5 + 0.5 * amount)
+ ctx.beginPath()
+ ctx.roundRect(x - 1.25, (height - bar) / 2, 2.5, bar, 1.25)
+ ctx.fill()
+ }
+ frame = requestAnimationFrame(draw)
+ }
+ frame = requestAnimationFrame(draw)
+ return () => cancelAnimationFrame(frame)
+ }, [scale, width, height])
+
+ return <canvas ref={canvas} role="img" aria-label={label} className={className} style={{ width, height }} />
+}
diff --git a/apps/desktop/src/renderer/components/RecordingPill.tsx b/apps/desktop/src/renderer/components/RecordingPill.tsx
new file mode 100644
index 00000000..1d13c267
--- /dev/null
+++ b/apps/desktop/src/renderer/components/RecordingPill.tsx
@@ -0,0 +1,39 @@
+import { useEffect, useState } from 'react'
+import { Icon } from './Icon'
+import { LevelDots, type LevelSample } from './LevelDots'
+import { Text } from './Text'
+import './recording-pill.css'
+
+/** m:ss since `startedAt`, updated every second. */
+function useElapsed(startedAt: number) {
+ const [now, setNow] = useState(() => Date.now())
+ useEffect(() => {
+ const timer = setInterval(() => setNow(Date.now()), 1000)
+ return () => clearInterval(timer)
+ }, [])
+ const seconds = Math.max(0, Math.floor((now - startedAt) / 1000))
+ return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, '0')}`
+}
+
+/**
+ * Dictation in progress, in place of the mic: stop, the time since it started, and dots that
+ * follow the microphone's level. `color` null is the text color.
+ */
+export function RecordingPill({ startedAt, sample, onStop, color = null, className = '' }: {
+ startedAt: number
+ sample: () => LevelSample
+ onStop: () => void
+ color?: string | null
+ className?: string
+}) {
+ const elapsed = useElapsed(startedAt)
+ return (
+ <span className={`recording-pill ${className}`}>
+ <button type="button" className="recording-stop" aria-label="Stop dictating" title="Stop dictating" onClick={onStop}>
+ <Icon name="stop.fill" size={11} scaled={false} />
+ </button>
+ <Text tabular tone="text">{elapsed}</Text>
+ <LevelDots sample={sample} color={color} width={32} height={12} label="Listening" />
+ </span>
+ )
+}
diff --git a/apps/desktop/src/renderer/components/recording-pill.css b/apps/desktop/src/renderer/components/recording-pill.css
new file mode 100644
index 00000000..037e0c85
--- /dev/null
+++ b/apps/desktop/src/renderer/components/recording-pill.css
@@ -0,0 +1,26 @@
+/* Dictation in progress: a quiet pill with stop, the time and the level dots. */
+.recording-pill {
+ flex: none;
+ display: inline-flex;
+ align-items: center;
+ gap: 6px;
+ height: 28px;
+ box-sizing: border-box;
+ padding: 0 10px 0 4px;
+ border-radius: 14px;
+ background: rgba(var(--text-rgb), 0.08);
+ font-size: var(--text-body);
+ animation: fade-in var(--fade) both;
+}
+.recording-stop {
+ width: 22px;
+ height: 22px;
+ border-radius: 50%;
+ display: grid;
+ place-items: center;
+ color: var(--text);
+ transition: background-color var(--hover);
+}
+.recording-stop:hover {
+ background: rgba(var(--text-rgb), 0.1);
+}
diff --git a/apps/desktop/src/renderer/demo/bridge.ts b/apps/desktop/src/renderer/demo/bridge.ts
index 93636b3f..0dfc6d63 100644
--- a/apps/desktop/src/renderer/demo/bridge.ts
+++ b/apps/desktop/src/renderer/demo/bridge.ts
@@ -57,7 +57,7 @@ const bridge: CodyncBridge = {
signOut: async () => {},
token: unavailable,
},
- speech: { available: false, locales: () => [], start() {}, stop() {}, onEvent: none },
+ speech: { available: false, locales: () => [], start() {}, stop() {}, onEvent: none, shortcut: async () => 'off', setShortcut: async () => 'off', setCalling() {} },
ssh: {
state: async () => ({ profiles: [], status: {}, attachments: [] }),
onChange: none,
@@ -81,6 +81,7 @@ const bridge: CodyncBridge = {
setFocusable() {},
setKeys() {},
onKey: none,
+ onVoiceKey: none,
respond() {},
send() {},
openBot() {},
diff --git a/apps/desktop/src/renderer/views/ChatWindow.tsx b/apps/desktop/src/renderer/views/ChatWindow.tsx
index 63ac3f30..71323599 100644
--- a/apps/desktop/src/renderer/views/ChatWindow.tsx
+++ b/apps/desktop/src/renderer/views/ChatWindow.tsx
@@ -20,6 +20,7 @@ import { SidebarFooter } from './SidebarFooter'
import { UpdateBadge } from './UpdateButton'
import { ComputerFilterHeader, computerSelection } from './ComputerFilterHeader'
import { ThreadView } from './thread/ThreadView'
+import { queueVoiceKey } from './thread/voice-shortcut-keys'
import { NewBotView, NewGroupView } from './bots/NewBotView'
import { BotEditorView } from './bots/BotEditorView'
import { MarketplaceView } from './marketplace/MarketplaceView'
@@ -227,6 +228,11 @@ function ChatSplitView() {
if (c.kind === 'newBot' && onlineStores.length) create('bot')
if (c.kind === 'search') setSearching(true)
if (c.kind === 'toggleSidebar') setCompact(!prefs.sidebarCompact.get())
+ if (c.kind === 'voiceKey') {
+ // No chat open: talk to the top bot in the sidebar.
+ if (c.key === 'double' && !app.selection) app.select((roster.navigable[0] ?? roster.items[0])?.ref ?? null)
+ queueVoiceKey(c.key)
+ }
}),
)
diff --git a/apps/desktop/src/renderer/views/call/CallView.tsx b/apps/desktop/src/renderer/views/call/CallView.tsx
index 5b21b23e..c83ae6b5 100644
--- a/apps/desktop/src/renderer/views/call/CallView.tsx
+++ b/apps/desktop/src/renderer/views/call/CallView.tsx
@@ -3,10 +3,11 @@ import { isWorking } from '@shared/models'
import { createContext, useContext, useEffect, useRef, useState } from 'react'
import { BotAvatar } from '../../components/Avatar'
import { Icon } from '../../components/Icon'
+import { LevelDots } from '../../components/LevelDots'
import { Sheet } from '../../components/Overlay'
import { Text } from '../../components/Text'
import { useModel } from '../../lib/observable'
-import { avatarColor, useDark, usePixelRatio, useReduceMotion, withAlpha } from '../../lib/theme'
+import { avatarColor } from '../../lib/theme'
import { analytics } from '../../store/analytics'
import { useStore } from '../../store/context'
import { VoiceChatSettingsView } from '../settings/VoiceChatSettingsView'
@@ -143,8 +144,10 @@ export function CallView({ botId, onSpeaking, interrupt, onEnd }: {
useEffect(() => {
const startedAt = Date.now()
+ window.codync.speech.setCalling(true)
begin(VoiceSettings.provider())
return () => {
+ window.codync.speech.setCalling(false)
closed.current = true
engine.current?.end()
countCloudMinutes()
@@ -200,7 +203,13 @@ export function CallView({ botId, onSpeaking, interrupt, onEnd }: {
>
{bot ? <BotAvatar bot={bot} size={34} /> : <span className="call-avatar-empty" />}
</button>
- <CallLevelDots engine={session} working={working} color={color} label={statusText} />
+ <LevelDots
+ sample={() => ({ level: session?.level ?? 0, speaking: session?.phase === 'speaking', working })}
+ color={color}
+ width={120}
+ label={statusText}
+ className="call-dots"
+ />
{/* Audio leaves the computer: say where. */}
{session?.provider ? (
<Text size="caption2" weight="semibold" tone="secondary" aria-label={`Voice by ${providerName(session.provider)}`}>
@@ -237,55 +246,3 @@ export function CallView({ botId, onSpeaking, interrupt, onEnd }: {
</div>
)
}
-
-const DOTS_WIDTH = 120
-const DOTS_HEIGHT = 16
-
-/**
- * The dotted line in the call bar: it follows your voice while listening, ripples while the bot
- * talks, breathes while it works, and rests as faint dots otherwise.
- */
-function CallLevelDots({ engine, working, color, label }: { engine: VoiceEngine | null; working: boolean; color: string | null; label: string }) {
- const canvas = useRef<HTMLCanvasElement>(null)
- const dark = useDark()
- const reduceMotion = useReduceMotion()
- const scale = usePixelRatio()
- const live = useRef({ engine, working, color, dark, reduceMotion })
- live.current = { engine, working, color, dark, reduceMotion }
-
- useEffect(() => {
- const el = canvas.current
- const ctx = el?.getContext('2d')
- if (!el || !ctx) return
- el.width = DOTS_WIDTH * scale
- el.height = DOTS_HEIGHT * scale
- let frame = 0
- const draw = (now: number) => {
- const { engine, working, color, dark, reduceMotion } = live.current
- const t = reduceMotion ? 0 : now / 1000
- const speaking = engine?.phase === 'speaking'
- const level = engine?.level ?? 0
- const accent = dark ? '#ffffff' : '#000000'
- const tint = speaking ? accent : (color ?? accent)
- ctx.setTransform(scale, 0, 0, scale, 0, 0)
- ctx.clearRect(0, 0, DOTS_WIDTH, DOTS_HEIGHT)
- const spacing = 8
- const count = Math.max(1, Math.floor(DOTS_WIDTH / spacing))
- for (let i = 0; i < count; i++) {
- const x = i * spacing + spacing / 2
- const wave = (Math.sin(i * 0.55 - t * 6) + 1) / 2
- const amount = speaking ? 0.5 + 0.5 * wave : level > 0.05 ? Math.max(0.2, level) * (0.45 + 0.55 * wave) : working ? 0.25 * ((Math.sin(t * 2.4) + 1) / 2) : 0.08
- const height = 4 + amount * (DOTS_HEIGHT - 4)
- ctx.fillStyle = withAlpha(tint, 0.5 + 0.5 * amount)
- ctx.beginPath()
- ctx.roundRect(x - 1.25, (DOTS_HEIGHT - height) / 2, 2.5, height, 1.25)
- ctx.fill()
- }
- frame = requestAnimationFrame(draw)
- }
- frame = requestAnimationFrame(draw)
- return () => cancelAnimationFrame(frame)
- }, [scale])
-
- return <canvas ref={canvas} role="img" aria-label={label} className="call-dots" style={{ width: DOTS_WIDTH, height: DOTS_HEIGHT }} />
-}
diff --git a/apps/desktop/src/renderer/views/island/IslandViews.tsx b/apps/desktop/src/renderer/views/island/IslandViews.tsx
index b6b9c077..e050eb98 100644
--- a/apps/desktop/src/renderer/views/island/IslandViews.tsx
+++ b/apps/desktop/src/renderer/views/island/IslandViews.tsx
@@ -1,9 +1,11 @@
-import { useRef, useState } from 'react'
+import { useRef } from 'react'
import type { IslandBot, IslandPermission } from '@shared/island'
import { CharacterAvatar } from '../../components/Avatar'
import { Button, Spinner } from '../../components/Controls'
import { Icon } from '../../components/Icon'
+import { RecordingPill } from '../../components/RecordingPill'
import { ThinkingOrb } from '../../components/ThinkingOrb'
+import { useModel } from '../../lib/observable'
import { avatarColor, withAlpha } from '../../lib/theme'
import type { IslandModel } from './island-model'
import { useArrivals, useFlip, useSwap } from './island-motion'
@@ -182,8 +184,10 @@ export function Notice({ model, bot, failed }: { model: IslandModel; bot: Island
/** Writes to a bot without opening Codync: the chip picks the bot, Enter sends, Esc closes. */
export function Message({ model }: { model: IslandModel }) {
- const [text, setText] = useState('')
+ const dictation = useModel(model.dictation)
const bot = model.messageBot
+ /** Why dictation stopped, else how to talk from any app. */
+ const hint = dictation.error ?? model.voiceTip
const swap = useSwap(bot?.ref)
if (!bot) return null
return (
@@ -211,24 +215,31 @@ export function Message({ model }: { model: IslandModel }) {
className="island-field"
onSubmit={(e) => {
e.preventDefault()
- model.send(text)
- setText('')
+ model.send()
}}
>
<input
autoFocus
aria-label={`Message ${bot.name}`}
placeholder={`Message ${bot.name}…`}
- value={text}
- onChange={(e) => setText(e.target.value)}
+ value={model.draft}
+ onChange={(e) => model.setDraft(e.target.value)}
onKeyDown={(e) => {
if (e.key === 'Escape') model.close()
}}
/>
- <button type="submit" className="island-send" aria-label="Send" title="Send" disabled={!text.trim()}>
+ {dictation.listening ? (
+ <RecordingPill startedAt={dictation.startedAt} sample={() => dictation} onStop={() => dictation.finish()} color="#ffffff" />
+ ) : dictation.available ? (
+ <button type="button" className="island-mic" aria-label="Dictate" title="Dictate" onClick={() => dictation.start(model.draft)}>
+ <Icon name="mic" size={12} weight="medium" scaled={false} />
+ </button>
+ ) : null}
+ <button type="submit" className="island-send" aria-label="Send" title="Send" disabled={!model.draft.trim()}>
<Icon name="arrow.up" size={12} weight="bold" scaled={false} />
</button>
</form>
+ {hint ? <span className="island-hint">{hint}</span> : null}
</div>
</div>
<MiniColumn bots={model.others(3).filter((b) => b.ref !== bot.ref)} onOpen={(ref) => model.openBot(ref)} />
diff --git a/apps/desktop/src/renderer/views/island/island-model.ts b/apps/desktop/src/renderer/views/island/island-model.ts
index 00a11038..ef45d1d1 100644
--- a/apps/desktop/src/renderer/views/island/island-model.ts
+++ b/apps/desktop/src/renderer/views/island/island-model.ts
@@ -2,7 +2,10 @@ import {
FINISHED_MS, focusOf, isActive, islandFrame, islandMode, islandRect, othersOf, transitions,
type IslandBot, type IslandNotice, type IslandPermission, type IslandScreen, type IslandSummary, type IslandTab,
} from '@shared/island'
+import { VOICE_SHORTCUT_KEYS, type VoiceKey, type VoiceShortcut } from '@shared/voice-shortcut'
import { Observable } from '../../lib/observable'
+import { pref } from '../../lib/prefs'
+import { Dictation } from '../thread/dictation'
/** The bare notch peeks at once and opens after a rest; the compact island opens sooner. */
const OPEN_FROM_PEEK_MS = 650
@@ -11,6 +14,8 @@ const UNPEEK_MS = 600
/** Opened by hover, it folds soon after the pointer leaves; opened by a click, after a countdown. */
const LEAVE_MS = 800
export const PINNED_LEAVE_MS = 10_000
+/** Set once the voice input shortcut has been used; until then the message field shows how. */
+const voiceShortcutUsed = pref('islandVoiceShortcutUsed', false)
/** The island window's state: the chat window's summary, the pointer, the header's tab and what was put off. */
export class IslandModel extends Observable {
@@ -23,6 +28,12 @@ export class IslandModel extends Observable {
closing = false
/** The bot the message field writes to. */
messageRef: string | null = null
+ /** The message field's text. */
+ draft = ''
+ /** Talking into the message field (the field's mic, or the voice input shortcut). */
+ readonly dictation = new Dictation((text) => this.update(() => (this.draft = text)))
+ /** The voice input shortcut, as Settings → General has it. */
+ shortcut: VoiceShortcut = 'off'
/** The shape grew on its last change: open with a spring, close without one. */
growing = true
private peek = false
@@ -43,6 +54,7 @@ export class IslandModel extends Observable {
island.onScreen((s) => this.update(() => (this.screen = s)))
island.onHover((inside) => this.hover(inside))
island.onKey((answer) => this.answerByKey(answer))
+ island.onVoiceKey((key) => this.voiceKey(key))
window.addEventListener('blur', () => {
if (this.tab === 'message') this.close()
})
@@ -96,6 +108,12 @@ export class IslandModel extends Observable {
return this.bot(this.messageRef) ?? this.focus
}
+ /** How to talk from any app, under the message field until the shortcut is first used. */
+ get voiceTip() {
+ const key = VOICE_SHORTCUT_KEYS[this.shortcut]
+ return key && !voiceShortcutUsed.get() ? `Tip: double-tap ${key} to talk from any app, then tap it once to send.` : null
+ }
+
/** A click on the closed island opens it and keeps it open. */
openNow() {
this.update(() => {
@@ -120,6 +138,7 @@ export class IslandModel extends Observable {
if (tab === 'message') {
this.keyboard = true
window.codync.island.setFocusable(true)
+ this.readShortcut()
}
this.foldWhenAway()
}
@@ -132,10 +151,14 @@ export class IslandModel extends Observable {
this.update(() => (this.messageRef = bots[(at + 1) % bots.length]!.ref))
}
- send(text: string) {
+ setDraft(text: string) {
+ this.update(() => (this.draft = text))
+ }
+
+ send() {
const bot = this.messageBot
- if (!bot || !text.trim()) return
- window.codync.island.send(bot.ref, text)
+ if (!bot || !this.draft.trim()) return
+ window.codync.island.send(bot.ref, this.draft)
this.close()
}
@@ -167,7 +190,9 @@ export class IslandModel extends Observable {
close() {
clearTimeout(this.hoverTimer)
+ this.dictation.cancel()
this.update(() => {
+ this.draft = ''
this.open = false
this.pinned = false
this.peek = false
@@ -178,6 +203,18 @@ export class IslandModel extends Observable {
this.keyboard = false
}
+ /** A double tap opens the message field and listens; the next tap sends what was said once the last words are in. */
+ private voiceKey(key: VoiceKey) {
+ if (!this.dictation.shortcutKey(key, () => this.send())) return
+ voiceShortcutUsed.set(true)
+ this.setTab('message')
+ if (this.messageBot) this.dictation.start(this.draft, true)
+ }
+
+ private readShortcut() {
+ void window.codync.speech.shortcut().then((shortcut) => this.update(() => (this.shortcut = shortcut)))
+ }
+
private answerByKey(answer: 'allow' | 'deny') {
const card = this.mode === 'approval' ? this.permission : null
const option = card?.options.find((o) => o.kind === (answer === 'allow' ? 'allow_once' : 'reject_once'))
diff --git a/apps/desktop/src/renderer/views/island/island-views.css b/apps/desktop/src/renderer/views/island/island-views.css
index 06d140bf..10115290 100644
--- a/apps/desktop/src/renderer/views/island/island-views.css
+++ b/apps/desktop/src/renderer/views/island/island-views.css
@@ -413,6 +413,26 @@
.island-send:disabled {
opacity: 0.35;
}
+.island-mic {
+ width: 30px;
+ height: 30px;
+ border-radius: 15px;
+ display: grid;
+ place-items: center;
+ flex: none;
+ background: rgba(var(--text-rgb), 0.08);
+ color: var(--secondary);
+ transition: background-color var(--morph), color var(--morph);
+}
+.island-mic:hover {
+ color: var(--text);
+}
+/* How to talk from any app, until the voice input shortcut is first used. */
+.island-hint {
+ padding-left: 16px;
+ font-size: 12px;
+ color: var(--island-quiet);
+}
/* Folding soon: a line that runs out. */
.island-countdown {
diff --git a/apps/desktop/src/renderer/views/settings/SettingsView.tsx b/apps/desktop/src/renderer/views/settings/SettingsView.tsx
index 834055ff..32489f3e 100644
--- a/apps/desktop/src/renderer/views/settings/SettingsView.tsx
+++ b/apps/desktop/src/renderer/views/settings/SettingsView.tsx
@@ -1,6 +1,7 @@
import { Fragment, useEffect, useState } from 'react'
import type { Screen } from '@shared/analytics'
import { islandLabel } from '@shared/island'
+import { VOICE_SHORTCUTS, type VoiceShortcut } from '@shared/voice-shortcut'
import { Button, CardForm, CardSection, ChoicePicker, IconButton, ProgressBar, Switch, ValueRow } from '../../components/Controls'
import { Icon } from '../../components/Icon'
import { useDismiss } from '../../components/Overlay'
@@ -120,6 +121,8 @@ function GeneralSettings() {
useEffect(() => void window.codync.app.launchAtLogin().then(setLaunchAtLogin), [])
const [island, setIsland] = useState(false)
useEffect(() => void window.codync.island.enabled().then(setIsland), [])
+ const [voiceShortcut, setVoiceShortcut] = useState<VoiceShortcut>('off')
+ useEffect(() => void window.codync.speech.shortcut().then(setVoiceShortcut), [])
return (
<CardForm>
@@ -172,6 +175,12 @@ function GeneralSettings() {
{islandLabel(window.codync.platform)}
</Switch>
) : null}
+ {/* On-device speech, so macOS only. */}
+ {window.codync.speech.available ? (
+ <ValueRow label="Voice input shortcut" detail="Talk to a bot from any app, then tap the key once more to send.">
+ <ChoicePicker selection={voiceShortcut} options={VOICE_SHORTCUTS} onChange={(s) => void window.codync.speech.setShortcut(s).then(setVoiceShortcut)} />
+ </ValueRow>
+ ) : null}
<Switch on={launchAtLogin} onChange={(on) => void window.codync.app.setLaunchAtLogin(on).then(setLaunchAtLogin)}>
Open at login
</Switch>
diff --git a/apps/desktop/src/renderer/views/thread/Composer.tsx b/apps/desktop/src/renderer/views/thread/Composer.tsx
index a33daae1..d560d72d 100644
--- a/apps/desktop/src/renderer/views/thread/Composer.tsx
+++ b/apps/desktop/src/renderer/views/thread/Composer.tsx
@@ -4,6 +4,7 @@ import { isGroup, isWorkingIn, quoteOf, type Bot, type Entry } from '@shared/mod
import { BotAvatar } from '../../components/Avatar'
import { Icon } from '../../components/Icon'
import { IconButton } from '../../components/Controls'
+import { RecordingPill } from '../../components/RecordingPill'
import { usePresence } from '../../components/Overlay'
import { HStack, VStack } from '../../components/Stack'
import { Text } from '../../components/Text'
@@ -17,6 +18,7 @@ import { ComposerAttachment } from './ComposerAttachment'
import { QuoteRow } from './ChatRows'
import { useDictation } from './dictation'
import { builtInCommand, slashCommands, slashQuery, type SlashCommand } from './slash-commands'
+import { useVoiceShortcut } from './voice-shortcut-keys'
type Trailing = 'interrupt' | 'stop' | 'call' | 'send'
@@ -50,7 +52,7 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:
const draft = useSyncExternalStore(drafts.subscribe, () => drafts.get(botId))
const setDraft = (text: string) => drafts.set(botId, text)
const [files, setFiles] = useState<OutgoingFile[]>([])
- const dictation = useDictation(draft, setDraft)
+ const dictation = useDictation(setDraft)
const field = useRef<HTMLTextAreaElement>(null)
const measure = useRef<HTMLDivElement>(null)
const composing = useRef(false)
@@ -157,8 +159,10 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:
const submit = () => {
const command = files.length ? null : builtInCommand(bot, draft, members)
+ if (!command && !canSend) return
+ // Words still on their way would land in the emptied box (and the shortcut's tap would send them again).
+ dictation.cancel()
if (command) return run(command)
- if (!canSend) return
store.send(draft, botId, { files, pasted: pasted.current, replyTo: reply && quoteOf(reply) })
cancelReply?.()
pasted.current = []
@@ -166,6 +170,10 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:
setFiles([])
field.current?.focus()
}
+ useVoiceShortcut(dictation, draft, submit, () => field.current?.focus())
+ useEffect(() => {
+ if (dictation.error) store.setError(dictation.error)
+ }, [dictation.error, store])
const trailing: Trailing = onInterrupt && !draft && !files.length ? 'interrupt' : working && !draft && !files.length ? 'stop' : onCall && !draft && !files.length ? 'call' : 'send'
const [symbol, label] = ({
@@ -271,15 +279,11 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:
}}
/>
</div>
- {dictation.available ? (
- <button
- className={`press composer-mic ${dictation.listening ? 'on' : ''}`}
- aria-label={dictation.listening ? 'Stop dictating' : 'Dictate'}
- title={dictation.listening ? 'Stop dictating' : 'Dictate'}
- aria-pressed={dictation.listening}
- onClick={dictation.toggle}
- >
- <Icon name={dictation.listening ? 'mic.fill' : 'mic'} size={13} weight="medium" />
+ {dictation.listening ? (
+ <RecordingPill startedAt={dictation.startedAt} sample={() => dictation} onStop={() => dictation.finish()} className="composer-recording" />
+ ) : dictation.available ? (
+ <button className="press composer-mic" aria-label="Dictate" title="Dictate" onClick={() => dictation.start(draft)}>
+ <Icon name="mic" size={13} weight="medium" />
</button>
) : null}
<button
diff --git a/apps/desktop/src/renderer/views/thread/composer.css b/apps/desktop/src/renderer/views/thread/composer.css
index 91defb47..89fd3be1 100644
--- a/apps/desktop/src/renderer/views/thread/composer.css
+++ b/apps/desktop/src/renderer/views/thread/composer.css
@@ -88,7 +88,7 @@
display: grid;
place-items: center;
flex: none;
- margin-bottom: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));
+ margin-bottom: var(--first-line-inset);
}
.composer-surface {
position: relative;
@@ -102,6 +102,8 @@
flex-direction: column;
}
.composer-box {
+ /* Centers a 25 px control on the field's first line. */
+ --first-line-inset: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));
display: flex;
align-items: flex-end;
gap: 8px;
@@ -154,7 +156,7 @@
transition: opacity var(--morph), transform var(--morph);
}
.composer-symbol.active { opacity: 1; transform: scale(1) rotate(0); }
-/* Dictation, beside send / call (Grok Bot's): a light circle; filled while it listens. */
+/* Dictation, beside send / call (Grok Bot's): a light circle; a recording pill while it listens. */
.composer-mic {
width: 25px;
height: 25px;
@@ -164,15 +166,16 @@
flex: none;
color: var(--secondary);
background: color-mix(in srgb, var(--text) 7%, var(--composer-fill));
- margin-bottom: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));
+ margin-bottom: var(--first-line-inset);
transition: background-color var(--morph), color var(--morph), transform var(--press);
}
.composer-mic:hover {
color: var(--text);
}
-.composer-mic.on {
- background: var(--danger-fill);
- color: var(--on-danger);
+/* Centered on the first line, like the mic it replaces. */
+.composer-recording {
+ height: 25px;
+ margin-bottom: var(--first-line-inset);
}
.composer-send {
width: 25px;
@@ -181,7 +184,7 @@
display: grid;
place-items: center;
flex: none;
- margin-bottom: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));
+ margin-bottom: var(--first-line-inset);
background: var(--accent-fill);
color: var(--on-accent);
transition: background-color var(--morph), color var(--morph), transform var(--press);
diff --git a/apps/desktop/src/renderer/views/thread/dictation.ts b/apps/desktop/src/renderer/views/thread/dictation.ts
index ff8dd23a..77461cb6 100644
--- a/apps/desktop/src/renderer/views/thread/dictation.ts
+++ b/apps/desktop/src/renderer/views/thread/dictation.ts
@@ -1,11 +1,14 @@
-import { useCallback, useEffect, useRef, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
+import type { SpeechEvent } from '@shared/ipc'
+import { VoiceKeyRouter, type VoiceKey } from '@shared/voice-shortcut'
+import { Observable, useModel } from '../../lib/observable'
import { VoiceSettings } from '../call/voice'
-/** The running dictation's stop: one recognizer at a time, and a call stops dictation first. */
-let stopActive: (() => void) | null = null
+/** The running dictation: one recognizer at a time, and a call stops dictation first. */
+let active: Dictation | null = null
export function stopDictation() {
- stopActive?.()
+ active?.cancel()
}
/** `said` after what's already in the box, with one space between. */
@@ -16,41 +19,111 @@ export function joinDictated(before: string, said: string) {
}
/**
- * Dictation into the composer with on-device speech (the speech helper; macOS). Words appear in
- * the box as they're recognized, after what was already typed; the mic again, or the recognizer
- * ending on a pause, stops it.
+ * Dictation with on-device speech (the speech helper; macOS). Words reach `write` as they're
+ * recognized, after what was already typed. The helper doesn't stop on a pause: it ends on
+ * `finish()` once the last words are in, on `cancel()`, on an error, or when another window takes it.
*/
-export function useDictation(draft: string, setDraft: (text: string) => void) {
- const [listening, setListening] = useState(false)
+export class Dictation extends Observable {
+ readonly available = window.codync.speech.available
+ listening = false
+ /** When it started listening (ms since the epoch). */
+ startedAt = 0
+ /** The microphone's level while listening, 0…1. Read it each frame: it doesn't re-render. */
+ level = 0
+ /** Why the last dictation ended early (no microphone, speech recognition not allowed), until the next. */
+ error: string | null = null
/** The box up to the words still being recognized. */
- const before = useRef('')
- const off = useRef<(() => void) | null>(null)
- const write = useRef(setDraft)
- write.current = setDraft
+ private before = ''
+ private off: (() => void) | null = null
+ private done: (() => void) | null = null
+ /** Started by the voice input shortcut's double tap: its next tap sends what's in the field. */
+ private byShortcut = false
+ private readonly keys = new VoiceKeyRouter()
- const finish = useCallback(() => {
- off.current?.()
- off.current = null
- window.codync.speech.stop()
- setListening(false)
- if (stopActive === finish) stopActive = null
- }, [])
-
- const start = () => {
- stopActive?.()
- before.current = draft
- off.current = window.codync.speech.onEvent((e) => {
- if (e.type === 'partial' || e.type === 'final') write.current(joinDictated(before.current, e.text))
- if (e.type === 'final') before.current = joinDictated(before.current, e.text)
- if (e.type === 'stopped' || e.type === 'error') finish()
- })
+ constructor(private write: (text: string) => void) {
+ super()
+ }
+
+ /** Listening, or stopped and waiting for the last words. */
+ get busy() {
+ return this.off !== null
+ }
+
+ start(draft: string, byShortcut = false) {
+ active?.cancel()
+ active = this
+ this.byShortcut = byShortcut
+ this.error = null
+ this.before = draft
+ this.off = window.codync.speech.onEvent((e) => this.event(e))
window.codync.speech.start(VoiceSettings.language() || null)
- stopActive = finish
- setListening(true)
+ this.startedAt = Date.now()
+ this.listening = true
+ this.changed()
}
- // A composer that goes away (another chat) stops listening.
- useEffect(() => () => { if (off.current) finish() }, [finish])
+ /** Stops listening; `done` runs once the recognizer has delivered its last words. */
+ finish(done?: () => void) {
+ this.byShortcut = false
+ if (!this.off) return done?.()
+ this.done = done ?? null
+ this.listening = false
+ this.level = 0
+ this.changed()
+ window.codync.speech.stop()
+ }
- return { available: window.codync.speech.available, listening, toggle: () => (listening ? finish() : start()) }
+ /** Stops listening now and drops words still on their way. */
+ cancel() {
+ this.byShortcut = false
+ this.error = null
+ if (!this.off) return
+ window.codync.speech.stop()
+ this.end()
+ }
+
+ /**
+ * A tap of the voice input shortcut: a tap after its double tap finishes and runs `send`. True
+ * when it's a double tap that should start: the caller shows its field, then `start(draft, true)`.
+ */
+ shortcutKey(key: VoiceKey, send: () => void): boolean {
+ const action = this.keys.route(key, { byShortcut: this.byShortcut, busy: this.busy })
+ if (action === 'send') this.finish(send)
+ return action === 'start'
+ }
+
+ private event(e: SpeechEvent) {
+ if (e.type === 'level') this.level = e.value
+ if (e.type === 'partial' || e.type === 'final') this.write(joinDictated(this.before, e.text))
+ if (e.type === 'final') this.before = joinDictated(this.before, e.text)
+ if (e.type === 'error') {
+ // Nothing to send, and the helper (which may still hold the microphone) goes too.
+ this.byShortcut = false
+ this.error = e.message
+ window.codync.speech.stop()
+ }
+ if (e.type === 'stopped' || e.type === 'error') this.end()?.()
+ }
+
+ /** Returns what was waiting for the last words. */
+ private end() {
+ this.off?.()
+ this.off = null
+ this.listening = false
+ this.level = 0
+ if (active === this) active = null
+ const done = this.done
+ this.done = null
+ this.changed()
+ return done
+ }
+}
+
+/** A composer's dictation; it stops when the composer goes away (another chat). */
+export function useDictation(setDraft: (text: string) => void) {
+ const write = useRef(setDraft)
+ write.current = setDraft
+ const [dictation] = useState(() => new Dictation((text) => write.current(text)))
+ useEffect(() => () => dictation.cancel(), [dictation])
+ return useModel(dictation)
}
diff --git a/apps/desktop/src/renderer/views/thread/voice-shortcut-keys.ts b/apps/desktop/src/renderer/views/thread/voice-shortcut-keys.ts
new file mode 100644
index 00000000..817c7def
--- /dev/null
+++ b/apps/desktop/src/renderer/views/thread/voice-shortcut-keys.ts
@@ -0,0 +1,44 @@
+import { useEffect, useRef, useState } from 'react'
+import type { VoiceKey } from '@shared/voice-shortcut'
+import type { Dictation } from './dictation'
+
+/** A tap waits this long for a composer to take it (one may be opening). */
+const KEY_WAIT_MS = 2000
+/** The voice input shortcut's latest tap for the chat window's composer. */
+let pendingKey: { key: VoiceKey; at: number } | null = null
+const keyTakers = new Set<() => void>()
+
+/** The chat window got a tap of the voice input shortcut (the island is off). */
+export function queueVoiceKey(key: VoiceKey) {
+ pendingKey = { key, at: Date.now() }
+ for (const take of keyTakers) take()
+}
+
+/**
+ * The voice input shortcut in the composer: a double tap dictates into it, the next tap sends
+ * what was said once the last words are in.
+ */
+export function useVoiceShortcut(dictation: Dictation, draft: string, submit: () => void, focus: () => void) {
+ const latest = useRef({ draft, focus })
+ latest.current = { draft, focus }
+ const [sending, setSending] = useState(false)
+ useEffect(() => {
+ const take = () => {
+ const pending = pendingKey
+ pendingKey = null
+ if (!pending || Date.now() - pending.at > KEY_WAIT_MS) return
+ if (!dictation.shortcutKey(pending.key, () => setSending(true))) return
+ latest.current.focus()
+ dictation.start(latest.current.draft, true)
+ }
+ keyTakers.add(take)
+ take()
+ return () => void keyTakers.delete(take)
+ }, [dictation])
+ // Sent after the render that has the last words in the draft.
+ useEffect(() => {
+ if (!sending) return
+ setSending(false)
+ submit()
+ }, [sending, submit])
+}
diff --git a/apps/desktop/src/shared/ipc.ts b/apps/desktop/src/shared/ipc.ts
index 4c3c0e3a..52c0ccd9 100644
--- a/apps/desktop/src/shared/ipc.ts
+++ b/apps/desktop/src/shared/ipc.ts
@@ -3,6 +3,7 @@ import type { SharedFile } from './models'
import type { SSHBridge } from './ssh'
import type { ScreenAgentState } from './computer-access'
import type { IslandRect, IslandScreen, IslandSummary } from './island'
+import type { VoiceKey, VoiceShortcut } from './voice-shortcut'
import type { ErrorReport } from './error-report'
// The bridge between the main process (host lifecycle, loopback HTTP, tray, windows)
@@ -71,6 +72,8 @@ export type WindowCommand =
| { kind: 'refresh' }
| { kind: 'respondPermission'; ref: string; entryId: string; optionId: string }
| { kind: 'sendMessage'; ref: string; text: string }
+ /** The voice input shortcut, while the island is off: a double tap starts dictating into the composer, a tap sends. */
+ | { kind: 'voiceKey'; key: VoiceKey }
/** The chat window was shown again after being hidden (`window_opened`). */
| { kind: 'windowOpened' }
@@ -149,6 +152,11 @@ export interface CodyncBridge {
start(locale: string | null): void
stop(): void
onEvent(cb: (e: SpeechEvent) => void): () => void
+ /** The voice input shortcut (Settings → General); always `off` without on-device speech. */
+ shortcut(): Promise<VoiceShortcut>
+ setShortcut(shortcut: VoiceShortcut): Promise<VoiceShortcut>
+ /** A call is on (or over): it has the microphone, so the voice input shortcut does nothing meanwhile. */
+ setCalling(on: boolean): void
}
ssh: SSHBridge
/** The notch island (shared/island.ts): its setting, the chat window's summary, and the island window's side. */
@@ -169,6 +177,8 @@ export interface CodyncBridge {
/** ⌥Y / ⌥N answer the approval on screen, from any app, while it shows. */
setKeys(on: boolean): void
onKey(cb: (answer: 'allow' | 'deny') => void): () => void
+ /** The voice input shortcut's taps, from any app, while the island is on. */
+ onVoiceKey(cb: (key: VoiceKey) => void): () => void
respond(ref: string, entryId: string, optionId: string): void
send(ref: string, text: string): void
openBot(ref: string): void
diff --git a/apps/desktop/src/shared/island.ts b/apps/desktop/src/shared/island.ts
index 01d9e168..95c5ce68 100644
--- a/apps/desktop/src/shared/island.ts
+++ b/apps/desktop/src/shared/island.ts
@@ -199,4 +199,4 @@ export function notchOf(screens: NotchScreen[], bounds: { x: number; width: numb
}
/** The setting's name: a notch on the Mac, the top edge elsewhere. */
-export const islandLabel = (platform: string) => (platform === 'darwin' ? 'Show in notch' : 'Show at top of screen')
+export const islandLabel = (platform: string) => (platform === 'darwin' ? 'Show in Notch' : 'Show at top of screen')
diff --git a/apps/desktop/src/shared/voice-shortcut.test.ts b/apps/desktop/src/shared/voice-shortcut.test.ts
new file mode 100644
index 00000000..f189bff0
--- /dev/null
+++ b/apps/desktop/src/shared/voice-shortcut.test.ts
@@ -0,0 +1,54 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { CommandTaps, VoiceKeyRouter, type ModifierChange } from './voice-shortcut.ts'
+
+const COMMAND = 0x100000
+const RIGHT = 0x10
+const LEFT = 0x08
+const down = (at: number, sinceInput = 60, flags = COMMAND | RIGHT): ModifierChange => ({ key: 54, flags, at, sinceInput })
+const up = (at: number, sinceInput = 60, flags = 0): ModifierChange => ({ key: 54, flags, at, sinceInput })
+const feed = (changes: ModifierChange[], taps = new CommandTaps('rightCommand')) => changes.map((c) => taps.feed(c))
+
+test('a quick tap is a tap, and a second one right after is a double tap', () => {
+ assert.deepEqual(feed([down(10), up(10.08), down(10.25), up(10.33)]), [null, 'tap', null, 'double'])
+ // The next tap starts over: it sends what was said.
+ assert.deepEqual(feed([down(10), up(10.08), down(10.25), up(10.33), down(12), up(12.08)]).at(-1), 'tap')
+ // Double-tapping to stop: the tap that sends and the next pair up (VoiceKeyRouter ignores that double).
+ assert.deepEqual(feed([down(10), up(10.08), down(10.25), up(10.33), down(12), up(12.08), down(12.3), up(12.38)]).slice(3), ['double', null, 'tap', null, 'double'])
+})
+
+test('slow taps are two taps, and a long press is none', () => {
+ assert.deepEqual(feed([down(10), up(10.08), down(11), up(11.08)]), [null, 'tap', null, 'tap'])
+ assert.deepEqual(feed([down(10), up(10.5), down(10.6), up(10.68)]), [null, null, null, 'tap'])
+})
+
+test('a key pressed with the command, or between the taps, is a shortcut', () => {
+ // ⌘C then ⌘V with the right command.
+ assert.deepEqual(feed([down(10), up(10.2, 0.1), down(10.3), up(10.45, 0.05)]), [null, null, null, null])
+ // A key between the taps.
+ assert.equal(feed([down(10), up(10.08), down(10.25), up(10.33, 0.15)]).at(-1), 'tap')
+})
+
+test('the other command and other modifiers break a tap', () => {
+ assert.deepEqual(feed([down(10, 60, COMMAND | RIGHT | LEFT), up(10.08, 60, COMMAND | LEFT)]), [null, null])
+ assert.equal(feed([down(10), up(10.08), { key: 56, flags: 0x20002, at: 10.15, sinceInput: 60 }, down(10.25), up(10.33)]).at(-1), 'tap')
+ assert.equal(feed([down(10), up(10.08), down(10.25, 60, COMMAND | RIGHT | 0x20000), up(10.33)]).at(-1), null)
+})
+
+test('the left command works the same way, and the right one then does nothing', () => {
+ const left = (at: number, isDown: boolean): ModifierChange => ({ key: 55, flags: isDown ? COMMAND | LEFT : 0, at, sinceInput: 60 })
+ assert.deepEqual(feed([left(10, true), left(10.08, false), left(10.25, true), left(10.33, false)], new CommandTaps('leftCommand')), [null, 'tap', null, 'double'])
+ assert.deepEqual(feed([down(10), up(10.08)], new CommandTaps('leftCommand')), [null, null])
+})
+
+test('a double tap starts dictation, the next tap sends, and the double tap that ends on it starts nothing', () => {
+ const keys = new VoiceKeyRouter()
+ const idle = { byShortcut: false, busy: false }
+ assert.equal(keys.route('double', idle), 'start')
+ assert.equal(keys.route('tap', { byShortcut: true, busy: true }), 'send')
+ assert.equal(keys.route('double', idle), null)
+ assert.equal(keys.route('double', idle), 'start')
+ // A tap with no shortcut start (the mic, or after the send) does nothing; nor does a double tap while busy.
+ assert.equal(keys.route('tap', { byShortcut: false, busy: true }), null)
+ assert.equal(keys.route('double', { byShortcut: false, busy: true }), null)
+})
diff --git a/apps/desktop/src/shared/voice-shortcut.ts b/apps/desktop/src/shared/voice-shortcut.ts
new file mode 100644
index 00000000..de2588f1
--- /dev/null
+++ b/apps/desktop/src/shared/voice-shortcut.ts
@@ -0,0 +1,95 @@
+// The voice input shortcut (macOS): a double tap of a ⌘ key starts talking to a bot from any app,
+// one more tap sends what was said. The key watcher (main/voice-shortcut.ts) reports every
+// modifier change; this decides which ones are taps.
+
+/** Settings → General: which ⌘ key, tapped twice, starts voice input. */
+export type VoiceShortcut = 'off' | 'rightCommand' | 'leftCommand'
+
+/** A tap of the shortcut's key, or the tap that completes a double tap. */
+export type VoiceKey = 'tap' | 'double'
+
+export const DEFAULT_VOICE_SHORTCUT: VoiceShortcut = 'rightCommand'
+
+export const VOICE_SHORTCUTS: { id: VoiceShortcut; label: string }[] = [
+ { id: 'off', label: 'Off' },
+ { id: 'rightCommand', label: 'Right ⌘ twice' },
+ { id: 'leftCommand', label: 'Left ⌘ twice' },
+]
+
+/** How the tip names the key ("the right ⌘"), or null when the shortcut is off. */
+export const VOICE_SHORTCUT_KEYS: Record<VoiceShortcut, string | null> = { off: null, rightCommand: 'the right ⌘', leftCommand: 'the left ⌘' }
+
+/** One modifier change, as an `NSEvent` reports it. */
+export interface ModifierChange {
+ /** The virtual key code of the modifier that changed. */
+ key: number
+ /** `NSEvent.modifierFlags` after the change, with the device bits that tell left from right. */
+ flags: number
+ /** The event's time, in seconds. */
+ at: number
+ /** Seconds since the last non-modifier key press or mouse click, in any app. */
+ sinceInput: number
+}
+
+/** Each ⌘ key's code and its device bit (`NX_DEVICERCMDKEYMASK`, `NX_DEVICELCMDKEYMASK`). */
+const COMMANDS = { rightCommand: { key: 54, down: 0x10, other: 0x08 }, leftCommand: { key: 55, down: 0x08, other: 0x10 } }
+/** ⇧, ⌃, ⌥ and fn: a chord, not a tap. */
+const MODIFIERS = 0x20000 | 0x40000 | 0x80000 | 0x800000
+/** The longest press that still counts as a tap. */
+const TAP_S = 0.3
+/** The longest pause between the first tap's release and the second press. */
+const GAP_S = 0.4
+
+/** Taps of one ⌘ key alone: short, nothing else held, no other key pressed or click meanwhile (⌘-click). */
+export class CommandTaps {
+ private readonly command: (typeof COMMANDS)[keyof typeof COMMANDS]
+ private pressed: number | null = null
+ private first: { pressed: number; released: number } | null = null
+
+ constructor(shortcut: Exclude<VoiceShortcut, 'off'>) {
+ this.command = COMMANDS[shortcut]
+ }
+
+ /** Takes the next change: a tap, the second of two quick taps, or nothing. */
+ feed(change: ModifierChange): VoiceKey | null {
+ const { key, flags, at, sinceInput } = change
+ const others = MODIFIERS | this.command.other
+ if (key !== this.command.key) return this.reset()
+ if (flags & this.command.down) {
+ this.pressed = flags & others ? null : at
+ return null
+ }
+ const pressed = this.pressed
+ this.pressed = null
+ if (pressed === null || flags & others || at - pressed > TAP_S || sinceInput < at - pressed) return this.reset()
+ const first = this.first
+ // A double tap also needs no key pressed or click between the taps.
+ if (first && pressed - first.released <= GAP_S && sinceInput >= at - first.pressed) {
+ this.first = null
+ return 'double'
+ }
+ this.first = { pressed, released: at }
+ return 'tap'
+ }
+
+ private reset() {
+ this.pressed = null
+ this.first = null
+ return null
+ }
+}
+
+/**
+ * What a tap of the shortcut does to dictation: a double tap starts it, a tap after a start by the
+ * shortcut sends. Double-tapping to send arrives as that tap then a double, and the double starts nothing.
+ */
+export class VoiceKeyRouter {
+ private sentByTap = false
+
+ route(key: VoiceKey, dictation: { byShortcut: boolean; busy: boolean }): 'start' | 'send' | null {
+ const pairsWithSend = this.sentByTap
+ this.sentByTap = key === 'tap' && dictation.byShortcut
+ if (this.sentByTap) return 'send'
+ return key === 'double' && !pairsWithSend && !dictation.busy ? 'start' : null
+ }
+}
diff --git a/docs/architecture/desktop-app.md b/docs/architecture/desktop-app.md
index e44b321c..c828e160 100644
--- a/docs/architecture/desktop-app.md
+++ b/docs/architecture/desktop-app.md
@@ -22,7 +22,7 @@ reaching the end of a history response determines when there are no earlier mess
| Path | |
|---|---|
-| `src/main/` | Main process: host service (`host-controller.ts`), loopback HTTP/SSE proxy (`host-proxy.ts`), menu bar/tray, windows, account sign-in (Clerk Frontend API, `account.ts`), cloud API and device key (`cloud.ts`), updates (`updates.ts`), SSH tunnels (`ssh.ts`), Codync Screen agent (`screen.ts`), on-device speech (`speech.ts`), browser sign-in callbacks (`auth.ts`), the [notch island](../features/notch-island.md) window (`island.ts`), [error tracking](../features/analytics.md#error-tracking) (`error-reports.ts`) |
+| `src/main/` | Main process: host service (`host-controller.ts`), loopback HTTP/SSE proxy (`host-proxy.ts`), menu bar/tray, windows, account sign-in (Clerk Frontend API, `account.ts`), cloud API and device key (`cloud.ts`), updates (`updates.ts`), SSH tunnels (`ssh.ts`), Codync Screen agent (`screen.ts`), on-device speech (`speech.ts`) and its [voice input shortcut](../features/voice-call.md#voice-input-shortcut) (`voice-shortcut.ts`), browser sign-in callbacks (`auth.ts`), the [notch island](../features/notch-island.md) window (`island.ts`), [error tracking](../features/analytics.md#error-tracking) (`error-reports.ts`) |
| `src/preload/` | The `window.codync` bridge (types in `src/shared/ipc.ts`) |
| `src/renderer/store/` | `BotStore` (port of kit's; `bot-store.ts` actions over `bot-sync.ts` events stream and `bot-mirror.ts` state), `AppModel` (window state, approvals), `CloudModel` (claims, cloud default), `AccountSession` |
| `src/renderer/components/` | Design system: controls, overlays (sheets, dialogs, menus), SF Symbol icons, character avatars, thinking orbs |
diff --git a/docs/features/notch-island.md b/docs/features/notch-island.md
index 9700eb82..468b3456 100644
--- a/docs/features/notch-island.md
+++ b/docs/features/notch-island.md
@@ -5,7 +5,7 @@ MacBook it grows out of the notch; on a Mac without a notch, and on Windows and
from the top edge. It draws Codync's own bot characters. Design inspired by
[@novra.design](https://www.instagram.com/novra.design/).
-Turn it on or off in **Settings → General → System** (*Show in notch* on macOS, *Show at top
+Turn it on or off in **Settings → General → System** (*Show in Notch* on macOS, *Show at top
of screen* elsewhere) or the menu bar / tray menu's **Settings**. It is off by default while it
is experimental. The setting belongs to the desktop installation (`showIsland` in `host-prefs.json`).
@@ -31,7 +31,7 @@ bar). A bot can be messaged from the island, and ⌥Y / ⌥N answer an approval
| Approval | A permission card is pending | The command and the card's options, deny left and Allow once right (⌥N / ⌥Y from any app while it shows) |
| Finished | A bot stops working | "*Bot* is done" with its last message, Open conversation / OK, for 5.2 s |
| Failed | A bot stops on an error | "*Bot* stopped." with the error in red, until OK |
-| Message a bot | The header's bubble | The bot chip (click for the next bot) and a field; Enter sends, Esc closes |
+| Message a bot | The header's bubble, or the [voice input shortcut](voice-call.md#voice-input-shortcut) | The bot chip (click for the next bot) and a field with the mic (macOS); Enter sends, Esc closes. Until the shortcut is first used, a tip under the field says how to talk from any app |
| Usage | The header's gauge or the usage meter | Claude and Codex limits side by side: each window's bar, percent and reset |
Opened by hover, the island folds 0.8 s after the pointer leaves; opened by a click or a tab,
diff --git a/docs/features/voice-call.md b/docs/features/voice-call.md
index c1013b82..7854d219 100644
--- a/docs/features/voice-call.md
+++ b/docs/features/voice-call.md
@@ -44,16 +44,52 @@ The bot is the same agent on the computer; the call only changes how you talk to
The composer's mic (beside send / call, as in Grok Bot) types what you say into the box with the
same on-device recognizer and the call's language setting; nothing is sent until you send it.
-Words appear as they're recognized, after what's already typed; tapping the mic again, or the
-recognizer finishing on a pause, stops it, and a call takes the microphone from it.
+Words appear as they're recognized, after what's already typed. While it listens the mic becomes
+a recording pill (stop, the time, dots that follow your voice), on the iPhone and the desktop, and
+a call takes the microphone from it. On the desktop, dictation ends only on the pill's stop (or the
+shortcut's tap), which waits for the last words, on an error (shown in the window), or when another
+window takes the speech helper; it doesn't stop on a pause. On the iPhone the pill's stop keeps what
+was recognized so far. The desktop island's message field has the same mic and pill.
- Desktop (`views/thread/dictation.ts`): the speech helper, macOS only, so Linux and Windows show
no mic. An unpackaged `npm run dev` can't use it: macOS stops the helper because the dev Electron
has no speech usage description, so test it in a packaged build (`dist:mac:dev`), like calls.
-- iPhone (`Thread/Dictation.swift`): `SFSpeechRecognizer` on device; with no microphone route
- (some simulators) the mic does nothing.
+- Desktop: the speech helper serves one window at a time. When another window starts it (the
+ island while the chat window dictates or calls, or the other way round), `main/speech.ts` tells
+ the window that lost it (`stopped`) and ignores that window's stop.
+- iPhone (`Thread/Dictation.swift`, pill in `Thread/RecordingPill.swift`, dots shared with the call
+ bar in `LevelDots.swift`): `SFSpeechRecognizer` on device; with no microphone route (some
+ simulators) the mic does nothing.
+- Android has no composer mic yet.
- The terminal UI has no microphone, so no mic.
+### Voice input shortcut
+
+Desktop on macOS, set in **Settings → General → System** (*Voice input shortcut*: off, the right ⌘
+twice (the default) or the left ⌘ twice; `voiceShortcut` in `host-prefs.json`). From any app, a
+double tap of that key starts dictation into the [notch island](notch-island.md)'s message field
+while the island is on, else into the chat window's composer (the top bot when no chat is open).
+One more tap stops it and sends what was said once the last words are in; the pill's stop leaves
+the words in the field unsent, and an error leaves nothing for the tap to send.
+Double-tapping to stop counts as that one tap, not a new start. While a call is on (any engine),
+the shortcut does nothing: the call has the microphone.
+
+- `apps/desktop/src/main/voice-shortcut.ts` keeps an `osascript -l JavaScript` process with an
+ `NSEvent` global monitor for modifier changes, plus the seconds since the last key press or
+ mouse click. Neither needs the Accessibility or Input Monitoring grant, and the app never sees
+ what is typed. The chat window reports calls (`voice:calling`) so taps are dropped meanwhile.
+- `shared/voice-shortcut.ts` decides what counts: a press under 0.3 s with no other modifier held
+ and no key pressed or click meanwhile is a tap; a second tap starting within 0.4 s of the first,
+ with no key pressed or click since the first, is a double tap. ⌘C, ⌘V and ⌘-clicks with the
+ same key are neither.
+- The renderer side is `Dictation.shortcutKey` (`views/thread/dictation.ts`), shared by the
+ composer and the island's model; `VoiceKeyRouter` (`shared/voice-shortcut.ts`) decides whether
+ a tap starts, sends or does nothing. The chat window queues taps for its composer in
+ `views/thread/voice-shortcut-keys.ts`.
+- macOS only: Windows and Linux have no ⌘ or on-device speech, and a modifier-only shortcut
+ there would need a native keyboard hook. The iPhone, Android and the terminal UI have no global
+ keyboard shortcut.
+
## Background verification
Store regression tests cover sending a recognized utterance after backgrounding,