Open source Grok Bot alternative, Bring your own agent and subscription: message Claude Code, Codex, Gemini CLI, Cursor, Copilot and 40+ AI agents as bots on your own computer, from iPhone, Mac, Windows, Linux or terminal. Agent-agnostic and free. Also a Muse and Dots alternative. Built in Rust.
Rust acpagent-client-protocolai-agentsai-assistantclaudeclaude-codecodexcoding-agentscursorgeminigemini-cligrok-bot-alternativeiosiphonelinuxmacosmcpopen-sourcerustself-hosted
feat(desktop): talk to a bot from any app with a double tap of ⌘
leepokai89b749257595
diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.tsindex e34638e1..22c23868 100644--- a/apps/desktop/src/main/index.ts+++ b/apps/desktop/src/main/index.ts@@ -14,6 +14,7 @@ import { AccountService } from './account'import { Updates, yieldsToInstall } from './updates'import { handleURL, registerAuthIPC, registerSchemes, urlFromArgv } from './auth'import { registerSpeech } from './speech'+import { VoiceShortcutKeys } from './voice-shortcut'import { registerScreenIPC } from './screen'import { registerCloud } from './cloud'import { registerSSH } from './ssh'@@ -227,6 +228,8 @@ if (yieldsToInstall()) {updates.on('change', () => tray.hostChanged())registerIPC(tray)island.register()+ // Voice input goes to the island while it's on, else to the chat window's composer.+ new VoiceShortcutKeys((key) => (island.enabled ? island.voiceKey(key) : send({ kind: 'voiceKey', key }, key === 'double'))).register()registerAuthIPC()registerSpeech()registerScreenIPC()diff --git a/apps/desktop/src/main/island.ts b/apps/desktop/src/main/island.tsindex 3720de1a..f0b730f0 100644--- a/apps/desktop/src/main/island.ts+++ b/apps/desktop/src/main/island.ts@@ -1,6 +1,7 @@import { execFile } from 'node:child_process'import { BrowserWindow, globalShortcut, ipcMain, screen } from 'electron'import type { WindowCommand } from '../shared/ipc'+import type { VoiceKey } from '../shared/voice-shortcut'import { hitRect, hits, notchOf, PANEL, type IslandRect, type IslandScreen, type IslandSummary, type NotchScreen } from '../shared/island'import { prefs } from './host-controller'@@ -202,6 +203,11 @@ export class Island {}}+ /** The voice input shortcut's taps, while the island is on. */+ voiceKey(key: VoiceKey) {+ this.window?.webContents.send('island:voiceKey', key)+ }+/** Top center of the display with the notch, else the main display. */private async place() {if (!this.window) returndiff --git a/apps/desktop/src/main/speech.ts b/apps/desktop/src/main/speech.tsindex ace32d5d..35a9ad68 100644--- a/apps/desktop/src/main/speech.ts+++ b/apps/desktop/src/main/speech.ts@@ -16,6 +16,9 @@ function helperPath() {return candidates.find((c): c is string => !!c && existsSync(c)) ?? null}+/** Whether this build can transcribe on device. */+export const speechAvailable = () => process.platform === 'darwin' && helperPath() !== null+let child: ChildProcessWithoutNullStreams | null = nulllet listener: WebContents | null = null/** Bumped per start: a stopping process may still deliver its final result, an older one may not. */@@ -27,6 +30,8 @@ function emit(event: SpeechEvent) {function start(sender: WebContents, locale: string | null) {stop()+ // Another window's dictation or call loses the helper, and its exit (an older generation) won't say so.+ if (listener && listener !== sender) emit({ type: 'stopped' })listener = senderconst path = helperPath()if (!path) return emit({ type: 'error', message: "On-device speech isn't available in this build." })@@ -73,9 +78,12 @@ function locales() {}export function registerSpeech() {- ipcMain.on('speech:available', (e) => (e.returnValue = process.platform === 'darwin' && helperPath() !== null))+ ipcMain.on('speech:available', (e) => (e.returnValue = speechAvailable()))ipcMain.on('speech:locales', (e) => (e.returnValue = locales()))ipcMain.on('speech:start', (e, locale: string | null) => start(e.sender, locale))- ipcMain.on('speech:stop', () => stop())+ // Only the window using the helper stops it: one that lost it to another window has nothing to stop.+ ipcMain.on('speech:stop', (e) => {+ if (e.sender === listener) stop()+ })app.on('before-quit', stop)}diff --git a/apps/desktop/src/main/voice-shortcut.ts b/apps/desktop/src/main/voice-shortcut.tsnew file mode 100644index 00000000..df01d3d6--- /dev/null+++ b/apps/desktop/src/main/voice-shortcut.ts@@ -0,0 +1,83 @@+import { spawn } from 'node:child_process'+import { createInterface } from 'node:readline'+import { app, ipcMain, type WebContents } from 'electron'+import { CommandTaps, DEFAULT_VOICE_SHORTCUT, VOICE_SHORTCUTS, type VoiceKey, type VoiceShortcut } from '../shared/voice-shortcut'+import { prefs } from './host-controller'+import { speechAvailable } from './speech'++// Every modifier change in any app, from an NSEvent global monitor, with the seconds since the+// last key press or click. Neither needs the Accessibility or Input Monitoring grant (other keys+// would), so the app learns only that a modifier moved, never what was typed. JXA reaches AppKit+// without a helper binary.+const MONITOR = `ObjC.import('AppKit'); ObjC.import('CoreGraphics')+const out = $.NSFileHandle.fileHandleWithStandardOutput+const since = (type) => $.CGEventSourceSecondsSinceLastEventType($.kCGEventSourceStateCombinedSessionState, type)+$.NSApplication.sharedApplication.setActivationPolicy($.NSApplicationActivationPolicyProhibited)+$.NSEvent.addGlobalMonitorForEventsMatchingMaskHandler($.NSEventMaskFlagsChanged, (e) => {+ const sinceInput = Math.min(since($.kCGEventKeyDown), since($.kCGEventLeftMouseDown), since($.kCGEventRightMouseDown), since($.kCGEventOtherMouseDown))+ out.writeData($(e.keyCode + ' ' + e.modifierFlags + ' ' + e.timestamp + ' ' + sinceInput + '\\n').dataUsingEncoding($.NSUTF8StringEncoding))+})+$.NSApp.run`++/** Reports the shortcut key's taps from any app until the returned stop. */+function watchTaps(shortcut: Exclude<VoiceShortcut, 'off'>, onKey: (key: VoiceKey) => void): () => void {+ const taps = new CommandTaps(shortcut)+ const proc = spawn('/usr/bin/osascript', ['-l', 'JavaScript', '-e', MONITOR], { stdio: ['ignore', 'pipe', 'pipe'] })+ createInterface({ input: proc.stdout }).on('line', (line) => {+ const [key = 0, flags = 0, at = 0, sinceInput = 0] = line.split(' ').map(Number)+ const tap = taps.feed({ key, flags, at, sinceInput })+ if (tap) onKey(tap)+ })+ proc.stderr.on('data', (chunk: Buffer) => console.error(`voice shortcut: ${chunk.toString('utf8').trim()}`))+ proc.on('error', (error) => console.error(`voice shortcut: couldn't watch the keyboard: ${error.message}`))+ let stopping = false+ // ponytail: logged only, not restarted; the setting or a relaunch starts it again.+ proc.on('exit', (code, signal) => {+ if (!stopping) console.error(`voice shortcut: the keyboard watcher exited (${code ?? signal})`)+ })+ return () => {+ stopping = true+ proc.kill()+ }+}++/**+ * The voice input shortcut (macOS, with on-device speech): watches the chosen ⌘ key while it's on+ * and hands its taps to `onKey`. The setting belongs to the desktop installation (`voiceShortcut`+ * in `host-prefs.json`).+ */+export class VoiceShortcutKeys {+ private stop: (() => void) | null = null+ /** The window with a call on: the call has the microphone, so taps do nothing meanwhile. */+ private call: WebContents | null = null++ constructor(private onKey: (key: VoiceKey) => void) {}++ get shortcut(): VoiceShortcut {+ if (!speechAvailable()) return 'off'+ const saved = prefs.get('voiceShortcut')+ return VOICE_SHORTCUTS.find((s) => s.id === saved)?.id ?? DEFAULT_VOICE_SHORTCUT+ }++ register() {+ ipcMain.handle('voice:shortcut', () => this.shortcut)+ ipcMain.handle('voice:setShortcut', (_e, shortcut: VoiceShortcut) => {+ prefs.set('voiceShortcut', shortcut)+ this.sync()+ return this.shortcut+ })+ ipcMain.on('voice:calling', (e, on: boolean) => (this.call = on ? e.sender : null))+ app.on('will-quit', () => this.stop?.())+ this.sync()+ }++ private sync() {+ this.stop?.()+ this.stop = null+ const shortcut = this.shortcut+ if (shortcut === 'off') return+ this.stop = watchTaps(shortcut, (key) => {+ if (!this.call || this.call.isDestroyed()) this.onKey(key)+ })+ }+}diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.tsindex 453d7d0f..48939706 100644--- a/apps/desktop/src/preload/index.ts+++ b/apps/desktop/src/preload/index.ts@@ -62,6 +62,9 @@ const bridge: CodyncBridge = {start: (locale) => ipcRenderer.send('speech:start', locale),stop: () => ipcRenderer.send('speech:stop'),onEvent: (cb) => listen('speech:event', cb),+ shortcut: () => ipcRenderer.invoke('voice:shortcut'),+ setShortcut: (shortcut) => ipcRenderer.invoke('voice:setShortcut', shortcut),+ setCalling: (on) => ipcRenderer.send('voice:calling', on),},ssh: sshBridge,island: {@@ -77,6 +80,7 @@ const bridge: CodyncBridge = {setFocusable: (on) => ipcRenderer.send('island:focusable', on),setKeys: (on) => ipcRenderer.send('island:keys', on),onKey: (cb) => listen('island:key', cb),+ onVoiceKey: (cb) => listen('island:voiceKey', cb),respond: (ref, entryId, optionId) => ipcRenderer.send('island:respond', ref, entryId, optionId),send: (ref, text) => ipcRenderer.send('island:send', ref, text),openBot: (ref) => ipcRenderer.send('island:openBot', ref),diff --git a/apps/desktop/src/renderer/components/LevelDots.tsx b/apps/desktop/src/renderer/components/LevelDots.tsxnew file mode 100644index 00000000..0f69f410--- /dev/null+++ b/apps/desktop/src/renderer/components/LevelDots.tsx@@ -0,0 +1,67 @@+import { useEffect, useRef } from 'react'+import { useDark, usePixelRatio, useReduceMotion, withAlpha } from '../lib/theme'++/** What the dots show this frame: your voice's level (0…1), the other side talking, or work going on. */+export interface LevelSample {+ level: number+ speaking?: boolean+ working?: boolean+}++/** The gap between dots, in points. */+const SPACING = 8++/**+ * A dotted line that follows your voice while listening, ripples while the other side talks,+ * breathes while work goes on, and rests as faint dots otherwise. `sample` is read every frame.+ * `color` null is the text color.+ */+export function LevelDots({ sample, color, width, height = 16, label, className }: {+ sample: () => LevelSample+ color: string | null+ width: number+ height?: number+ label: string+ className?: string+}) {+ const canvas = useRef<HTMLCanvasElement>(null)+ const dark = useDark()+ const reduceMotion = useReduceMotion()+ const scale = usePixelRatio()+ const live = useRef({ sample, color, dark, reduceMotion })+ live.current = { sample, color, dark, reduceMotion }++ useEffect(() => {+ const el = canvas.current+ const ctx = el?.getContext('2d')+ if (!el || !ctx) return+ el.width = width * scale+ el.height = height * scale+ let frame = 0+ const draw = (now: number) => {+ const { sample, color, dark, reduceMotion } = live.current+ const { level, speaking = false, working = false } = sample()+ const t = reduceMotion ? 0 : now / 1000+ const accent = dark ? '#ffffff' : '#000000'+ const tint = speaking ? accent : (color ?? accent)+ ctx.setTransform(scale, 0, 0, scale, 0, 0)+ ctx.clearRect(0, 0, width, height)+ const count = Math.max(1, Math.floor(width / SPACING))+ for (let i = 0; i < count; i++) {+ const x = i * SPACING + SPACING / 2+ const wave = (Math.sin(i * 0.55 - t * 6) + 1) / 2+ const amount = speaking ? 0.5 + 0.5 * wave : level > 0.05 ? Math.max(0.2, level) * (0.45 + 0.55 * wave) : working ? 0.25 * ((Math.sin(t * 2.4) + 1) / 2) : 0.08+ const bar = 4 + amount * (height - 4)+ ctx.fillStyle = withAlpha(tint, 0.5 + 0.5 * amount)+ ctx.beginPath()+ ctx.roundRect(x - 1.25, (height - bar) / 2, 2.5, bar, 1.25)+ ctx.fill()+ }+ frame = requestAnimationFrame(draw)+ }+ frame = requestAnimationFrame(draw)+ return () => cancelAnimationFrame(frame)+ }, [scale, width, height])++ return <canvas ref={canvas} role="img" aria-label={label} className={className} style={{ width, height }} />+}diff --git a/apps/desktop/src/renderer/components/RecordingPill.tsx b/apps/desktop/src/renderer/components/RecordingPill.tsxnew file mode 100644index 00000000..1d13c267--- /dev/null+++ b/apps/desktop/src/renderer/components/RecordingPill.tsx@@ -0,0 +1,39 @@+import { useEffect, useState } from 'react'+import { Icon } from './Icon'+import { LevelDots, type LevelSample } from './LevelDots'+import { Text } from './Text'+import './recording-pill.css'++/** m:ss since `startedAt`, updated every second. */+function useElapsed(startedAt: number) {+ const [now, setNow] = useState(() => Date.now())+ useEffect(() => {+ const timer = setInterval(() => setNow(Date.now()), 1000)+ return () => clearInterval(timer)+ }, [])+ const seconds = Math.max(0, Math.floor((now - startedAt) / 1000))+ return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, '0')}`+}++/**+ * Dictation in progress, in place of the mic: stop, the time since it started, and dots that+ * follow the microphone's level. `color` null is the text color.+ */+export function RecordingPill({ startedAt, sample, onStop, color = null, className = '' }: {+ startedAt: number+ sample: () => LevelSample+ onStop: () => void+ color?: string | null+ className?: string+}) {+ const elapsed = useElapsed(startedAt)+ return (+ <span className={`recording-pill ${className}`}>+ <button type="button" className="recording-stop" aria-label="Stop dictating" title="Stop dictating" onClick={onStop}>+ <Icon name="stop.fill" size={11} scaled={false} />+ </button>+ <Text tabular tone="text">{elapsed}</Text>+ <LevelDots sample={sample} color={color} width={32} height={12} label="Listening" />+ </span>+ )+}diff --git a/apps/desktop/src/renderer/components/recording-pill.css b/apps/desktop/src/renderer/components/recording-pill.cssnew file mode 100644index 00000000..037e0c85--- /dev/null+++ b/apps/desktop/src/renderer/components/recording-pill.css@@ -0,0 +1,26 @@+/* Dictation in progress: a quiet pill with stop, the time and the level dots. */+.recording-pill {+ flex: none;+ display: inline-flex;+ align-items: center;+ gap: 6px;+ height: 28px;+ box-sizing: border-box;+ padding: 0 10px 0 4px;+ border-radius: 14px;+ background: rgba(var(--text-rgb), 0.08);+ font-size: var(--text-body);+ animation: fade-in var(--fade) both;+}+.recording-stop {+ width: 22px;+ height: 22px;+ border-radius: 50%;+ display: grid;+ place-items: center;+ color: var(--text);+ transition: background-color var(--hover);+}+.recording-stop:hover {+ background: rgba(var(--text-rgb), 0.1);+}diff --git a/apps/desktop/src/renderer/demo/bridge.ts b/apps/desktop/src/renderer/demo/bridge.tsindex 93636b3f..0dfc6d63 100644--- a/apps/desktop/src/renderer/demo/bridge.ts+++ b/apps/desktop/src/renderer/demo/bridge.ts@@ -57,7 +57,7 @@ const bridge: CodyncBridge = {signOut: async () => {},token: unavailable,},- speech: { available: false, locales: () => [], start() {}, stop() {}, onEvent: none },+ speech: { available: false, locales: () => [], start() {}, stop() {}, onEvent: none, shortcut: async () => 'off', setShortcut: async () => 'off', setCalling() {} },ssh: {state: async () => ({ profiles: [], status: {}, attachments: [] }),onChange: none,@@ -81,6 +81,7 @@ const bridge: CodyncBridge = {setFocusable() {},setKeys() {},onKey: none,+ onVoiceKey: none,respond() {},send() {},openBot() {},diff --git a/apps/desktop/src/renderer/views/ChatWindow.tsx b/apps/desktop/src/renderer/views/ChatWindow.tsxindex 63ac3f30..71323599 100644--- a/apps/desktop/src/renderer/views/ChatWindow.tsx+++ b/apps/desktop/src/renderer/views/ChatWindow.tsx@@ -20,6 +20,7 @@ import { SidebarFooter } from './SidebarFooter'import { UpdateBadge } from './UpdateButton'import { ComputerFilterHeader, computerSelection } from './ComputerFilterHeader'import { ThreadView } from './thread/ThreadView'+import { queueVoiceKey } from './thread/voice-shortcut-keys'import { NewBotView, NewGroupView } from './bots/NewBotView'import { BotEditorView } from './bots/BotEditorView'import { MarketplaceView } from './marketplace/MarketplaceView'@@ -227,6 +228,11 @@ function ChatSplitView() {if (c.kind === 'newBot' && onlineStores.length) create('bot')if (c.kind === 'search') setSearching(true)if (c.kind === 'toggleSidebar') setCompact(!prefs.sidebarCompact.get())+ if (c.kind === 'voiceKey') {+ // No chat open: talk to the top bot in the sidebar.+ if (c.key === 'double' && !app.selection) app.select((roster.navigable[0] ?? roster.items[0])?.ref ?? null)+ queueVoiceKey(c.key)+ }}),)diff --git a/apps/desktop/src/renderer/views/call/CallView.tsx b/apps/desktop/src/renderer/views/call/CallView.tsxindex 5b21b23e..c83ae6b5 100644--- a/apps/desktop/src/renderer/views/call/CallView.tsx+++ b/apps/desktop/src/renderer/views/call/CallView.tsx@@ -3,10 +3,11 @@ import { isWorking } from '@shared/models'import { createContext, useContext, useEffect, useRef, useState } from 'react'import { BotAvatar } from '../../components/Avatar'import { Icon } from '../../components/Icon'+import { LevelDots } from '../../components/LevelDots'import { Sheet } from '../../components/Overlay'import { Text } from '../../components/Text'import { useModel } from '../../lib/observable'-import { avatarColor, useDark, usePixelRatio, useReduceMotion, withAlpha } from '../../lib/theme'+import { avatarColor } from '../../lib/theme'import { analytics } from '../../store/analytics'import { useStore } from '../../store/context'import { VoiceChatSettingsView } from '../settings/VoiceChatSettingsView'@@ -143,8 +144,10 @@ export function CallView({ botId, onSpeaking, interrupt, onEnd }: {useEffect(() => {const startedAt = Date.now()+ window.codync.speech.setCalling(true)begin(VoiceSettings.provider())return () => {+ window.codync.speech.setCalling(false)closed.current = trueengine.current?.end()countCloudMinutes()@@ -200,7 +203,13 @@ export function CallView({ botId, onSpeaking, interrupt, onEnd }: {>{bot ? <BotAvatar bot={bot} size={34} /> : <span className="call-avatar-empty" />}</button>- <CallLevelDots engine={session} working={working} color={color} label={statusText} />+ <LevelDots+ sample={() => ({ level: session?.level ?? 0, speaking: session?.phase === 'speaking', working })}+ color={color}+ width={120}+ label={statusText}+ className="call-dots"+ />{/* Audio leaves the computer: say where. */}{session?.provider ? (<Text size="caption2" weight="semibold" tone="secondary" aria-label={`Voice by ${providerName(session.provider)}`}>@@ -237,55 +246,3 @@ export function CallView({ botId, onSpeaking, interrupt, onEnd }: {</div>)}--const DOTS_WIDTH = 120-const DOTS_HEIGHT = 16--/**- * The dotted line in the call bar: it follows your voice while listening, ripples while the bot- * talks, breathes while it works, and rests as faint dots otherwise.- */-function CallLevelDots({ engine, working, color, label }: { engine: VoiceEngine | null; working: boolean; color: string | null; label: string }) {- const canvas = useRef<HTMLCanvasElement>(null)- const dark = useDark()- const reduceMotion = useReduceMotion()- const scale = usePixelRatio()- const live = useRef({ engine, working, color, dark, reduceMotion })- live.current = { engine, working, color, dark, reduceMotion }-- useEffect(() => {- const el = canvas.current- const ctx = el?.getContext('2d')- if (!el || !ctx) return- el.width = DOTS_WIDTH * scale- el.height = DOTS_HEIGHT * scale- let frame = 0- const draw = (now: number) => {- const { engine, working, color, dark, reduceMotion } = live.current- const t = reduceMotion ? 0 : now / 1000- const speaking = engine?.phase === 'speaking'- const level = engine?.level ?? 0- const accent = dark ? '#ffffff' : '#000000'- const tint = speaking ? accent : (color ?? accent)- ctx.setTransform(scale, 0, 0, scale, 0, 0)- ctx.clearRect(0, 0, DOTS_WIDTH, DOTS_HEIGHT)- const spacing = 8- const count = Math.max(1, Math.floor(DOTS_WIDTH / spacing))- for (let i = 0; i < count; i++) {- const x = i * spacing + spacing / 2- const wave = (Math.sin(i * 0.55 - t * 6) + 1) / 2- const amount = speaking ? 0.5 + 0.5 * wave : level > 0.05 ? Math.max(0.2, level) * (0.45 + 0.55 * wave) : working ? 0.25 * ((Math.sin(t * 2.4) + 1) / 2) : 0.08- const height = 4 + amount * (DOTS_HEIGHT - 4)- ctx.fillStyle = withAlpha(tint, 0.5 + 0.5 * amount)- ctx.beginPath()- ctx.roundRect(x - 1.25, (DOTS_HEIGHT - height) / 2, 2.5, height, 1.25)- ctx.fill()- }- frame = requestAnimationFrame(draw)- }- frame = requestAnimationFrame(draw)- return () => cancelAnimationFrame(frame)- }, [scale])-- return <canvas ref={canvas} role="img" aria-label={label} className="call-dots" style={{ width: DOTS_WIDTH, height: DOTS_HEIGHT }} />-}diff --git a/apps/desktop/src/renderer/views/island/IslandViews.tsx b/apps/desktop/src/renderer/views/island/IslandViews.tsxindex b6b9c077..e050eb98 100644--- a/apps/desktop/src/renderer/views/island/IslandViews.tsx+++ b/apps/desktop/src/renderer/views/island/IslandViews.tsx@@ -1,9 +1,11 @@-import { useRef, useState } from 'react'+import { useRef } from 'react'import type { IslandBot, IslandPermission } from '@shared/island'import { CharacterAvatar } from '../../components/Avatar'import { Button, Spinner } from '../../components/Controls'import { Icon } from '../../components/Icon'+import { RecordingPill } from '../../components/RecordingPill'import { ThinkingOrb } from '../../components/ThinkingOrb'+import { useModel } from '../../lib/observable'import { avatarColor, withAlpha } from '../../lib/theme'import type { IslandModel } from './island-model'import { useArrivals, useFlip, useSwap } from './island-motion'@@ -182,8 +184,10 @@ export function Notice({ model, bot, failed }: { model: IslandModel; bot: Island/** Writes to a bot without opening Codync: the chip picks the bot, Enter sends, Esc closes. */export function Message({ model }: { model: IslandModel }) {- const [text, setText] = useState('')+ const dictation = useModel(model.dictation)const bot = model.messageBot+ /** Why dictation stopped, else how to talk from any app. */+ const hint = dictation.error ?? model.voiceTipconst swap = useSwap(bot?.ref)if (!bot) return nullreturn (@@ -211,24 +215,31 @@ export function Message({ model }: { model: IslandModel }) {className="island-field"onSubmit={(e) => {e.preventDefault()- model.send(text)- setText('')+ model.send()}}><inputautoFocusaria-label={`Message ${bot.name}`}placeholder={`Message ${bot.name}…`}- value={text}- onChange={(e) => setText(e.target.value)}+ value={model.draft}+ onChange={(e) => model.setDraft(e.target.value)}onKeyDown={(e) => {if (e.key === 'Escape') model.close()}}/>- <button type="submit" className="island-send" aria-label="Send" title="Send" disabled={!text.trim()}>+ {dictation.listening ? (+ <RecordingPill startedAt={dictation.startedAt} sample={() => dictation} onStop={() => dictation.finish()} color="#ffffff" />+ ) : dictation.available ? (+ <button type="button" className="island-mic" aria-label="Dictate" title="Dictate" onClick={() => dictation.start(model.draft)}>+ <Icon name="mic" size={12} weight="medium" scaled={false} />+ </button>+ ) : null}+ <button type="submit" className="island-send" aria-label="Send" title="Send" disabled={!model.draft.trim()}><Icon name="arrow.up" size={12} weight="bold" scaled={false} /></button></form>+ {hint ? <span className="island-hint">{hint}</span> : null}</div></div><MiniColumn bots={model.others(3).filter((b) => b.ref !== bot.ref)} onOpen={(ref) => model.openBot(ref)} />diff --git a/apps/desktop/src/renderer/views/island/island-model.ts b/apps/desktop/src/renderer/views/island/island-model.tsindex 00a11038..ef45d1d1 100644--- a/apps/desktop/src/renderer/views/island/island-model.ts+++ b/apps/desktop/src/renderer/views/island/island-model.ts@@ -2,7 +2,10 @@ import {FINISHED_MS, focusOf, isActive, islandFrame, islandMode, islandRect, othersOf, transitions,type IslandBot, type IslandNotice, type IslandPermission, type IslandScreen, type IslandSummary, type IslandTab,} from '@shared/island'+import { VOICE_SHORTCUT_KEYS, type VoiceKey, type VoiceShortcut } from '@shared/voice-shortcut'import { Observable } from '../../lib/observable'+import { pref } from '../../lib/prefs'+import { Dictation } from '../thread/dictation'/** The bare notch peeks at once and opens after a rest; the compact island opens sooner. */const OPEN_FROM_PEEK_MS = 650@@ -11,6 +14,8 @@ const UNPEEK_MS = 600/** Opened by hover, it folds soon after the pointer leaves; opened by a click, after a countdown. */const LEAVE_MS = 800export const PINNED_LEAVE_MS = 10_000+/** Set once the voice input shortcut has been used; until then the message field shows how. */+const voiceShortcutUsed = pref('islandVoiceShortcutUsed', false)/** The island window's state: the chat window's summary, the pointer, the header's tab and what was put off. */export class IslandModel extends Observable {@@ -23,6 +28,12 @@ export class IslandModel extends Observable {closing = false/** The bot the message field writes to. */messageRef: string | null = null+ /** The message field's text. */+ draft = ''+ /** Talking into the message field (the field's mic, or the voice input shortcut). */+ readonly dictation = new Dictation((text) => this.update(() => (this.draft = text)))+ /** The voice input shortcut, as Settings → General has it. */+ shortcut: VoiceShortcut = 'off'/** The shape grew on its last change: open with a spring, close without one. */growing = trueprivate peek = false@@ -43,6 +54,7 @@ export class IslandModel extends Observable {island.onScreen((s) => this.update(() => (this.screen = s)))island.onHover((inside) => this.hover(inside))island.onKey((answer) => this.answerByKey(answer))+ island.onVoiceKey((key) => this.voiceKey(key))window.addEventListener('blur', () => {if (this.tab === 'message') this.close()})@@ -96,6 +108,12 @@ export class IslandModel extends Observable {return this.bot(this.messageRef) ?? this.focus}+ /** How to talk from any app, under the message field until the shortcut is first used. */+ get voiceTip() {+ const key = VOICE_SHORTCUT_KEYS[this.shortcut]+ return key && !voiceShortcutUsed.get() ? `Tip: double-tap ${key} to talk from any app, then tap it once to send.` : null+ }+/** A click on the closed island opens it and keeps it open. */openNow() {this.update(() => {@@ -120,6 +138,7 @@ export class IslandModel extends Observable {if (tab === 'message') {this.keyboard = truewindow.codync.island.setFocusable(true)+ this.readShortcut()}this.foldWhenAway()}@@ -132,10 +151,14 @@ export class IslandModel extends Observable {this.update(() => (this.messageRef = bots[(at + 1) % bots.length]!.ref))}- send(text: string) {+ setDraft(text: string) {+ this.update(() => (this.draft = text))+ }++ send() {const bot = this.messageBot- if (!bot || !text.trim()) return- window.codync.island.send(bot.ref, text)+ if (!bot || !this.draft.trim()) return+ window.codync.island.send(bot.ref, this.draft)this.close()}@@ -167,7 +190,9 @@ export class IslandModel extends Observable {close() {clearTimeout(this.hoverTimer)+ this.dictation.cancel()this.update(() => {+ this.draft = ''this.open = falsethis.pinned = falsethis.peek = false@@ -178,6 +203,18 @@ export class IslandModel extends Observable {this.keyboard = false}+ /** A double tap opens the message field and listens; the next tap sends what was said once the last words are in. */+ private voiceKey(key: VoiceKey) {+ if (!this.dictation.shortcutKey(key, () => this.send())) return+ voiceShortcutUsed.set(true)+ this.setTab('message')+ if (this.messageBot) this.dictation.start(this.draft, true)+ }++ private readShortcut() {+ void window.codync.speech.shortcut().then((shortcut) => this.update(() => (this.shortcut = shortcut)))+ }+private answerByKey(answer: 'allow' | 'deny') {const card = this.mode === 'approval' ? this.permission : nullconst option = card?.options.find((o) => o.kind === (answer === 'allow' ? 'allow_once' : 'reject_once'))diff --git a/apps/desktop/src/renderer/views/island/island-views.css b/apps/desktop/src/renderer/views/island/island-views.cssindex 06d140bf..10115290 100644--- a/apps/desktop/src/renderer/views/island/island-views.css+++ b/apps/desktop/src/renderer/views/island/island-views.css@@ -413,6 +413,26 @@.island-send:disabled {opacity: 0.35;}+.island-mic {+ width: 30px;+ height: 30px;+ border-radius: 15px;+ display: grid;+ place-items: center;+ flex: none;+ background: rgba(var(--text-rgb), 0.08);+ color: var(--secondary);+ transition: background-color var(--morph), color var(--morph);+}+.island-mic:hover {+ color: var(--text);+}+/* How to talk from any app, until the voice input shortcut is first used. */+.island-hint {+ padding-left: 16px;+ font-size: 12px;+ color: var(--island-quiet);+}/* Folding soon: a line that runs out. */.island-countdown {diff --git a/apps/desktop/src/renderer/views/settings/SettingsView.tsx b/apps/desktop/src/renderer/views/settings/SettingsView.tsxindex 834055ff..32489f3e 100644--- a/apps/desktop/src/renderer/views/settings/SettingsView.tsx+++ b/apps/desktop/src/renderer/views/settings/SettingsView.tsx@@ -1,6 +1,7 @@import { Fragment, useEffect, useState } from 'react'import type { Screen } from '@shared/analytics'import { islandLabel } from '@shared/island'+import { VOICE_SHORTCUTS, type VoiceShortcut } from '@shared/voice-shortcut'import { Button, CardForm, CardSection, ChoicePicker, IconButton, ProgressBar, Switch, ValueRow } from '../../components/Controls'import { Icon } from '../../components/Icon'import { useDismiss } from '../../components/Overlay'@@ -120,6 +121,8 @@ function GeneralSettings() {useEffect(() => void window.codync.app.launchAtLogin().then(setLaunchAtLogin), [])const [island, setIsland] = useState(false)useEffect(() => void window.codync.island.enabled().then(setIsland), [])+ const [voiceShortcut, setVoiceShortcut] = useState<VoiceShortcut>('off')+ useEffect(() => void window.codync.speech.shortcut().then(setVoiceShortcut), [])return (<CardForm>@@ -172,6 +175,12 @@ function GeneralSettings() {{islandLabel(window.codync.platform)}</Switch>) : null}+ {/* On-device speech, so macOS only. */}+ {window.codync.speech.available ? (+ <ValueRow label="Voice input shortcut" detail="Talk to a bot from any app, then tap the key once more to send.">+ <ChoicePicker selection={voiceShortcut} options={VOICE_SHORTCUTS} onChange={(s) => void window.codync.speech.setShortcut(s).then(setVoiceShortcut)} />+ </ValueRow>+ ) : null}<Switch on={launchAtLogin} onChange={(on) => void window.codync.app.setLaunchAtLogin(on).then(setLaunchAtLogin)}>Open at login</Switch>diff --git a/apps/desktop/src/renderer/views/thread/Composer.tsx b/apps/desktop/src/renderer/views/thread/Composer.tsxindex a33daae1..d560d72d 100644--- a/apps/desktop/src/renderer/views/thread/Composer.tsx+++ b/apps/desktop/src/renderer/views/thread/Composer.tsx@@ -4,6 +4,7 @@ import { isGroup, isWorkingIn, quoteOf, type Bot, type Entry } from '@shared/modimport { BotAvatar } from '../../components/Avatar'import { Icon } from '../../components/Icon'import { IconButton } from '../../components/Controls'+import { RecordingPill } from '../../components/RecordingPill'import { usePresence } from '../../components/Overlay'import { HStack, VStack } from '../../components/Stack'import { Text } from '../../components/Text'@@ -17,6 +18,7 @@ import { ComposerAttachment } from './ComposerAttachment'import { QuoteRow } from './ChatRows'import { useDictation } from './dictation'import { builtInCommand, slashCommands, slashQuery, type SlashCommand } from './slash-commands'+import { useVoiceShortcut } from './voice-shortcut-keys'type Trailing = 'interrupt' | 'stop' | 'call' | 'send'@@ -50,7 +52,7 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:const draft = useSyncExternalStore(drafts.subscribe, () => drafts.get(botId))const setDraft = (text: string) => drafts.set(botId, text)const [files, setFiles] = useState<OutgoingFile[]>([])- const dictation = useDictation(draft, setDraft)+ const dictation = useDictation(setDraft)const field = useRef<HTMLTextAreaElement>(null)const measure = useRef<HTMLDivElement>(null)const composing = useRef(false)@@ -157,8 +159,10 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:const submit = () => {const command = files.length ? null : builtInCommand(bot, draft, members)+ if (!command && !canSend) return+ // Words still on their way would land in the emptied box (and the shortcut's tap would send them again).+ dictation.cancel()if (command) return run(command)- if (!canSend) returnstore.send(draft, botId, { files, pasted: pasted.current, replyTo: reply && quoteOf(reply) })cancelReply?.()pasted.current = []@@ -166,6 +170,10 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:setFiles([])field.current?.focus()}+ useVoiceShortcut(dictation, draft, submit, () => field.current?.focus())+ useEffect(() => {+ if (dictation.error) store.setError(dictation.error)+ }, [dictation.error, store])const trailing: Trailing = onInterrupt && !draft && !files.length ? 'interrupt' : working && !draft && !files.length ? 'stop' : onCall && !draft && !files.length ? 'call' : 'send'const [symbol, label] = ({@@ -271,15 +279,11 @@ function DestinationComposer({ botId, reply, cancelReply, onCall, onInterrupt }:}}/></div>- {dictation.available ? (- <button- className={`press composer-mic ${dictation.listening ? 'on' : ''}`}- aria-label={dictation.listening ? 'Stop dictating' : 'Dictate'}- title={dictation.listening ? 'Stop dictating' : 'Dictate'}- aria-pressed={dictation.listening}- onClick={dictation.toggle}- >- <Icon name={dictation.listening ? 'mic.fill' : 'mic'} size={13} weight="medium" />+ {dictation.listening ? (+ <RecordingPill startedAt={dictation.startedAt} sample={() => dictation} onStop={() => dictation.finish()} className="composer-recording" />+ ) : dictation.available ? (+ <button className="press composer-mic" aria-label="Dictate" title="Dictate" onClick={() => dictation.start(draft)}>+ <Icon name="mic" size={13} weight="medium" /></button>) : null}<buttondiff --git a/apps/desktop/src/renderer/views/thread/composer.css b/apps/desktop/src/renderer/views/thread/composer.cssindex 91defb47..89fd3be1 100644--- a/apps/desktop/src/renderer/views/thread/composer.css+++ b/apps/desktop/src/renderer/views/thread/composer.css@@ -88,7 +88,7 @@display: grid;place-items: center;flex: none;- margin-bottom: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));+ margin-bottom: var(--first-line-inset);}.composer-surface {position: relative;@@ -102,6 +102,8 @@flex-direction: column;}.composer-box {+ /* Centers a 25 px control on the field's first line. */+ --first-line-inset: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));display: flex;align-items: flex-end;gap: 8px;@@ -154,7 +156,7 @@transition: opacity var(--morph), transform var(--morph);}.composer-symbol.active { opacity: 1; transform: scale(1) rotate(0); }-/* Dictation, beside send / call (Grok Bot's): a light circle; filled while it listens. */+/* Dictation, beside send / call (Grok Bot's): a light circle; a recording pill while it listens. */.composer-mic {width: 25px;height: 25px;@@ -164,15 +166,16 @@flex: none;color: var(--secondary);background: color-mix(in srgb, var(--text) 7%, var(--composer-fill));- margin-bottom: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));+ margin-bottom: var(--first-line-inset);transition: background-color var(--morph), color var(--morph), transform var(--press);}.composer-mic:hover {color: var(--text);}-.composer-mic.on {- background: var(--danger-fill);- color: var(--on-danger);+/* Centered on the first line, like the mic it replaces. */+.composer-recording {+ height: 25px;+ margin-bottom: var(--first-line-inset);}.composer-send {width: 25px;@@ -181,7 +184,7 @@display: grid;place-items: center;flex: none;- margin-bottom: calc(max(0px, (round(up, calc(var(--conversation) * 1.25), 1px) + 10px - 25px) / 2));+ margin-bottom: var(--first-line-inset);background: var(--accent-fill);color: var(--on-accent);transition: background-color var(--morph), color var(--morph), transform var(--press);diff --git a/apps/desktop/src/renderer/views/thread/dictation.ts b/apps/desktop/src/renderer/views/thread/dictation.tsindex ff8dd23a..77461cb6 100644--- a/apps/desktop/src/renderer/views/thread/dictation.ts+++ b/apps/desktop/src/renderer/views/thread/dictation.ts@@ -1,11 +1,14 @@-import { useCallback, useEffect, useRef, useState } from 'react'+import { useEffect, useRef, useState } from 'react'+import type { SpeechEvent } from '@shared/ipc'+import { VoiceKeyRouter, type VoiceKey } from '@shared/voice-shortcut'+import { Observable, useModel } from '../../lib/observable'import { VoiceSettings } from '../call/voice'-/** The running dictation's stop: one recognizer at a time, and a call stops dictation first. */-let stopActive: (() => void) | null = null+/** The running dictation: one recognizer at a time, and a call stops dictation first. */+let active: Dictation | null = nullexport function stopDictation() {- stopActive?.()+ active?.cancel()}/** `said` after what's already in the box, with one space between. */@@ -16,41 +19,111 @@ export function joinDictated(before: string, said: string) {}/**- * Dictation into the composer with on-device speech (the speech helper; macOS). Words appear in- * the box as they're recognized, after what was already typed; the mic again, or the recognizer- * ending on a pause, stops it.+ * Dictation with on-device speech (the speech helper; macOS). Words reach `write` as they're+ * recognized, after what was already typed. The helper doesn't stop on a pause: it ends on+ * `finish()` once the last words are in, on `cancel()`, on an error, or when another window takes it.*/-export function useDictation(draft: string, setDraft: (text: string) => void) {- const [listening, setListening] = useState(false)+export class Dictation extends Observable {+ readonly available = window.codync.speech.available+ listening = false+ /** When it started listening (ms since the epoch). */+ startedAt = 0+ /** The microphone's level while listening, 0…1. Read it each frame: it doesn't re-render. */+ level = 0+ /** Why the last dictation ended early (no microphone, speech recognition not allowed), until the next. */+ error: string | null = null/** The box up to the words still being recognized. */- const before = useRef('')- const off = useRef<(() => void) | null>(null)- const write = useRef(setDraft)- write.current = setDraft+ private before = ''+ private off: (() => void) | null = null+ private done: (() => void) | null = null+ /** Started by the voice input shortcut's double tap: its next tap sends what's in the field. */+ private byShortcut = false+ private readonly keys = new VoiceKeyRouter()- const finish = useCallback(() => {- off.current?.()- off.current = null- window.codync.speech.stop()- setListening(false)- if (stopActive === finish) stopActive = null- }, [])-- const start = () => {- stopActive?.()- before.current = draft- off.current = window.codync.speech.onEvent((e) => {- if (e.type === 'partial' || e.type === 'final') write.current(joinDictated(before.current, e.text))- if (e.type === 'final') before.current = joinDictated(before.current, e.text)- if (e.type === 'stopped' || e.type === 'error') finish()- })+ constructor(private write: (text: string) => void) {+ super()+ }++ /** Listening, or stopped and waiting for the last words. */+ get busy() {+ return this.off !== null+ }++ start(draft: string, byShortcut = false) {+ active?.cancel()+ active = this+ this.byShortcut = byShortcut+ this.error = null+ this.before = draft+ this.off = window.codync.speech.onEvent((e) => this.event(e))window.codync.speech.start(VoiceSettings.language() || null)- stopActive = finish- setListening(true)+ this.startedAt = Date.now()+ this.listening = true+ this.changed()}- // A composer that goes away (another chat) stops listening.- useEffect(() => () => { if (off.current) finish() }, [finish])+ /** Stops listening; `done` runs once the recognizer has delivered its last words. */+ finish(done?: () => void) {+ this.byShortcut = false+ if (!this.off) return done?.()+ this.done = done ?? null+ this.listening = false+ this.level = 0+ this.changed()+ window.codync.speech.stop()+ }- return { available: window.codync.speech.available, listening, toggle: () => (listening ? finish() : start()) }+ /** Stops listening now and drops words still on their way. */+ cancel() {+ this.byShortcut = false+ this.error = null+ if (!this.off) return+ window.codync.speech.stop()+ this.end()+ }++ /**+ * A tap of the voice input shortcut: a tap after its double tap finishes and runs `send`. True+ * when it's a double tap that should start: the caller shows its field, then `start(draft, true)`.+ */+ shortcutKey(key: VoiceKey, send: () => void): boolean {+ const action = this.keys.route(key, { byShortcut: this.byShortcut, busy: this.busy })+ if (action === 'send') this.finish(send)+ return action === 'start'+ }++ private event(e: SpeechEvent) {+ if (e.type === 'level') this.level = e.value+ if (e.type === 'partial' || e.type === 'final') this.write(joinDictated(this.before, e.text))+ if (e.type === 'final') this.before = joinDictated(this.before, e.text)+ if (e.type === 'error') {+ // Nothing to send, and the helper (which may still hold the microphone) goes too.+ this.byShortcut = false+ this.error = e.message+ window.codync.speech.stop()+ }+ if (e.type === 'stopped' || e.type === 'error') this.end()?.()+ }++ /** Returns what was waiting for the last words. */+ private end() {+ this.off?.()+ this.off = null+ this.listening = false+ this.level = 0+ if (active === this) active = null+ const done = this.done+ this.done = null+ this.changed()+ return done+ }+}++/** A composer's dictation; it stops when the composer goes away (another chat). */+export function useDictation(setDraft: (text: string) => void) {+ const write = useRef(setDraft)+ write.current = setDraft+ const [dictation] = useState(() => new Dictation((text) => write.current(text)))+ useEffect(() => () => dictation.cancel(), [dictation])+ return useModel(dictation)}diff --git a/apps/desktop/src/renderer/views/thread/voice-shortcut-keys.ts b/apps/desktop/src/renderer/views/thread/voice-shortcut-keys.tsnew file mode 100644index 00000000..817c7def--- /dev/null+++ b/apps/desktop/src/renderer/views/thread/voice-shortcut-keys.ts@@ -0,0 +1,44 @@+import { useEffect, useRef, useState } from 'react'+import type { VoiceKey } from '@shared/voice-shortcut'+import type { Dictation } from './dictation'++/** A tap waits this long for a composer to take it (one may be opening). */+const KEY_WAIT_MS = 2000+/** The voice input shortcut's latest tap for the chat window's composer. */+let pendingKey: { key: VoiceKey; at: number } | null = null+const keyTakers = new Set<() => void>()++/** The chat window got a tap of the voice input shortcut (the island is off). */+export function queueVoiceKey(key: VoiceKey) {+ pendingKey = { key, at: Date.now() }+ for (const take of keyTakers) take()+}++/**+ * The voice input shortcut in the composer: a double tap dictates into it, the next tap sends+ * what was said once the last words are in.+ */+export function useVoiceShortcut(dictation: Dictation, draft: string, submit: () => void, focus: () => void) {+ const latest = useRef({ draft, focus })+ latest.current = { draft, focus }+ const [sending, setSending] = useState(false)+ useEffect(() => {+ const take = () => {+ const pending = pendingKey+ pendingKey = null+ if (!pending || Date.now() - pending.at > KEY_WAIT_MS) return+ if (!dictation.shortcutKey(pending.key, () => setSending(true))) return+ latest.current.focus()+ dictation.start(latest.current.draft, true)+ }+ keyTakers.add(take)+ take()+ return () => void keyTakers.delete(take)+ }, [dictation])+ // Sent after the render that has the last words in the draft.+ useEffect(() => {+ if (!sending) return+ setSending(false)+ submit()+ }, [sending, submit])+}diff --git a/apps/desktop/src/shared/ipc.ts b/apps/desktop/src/shared/ipc.tsindex 4c3c0e3a..52c0ccd9 100644--- a/apps/desktop/src/shared/ipc.ts+++ b/apps/desktop/src/shared/ipc.ts@@ -3,6 +3,7 @@ import type { SharedFile } from './models'import type { SSHBridge } from './ssh'import type { ScreenAgentState } from './computer-access'import type { IslandRect, IslandScreen, IslandSummary } from './island'+import type { VoiceKey, VoiceShortcut } from './voice-shortcut'import type { ErrorReport } from './error-report'// The bridge between the main process (host lifecycle, loopback HTTP, tray, windows)@@ -71,6 +72,8 @@ export type WindowCommand =| { kind: 'refresh' }| { kind: 'respondPermission'; ref: string; entryId: string; optionId: string }| { kind: 'sendMessage'; ref: string; text: string }+ /** The voice input shortcut, while the island is off: a double tap starts dictating into the composer, a tap sends. */+ | { kind: 'voiceKey'; key: VoiceKey }/** The chat window was shown again after being hidden (`window_opened`). */| { kind: 'windowOpened' }@@ -149,6 +152,11 @@ export interface CodyncBridge {start(locale: string | null): voidstop(): voidonEvent(cb: (e: SpeechEvent) => void): () => void+ /** The voice input shortcut (Settings → General); always `off` without on-device speech. */+ shortcut(): Promise<VoiceShortcut>+ setShortcut(shortcut: VoiceShortcut): Promise<VoiceShortcut>+ /** A call is on (or over): it has the microphone, so the voice input shortcut does nothing meanwhile. */+ setCalling(on: boolean): void}ssh: SSHBridge/** The notch island (shared/island.ts): its setting, the chat window's summary, and the island window's side. */@@ -169,6 +177,8 @@ export interface CodyncBridge {/** ⌥Y / ⌥N answer the approval on screen, from any app, while it shows. */setKeys(on: boolean): voidonKey(cb: (answer: 'allow' | 'deny') => void): () => void+ /** The voice input shortcut's taps, from any app, while the island is on. */+ onVoiceKey(cb: (key: VoiceKey) => void): () => voidrespond(ref: string, entryId: string, optionId: string): voidsend(ref: string, text: string): voidopenBot(ref: string): voiddiff --git a/apps/desktop/src/shared/island.ts b/apps/desktop/src/shared/island.tsindex 01d9e168..95c5ce68 100644--- a/apps/desktop/src/shared/island.ts+++ b/apps/desktop/src/shared/island.ts@@ -199,4 +199,4 @@ export function notchOf(screens: NotchScreen[], bounds: { x: number; width: numb}/** The setting's name: a notch on the Mac, the top edge elsewhere. */-export const islandLabel = (platform: string) => (platform === 'darwin' ? 'Show in notch' : 'Show at top of screen')+export const islandLabel = (platform: string) => (platform === 'darwin' ? 'Show in Notch' : 'Show at top of screen')diff --git a/apps/desktop/src/shared/voice-shortcut.test.ts b/apps/desktop/src/shared/voice-shortcut.test.tsnew file mode 100644index 00000000..f189bff0--- /dev/null+++ b/apps/desktop/src/shared/voice-shortcut.test.ts@@ -0,0 +1,54 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'+import { CommandTaps, VoiceKeyRouter, type ModifierChange } from './voice-shortcut.ts'++const COMMAND = 0x100000+const RIGHT = 0x10+const LEFT = 0x08+const down = (at: number, sinceInput = 60, flags = COMMAND | RIGHT): ModifierChange => ({ key: 54, flags, at, sinceInput })+const up = (at: number, sinceInput = 60, flags = 0): ModifierChange => ({ key: 54, flags, at, sinceInput })+const feed = (changes: ModifierChange[], taps = new CommandTaps('rightCommand')) => changes.map((c) => taps.feed(c))++test('a quick tap is a tap, and a second one right after is a double tap', () => {+ assert.deepEqual(feed([down(10), up(10.08), down(10.25), up(10.33)]), [null, 'tap', null, 'double'])+ // The next tap starts over: it sends what was said.+ assert.deepEqual(feed([down(10), up(10.08), down(10.25), up(10.33), down(12), up(12.08)]).at(-1), 'tap')+ // Double-tapping to stop: the tap that sends and the next pair up (VoiceKeyRouter ignores that double).+ assert.deepEqual(feed([down(10), up(10.08), down(10.25), up(10.33), down(12), up(12.08), down(12.3), up(12.38)]).slice(3), ['double', null, 'tap', null, 'double'])+})++test('slow taps are two taps, and a long press is none', () => {+ assert.deepEqual(feed([down(10), up(10.08), down(11), up(11.08)]), [null, 'tap', null, 'tap'])+ assert.deepEqual(feed([down(10), up(10.5), down(10.6), up(10.68)]), [null, null, null, 'tap'])+})++test('a key pressed with the command, or between the taps, is a shortcut', () => {+ // ⌘C then ⌘V with the right command.+ assert.deepEqual(feed([down(10), up(10.2, 0.1), down(10.3), up(10.45, 0.05)]), [null, null, null, null])+ // A key between the taps.+ assert.equal(feed([down(10), up(10.08), down(10.25), up(10.33, 0.15)]).at(-1), 'tap')+})++test('the other command and other modifiers break a tap', () => {+ assert.deepEqual(feed([down(10, 60, COMMAND | RIGHT | LEFT), up(10.08, 60, COMMAND | LEFT)]), [null, null])+ assert.equal(feed([down(10), up(10.08), { key: 56, flags: 0x20002, at: 10.15, sinceInput: 60 }, down(10.25), up(10.33)]).at(-1), 'tap')+ assert.equal(feed([down(10), up(10.08), down(10.25, 60, COMMAND | RIGHT | 0x20000), up(10.33)]).at(-1), null)+})++test('the left command works the same way, and the right one then does nothing', () => {+ const left = (at: number, isDown: boolean): ModifierChange => ({ key: 55, flags: isDown ? COMMAND | LEFT : 0, at, sinceInput: 60 })+ assert.deepEqual(feed([left(10, true), left(10.08, false), left(10.25, true), left(10.33, false)], new CommandTaps('leftCommand')), [null, 'tap', null, 'double'])+ assert.deepEqual(feed([down(10), up(10.08)], new CommandTaps('leftCommand')), [null, null])+})++test('a double tap starts dictation, the next tap sends, and the double tap that ends on it starts nothing', () => {+ const keys = new VoiceKeyRouter()+ const idle = { byShortcut: false, busy: false }+ assert.equal(keys.route('double', idle), 'start')+ assert.equal(keys.route('tap', { byShortcut: true, busy: true }), 'send')+ assert.equal(keys.route('double', idle), null)+ assert.equal(keys.route('double', idle), 'start')+ // A tap with no shortcut start (the mic, or after the send) does nothing; nor does a double tap while busy.+ assert.equal(keys.route('tap', { byShortcut: false, busy: true }), null)+ assert.equal(keys.route('double', { byShortcut: false, busy: true }), null)+})diff --git a/apps/desktop/src/shared/voice-shortcut.ts b/apps/desktop/src/shared/voice-shortcut.tsnew file mode 100644index 00000000..de2588f1--- /dev/null+++ b/apps/desktop/src/shared/voice-shortcut.ts@@ -0,0 +1,95 @@+// The voice input shortcut (macOS): a double tap of a ⌘ key starts talking to a bot from any app,+// one more tap sends what was said. The key watcher (main/voice-shortcut.ts) reports every+// modifier change; this decides which ones are taps.++/** Settings → General: which ⌘ key, tapped twice, starts voice input. */+export type VoiceShortcut = 'off' | 'rightCommand' | 'leftCommand'++/** A tap of the shortcut's key, or the tap that completes a double tap. */+export type VoiceKey = 'tap' | 'double'++export const DEFAULT_VOICE_SHORTCUT: VoiceShortcut = 'rightCommand'++export const VOICE_SHORTCUTS: { id: VoiceShortcut; label: string }[] = [+ { id: 'off', label: 'Off' },+ { id: 'rightCommand', label: 'Right ⌘ twice' },+ { id: 'leftCommand', label: 'Left ⌘ twice' },+]++/** How the tip names the key ("the right ⌘"), or null when the shortcut is off. */+export const VOICE_SHORTCUT_KEYS: Record<VoiceShortcut, string | null> = { off: null, rightCommand: 'the right ⌘', leftCommand: 'the left ⌘' }++/** One modifier change, as an `NSEvent` reports it. */+export interface ModifierChange {+ /** The virtual key code of the modifier that changed. */+ key: number+ /** `NSEvent.modifierFlags` after the change, with the device bits that tell left from right. */+ flags: number+ /** The event's time, in seconds. */+ at: number+ /** Seconds since the last non-modifier key press or mouse click, in any app. */+ sinceInput: number+}++/** Each ⌘ key's code and its device bit (`NX_DEVICERCMDKEYMASK`, `NX_DEVICELCMDKEYMASK`). */+const COMMANDS = { rightCommand: { key: 54, down: 0x10, other: 0x08 }, leftCommand: { key: 55, down: 0x08, other: 0x10 } }+/** ⇧, ⌃, ⌥ and fn: a chord, not a tap. */+const MODIFIERS = 0x20000 | 0x40000 | 0x80000 | 0x800000+/** The longest press that still counts as a tap. */+const TAP_S = 0.3+/** The longest pause between the first tap's release and the second press. */+const GAP_S = 0.4++/** Taps of one ⌘ key alone: short, nothing else held, no other key pressed or click meanwhile (⌘-click). */+export class CommandTaps {+ private readonly command: (typeof COMMANDS)[keyof typeof COMMANDS]+ private pressed: number | null = null+ private first: { pressed: number; released: number } | null = null++ constructor(shortcut: Exclude<VoiceShortcut, 'off'>) {+ this.command = COMMANDS[shortcut]+ }++ /** Takes the next change: a tap, the second of two quick taps, or nothing. */+ feed(change: ModifierChange): VoiceKey | null {+ const { key, flags, at, sinceInput } = change+ const others = MODIFIERS | this.command.other+ if (key !== this.command.key) return this.reset()+ if (flags & this.command.down) {+ this.pressed = flags & others ? null : at+ return null+ }+ const pressed = this.pressed+ this.pressed = null+ if (pressed === null || flags & others || at - pressed > TAP_S || sinceInput < at - pressed) return this.reset()+ const first = this.first+ // A double tap also needs no key pressed or click between the taps.+ if (first && pressed - first.released <= GAP_S && sinceInput >= at - first.pressed) {+ this.first = null+ return 'double'+ }+ this.first = { pressed, released: at }+ return 'tap'+ }++ private reset() {+ this.pressed = null+ this.first = null+ return null+ }+}++/**+ * What a tap of the shortcut does to dictation: a double tap starts it, a tap after a start by the+ * shortcut sends. Double-tapping to send arrives as that tap then a double, and the double starts nothing.+ */+export class VoiceKeyRouter {+ private sentByTap = false++ route(key: VoiceKey, dictation: { byShortcut: boolean; busy: boolean }): 'start' | 'send' | null {+ const pairsWithSend = this.sentByTap+ this.sentByTap = key === 'tap' && dictation.byShortcut+ if (this.sentByTap) return 'send'+ return key === 'double' && !pairsWithSend && !dictation.busy ? 'start' : null+ }+}diff --git a/docs/architecture/desktop-app.md b/docs/architecture/desktop-app.mdindex e44b321c..c828e160 100644--- a/docs/architecture/desktop-app.md+++ b/docs/architecture/desktop-app.md@@ -22,7 +22,7 @@ reaching the end of a history response determines when there are no earlier mess| Path | ||---|---|-| `src/main/` | Main process: host service (`host-controller.ts`), loopback HTTP/SSE proxy (`host-proxy.ts`), menu bar/tray, windows, account sign-in (Clerk Frontend API, `account.ts`), cloud API and device key (`cloud.ts`), updates (`updates.ts`), SSH tunnels (`ssh.ts`), Codync Screen agent (`screen.ts`), on-device speech (`speech.ts`), browser sign-in callbacks (`auth.ts`), the [notch island](../features/notch-island.md) window (`island.ts`), [error tracking](../features/analytics.md#error-tracking) (`error-reports.ts`) |+| `src/main/` | Main process: host service (`host-controller.ts`), loopback HTTP/SSE proxy (`host-proxy.ts`), menu bar/tray, windows, account sign-in (Clerk Frontend API, `account.ts`), cloud API and device key (`cloud.ts`), updates (`updates.ts`), SSH tunnels (`ssh.ts`), Codync Screen agent (`screen.ts`), on-device speech (`speech.ts`) and its [voice input shortcut](../features/voice-call.md#voice-input-shortcut) (`voice-shortcut.ts`), browser sign-in callbacks (`auth.ts`), the [notch island](../features/notch-island.md) window (`island.ts`), [error tracking](../features/analytics.md#error-tracking) (`error-reports.ts`) || `src/preload/` | The `window.codync` bridge (types in `src/shared/ipc.ts`) || `src/renderer/store/` | `BotStore` (port of kit's; `bot-store.ts` actions over `bot-sync.ts` events stream and `bot-mirror.ts` state), `AppModel` (window state, approvals), `CloudModel` (claims, cloud default), `AccountSession` || `src/renderer/components/` | Design system: controls, overlays (sheets, dialogs, menus), SF Symbol icons, character avatars, thinking orbs |diff --git a/docs/features/notch-island.md b/docs/features/notch-island.mdindex 9700eb82..468b3456 100644--- a/docs/features/notch-island.md+++ b/docs/features/notch-island.md@@ -5,7 +5,7 @@ MacBook it grows out of the notch; on a Mac without a notch, and on Windows andfrom the top edge. It draws Codync's own bot characters. Design inspired by[@novra.design](https://www.instagram.com/novra.design/).-Turn it on or off in **Settings → General → System** (*Show in notch* on macOS, *Show at top+Turn it on or off in **Settings → General → System** (*Show in Notch* on macOS, *Show at topof screen* elsewhere) or the menu bar / tray menu's **Settings**. It is off by default while itis experimental. The setting belongs to the desktop installation (`showIsland` in `host-prefs.json`).@@ -31,7 +31,7 @@ bar). A bot can be messaged from the island, and ⌥Y / ⌥N answer an approval| Approval | A permission card is pending | The command and the card's options, deny left and Allow once right (⌥N / ⌥Y from any app while it shows) || Finished | A bot stops working | "*Bot* is done" with its last message, Open conversation / OK, for 5.2 s || Failed | A bot stops on an error | "*Bot* stopped." with the error in red, until OK |-| Message a bot | The header's bubble | The bot chip (click for the next bot) and a field; Enter sends, Esc closes |+| Message a bot | The header's bubble, or the [voice input shortcut](voice-call.md#voice-input-shortcut) | The bot chip (click for the next bot) and a field with the mic (macOS); Enter sends, Esc closes. Until the shortcut is first used, a tip under the field says how to talk from any app || Usage | The header's gauge or the usage meter | Claude and Codex limits side by side: each window's bar, percent and reset |Opened by hover, the island folds 0.8 s after the pointer leaves; opened by a click or a tab,diff --git a/docs/features/voice-call.md b/docs/features/voice-call.mdindex c1013b82..7854d219 100644--- a/docs/features/voice-call.md+++ b/docs/features/voice-call.md@@ -44,16 +44,52 @@ The bot is the same agent on the computer; the call only changes how you talk toThe composer's mic (beside send / call, as in Grok Bot) types what you say into the box with thesame on-device recognizer and the call's language setting; nothing is sent until you send it.-Words appear as they're recognized, after what's already typed; tapping the mic again, or the-recognizer finishing on a pause, stops it, and a call takes the microphone from it.+Words appear as they're recognized, after what's already typed. While it listens the mic becomes+a recording pill (stop, the time, dots that follow your voice), on the iPhone and the desktop, and+a call takes the microphone from it. On the desktop, dictation ends only on the pill's stop (or the+shortcut's tap), which waits for the last words, on an error (shown in the window), or when another+window takes the speech helper; it doesn't stop on a pause. On the iPhone the pill's stop keeps what+was recognized so far. The desktop island's message field has the same mic and pill.- Desktop (`views/thread/dictation.ts`): the speech helper, macOS only, so Linux and Windows showno mic. An unpackaged `npm run dev` can't use it: macOS stops the helper because the dev Electronhas no speech usage description, so test it in a packaged build (`dist:mac:dev`), like calls.-- iPhone (`Thread/Dictation.swift`): `SFSpeechRecognizer` on device; with no microphone route- (some simulators) the mic does nothing.+- Desktop: the speech helper serves one window at a time. When another window starts it (the+ island while the chat window dictates or calls, or the other way round), `main/speech.ts` tells+ the window that lost it (`stopped`) and ignores that window's stop.+- iPhone (`Thread/Dictation.swift`, pill in `Thread/RecordingPill.swift`, dots shared with the call+ bar in `LevelDots.swift`): `SFSpeechRecognizer` on device; with no microphone route (some+ simulators) the mic does nothing.+- Android has no composer mic yet.- The terminal UI has no microphone, so no mic.+### Voice input shortcut++Desktop on macOS, set in **Settings → General → System** (*Voice input shortcut*: off, the right ⌘+twice (the default) or the left ⌘ twice; `voiceShortcut` in `host-prefs.json`). From any app, a+double tap of that key starts dictation into the [notch island](notch-island.md)'s message field+while the island is on, else into the chat window's composer (the top bot when no chat is open).+One more tap stops it and sends what was said once the last words are in; the pill's stop leaves+the words in the field unsent, and an error leaves nothing for the tap to send.+Double-tapping to stop counts as that one tap, not a new start. While a call is on (any engine),+the shortcut does nothing: the call has the microphone.++- `apps/desktop/src/main/voice-shortcut.ts` keeps an `osascript -l JavaScript` process with an+ `NSEvent` global monitor for modifier changes, plus the seconds since the last key press or+ mouse click. Neither needs the Accessibility or Input Monitoring grant, and the app never sees+ what is typed. The chat window reports calls (`voice:calling`) so taps are dropped meanwhile.+- `shared/voice-shortcut.ts` decides what counts: a press under 0.3 s with no other modifier held+ and no key pressed or click meanwhile is a tap; a second tap starting within 0.4 s of the first,+ with no key pressed or click since the first, is a double tap. ⌘C, ⌘V and ⌘-clicks with the+ same key are neither.+- The renderer side is `Dictation.shortcutKey` (`views/thread/dictation.ts`), shared by the+ composer and the island's model; `VoiceKeyRouter` (`shared/voice-shortcut.ts`) decides whether+ a tap starts, sends or does nothing. The chat window queues taps for its composer in+ `views/thread/voice-shortcut-keys.ts`.+- macOS only: Windows and Linux have no ⌘ or on-device speech, and a modifier-only shortcut+ there would need a native keyboard hook. The iPhone, Android and the terminal UI have no global+ keyboard shortcut.+## Background verificationStore regression tests cover sending a recognized utterance after backgrounding,